diff --git a/Cargo.lock b/Cargo.lock index 2868f3e1b..9c02f7883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "atty" version = "0.2.14" @@ -132,6 +144,20 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -313,6 +339,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -328,6 +360,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.4.0" @@ -430,6 +471,7 @@ name = "crypto" version = "0.1.0" dependencies = [ "bincode", + "blake3", "digest", "lambda-vm-syscalls", "libc", @@ -819,13 +861,14 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] name = "lambda-vm-prover" version = "0.1.0" dependencies = [ + "blake3", "criterion 0.5.1", "crypto", "digest", @@ -937,6 +980,7 @@ dependencies = [ name = "math-cuda" version = "0.1.0" dependencies = [ + "blake3", "crypto", "cudarc", "libloading 0.8.9", @@ -1527,7 +1571,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] diff --git a/Makefile b/Makefile index a4b05b507..53e5807d9 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-s clean-recursion-elfs clean test test-asm \ test-rust test-ethrex test-ethrex-offline test-executor test-syscalls test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \ test-profile-recursion-block recursion-profile-block-input \ -test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-cuda-integration test-cuda-fallback \ +test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-blake3-host-kat test-blake3-second-source test-cuda-integration test-cuda-fallback \ test-prover-cuda test-prover-comprehensive-cuda \ bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \ update-ethrex-fixture-checksums check-ethrex-fixture-checksums ethrex-real-block-fixture \ @@ -565,6 +565,65 @@ test-disk-spill: test-math-cuda: cargo test -p math-cuda --release +# Known-answer tests for the BLAKE3 device kernels, run on the HOST. No GPU, no +# nvcc, no cargo — a couple of seconds. +# +# This exists because `test-math-cuda` above, which is the authority on these +# kernels, runs only where a GPU does, and the per-PR CI runners have none (GPU +# jobs are merge_group-only). Without this the kernels have no per-PR gate: an +# edit to blake3.cu that broke the hash would reach the merge queue before +# anything caught it. `crypto/math-cuda/tests/host_kat/` compiles the real kernel +# source as host C++ through a shim and runs the official BLAKE3 vectors, the +# canonical 6-round table, the official multi-block vectors against the +# `Blake3Chain` construction, and every leaf kernel's byte stream through it. +# +# BOTH ROUND COUNTS are built and run. The 6-round arm is the one the campaign +# ships and the one no other CI job compiles (risk R10), and the round count is a +# compile-time knob, so a single-arm run would leave the shipping configuration +# ungated. The two arms differ only in `-DBLAKE3_ROUNDS`, exactly as build.rs +# drives the cubin from the `blake3-6round` feature. +# +# It checks arithmetic ONLY. Whether nvcc accepts the file, and everything about +# execution rather than arithmetic — grid indexing, the Merkle tail's barriers, +# device alignment, register pressure — stays with `test-math-cuda`. Necessary, +# never sufficient. +HOST_KAT_DIR := crypto/math-cuda/tests/host_kat +HOST_KAT_CXXFLAGS := -std=c++17 -O2 -Wall -Wno-unknown-pragmas \ + -I$(HOST_KAT_DIR) -Icrypto/math-cuda/kernels +test-blake3-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/blake3_host_kat $(HOST_KAT_DIR)/blake3_host_kat.cpp + ./target/host_kat/blake3_host_kat + @echo + @echo "=== rebuilding for the 6-round arm (BLAKE3_ROUNDS=6) ===" + $(CXX) $(HOST_KAT_CXXFLAGS) -DBLAKE3_ROUNDS=6 \ + -o target/host_kat/blake3_host_kat_6r $(HOST_KAT_DIR)/blake3_host_kat.cpp + ./target/host_kat/blake3_host_kat_6r + +# SECOND-SOURCE validation of the 6-round vectors the KAT above trusts. +# +# `test-blake3-host-kat` checks the KERNEL against the committed tables. This +# checks the TABLES, against upstream BLAKE3's own portable C with its round loop +# parameterised (`thoughts/blake3/reference-impl/`, a 2 KB reviewable diff in +# PARAMETERISATION.diff). That reference is a different language and author and — +# the part that matters — a different message-schedule CONSTRUCTION: it indexes a +# precomputed MSG_SCHEDULE table where our Rust and CUDA compose one permutation +# between rounds. A bug in the iterative composition is exactly what a single +# source cannot catch, and check [C] compares the two constructions directly. +# +# It exists because 6-round BLAKE3 is computed by nothing else in the world +# (assumption A6R), so the 6-round column of every table here rests on oracle +# agreement rather than on a published vector. +# +# ⚠ Checks [D] and [E] were SILENTLY DEAD from P-a Stage 1 — which moved +# CANONICAL_VECTORS out of `prover/src/lfm/blake3.rs` into `crypto` — until +# 2026-08-15, because nothing ever ran this: it had no target. That is why it has +# one now. A ~1 second C compile plus a few seconds of Python; no cargo, no GPU. +test-blake3-second-source: + thoughts/blake3/reference-impl/build.sh + python3 thoughts/blake3/reference-impl/check.py + # End-to-end cuda dispatch coverage (requires NVIDIA GPU + nvcc). # Asserts the R1-R4 GPU dispatch counters fired on a real prove. # --test-threads=1: these tests reset and assert on process-global GPU call @@ -625,6 +684,14 @@ clippy: cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --features lambda-vm-prover/disk-spill -- -D warnings -A clippy::op_ref + # BLAKE3 at 6 rounds. ONE pass, with BOTH crates' features set, because they + # are separate features that must be set in lockstep: crypto's moves the host + # primitive (the LFM chip, the socket and the commitment backends all read its + # BLAKE3_ROUNDS) and math-cuda's recompiles the cubin. Setting one alone means + # a GPU tree committing under a different hash than the CPU one, so linting + # them apart would certify a combination nothing should ever build. The + # prover's feature forwards to crypto's, so naming it covers both host halves. + cargo clippy --workspace --all-targets --features lambda-vm-prover/blake3-6round,math-cuda/blake3-6round -- -D warnings -A clippy::op_ref fmt: cargo fmt --all @@ -635,6 +702,14 @@ lint: cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref cargo clippy --workspace --all-targets --features lambda-vm-prover/disk-spill -- -D warnings -A clippy::op_ref + # BLAKE3 at 6 rounds. ONE pass, with BOTH crates' features set, because they + # are separate features that must be set in lockstep: crypto's moves the host + # primitive (the LFM chip, the socket and the commitment backends all read its + # BLAKE3_ROUNDS) and math-cuda's recompiles the cubin. Setting one alone means + # a GPU tree committing under a different hash than the CPU one, so linting + # them apart would certify a combination nothing should ever build. The + # prover's feature forwards to crypto's, so naming it covers both host halves. + cargo clippy --workspace --all-targets --features lambda-vm-prover/blake3-6round,math-cuda/blake3-6round -- -D warnings -A clippy::op_ref # The cuda feature gates whole modules + cuda-only integration tests. build.rs emits empty # cubin stubs when nvcc is absent, so this checks on a GPU-less host (CI lint runner, dev laptop) # too — no GPU required. Catches cuda-gated breakage that the non-cuda passes above miss. diff --git a/RESUME-MMCS.md b/RESUME-MMCS.md new file mode 100644 index 000000000..c7ffdb415 --- /dev/null +++ b/RESUME-MMCS.md @@ -0,0 +1,301 @@ +# Batched-MMCS primitives port (M-1 / M-2) — resume note + +**Branch** `mmcs-primitives` (worktree `/Users/maurofab/workspace/lambda_vm-mmcs`), based on +`blake3-real-hash` @ `3a0b8485`. Signed, **unpushed**, working tree clean. This is a +complete milestone, not a checkpoint of half-done work. + +> **Why this file is not `RESUME.md`.** The worktree root already holds `RESUME.md`, the +> **RATE-4 lane's** resume note, inherited from the base branch (`38c89d86`). The lead's +> standing order said "write RESUME.md at your worktree root"; taken literally that +> clobbers a sibling lane's handoff the moment `mmcs-primitives` is merged into +> `blake3-real-hash` (which is the plan — see the campaign's task list). Renaming costs +> nothing and loses nothing. **Do not "fix" this by overwriting `RESUME.md`.** + +Source of the port: `origin/feat/batched-fri-per-epoch` (PR #768), files +`crypto/stark/src/fri/mmcs.rs` (~1,015 lines), `crypto/stark/src/fri/batched.rs` (~499), +`crypto/stark/src/tests/bus_tests/batched_soundness_tests.rs` (~237). Scoping document: +`thoughts/shared/block-compression/MMCS-PLAN.md` (§2 the port verdict, §3.3 the streaming +constraint, §3.6 the item table, §M-10-RESULT the index-convention analysis). + +| commit | what | +|---|---| +| `13aac0fe` | `feat(fri)`: the two primitives, re-parameterized over `StarkHash`, with M-12 / M-13a / M-14 corrected on port | +| `472e7efd` | `test(fri)`: soundness negatives for the batched commitment primitives | +| this note | `docs(mmcs)` | + +``` + crypto/stark/src/fri/batched.rs | 921 +++++ NEW + crypto/stark/src/fri/mmcs.rs |1334 +++++ NEW + crypto/stark/src/fri/mod.rs | 2 + two `mod` lines only + crypto/stark/src/tests/batched_mmcs_soundness_tests.rs | 339 +++ NEW + crypto/stark/src/tests/mod.rs | 1 + one `mod` line only +``` + +## Scope guards — all honoured + +* **No prover/verifier integration.** M-3+ waits for P-a Stage 2. `prover.rs`, + `verifier.rs`, `continuation.rs`, `prover/src/lib.rs`, anything cuda: untouched. +* **No wire-type change.** `git diff 3a0b8485..HEAD -- crypto/stark/src/proof/ prover/src/` + is **empty**. `MixedOpening` lives inside `fri/mmcs.rs`, so `StarkProof` / `MultiProof` + rkyv layouts are byte-identical by construction, not by test. The wire types + `BatchedQueryOpening` / `BatchedTableData` / `BatchedMultiProof` were **not** needed and + did not come along. +* **#845 zero-copy view layer intact** (the silent deletion §2.1 warns a rebase would + cause): `EpochProofView` 10 hits, `ContinuationProofView` 7, `verify_continuation_view` + 5, `access_recursion_archive` 3, `verify_l2g_commitment_binding_view` 15. +* `fri/mod.rs`'s existing paths untouched beyond the two `mod` declarations. + +## Drift vs the June code, and how it was adapted + +**The hash path — this is M-1, and it is resolved differently from MMCS-PLAN §2.3.** +The June files hard-code `BatchedMerkleTreeBackend` in three places (`hash_group_leaf`, +`hash_group_openings`, `compress`). Both files are now generic over `H: StarkHash`, +reaching `>::hash_data` and `::hash_new_parent` — the same two functions the +existing per-table row-pair tree commits with. Types are `MixedMmcs`, matching the +`TableCommit` convention already in `prover.rs`. + +**I did not add a third `type Mmcs` member to `StarkHash`, and §2.3's recommendation to +add one should be treated as superseded.** Its stated obstacle — "a mixed-height MMCS leaf +is not a `Vec`; the tree builder must know the injection schedule; `IsStreamingLeafBackend` +has no vocabulary for it" — is about `MerkleTree::build`, which `MixedMmcs` never calls. It +builds its own layers and needs exactly two things, a leaf hash over a `Vec>` +and a 2-input compression, both of which `Batched` already has at the right shapes. A +third member whose keccak instance is literally `BatchKeccak256Backend` would be a second +encoding of the same leaf — precisely what PA-PLAN §1.4 forbids ("do not prove that two +independently-written encodings coincide; make them one function"). Going through `Batched` +makes "a single-matrix MMCS equals the per-table tree" true **by construction**; +`single_matrix_root_matches_existing_row_pair_tree` and +`single_matrix_fp3_root_matches_existing_row_pair_tree` pin that no second encoding crept +in, which is the whole backward-compatibility argument §2.3 wanted a test for. + +Everything else still exists with compatible signatures: `crate::par::par_map_collect`, +`commitment::commit_bit_reversed`, `proof::stark::PolynomialOpenings`, +`grinding::generate_nonce`, and `fri_functions::{fold_evaluations_in_place, +compute_coset_twiddles_inv, update_twiddles_in_place}`. The module docs' "Task 1 / Task 2 / +Task 7" scaffolding and the two stale doc comments §2.4 flagged (`batched.rs:87-89` +claiming termination mirrors the unbatched phase; `:186-188` claiming #729 is absent) are +gone — present tense, no migration references. + +## M-12 — the terminal-polynomial gap: FIXED, with a consequence the plan did not price + +`batched_commit_phase` used `num_committed_layers = h_max - 1`, folded to a scalar and +appended it. It now derives the fold count through the shared +`crate::fri::terminal::FriFoldLayout` and appends the terminal polynomial's coefficients, +exactly as `commit_phase_from_evaluations` does. **The saving is exactly `blowup_log + k` +committed layers** (8 at blowup 2 / k=7; the plan's "~9" is its own `k + blowup_log` +estimate rounded up). + +**★ The consequence, and M-3/M-4 must not re-derive the terminal from `h_max` alone.** +A bucket whose height is below the terminal would never be folded into the running +codeword — it would be silently dropped from the FRI, which is a soundness hole, not a +perf question. So `BatchedFriLayout::new(h_max, h_min, blowup_log, k)` floors the stop at +the **shortest** bucket: + +``` +terminal_log = min(blowup_log + k, h_min) +total_folds = h_max - terminal_log +num_committed = total_folds - 1 (saturating) +``` + +and the **final fold now injects the bucket at the terminal height** — #768's loop injects +only after the committed folds, so a bucket sitting exactly at the terminal would have been +missed. Rate is preserved by the injection (MMCS-PLAN §M-10.1), so the sum is still degree +`< 2^effective_k` and `coeffs_from_terminal_codeword` applies unchanged. At a real epoch +the shortest table sits well above `blowup_log + k`, so the floor is **inert** and the +layout is exactly the unbatched one — it costs nothing in the common case. + +Oracles: `single_bucket_terminal_matches_the_unbatched_commit_phase` asserts same layer +count, same coefficients, same layer roots and identical final transcript state against +`commit_phase_from_evaluations`. It is **non-vacuous** — under #768's construction that +input commits 9 layers where the unbatched one commits 3. +`terminal_is_floored_at_the_shortest_codeword` covers both the inert and the active branch. + +## M-13a — shape binding: DONE + +`absorb_height_histogram(transcript, heights)` → `absorb_shape_histogram(transcript, +heights, widths)`, binding `(height, width)` pairs; length-prefixed, fixed-width, +order-preserving, no sort or dedup. Renamed because "height histogram" would now be a stale +name and there was no existing call site to migrate. `derive_batched_fri_challenges` threads +widths through. Controls: `absorb_shape_histogram_binds_heights_and_widths_into_alpha` +(height, width and table-order changes each move α) and +`the_shape_encoding_separates_distinct_epochs` (the encoding is injective). + +**M-13b is NOT answered and is not mine.** MMCS-PLAN §M-10.4 splits M-13 into (a) add +widths to the round-4 histogram — done here — and (b) *answer whether any rounds-1-3 +challenge is shape-exploitable*, which gates §3.4's addendum ratification. That remains +open and needs the integration to be meaningful. + +## M-14 — the index convention: DOCUMENTED, CONTROLLED, and HARDENED + +`verify_batch` walks the path with `(iota >> level) & 1`, i.e. it consumes the **low** +`h_max - 1` bits, while a shorter matrix inside the tree is located by +`iota >> (h_max - h_m)`, i.e. the **high** bits. Consistent only when this MMCS's `h_max` +equals the FRI's — which §3.1/M-6's batched *preprocessed* round breaks (round h_max 21 vs +FRI h_max 23 at the real 2^21 epoch). + +Three dispositions: + +1. The module header states the caller's obligation as a hard precondition: + `iota_round = iota_fri >> (h_max_fri - h_max_round)`, with the reason it fails silently + (prover and verifier share the routine, so a wrong convention is self-consistent — + honest proofs verify and the short matrices end up authenticated at positions the + DEEP/FRI join never checks). +2. **Beyond the plan: `verify_batch` now rejects an `iota` outside `[0, 2^(h_max-1))`.** + A global index from a taller domain exceeds the round's leaf count most of the time, so + this converts most of the misuse class into a loud rejection at zero cost — honest + callers already pass in-range indices. It is a backstop, **not** a substitute for the + reduction: an index that happens to land in range is still accepted at the wrong leaf, + and the header says so. +3. The control the analysis demands: `short_round_low_bit_convention_is_exercised` — a + round with `h_max` 4 under a hypothetical FRI `h_max` 6, asserting (a) the honest + reduced index verifies [honest-path control, house rule], (b) a tampered row of the + **SHORT** (injected) matrix is rejected, (c) the un-reduced FRI index is rejected, and + (d) an in-range-but-wrong leaf is also rejected, so the guard is not the only thing + standing between the two conventions. A tamper control on the tallest matrix alone + passes under either convention and catches none of this. + +## ★ Two findings the integration must absorb + +**1. #768's soundness tests could not be ported, and the reason is structural.** +All 16 tests in `batched_soundness_tests.rs` build a `BatchedMultiProof` via +`multi_prove_batched_ram` and call `Verifier::batched_multi_verify`. Both are integration +surfaces that do not exist here, and porting them would mean porting the integration — +explicitly out of scope. What landed instead is +`crypto/stark/src/tests/batched_mmcs_soundness_tests.rs`, covering what the primitives can +actually decide: + +* *Reaching down from #768*: a tampered row in **every** height group (not only the tall + one — short matrices are bound through injection, and a tall-only control would miss a + wrong injection level entirely), a tampered or mis-sized authentication path, widths + disagreeing with the opening. +* *New here, not in #768's file*: an opening replayed at any other index is rejected + (asserted over the **whole** leaf range, not one sample); two same-shape matrices' + openings swapped inside a height group is rejected, so INPUT ORDER is part of the + commitment; a relabelled injection height is rejected; a root from another epoch shape is + rejected; tampering the FRI transcript (layer root, terminal coefficient, height, width) + moves the query indices. +* *Deferred with the integration (M-5)*: per-query FRI layer evaluations, OOD values, bus + balance, query count, grinding nonce. `batched_mmcs_soundness_tests.rs` is the named home + for those to grow into. + +**2. Streaming is per height GROUP, not per matrix — MMCS-PLAN §3.3's pseudocode does not +describe what this leaf layout does.** +`MixedMmcs::commit` takes a `LeafSource` and owns no evaluations; that property is +preserved and is made falsifiable rather than asserted in prose by +`commit_reads_each_height_group_in_one_contiguous_phase`, which traces access windows and +proves each matrix is read inside **one contiguous phase, in descending height order** — so +a caller may produce a height group's LDEs, commit, and drop them before the next group is +needed. + +But §3.3 assumes a per-matrix chained absorb (`acc[leaf] = absorb(acc[leaf], m's rows)`). +This layout does **not** do that: the group leaf is a single `hash_data` over the +concatenation, so **every matrix at a given height must be readable simultaneously**. Since +the tallest group is most of an epoch's tables, a caller serving rows from full in-RAM LDE +buffers still holds `O(N)` at the base layer. A `LeafSource` may serve from disk, device +memory or recomputation instead — that is the escape hatch — but true streaming *within* a +height group needs an incremental leaf hasher (absorb matrix by matrix into one sponge per +leaf, ~200 B of state per leaf, ≈200 MB at 2^20 leaves), and `IsStreamingLeafBackend` +exposes only `hash_bytes` and `hash_data_from_slices`, neither of which is a multi-update +API. This is documented in the `fri/mmcs.rs` module header under "Memory: what the caller +may drop, and when". + +**Consequence: M-4's peak-anon acceptance test will fail if the integration assumes §3.3's +chain is what the primitive provides.** Either serve the base group's rows without +materializing them, or extend the backend trait with an incremental hasher. + +On the FRI side the analogous concern *was* fixed: `HeightCombiner` absorbs codewords one +at a time (`combine_by_height` is now a thin materialized wrapper over it), so a prover +never has to hold every table's quotient at once — +`streaming_absorption_matches_materialized_combine` pins the equivalence. + +## Public API the integration will consume + +```rust +// crypto/stark/src/fri/mmcs.rs +pub trait LeafSource { + fn num_matrices(&self) -> usize; + fn log_height(&self, m: usize) -> usize; + fn width(&self, m: usize) -> usize; + fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>); +} +pub enum BorrowedMatrix<'a, E> { RowMajorNatural {..}, ColMajorNatural {..} } +pub struct MixedMmcs; +impl MixedMmcs { + pub fn commit + Sync>(source: &S) -> Self; + pub fn root(&self) -> Commitment; + pub fn h_max(&self) -> usize; // added: the round's index space + pub fn dims(&self) -> &[(usize, usize)];// added: the shape actually committed + pub fn open_batch>(&self, iota: usize, source: &S) -> MixedOpening; + pub fn verify_batch(root, iota, opening, heights, widths) -> bool; // never panics +} + +// crypto/stark/src/fri/batched.rs +pub struct HeightCombiner; // new: streaming absorption +pub fn combine_by_height(inputs, alpha) -> Vec>>>; +pub struct BatchedFriLayout { total_folds, num_committed, terminal_len, effective_k } +pub fn batched_commit_phase(combined, transcript, coset_offset, blowup_log, k) + -> (Vec>, Vec>); // coeffs, not last_value +pub fn absorb_shape_histogram(transcript, heights, widths); +pub fn derive_batched_fri_challenges(..) -> Option>; // None = reject +``` + +Two signature notes for whoever wires this up. `batched_commit_phase` returns the terminal +**coefficients**, not a `last_value` — the round-4 transcript sequence is +`shape histogram → α → (β, root)* → β_final → coeffs → grinding → iotas`, and +`derive_batched_fri_challenges` is the single routine both sides must call so they provably +agree (`batched_round4_prover_inline_matches_verifier_replay` checks the by-hand prover +sequence against it). `derive_batched_fri_challenges` returns `Option` rather than panicking: +`None` when the proof's layer-root count or coefficient count contradicts the layout the +epoch's shape implies, or when a height is out of range — heights come from proof-supplied +trace lengths, so a bogus one is a rejection, never a panic on the verifier's path. + +## Suite counts + +| | result | +|---|---| +| `stark` lib | **274 passed, 0 failed** (245 baseline + **29 new**) | +| `stark` other test binaries | 0 / 0 / 3 ignored — unchanged | +| `crypto` | **52 passed, 0 failed** | +| `lfm::` | **310 passed / 19 failed / 9 ignored** — exactly the `blake3-real-hash` baseline, untouched | +| `make lint` | clean on **all four** combos (default, no-default+debug-checks, disk-spill, cuda) | +| `make fmt` | applied; `--check` clean | + +The 29 new tests also pass in a **debug** build, which is what actually exercises the +`debug_assert`s — the terminal-length check and "every bucket was injected before the +terminal", the two invariants that would catch a wrong fold count. + +Reproduce: + +```sh +cd /Users/maurofab/workspace/lambda_vm-mmcs +cargo test --release -p stark +cargo test --release -p crypto +cargo test --release -p lambda-vm-prover --lib lfm:: # 310/19 is baseline, not a regression +cargo test -p stark --lib -- fri::batched fri::mmcs batched_mmcs_soundness # debug: fires the debug_asserts +make lint +``` + +## Open items + +* **M-13b** — is any rounds-1-3 challenge shape-exploitable? Gates §3.4's addendum. Not + answerable without the integration. +* **M-4's peak-anon test** — see finding 2. The base-layer group must not be served from + `O(N)` resident LDE buffers, or the win is given back in the same commit. +* **M-6 / the batched preprocessed round** — the one case where the round's `h_max` is + below the FRI's. The reduction is now documented and range-guarded, but M-6 must apply it + and keep a per-matrix tamper control on the `prep_root` comparison (MMCS-PLAN §3.3's + closing warning: consolidating a per-table soundness check into one comparison is exactly + where coverage quietly goes missing). +* **Leaf allocation** — `hash_group_leaf` builds one `Vec>` per leaf, as + #768 did. `IsStreamingLeafBackend` exists to avoid exactly that, but `LeafSource::append_row` + bakes the `Vec` into the trait. Left alone deliberately: it is a micro-opt that has not + earned a measured win, and changing it touches the trait every caller implements. +* **Not a batching blocker, for Mauro** — MMCS-PLAN §M-10.3's aside: under the most + conservative proximity-gaps form the batching term is already 2^−108 *today*, at parity + with the query term. Which theorem/constant the system claims is unstated anywhere. + Batching does not change the answer; it is the natural moment to write it down. + +## Do not + +Push. Merge. Wire any of this into `prover.rs` / `verifier.rs` — that is M-3+ and it waits +for P-a Stage 2. Overwrite the RATE-4 `RESUME.md`. diff --git a/RESUME.md b/RESUME.md new file mode 100644 index 000000000..a6113a6df --- /dev/null +++ b/RESUME.md @@ -0,0 +1,86 @@ +# RATE-4 leaf widening — resume note + +**Branch** `rate4-leaf` (worktree `/Users/maurofab/workspace/lambda_vm-rate4`), based on +`blake3-real-hash` @ `681b749c`. Signed, unpushed, **working tree clean**. This is a +complete milestone, not a checkpoint of half-done work. + +| commit | what | +|---|---| +| `75d3162e` | the construction: socket + chips + instr + hash trait + callers | +| `cbf834ff` | registry re-bless | +| `85473426` | KAT re-pin + the generator | +| `240a308c` | rider: derive the socket-vs-standalone cost figures | +| `6669c997` | this note | +| `8312bf58` | the H6 gate controls, and the `m[8]` doc corrections | + +## H-register — all nine done and verified + +| id | disposition | +|---|---| +| H1 | framing indices derived from `NUM_LANES` (`LANE_IDX`/`OUT_PIN_IDX`/`DIGEST_IDX`/`UNREAD_IDX`). Guard `every_hash_candidate_emits_each_constraint_index_exactly_once` **green**. Confirmed count-preserving: lanes +4, unread pins −4, `NUM_CONSTRAINTS` unmoved — exactly the silent shape H1 predicted. | +| H2 | `emit_unread_input_pins` skips a slot every mode reads; `NUM_UNREAD_INPUT_PINS` derived, 8 → 4. | +| H3 | 2nd `LfmMem` receive is `is_real()` (was `Sum3` excluding `MODE_L`). | +| H4 | `leaf_lo_lane`/`leaf_hi_lane` = `4 + 2i` / `4 + 2i + 1`; felt source is `cols::leaf_felt(i)`. | +| H5 | `lanes_from_cells` is the single hybrid split; trace filler and BITWISE histogram both call it. | +| H6 | lanes 0–3 gated on full `mu`, lanes 4–11 on `digest_mu`. | +| H7 | `admits` checks `lanes_of(acc)` **and** `leaf_lanes(felts)`, in the cells the AIR reads. | +| H8 | `LfmHasher::leaf(acc, felts)` on all three arms; Test/Poseidon default `compress_out(acc, felts)`. | +| H9 | `block_len` derived as `4*(NUM_LANES+1)`; all three domains re-pinned. | + +No tenth hazard found. + +## ★ Two things a reviewer must look at + +1. **Message layout differs from the task brief, follows the spec.** COMMIT.md §1.2 and + `commit_ref.py::lfml_chain_row` put the tag LAST: lanes at `m[0..12]`, tag at `m[12]`. + The brief (and §1.4.4 H6's parenthetical "m[9..13]") assumed the tag stays at `m[8]`. + H6's substantive argument is unaffected — the free pin comes from the lane→**column** + map (`IN0 + lane` landing on the third input cell), not from the message index. + **§1.4.4's aside needs a doc fix.** +2. **Registry drift is narrower than the brief predicted.** Only `FriToyV0` moved. The + brief expected all six `program_id`s to move; they don't, because the registry is + generated under `HasherKind::Test` and `program_id` is `f(roots, log_heights, chunks, + hasher)` — leaf hash *semantics* never enter it. + +## State + +* `lfm::` suite **310 passed / 19 failed**; the 19 are byte-identical to the + `blake3-real-hash` baseline (measured in that worktree: 307/19). **Zero new + failures**; the +3 are the tests this branch adds. +* Whole prover crate: **861 passed / 34 failed** = the 19 above plus 15 in + `tests::prove_elfs_tests` / `tests::recursion_*`, every one of which panics + with "run `make compile-programs-rust`" or "run `make compile-recursion-elfs`". + Nothing outside `prover/src/lfm/` references the changed code — the only + consumer is `bin/compute_lfm_registry.rs`. +* `FriToyV0` proves and verifies under BLAKE3 and under every hasher. +* `make lint` and `make fmt`: exit 0. +* Fresh worktrees need `make compile-programs-asm` (and the two above for the + full crate) before the suite means anything. + +## Projection (calibrated model, `lfm_census_2026-08-12/tower.py`) + +Gate D1 node — fixture wrap, 1 proof, 110 queries: **124 → 78 GiB**, against the +~93 GiB budget, so it **FITS**. §1.4.1 predicted ≈81. Priced at the socket's real +width rather than the standalone chip's (§1.4.3), 119 → 75 GiB. + +⚠ Two honesty caveats, both of which make the headline *less* good than it looks: + +1. **The realized factor is 1.60–1.65×, not 2.0×.** The 2.0× is on leaf + absorption alone (~75% of this node); Merkle parents and the FRI legs do not + move. §1.4.1's ≈81 GiB already accounts for this — its "2.0× cut in ~70% of + the cost" needs the reader to do the Amdahl step, and several downstream notes + quote the 2.0× as if it were the node factor. +2. **78 GiB is a one-proof-VERIFY node, not the smallest aggregating one.** The + arity-2 node is 155 GiB (fixture) / 232 GiB (real 2^21) and does not fit. The + campaign notes already flag the aggregating node as the binding memory + constraint, and that `tower.py`'s flat 6.5% non-hash residue is optimistic at + higher rates — the residue tracks felts absorbed, so it does not fall with the + compression count. + +## Regenerating + +``` +python3 thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py # re-pin KATs +python3 thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py --check # staleness gate +cargo run --bin compute_lfm_registry --release # re-bless registry +``` diff --git a/bench_vs/lambda/recursion/Cargo.lock b/bench_vs/lambda/recursion/Cargo.lock index c358f86ec..d90d9dbcf 100644 --- a/bench_vs/lambda/recursion/Cargo.lock +++ b/bench_vs/lambda/recursion/Cargo.lock @@ -129,8 +129,6 @@ dependencies = [ "digest", "lambda-vm-syscalls", "math", - "rand 0.8.6", - "rand_chacha 0.3.1", "rkyv", "serde", "sha3", @@ -400,7 +398,7 @@ dependencies = [ "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", - "rand 0.9.4", + "rand", "riscv", "thiserror", ] @@ -436,7 +434,6 @@ dependencies = [ "getrandom 0.2.17", "num-bigint", "num-traits", - "rand 0.8.6", "rayon", "rkyv", "serde", @@ -586,35 +583,16 @@ dependencies = [ "ptr_meta", ] -[[package]] -name = "rand" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs index a04e920db..9b3d6cd53 100644 --- a/bin/cli/src/main.rs +++ b/bin/cli/src/main.rs @@ -412,7 +412,7 @@ fn cmd_execute( // below (the flamegraph path drives execution inside the executor and does // not expose per-log data). `None` means "not counted", so the accel lines // are omitted rather than printed as misleading zeros. - let mut accel_counts: Option<(u64, u64)> = None; + let mut accel_counts: Option<(u64, u64, u64)> = None; let cycle_count = if let Some(ref output_path) = flamegraph.path { // Shared execute+flamegraph path (executor::flamegraph) instead of @@ -479,6 +479,7 @@ fn cmd_execute( let mut cycle_count: u64 = 0; let mut keccak_calls: u64 = 0; + let mut blake3_calls: u64 = 0; let mut ecsm_calls: u64 = 0; // Reused per chunk: `(current_pc, a7)` for logs whose a7 matches an // accelerator syscall number. This is a cheap superset — a non-ECALL @@ -511,6 +512,7 @@ fn cmd_execute( for (pc, a7) in accel_candidates.drain(..) { match accelerator_of(executor.instructions.get(pc), a7) { Some(Accelerator::Keccak) => keccak_calls += 1, + Some(Accelerator::Blake3) => blake3_calls += 1, Some(Accelerator::Ecsm) => ecsm_calls += 1, None => {} } @@ -526,15 +528,16 @@ fn cmd_execute( } if cycles { - accel_counts = Some((keccak_calls, ecsm_calls)); + accel_counts = Some((keccak_calls, blake3_calls, ecsm_calls)); } cycle_count }; if cycles { println!("Cycles: {}", cycle_count); - if let Some((keccak_calls, ecsm_calls)) = accel_counts { + if let Some((keccak_calls, blake3_calls, ecsm_calls)) = accel_counts { println!("Keccak calls: {}", keccak_calls); + println!("Blake3 calls: {}", blake3_calls); println!("Ecsm calls: {}", ecsm_calls); } } diff --git a/crypto/crypto/Cargo.toml b/crypto/crypto/Cargo.toml index 532d17e4b..91bd16458 100644 --- a/crypto/crypto/Cargo.toml +++ b/crypto/crypto/Cargo.toml @@ -35,6 +35,15 @@ lambda-vm-syscalls = { path = "../../syscalls" } math = { path = "../math", features = ["test-utils"] } sha2 = { version = "0.10", default-features = false } bincode = "1" +# The external anchor for `hash::blake3`: at 7 rounds the compression function +# and `Blake3Chain` up to 1024 bytes are both bit-for-bit this crate. Test-only +# on purpose, and 7-round-only, so it CANNOT become the implementation — the +# variant behind `blake3-6round` is computed by nothing else in the world. Same +# dev-only role it has in `prover` and `math-cuda`. +blake3 = { version = "1.8.5", default-features = false, features = [ + "std", + "pure", +] } [features] default = ["asm", "std"] @@ -44,4 +53,17 @@ serde = ["dep:serde"] parallel = ["dep:rayon"] disk-spill = ["std", "dep:memmap2", "dep:tempfile", "dep:libc"] alloc = [] -rkyv = ["dep:rkyv", "math/rkyv"] \ No newline at end of file +rkyv = ["dep:rkyv", "math/rkyv"] +# `hash::blake3` at the 6-round internal variant instead of the 7-round standard +# one. Off = 7 rounds: standard BLAKE3, externally anchored, carrying no +# unratified assumption. DO NOT INVERT THE POLARITY — every existing measurement +# and the A6R sign-off read "7-round instantiated baseline, 6 behind the feature". +# +# This is the host tree's single round-count knob: `lambda-vm-prover`'s feature +# of the same name forwards to it, so the `LFM_BLAKE3` chip, the `LFM_HASH` +# socket and the commitment backends move together and one build cannot produce +# two hashes. `math-cuda`'s is necessarily separate (it compiles a cubin) and has +# to be set in lockstep with this one; `make lint` has a combined pass that +# exercises both, and `math_cuda::blake3::device_rounds` makes the match +# assertable rather than discoverable as a wrong root. +blake3-6round = [] \ No newline at end of file diff --git a/crypto/crypto/src/fiat_shamir/default_transcript.rs b/crypto/crypto/src/fiat_shamir/default_transcript.rs index d64f805a2..e4149ee56 100644 --- a/crypto/crypto/src/fiat_shamir/default_transcript.rs +++ b/crypto/crypto/src/fiat_shamir/default_transcript.rs @@ -1,6 +1,8 @@ use crate::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; +use crate::fiat_shamir::transcript_hash::{ + Blake3TranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; -use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256; use core::marker::PhantomData; use digest::Digest; use math::{ @@ -16,8 +18,8 @@ use math::{ /// per squeeze). const SQUEEZE_LEN: usize = 32; -/// Keccak-sponge Fiat-Shamir transcript with a Plonky3-style duplex output -/// buffer. +/// Sponge Fiat-Shamir transcript with a Plonky3-style duplex output buffer, +/// over the hash `T` names. /// /// Challenges are derived by squeezing the sponge and rejection-sampling field /// coordinates directly from those bytes — there is **no CSPRNG**. Earlier this @@ -28,8 +30,13 @@ const SQUEEZE_LEN: usize = 32; /// free. The output buffer amortizes one squeeze across up to `SQUEEZE_LEN / 8` /// 64-bit candidates, so a cubic-extension element (3 coordinates) usually costs /// a single squeeze. -pub struct DefaultTranscript { - hasher: Keccak256, +/// +/// `T` defaults to [`KeccakTranscriptHash`], so `DefaultTranscript::::new(..)` +/// still names exactly the transcript this system has always produced — every +/// method body below is hash-agnostic, and the keccak configuration selects the +/// unbounded rejection schedule, so its bytes do not move. +pub struct DefaultTranscript { + hasher: T::Digest, /// Duplex output buffer: bytes squeezed from the sponge, consumed 8 at a /// time by field/`u64` sampling. Positions `[out_pos, SQUEEZE_LEN)` are the /// bytes not yet handed out; `out_pos == SQUEEZE_LEN` means "empty, squeeze @@ -37,10 +44,10 @@ pub struct DefaultTranscript { /// squeeze can never reflect input appended after it was produced. out_buf: [u8; SQUEEZE_LEN], out_pos: usize, - phantom: PhantomData, + phantom: PhantomData<(F, T)>, } -impl Clone for DefaultTranscript { +impl Clone for DefaultTranscript { fn clone(&self) -> Self { Self { hasher: self.hasher.clone(), @@ -51,14 +58,15 @@ impl Clone for DefaultTranscript { } } -impl DefaultTranscript +impl DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { pub fn new(data: &[u8]) -> Self { let mut res = Self { - hasher: Keccak256::new(), + hasher: T::Digest::new(), out_buf: [0u8; SQUEEZE_LEN], // Empty: the first sample forces a squeeze. out_pos: SQUEEZE_LEN, @@ -93,11 +101,55 @@ where self.out_pos += 8; u64::from_be_bytes(bytes) } + + /// One base coordinate's worth of candidates under a FIXED schedule: draw + /// exactly `n`, hand back the first that `F` would accept. + /// + /// All `n` are drawn whichever one lands in range — that is the entire + /// point. Returning early on the first hit would restore the data-dependent + /// schedule this exists to remove. + /// + /// The value handed back is one `F::sample_field_element_from` accepts, so + /// its own rejection loop exits after a single call and consumption is + /// exactly `n` per coordinate. When every candidate misses (≈ 2⁻³²ⁿ) the + /// last one is returned, `F` rejects it, and the loop draws another `n` — + /// the schedule stays a multiple of `n` and the distribution stays exactly + /// uniform, because nothing is ever reduced into range. + fn next_candidate_fixed(&mut self, n: usize) -> u64 { + candidate_under_fixed_schedule::(n, || self.next_sample_u64()) + } } -impl Default for DefaultTranscript +/// One base coordinate's worth of candidates under a FIXED schedule: pull +/// exactly `n` from `next`, hand back the first that `F` would accept. +/// +/// Free-standing rather than a method so the schedule can be driven by a +/// counting closure in a test — "consumes exactly `n`" is the whole property, +/// and it is not observable from the transcript's outputs. +pub(crate) fn candidate_under_fixed_schedule( + n: usize, + mut next: impl FnMut() -> u64, +) -> u64 { + let mut chosen: Option = None; + let mut last = 0u64; + for _ in 0..n { + let candidate = next(); + last = candidate; + if chosen.is_none() && F::candidate_in_range(candidate) { + chosen = Some(candidate); + } + } + chosen.unwrap_or(last) +} + +/// The BLAKE3 Fiat-Shamir transcript: `Blake3Chain` in the sponge, and rider +/// 1's constant-consumption sampling. +pub type Blake3Transcript = DefaultTranscript; + +impl Default for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { fn default() -> Self { @@ -105,9 +157,10 @@ where } } -impl IsTranscript for DefaultTranscript +impl IsTranscript for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { fn append_bytes(&mut self, new_bytes: &[u8]) { @@ -130,9 +183,17 @@ where } fn sample_field_element(&mut self) -> FieldElement { - F::sample_field_element_from(|| self.next_sample_u64()) + match T::CANDIDATES_PER_COORDINATE { + None => F::sample_field_element_from(|| self.next_sample_u64()), + Some(n) => F::sample_field_element_from(|| self.next_candidate_fixed(n.get())), + } } + /// Note this loop is already fixed-consumption where it matters. Its only + /// production caller samples query indices against `domain_size >> 1`, a + /// power of two, and for `upper_bound = 2^k` the threshold is + /// `(-2^k) mod 2^k = 0` — so no candidate is ever rejected. The loop is here + /// for non-power-of-two bounds, which the protocol does not use. fn sample_u64(&mut self, upper_bound: u64) -> u64 { assert!(upper_bound > 0, "upper_bound must be greater than 0"); let threshold = upper_bound.wrapping_neg() % upper_bound; @@ -145,9 +206,10 @@ where } } -impl IsStarkTranscript for DefaultTranscript +impl IsStarkTranscript for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, S: IsField + IsSubFieldOf, { diff --git a/crypto/crypto/src/fiat_shamir/mod.rs b/crypto/crypto/src/fiat_shamir/mod.rs index a16f61b62..27a518d0b 100644 --- a/crypto/crypto/src/fiat_shamir/mod.rs +++ b/crypto/crypto/src/fiat_shamir/mod.rs @@ -6,3 +6,4 @@ pub mod default_transcript; pub mod is_transcript; +pub mod transcript_hash; diff --git a/crypto/crypto/src/fiat_shamir/transcript_hash.rs b/crypto/crypto/src/fiat_shamir/transcript_hash.rs new file mode 100644 index 000000000..3b8908c7e --- /dev/null +++ b/crypto/crypto/src/fiat_shamir/transcript_hash.rs @@ -0,0 +1,92 @@ +//! The hash a Fiat-Shamir transcript runs on, and the sampling schedule that +//! travels with it. +//! +//! `DefaultTranscript` is a thin `digest::Digest` wrapper, so swapping the hash +//! is a type substitution. What this trait adds beyond the digest is the +//! *challenge-consumption schedule*, because the two are decided together: a +//! proof's transcript is named by one configuration, and the schedule is part of +//! what a replaying verifier — host or in-machine — has to reproduce. + +use core::num::NonZeroUsize; +use digest::{Digest, FixedOutputReset, OutputSizeUser, typenum::U32}; + +use crate::hash::blake3::chain::Blake3Chain; +use crate::hash::platform_keccak::PlatformKeccak256; + +/// One Fiat-Shamir configuration: the digest the sponge runs on, plus how many +/// candidates a field-coordinate draw consumes. +pub trait TranscriptHash: 'static { + /// The sponge's hash. + /// + /// `Clone` because the transcript is snapshotted (the GPU FRI path restores + /// it) and because `state()` finalizes a clone. `FixedOutputReset` because + /// the squeeze is `finalize_reset`. The 32-byte output size is pinned rather + /// than left associated: `state()` returns `[u8; 32]`, and that is what + /// seeds grinding, so a configuration with a different digest width would + /// not be a drop-in anywhere it is consumed. + type Digest: Digest + FixedOutputReset + OutputSizeUser + Clone; + + /// How many 64-bit candidates one *base coordinate* draws. + /// + /// `None` — draw until one lands in the field's canonical range. The + /// expected cost is one candidate (rejection probability ≈ 2⁻³²), but the + /// count is data-dependent. + /// + /// `Some(n)` — always draw exactly `n` and take the first in range. This is + /// the property a straight-line machine needs: the LFM transcript replay + /// encodes one consumption schedule, and a transcript whose draw count + /// varies is unprovable against it (`SOUNDNESS.md` §6.3, and + /// `others/lfm-migration-riders.md` rider 1). + /// + /// ⚠ `Some(n)` is constant-consumption *up to a tail*: if all `n` candidates + /// miss — probability ≈ 2⁻³²ⁿ per coordinate — the draw continues rather + /// than failing. Failing would make challenge sampling fallible on the + /// verifier's replay path, which the no-panic policy forbids and which would + /// make `sample_field_element` return an `Option` everywhere. Continuing + /// keeps the distribution *exactly* uniform (no modular-reduction bias, + /// which at 2⁻³² per draw would dominate the proof system's soundness + /// error), and leaves a fixed schedule that holds except on that tail. + const CANDIDATES_PER_COORDINATE: Option; + + /// Name for KATs and diagnostics. + const NAME: &'static str; +} + +/// The keccak-256 configuration — what every `DefaultTranscript` is unless a +/// caller says otherwise, and byte-for-byte the transcript this system has +/// always produced. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeccakTranscriptHash; + +impl TranscriptHash for KeccakTranscriptHash { + type Digest = PlatformKeccak256; + + /// Deliberately `None`. Rider 1 is adopted for the BLAKE3 configuration + /// only: changing the keccak schedule would move every existing proof's + /// challenges, which is the one thing P-a's staging keeps still until the + /// flip. + const CANDIDATES_PER_COORDINATE: Option = None; + + const NAME: &'static str = "keccak256"; +} + +/// The BLAKE3 configuration — `Blake3Chain` over the same sponge, with rider +/// 1's constant-consumption sampling adopted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Blake3TranscriptHash; + +impl TranscriptHash for Blake3TranscriptHash { + type Digest = Blake3Chain; + + /// Two candidates per coordinate. + /// + /// One would be free — it is what the current schedule costs in the modal + /// case — but a single candidate that misses has nowhere to go, so the tail + /// would sit at ≈ 2⁻³² per coordinate, i.e. once in a few hundred thousand + /// proofs at production draw counts. That is not negligible enough to call + /// the schedule fixed. Two puts the tail at ≈ 2⁻⁶⁴ per coordinate, for one + /// extra candidate per coordinate — see the cost note in PA-PLAN §2.3. + const CANDIDATES_PER_COORDINATE: Option = NonZeroUsize::new(2); + + const NAME: &'static str = "blake3-chain"; +} diff --git a/crypto/crypto/src/hash/blake3/chain.rs b/crypto/crypto/src/hash/blake3/chain.rs new file mode 100644 index 000000000..1236cd1ff --- /dev/null +++ b/crypto/crypto/src/hash/blake3/chain.rs @@ -0,0 +1,722 @@ +//! [`Blake3Chain`] — the byte hash the RV64 prover's commitments are built from. +//! +//! **Specified in `thoughts/shared/block-compression/PA-PLAN.md` §1.7, which is +//! the normative text; this is the implementation of it.** The construction is +//! DRAFT: it is the working default by standing decision, and formally awaits +//! ratification. §1.7.3 lists the forks that are still open. +//! +//! # The construction, in one sentence +//! +//! **Standard BLAKE3 restricted to a single chunk that never ends.** The message +//! is split into 64-byte blocks, the last zero-padded; the chaining value starts +//! at [`BLAKE3_IV`] and each block compresses it forward with `t = 0`; the first +//! block carries `CHUNK_START`, the last carries `CHUNK_END | ROOT` and the true +//! byte count as its `block_len`. The digest is the low 8 output words, +//! little-endian. +//! +//! # Why this shape +//! +//! It is chosen so that two other things are true by construction rather than by +//! agreement, which is the whole reason to prefer it to a bare chain with one +//! flag constant: +//! +//! - **For any message of at most 1024 bytes, at 7 rounds, this IS +//! `blake3::hash`.** Standard BLAKE3's first chunk is exactly this chain, and a +//! message of at most one chunk has that chunk's output as its root — so `ROOT` +//! lands on the same compression. The official crate is therefore a direct +//! known-answer test for the *framing*, not merely for the round function, over +//! the entire range that matters: leaves, FRI pairs and parents are all far +//! inside it. `seven_round_chain_is_the_blake3_crate` is that test. +//! - **A 64-byte message is exactly a Merkle parent.** One block, first and last, +//! so `flags = 0x0B`, `block_len = 64`, `h = IV`, `t = 0` — which is what +//! `hash_new_parent` compresses and what the device kernel +//! (`math-cuda/kernels/blake3.cu`) implements. So the `StarkHash` invariant +//! that `Batched::hash_data(&vec![a, b]) == Pair::hash_data(&[a, b])` holds +//! because both are the same 64 bytes through this one function. +//! +//! Above 1024 bytes it deliberately leaves the standard: BLAKE3 would start a +//! second chunk (`t = 1`, chaining value reset to `IV`) and build a tree over +//! chunk chaining values. Keeping one unbounded chunk costs nothing at 6 rounds, +//! where no external verifier exists in any case, and saves both a chunk-tree +//! state machine in every CUDA kernel and the same state machine again in the +//! wrap's eDSL emitter. `past_one_chunk_leaves_the_blake3_crate` pins that the +//! divergence is real, so the claim is falsifiable rather than decorative. + +use digest::{FixedOutput, FixedOutputReset, HashMarker, Output, OutputSizeUser, Reset, Update}; + +use super::{BLAKE3_IV, BLAKE3_ROUNDS, blake3_compress_rounds}; + +/// Bytes in one BLAKE3 message block. +pub const BLOCK_LEN: usize = 64; + +/// Dwords in the accelerator's state region: `h[8] | m[16] | t | +/// (block_len, flags) | out[16]`, two little-endian `u32` words per dword. +/// Mirrors `BLAKE3_STATE_BYTES / 8` in +/// `executor::vm::instruction::execution`. +pub const SYSCALL_STATE_DWORDS: usize = 22; + +/// First dword of the output region — the accelerator reads dwords `0..14` and +/// writes `out[0..16]` into `14..22`. Mirrors the executor's +/// `BLAKE3_OUT_DWORDS`. +pub const SYSCALL_OUT_DWORD: usize = 14; + +/// This block begins the chunk. Set on the first block only. +const CHUNK_START: u32 = 1; +/// This block ends the chunk. Set on the last block only. +const CHUNK_END: u32 = 2; +/// This compression produces the root output. Set on the last block only — +/// a single-chunk message's chunk output *is* its root output. +const ROOT: u32 = 8; + +/// The flags of a message that is one block long: first and last at once. Equal +/// to `CHUNK_START | CHUNK_END | ROOT`, and the framing every Merkle parent uses. +pub const FLAGS_ONE_BLOCK: u32 = CHUNK_START | CHUNK_END | ROOT; + +/// [`Blake3Chain`] as a one-shot over a byte slice. +/// +/// The streaming type and this agree by construction — this *is* the streaming +/// type, fed once. +pub fn blake3_chain(data: &[u8]) -> [u8; 32] { + blake3_chain_rounds(data, BLAKE3_ROUNDS) +} + +/// [`blake3_chain`] with the round count as an argument. +/// +/// The round count is the only parameter, exactly as in +/// [`blake3_compress_rounds`] and for the same reason: at +/// [`BLAKE3_STANDARD_ROUNDS`](super::BLAKE3_STANDARD_ROUNDS) the result is the +/// `blake3` crate's, so that arm certifies this whole code path — the block +/// splitting, the padding, the flag schedule, the `block_len` — and the 6-round +/// arm differs from it by a loop bound alone. +pub fn blake3_chain_rounds(data: &[u8], rounds: usize) -> [u8; 32] { + let mut chain = Blake3Chain::with_rounds(rounds); + chain.update(data); + chain.finalize_digest() +} + +/// Lay a compression's inputs out as the accelerator's 22-dword state region. +/// +/// The guest side of the syscall ABI, and the *only* place this crate encodes +/// it. The output dwords are left zero; the accelerator fills `14..22`. +/// +/// Compiled on every target although only the riscv64 arm of [`compress_block`] +/// calls it, so the packing can be checked on the host against the executor's +/// handler — which is ordinary host code — rather than only inside a guest. The +/// executor's unpacking is the mirror of this, and +/// `prover::tables::blake3::executor_syscall_packing` drives the two against +/// each other through a real `EcallEbreak`. +pub fn pack_syscall_state( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> [u64; SYSCALL_STATE_DWORDS] { + let mut words = [0u32; 2 * SYSCALL_OUT_DWORD]; + words[0..8].copy_from_slice(h); + words[8..24].copy_from_slice(m); + words[24] = t as u32; + words[25] = (t >> 32) as u32; + words[26] = block_len; + words[27] = flags; + + let mut state = [0u64; SYSCALL_STATE_DWORDS]; + for (k, dword) in state[..SYSCALL_OUT_DWORD].iter_mut().enumerate() { + *dword = (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32); + } + state +} + +/// Read the 16 output words the accelerator wrote into [`pack_syscall_state`]'s +/// region. The inverse of the low half of that layout, over dwords `14..22`. +pub fn unpack_syscall_out(state: &[u64; SYSCALL_STATE_DWORDS]) -> [u32; 16] { + core::array::from_fn(|i| { + let dword = state[SYSCALL_OUT_DWORD + i / 2]; + if i.is_multiple_of(2) { + dword as u32 + } else { + (dword >> 32) as u32 + } + }) +} + +/// ★ The chain's single entry into the compression function. +/// +/// Both the interior step and the finalization go through here, so there is one +/// place where a guest reaches the accelerator and one framing above it. Adding +/// a second call to [`blake3_compress_rounds`] in this file would put a +/// compression outside the accelerator's reach on the guest and split the two +/// paths silently — the trap PA-PLAN §1.4 names. +/// +/// `t` is not a parameter: the construction is a single chunk that never ends, +/// so the counter is 0 at every block (§1.7). The syscall ABI still carries a +/// full 64-bit counter, and this is where it is pinned to zero. +#[cfg(all(target_arch = "riscv64", feature = "blake3-6round"))] +fn compress_block( + cv: &[u32; 8], + block: &[u32; 16], + block_len: u32, + flags: u32, + rounds: usize, +) -> [u32; 16] { + // `with_rounds` can hand this any count — it exists so the 7-round anchor + // is reachable from one build — while the accelerator implements six and + // nothing else. Anything but the crate-global count takes the software + // path, so the anchor constructor cannot be answered at the wrong round + // count by a machine that has the precompile. + if rounds != BLAKE3_ROUNDS { + return blake3_compress_rounds(cv, block, 0, block_len, flags, rounds); + } + let mut state = pack_syscall_state(cv, block, 0, block_len, flags); + lambda_vm_syscalls::syscalls::blake3_compress_6round(&mut state); + unpack_syscall_out(&state) +} + +/// The chain's single entry into the compression function, in software. +/// +/// See the riscv64 arm above for what this is one of two of. Every host build +/// takes this path, and so does a guest built without `blake3-6round`: the +/// accelerator is 6-round only, so at 7 rounds there is nothing to dispatch to. +#[cfg(not(all(target_arch = "riscv64", feature = "blake3-6round")))] +fn compress_block( + cv: &[u32; 8], + block: &[u32; 16], + block_len: u32, + flags: u32, + rounds: usize, +) -> [u32; 16] { + blake3_compress_rounds(cv, block, 0, block_len, flags, rounds) +} + +/// The accelerator is **six rounds, hard-coded**: `BLAKE3_ROUNDS` in +/// `executor::vm::instruction::execution` is a plain `6` with no feature behind +/// it, and the chip's columns are laid out for that width. So the syscall arm +/// above computes the host prover's hash only while this crate is at six rounds +/// too, and the coupling is compile-time rather than a comment: inverting +/// `blake3-6round`'s polarity must fail the build, not surface later as a root +/// the verifier rejects. +/// +/// Gated on the feature alone, not on the target, although only the riscv64 arm +/// dispatches to the accelerator: a `target_arch` gate would put it out of reach +/// of every host build, including `make lint`'s `blake3-6round` pass, which is +/// the one place CI compiles this feature at all. +#[cfg(feature = "blake3-6round")] +const _: () = assert!( + BLAKE3_ROUNDS == super::BLAKE3_SIX_ROUNDS, + "the BLAKE3 accelerator implements 6 rounds only, but `blake3-6round` did \ + not select 6 — the guest would hash differently from the host prover" +); + +/// The single-chunk BLAKE3 chain as an incremental hasher. +/// +/// Implements the `digest` traits, so it drops into the Merkle backends and the +/// transcript anywhere a `D: Digest` is expected — the same way +/// [`PlatformKeccak256`](crate::hash::platform_keccak::PlatformKeccak256) does. +/// That is what lets the batched and paired backends be one hash rather than two +/// implementations that have to be shown to coincide. +/// +/// A full block is held rather than compressed until more input arrives, because +/// the final block's flags and `block_len` differ from every other block's and +/// whether a block is final is not known until the message ends. +#[derive(Clone)] +pub struct Blake3Chain { + /// The chaining value: `IV`, then the truncated output of each compressed + /// block. Never reset — that is the "single chunk" of the construction. + cv: [u32; 8], + /// The pending block, zero-padded. Zeroing on reset is what pads the final + /// partial block. + block: [u8; BLOCK_LEN], + /// Bytes of `block` that are message, `0..=BLOCK_LEN`. + block_len: usize, + /// Whether any block has been compressed yet — i.e. whether the pending + /// block still carries `CHUNK_START`. + started: bool, + /// Rounds. [`BLAKE3_ROUNDS`] for every production instance; see + /// [`Self::with_rounds`]. + rounds: usize, +} + +impl Default for Blake3Chain { + fn default() -> Self { + Self::with_rounds(BLAKE3_ROUNDS) + } +} + +impl Blake3Chain { + /// A hasher at the crate-global [`BLAKE3_ROUNDS`]. The only constructor any + /// production path uses; [`Default`] and `Digest::new` are this. + pub fn new() -> Self { + Self::default() + } + + /// A hasher at an explicit round count. + /// + /// **For anchoring and known-answer tests only.** The production round count + /// is a compile-time crate-global on purpose — a per-instance one would let a + /// single build commit under two different hashes, which is the failure the + /// `SOCKET_ROUNDS == BLAKE3_ROUNDS` assertion in `prover` exists to prevent. + /// It is exposed because the 7-round arm is the external anchor for the + /// 6-round one, so both must be reachable from one build's tests. + pub fn with_rounds(rounds: usize) -> Self { + Self { + cv: BLAKE3_IV, + block: [0u8; BLOCK_LEN], + block_len: 0, + started: false, + rounds, + } + } + + /// The pending block as 16 little-endian message words. + fn block_words(&self) -> [u32; 16] { + core::array::from_fn(|i| { + u32::from_le_bytes([ + self.block[4 * i], + self.block[4 * i + 1], + self.block[4 * i + 2], + self.block[4 * i + 3], + ]) + }) + } + + /// The pending block's flags. `CHUNK_START` while nothing has been + /// compressed yet; `CHUNK_END | ROOT` when this is the message's last block. + fn flags(&self, is_final: bool) -> u32 { + let start = if self.started { 0 } else { CHUNK_START }; + let end = if is_final { CHUNK_END | ROOT } else { 0 }; + start | end + } + + /// Fold the pending block — known not to be the last — into the chaining + /// value, and clear the block so the next one is zero-padded. + fn compress_pending(&mut self) { + let out = compress_block( + &self.cv, + &self.block_words(), + BLOCK_LEN as u32, + self.flags(false), + self.rounds, + ); + self.cv.copy_from_slice(&out[..8]); + self.block = [0u8; BLOCK_LEN]; + self.block_len = 0; + self.started = true; + } + + /// Absorb more message. Identical results for any split of the same bytes — + /// `streaming_splits_agree_with_one_shot`. + pub fn update(&mut self, mut input: &[u8]) { + while !input.is_empty() { + // Only now is the pending block known not to be the last one. + if self.block_len == BLOCK_LEN { + self.compress_pending(); + } + let take = (BLOCK_LEN - self.block_len).min(input.len()); + self.block[self.block_len..self.block_len + take].copy_from_slice(&input[..take]); + self.block_len += take; + input = &input[take..]; + } + } + + /// The 32-byte digest: one final compression over the pending block, with + /// the true byte count as `block_len` and `CHUNK_END | ROOT` set. + /// + /// The empty message takes this path with an all-zero block and + /// `block_len = 0`, which is one compression, not zero — and is what + /// `blake3::hash(b"")` is at 7 rounds. + pub fn finalize_digest(&self) -> [u8; 32] { + let out = compress_block( + &self.cv, + &self.block_words(), + self.block_len as u32, + self.flags(true), + self.rounds, + ); + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest + } +} + +impl HashMarker for Blake3Chain {} + +impl OutputSizeUser for Blake3Chain { + type OutputSize = digest::typenum::U32; +} + +impl Update for Blake3Chain { + fn update(&mut self, data: &[u8]) { + Blake3Chain::update(self, data); + } +} + +impl FixedOutput for Blake3Chain { + fn finalize_into(self, out: &mut Output) { + out.copy_from_slice(&self.finalize_digest()); + } +} + +impl Reset for Blake3Chain { + fn reset(&mut self) { + *self = Self::with_rounds(self.rounds); + } +} + +impl FixedOutputReset for Blake3Chain { + fn finalize_into_reset(&mut self, out: &mut Output) { + out.copy_from_slice(&self.finalize_digest()); + Reset::reset(self); + } +} + +/// The message the KAT table is taken over, at a given length. +/// +/// Byte `i` is `37i + 11 (mod 256)`: every length is a different message, no +/// byte value repeats within a block, and it is the same generator the existing +/// compression-level anchor in `prover::lfm::blake3` uses. +pub const fn kat_message_byte(i: usize) -> u8 { + (i as u8).wrapping_mul(37).wrapping_add(11) +} + +/// The lengths [`CHAIN_KAT_6ROUND`] covers, in order. PA-PLAN §1.7.4 says what +/// each one discriminates: the empty message is one block (0); `block_len` is +/// the true length and the tail is zero-padded (1, 31, 63); a 64-byte message is +/// the parent form (64); the chain's first step moves `CHUNK_END | ROOT` off +/// block 0 (65); an exact multiple of 64 emits no spurious final block (128); +/// interior blocks carry no flags (192, 256, 1024); and 1088 is the first length +/// past one chunk, where this construction leaves standard BLAKE3 (1088). +pub const CHAIN_KAT_LENS: [usize; 12] = [0, 1, 31, 63, 64, 65, 127, 128, 192, 256, 1024, 1088]; + +/// [`blake3_chain`] at **6 rounds** over `kat_message_byte` messages of each +/// [`CHAIN_KAT_LENS`] length. +/// +/// # What this table is, and how strong its provenance actually is +/// +/// It is a regression pin — generated from this implementation and committed, so +/// a later refactor cannot change the construction silently. But it is more than +/// that, and the difference is worth stating precisely. +/// +/// It is not "stronger than the compression vectors next door" either: the two +/// cover different axes. This table pins the FRAMING across blocks — the flag +/// schedule, the chaining value, the final block's `block_len`. It cannot pin the +/// counter split, because every message here is hashed with `t = 0`; only +/// [`CANONICAL_VECTORS`](super::CANONICAL_VECTORS), whose ten vectors all carry +/// `t >= 2^32`, does that. Neither table is redundant with the other. +/// +/// Every entry from length 0 to 1024 was **independently reproduced** by #903's +/// Python oracle (`thoughts/blake3/blake3-oracle/blake3_ref.py`) evaluated at +/// `rounds = 6`, on 2026-08-14. That oracle is a full standard-BLAKE3 +/// implementation with the round count as a parameter, written by another author +/// for a different purpose, and at `rounds = 7` it reproduces the official +/// `blake3` package bit-for-bit at every length checked — including the +/// multi-chunk ones. So for the whole ≤1-chunk range these digests are not a +/// self-consistency check: two implementations that share no code agree, and +/// the conventions they agree on are pinned to the published hash from outside. +/// +/// Length 1088 is where they part, and that is the point of including it: the +/// oracle stays standard past one chunk and this construction does not (P3). +/// Being able to say the divergence is *the chunking* rather than the round +/// count needs a reference that is standard at 6 rounds too, which is exactly +/// what the oracle is. +/// +/// Everything that cross-check needs is tracked, so it is reproducible rather +/// than merely recorded. `thoughts/blake3/blake3-oracle/` holds the +/// round-parameterized reference (`blake3_ref.py`, vendored at commit +/// `65025095`), which exposes raw compression entry points — `compress`, +/// `compress_cv`, `compress_6round` — as well as `blake3_hash`, alongside +/// `canonical_6round_vectors.json`, `official_test_vectors.json` and +/// `test_oracle.py`. +/// +/// A **second source** sits beside it, and it is the stronger of the two: +/// `thoughts/blake3/reference-impl/` is upstream BLAKE3 1.8.5's own portable C +/// with its round loop parameterized, the whole edit being +/// `PARAMETERISATION.diff`. It reproduces this table at 6 rounds over every +/// length up to one chunk, and it encodes the message schedule as an indexed +/// `MSG_SCHEDULE[r]` table where this crate composes a single permutation +/// between rounds. Those are structurally different expressions of the same +/// convention, so its agreement cross-validates the schedule instead of +/// restating it — a bug in the iterative composition is precisely what one +/// source cannot catch. `make test-blake3-second-source` runs both against these +/// digests: a ~1 second C compile plus a randomised differential, no cargo and +/// no GPU. +/// +/// The 7-round arm remains the primary anchor and needs none of this: +/// `blake3_chain_rounds(m, 7)` is checked directly against the `blake3` crate +/// over all 1025 lengths, with no table in between. +pub const CHAIN_KAT_6ROUND: [[u8; 32]; 12] = [ + // len 0 + [ + 0x3C, 0x3B, 0xBB, 0x1F, 0x33, 0x5A, 0x31, 0xEA, 0x86, 0x46, 0x4B, 0x65, 0x1C, 0x02, 0x06, + 0xFC, 0x81, 0xD3, 0x32, 0x62, 0xAE, 0x00, 0xEA, 0x1A, 0x65, 0xF3, 0xD1, 0xD0, 0x4A, 0xFA, + 0xEF, 0xC9, + ], + // len 1 + [ + 0x2A, 0x50, 0xE4, 0x5B, 0x89, 0x21, 0xF9, 0xEF, 0xA0, 0x08, 0xD9, 0xF3, 0x9F, 0x71, 0x65, + 0x60, 0x0C, 0xF4, 0x8A, 0x7F, 0x0E, 0x85, 0x9C, 0x21, 0x22, 0xE3, 0xCC, 0xB6, 0xB9, 0x67, + 0x7E, 0xE5, + ], + // len 31 + [ + 0xC3, 0x8B, 0xF6, 0x2F, 0x50, 0x60, 0x40, 0xB2, 0x60, 0x02, 0x73, 0x77, 0x8D, 0x28, 0x1B, + 0x89, 0x43, 0x62, 0x1E, 0x2B, 0x8A, 0x9F, 0x59, 0xE2, 0x37, 0x9F, 0x8F, 0xD7, 0xE5, 0xC8, + 0x51, 0x25, + ], + // len 63 + [ + 0xC3, 0x73, 0xF5, 0x1A, 0x5E, 0xB8, 0xB2, 0x7E, 0xA0, 0x5B, 0xB1, 0xF6, 0xF4, 0xE6, 0x2E, + 0x92, 0x4F, 0xF4, 0xD8, 0xA2, 0x79, 0xF0, 0xD0, 0x5A, 0xFA, 0x5C, 0xD5, 0x19, 0x39, 0x1D, + 0x63, 0x89, + ], + // len 64 + [ + 0x59, 0x00, 0xA1, 0xE3, 0x98, 0xBB, 0x2B, 0xF6, 0xD3, 0xBA, 0x7F, 0x1A, 0x29, 0x19, 0x7B, + 0x79, 0xC8, 0x6B, 0x71, 0xAD, 0x2C, 0x26, 0x31, 0xF4, 0xAC, 0x73, 0x6C, 0x82, 0xDB, 0x04, + 0x3C, 0xB5, + ], + // len 65 + [ + 0x53, 0x95, 0x3F, 0xCA, 0xDC, 0x39, 0xB8, 0x62, 0x39, 0x01, 0xAF, 0x7B, 0x53, 0x4F, 0x2F, + 0x69, 0x33, 0xE3, 0x12, 0xF5, 0x02, 0x99, 0x33, 0x13, 0x34, 0xE6, 0xC0, 0xA7, 0xC9, 0xDB, + 0xC2, 0xBE, + ], + // len 127 + [ + 0x9E, 0x0D, 0xD8, 0x16, 0x8D, 0x19, 0x9A, 0x04, 0x59, 0x0C, 0x2C, 0xBA, 0x43, 0x9B, 0x27, + 0x07, 0x76, 0xE4, 0x27, 0x15, 0xD5, 0x18, 0xF6, 0x86, 0x55, 0xE5, 0x66, 0x92, 0x48, 0x3E, + 0x50, 0x5E, + ], + // len 128 + [ + 0x5C, 0xAF, 0xFC, 0x87, 0x84, 0xE8, 0x17, 0xBB, 0xBA, 0x99, 0x1B, 0x21, 0x08, 0xC2, 0x6A, + 0x3D, 0xFD, 0xF8, 0x04, 0x24, 0x5E, 0xF6, 0x3A, 0xE1, 0x04, 0x0A, 0x3C, 0x34, 0xF1, 0xB3, + 0x62, 0xFF, + ], + // len 192 + [ + 0x39, 0x9D, 0x6B, 0x9A, 0xDE, 0xB2, 0xF8, 0x84, 0x50, 0x77, 0x5F, 0x77, 0x3E, 0x9D, 0xEC, + 0x08, 0x83, 0x6C, 0x13, 0x57, 0x13, 0xC2, 0xC5, 0xDD, 0x09, 0xF4, 0xCE, 0xCE, 0xB0, 0xED, + 0x38, 0x88, + ], + // len 256 + [ + 0xFB, 0xCA, 0xB3, 0x69, 0x9A, 0x49, 0x59, 0xFA, 0x37, 0x19, 0x0E, 0x98, 0xCA, 0x51, 0x42, + 0xDD, 0xBC, 0x88, 0x33, 0x0F, 0x2E, 0x7D, 0x12, 0x33, 0x5D, 0xB9, 0xC6, 0xC8, 0x88, 0x1A, + 0x0B, 0x87, + ], + // len 1024 + [ + 0xF3, 0x95, 0xE7, 0xE2, 0x15, 0x03, 0x63, 0xB6, 0xD2, 0x00, 0x48, 0x75, 0x15, 0x42, 0x5B, + 0x02, 0x04, 0xEE, 0xA4, 0x24, 0x07, 0x21, 0x83, 0xB7, 0x01, 0x17, 0x6E, 0xCC, 0xBE, 0x0F, + 0xFE, 0x1B, + ], + // len 1088 + [ + 0xB4, 0x73, 0x8E, 0xDE, 0x77, 0xA6, 0xEC, 0x16, 0x6E, 0xE9, 0x76, 0x67, 0x11, 0x8D, 0x47, + 0x93, 0xCB, 0xF2, 0xB0, 0x8B, 0x45, 0xAA, 0xC7, 0xC6, 0xD5, 0x29, 0x43, 0xB5, 0xD2, 0x98, + 0xC6, 0x88, + ], +]; + +#[cfg(test)] +mod tests { + use super::*; + use crate::hash::blake3::{BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS}; + use alloc::vec::Vec; + + fn message(len: usize) -> Vec { + (0..len).map(kat_message_byte).collect() + } + + /// ★ **The external anchor.** At 7 rounds this construction is the `blake3` + /// crate's hash, for every message length up to one full chunk — no oracle, + /// no JSON, no transcription. + /// + /// The range is what makes it worth more than a compression-level anchor: + /// it pins the block splitting, the zero padding, the `block_len` of the + /// final block, the `CHUNK_START`/`CHUNK_END`/`ROOT` schedule and the + /// little-endian digest read-back, at every boundary those can be wrong at. + #[test] + fn seven_round_chain_is_the_blake3_crate() { + for len in 0..=1024usize { + let msg = message(len); + assert_eq!( + blake3_chain_rounds(&msg, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&msg).as_bytes(), + "the 7-round chain must equal the blake3 crate at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the anchor: at 6 rounds it must not match, or the + /// test above would pass just as well with `rounds` ignored — the one bug + /// that would make the whole external-anchor argument vacuous. + #[test] + fn six_round_chain_is_not_the_blake3_crate() { + for len in [0usize, 1, 64, 65, 128, 1024] { + let msg = message(len); + assert_ne!( + blake3_chain_rounds(&msg, BLAKE3_SIX_ROUNDS), + *blake3::hash(&msg).as_bytes(), + "length {len}" + ); + } + } + + /// ★ **P3, stated as a test.** Past one chunk the construction deliberately + /// leaves standard BLAKE3 — the standard would start chunk 1 and build a + /// tree, this keeps chaining. Without this, "we implement the single-chunk + /// chain" would be an unfalsifiable claim: the anchor above would pass + /// identically if we had implemented the whole chunk tree instead. + /// + /// 1024 is the last length where they agree and 1088 the first block past + /// it, so the two assertions together locate the divergence exactly. + #[test] + fn past_one_chunk_leaves_the_blake3_crate() { + let last_agreeing = message(1024); + assert_eq!( + blake3_chain_rounds(&last_agreeing, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&last_agreeing).as_bytes(), + "1024 bytes is still one chunk and must agree" + ); + for len in [1025usize, 1088, 2048] { + let msg = message(len); + assert_ne!( + blake3_chain_rounds(&msg, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&msg).as_bytes(), + "past one chunk the constructions must differ, at length {len}" + ); + } + } + + /// ★ **P2** — a 64-byte message is exactly the Merkle parent form: one + /// compression, `h = IV`, `t = 0`, `block_len = 64`, `flags = 0x0B`. + /// + /// This is what makes the `StarkHash` two-element invariant hold by + /// construction, and it is the framing the device kernel implements. Written + /// out as an explicit compression rather than as "whatever the code does", + /// so it fails if the flag schedule or the counter moves. + #[test] + fn a_sixty_four_byte_message_is_the_parent_form() { + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(7)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(3)); + let mut msg = [0u8; 64]; + msg[..32].copy_from_slice(&left); + msg[32..].copy_from_slice(&right); + + for rounds in [BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS] { + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(msg[4 * i..4 * i + 4].try_into().unwrap()) + }); + let out = blake3_compress_rounds(&BLAKE3_IV, &words, 0, 64, FLAGS_ONE_BLOCK, rounds); + let mut expected = [0u8; 32]; + for i in 0..8 { + expected[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_eq!( + blake3_chain_rounds(&msg, rounds), + expected, + "a 64-byte message must be one parent compression, at {rounds} rounds" + ); + } + } + + /// The `Update` contract: the digest depends on the bytes, not on how they + /// were handed over. Splits are taken at and either side of every block + /// boundary, which is where a mis-set `CHUNK_START` or a prematurely + /// compressed final block would show. + #[test] + fn streaming_splits_agree_with_one_shot() { + for len in [0usize, 1, 63, 64, 65, 127, 128, 129, 200] { + let msg = message(len); + let want = blake3_chain(&msg); + for split in 0..=len { + let mut chain = Blake3Chain::new(); + chain.update(&msg[..split]); + chain.update(&msg[split..]); + assert_eq!( + chain.finalize_digest(), + want, + "length {len} split at {split}" + ); + } + // Byte at a time, which crosses every boundary in the smallest + // possible increments. + let mut chain = Blake3Chain::new(); + for b in &msg { + chain.update(&[*b]); + } + assert_eq!(chain.finalize_digest(), want, "length {len} byte at a time"); + } + } + + /// The committed 6-round regression pin. See [`CHAIN_KAT_6ROUND`] for what + /// this does and does not establish. + #[test] + fn six_round_chain_matches_the_committed_table() { + for (i, &len) in CHAIN_KAT_LENS.iter().enumerate() { + assert_eq!( + blake3_chain_rounds(&message(len), BLAKE3_SIX_ROUNDS), + CHAIN_KAT_6ROUND[i], + "6-round chain KAT at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the table: the entries must be distinct data, or a + /// generation bug that wrote one digest twelve times would leave the test + /// above passing and pinning nothing. + #[test] + fn the_committed_table_entries_are_distinct() { + for (i, a) in CHAIN_KAT_6ROUND.iter().enumerate() { + for (j, b) in CHAIN_KAT_6ROUND.iter().enumerate().skip(i + 1) { + assert_ne!(a, b, "KAT entries {i} and {j} are the same digest"); + } + } + } + + /// **P4** in its cheapest observable form: lengths that share a padded block + /// must not share a digest. A construction that ignored `block_len` would + /// collide 31 with 32, and one that ignored the flag schedule would collide + /// 64 with 65's first block. + #[test] + fn lengths_sharing_a_padded_block_do_not_collide() { + let mut seen: Vec<[u8; 32]> = Vec::new(); + for len in 0..=130usize { + let digest = blake3_chain(&message(len)); + assert!( + !seen.contains(&digest), + "length {len} collides with a shorter message" + ); + seen.push(digest); + } + } + + /// `Reset` really returns to the initial state, including the pending block + /// and the `CHUNK_START` flag — a reset that kept `started` set would hash + /// the next message under the wrong flags. + #[test] + fn reset_returns_to_the_initial_state() { + let mut chain = Blake3Chain::new(); + chain.update(&message(100)); + Reset::reset(&mut chain); + chain.update(&message(7)); + assert_eq!(chain.finalize_digest(), blake3_chain(&message(7))); + } + + /// The `digest` route and the free function are the same hash — the backends + /// reach this type through `Digest`, the KATs above through `blake3_chain`. + #[test] + fn the_digest_trait_route_agrees_with_the_free_function() { + use digest::Digest; + for len in [0usize, 1, 64, 65, 200] { + let msg = message(len); + let mut hasher = ::new(); + Digest::update(&mut hasher, &msg); + let via_digest: [u8; 32] = Digest::finalize(hasher).into(); + assert_eq!(via_digest, blake3_chain(&msg), "length {len}"); + } + } +} diff --git a/crypto/crypto/src/hash/blake3/mod.rs b/crypto/crypto/src/hash/blake3/mod.rs new file mode 100644 index 000000000..4fdfc11c0 --- /dev/null +++ b/crypto/crypto/src/hash/blake3/mod.rs @@ -0,0 +1,216 @@ +//! The BLAKE3 compression function with the round count as a parameter, and the +//! byte hash the RV64 prover's commitments are built from. +//! +//! # Why this lives in `crypto` +//! +//! It has three callers that cannot share a copy any other way. The Merkle +//! backends in [`crate::merkle_tree::backends`] are in this crate; the +//! `LFM_BLAKE3` chip and the `LFM_HASH` socket are in `prover`, which depends on +//! this crate; and the CUDA kernels are checked against it from `math-cuda`, +//! which `prover` depends on. `crypto` is the only place all three can reach, so +//! the compression function is defined here once and re-exported upward — +//! `prover::lfm::blake3` is a re-export of this module, not a second +//! implementation. A chip and a commitment backend that hash identically because +//! they call one function is a different claim from two that agree today. +//! +//! # The round count +//! +//! [`BLAKE3_ROUNDS`] is 7 — standard BLAKE3 — unless the crate's +//! `blake3-6round` feature is on, and then it is 6. The polarity is deliberate +//! and must not be inverted: every existing measurement and sign-off reads +//! "7-round instantiated baseline, 6 behind the feature". +//! +//! The knob is crate-global rather than a generic parameter so that one build +//! cannot produce two hashes. Crates above re-export it rather than defining +//! their own, and `prover`'s `blake3-6round` feature forwards to this one, so +//! the chip's round count and the commitment's round count are the same symbol. +//! `math-cuda` necessarily has its own (it compiles a cubin), which is why it +//! exports the compiled-in count for a caller to assert instead of discover. +//! +//! # Provenance of the primitive, and why no external KAT exists at 6 rounds +//! +//! Vendored from PR #903 (`yetanotherco/lambda_vm`, head +//! `89aeeb8c2b0389e9d21a861c9e3a10a7b1b5704e`). Standing-decisions rule 9 +//! requires pinning a new primitive against an external known-answer vector that +//! nothing in this repository produced. That is *impossible in the usual form* +//! for the 6-round variant: it is not standard BLAKE3, so no published vector +//! and no crate exposes it. The provenance chain #903 supplies instead: +//! +//! 1. A z3-proved model of the compression dataflow +//! (`thoughts/blake3/blake3-chip/z3_blake_verify.py`). +//! 2. A Python oracle (`thoughts/blake3/blake3-oracle/blake3_ref.py`) whose +//! **7-round** instantiation is pinned against the official `blake3` crate's +//! published test vectors — so the oracle's G-function, message schedule, +//! counter split and feed-forward are all externally validated; only the +//! round count is varied. +//! 3. That oracle at `rounds = 6` emitted the 10 canonical vectors in +//! [`CANONICAL_VECTORS`], which pin this port. +//! +//! So the external anchor is one step removed: the *conventions* are pinned by +//! the official crate through the oracle, and the round count is the single +//! degree of freedom the canonical vectors add. That is weaker than a direct +//! KAT and is recorded as such — but [`CANONICAL_VECTORS`] still discriminates +//! every convention a wrong port could get wrong, which the falsification tests +//! in `prover::lfm::blake3` demonstrate one convention at a time. +//! +//! [`chain`] extends the anchor considerably: at 7 rounds [`Blake3Chain`] is the +//! `blake3` crate's full hash for every message up to 1024 bytes, so the framing +//! — not just the round function — is externally checked. +//! +//! ⚠ Security assumption **A6R**: collision resistance of the 6-round variant +//! is a named, unratified assumption (#903's `IMPLEMENTATION.md`). Nothing here +//! ratifies it. + +pub mod chain; +mod vectors; + +pub use chain::{Blake3Chain, blake3_chain}; +pub use vectors::{CANONICAL_OUT_7ROUND, CANONICAL_VECTORS, Vector}; + +/// The BLAKE3 IV (identical to SHA-256's initial state). `IV[0..4]` seeds +/// `v[8..12]` of the compression working state. +pub const BLAKE3_IV: [u32; 8] = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +]; + +/// The BLAKE3 message-schedule permutation, applied between rounds +/// (`m'[i] = m[MSG_PERMUTATION[i]]`). +pub const BLAKE3_MSG_PERMUTATION: [usize; 16] = + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8]; + +/// Rounds of *standard* BLAKE3. At this value [`blake3_compress_rounds`] is +/// bit-for-bit the published compression function — the property the whole +/// external-anchor argument rests on, pinned by +/// `tests::seven_rounds_is_the_blake3_crate`. +pub const BLAKE3_STANDARD_ROUNDS: usize = 7; + +/// Rounds of the 6-round internal variant. Reachable only through the +/// `blake3-6round` feature; [`CANONICAL_VECTORS`] pin it unconditionally. +pub const BLAKE3_SIX_ROUNDS: usize = 6; + +/// The round count every BLAKE3 chip in this tree is compiled for — the +/// standalone `LFM_BLAKE3` probe and the `LFM_HASH` socket arm alike. They share +/// one knob deliberately: two would let a sweep leave the two chips describing +/// different hashes. +/// +/// **7 by default**, i.e. standard BLAKE3, which is what the A6R sign-off +/// instantiates. At 7 rounds the `blake3` crate is a direct known-answer test +/// for the primitive *and* for the socket, and no unratified assumption is +/// carried. `--features blake3-6round` selects the 6-round internal variant: +/// the measured performance variant, resting on **A6R**. +/// +/// It is a compile-time constant rather than a parameter because both chips' +/// column layouts are `8 · rounds` G-blocks wide and their width functions are +/// `const fn`. The round count is the ONLY thing it varies — the G function, the +/// message schedule, the counter split and the feed-forward are fixed — which is +/// what lets the 7-round anchor certify the whole code path rather than a +/// separate 7-round copy of it. +#[cfg(not(feature = "blake3-6round"))] +pub const BLAKE3_ROUNDS: usize = BLAKE3_STANDARD_ROUNDS; +#[cfg(feature = "blake3-6round")] +pub const BLAKE3_ROUNDS: usize = BLAKE3_SIX_ROUNDS; + +/// The BLAKE3 quarter-round G (spec §2.1). +#[inline] +fn blake3_g(v: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, mx: u32, my: u32) { + v[a] = v[a].wrapping_add(v[b]).wrapping_add(mx); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(12); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(my); + v[d] = (v[d] ^ v[a]).rotate_right(8); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(7); +} + +/// The BLAKE3 compression function `f` at 6 rounds (spec §2.2, oracle §2.4). +/// +/// State init: `v[0..8] = h`, `v[8..12] = IV[0..4]`, `v[12] = t as u32`, +/// `v[13] = (t >> 32) as u32`, `v[14] = block_len`, `v[15] = flags`. Six rounds +/// of 8 G-calls (4 columns then 4 diagonals), permuting the message schedule +/// between rounds (`r < rounds - 1`, i.e. 5 permutes — the trailing permute is +/// never consumed). Feed-forward: `out[i] = v[i] ^ v[i+8]`, +/// `out[i+8] = v[i+8] ^ h[i]`. The truncated chaining value is `out[0..8]`. +pub fn blake3_compress_6round( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> [u32; 16] { + blake3_compress_rounds(h, m, t, block_len, flags, BLAKE3_SIX_ROUNDS) +} + +/// [`blake3_compress_6round`] with the round count as an argument. +/// +/// The round count is the *only* parameter: everything else — the G function, +/// the message schedule, the counter split, the feed-forward — is fixed. That +/// is what makes `rounds = BLAKE3_STANDARD_ROUNDS` an external anchor for the +/// whole code path rather than for a separate 7-round copy of it, and it is why +/// this is one function with a loop bound instead of two functions. +pub fn blake3_compress_rounds( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, + rounds: usize, +) -> [u32; 16] { + let mut v: [u32; 16] = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + t as u32, + (t >> 32) as u32, + block_len, + flags, + ]; + + let mut m = *m; + for r in 0..rounds { + // Mix the columns. + blake3_g(&mut v, 0, 4, 8, 12, m[0], m[1]); + blake3_g(&mut v, 1, 5, 9, 13, m[2], m[3]); + blake3_g(&mut v, 2, 6, 10, 14, m[4], m[5]); + blake3_g(&mut v, 3, 7, 11, 15, m[6], m[7]); + // Mix the diagonals. + blake3_g(&mut v, 0, 5, 10, 15, m[8], m[9]); + blake3_g(&mut v, 1, 6, 11, 12, m[10], m[11]); + blake3_g(&mut v, 2, 7, 8, 13, m[12], m[13]); + blake3_g(&mut v, 3, 4, 9, 14, m[14], m[15]); + // Permute between rounds; the permute after the last round is never + // consumed (oracle: `r < rounds - 1`). + if r < rounds - 1 { + let prev = m; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + m[i] = prev[p]; + } + } + } + + let mut out = [0u32; 16]; + for i in 0..8 { + out[i] = v[i] ^ v[i + 8]; + out[i + 8] = v[i + 8] ^ h[i]; + } + out +} + +/// The 16-word output of `CANONICAL_VECTORS[i]` at the compiled-in +/// [`BLAKE3_ROUNDS`] — what a chip built from this module must produce. +pub const fn canonical_expected_out(i: usize) -> [u32; 16] { + if BLAKE3_ROUNDS == BLAKE3_STANDARD_ROUNDS { + CANONICAL_OUT_7ROUND[i] + } else { + CANONICAL_VECTORS[i].out + } +} diff --git a/crypto/crypto/src/hash/blake3/vectors.rs b/crypto/crypto/src/hash/blake3/vectors.rs new file mode 100644 index 000000000..450101ba1 --- /dev/null +++ b/crypto/crypto/src/hash/blake3/vectors.rs @@ -0,0 +1,285 @@ +//! The canonical known-answer vectors for [`super::blake3_compress_rounds`], +//! at both round counts. +//! +//! Provenance is recorded in the parent module's header: the 7-round table is +//! what the official `blake3` crate produces (and is checked against it +//! directly), and the 6-round table came from #903's Python oracle, whose +//! conventions the 7-round arm pins from outside. + +/// One canonical 6-round vector: the oracle's inputs and its 16-word output. +#[derive(Debug, Clone, Copy)] +pub struct Vector { + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, + pub out: [u32; 16], +} + +/// The 10 canonical 6-round vectors, transcribed verbatim from #903's +/// `thoughts/blake3/blake3-oracle/canonical_6round_vectors.json` (head +/// `89aeeb8c`). Seeds 0..9 of the oracle's generator; `t` is full-width, which +/// matters — the counter-split order is load-bearing and was behaviourally +/// verified against the official crate. +pub const CANONICAL_VECTORS: [Vector; 10] = [ + Vector { + h: [ + 0xD82C07CD, 0x6BAA9455, 0x82E2E662, 0x7A024204, 0xE87A1613, 0x81332876, 0x48268673, + 0xC17C6279, + ], + m: [ + 0xE6F4590B, 0x4F65D4D9, 0xBAD640FB, 0xAF19922A, 0x19C78DF4, 0x6F25E2A2, 0xE9BB17BC, + 0x7A1D5006, 0x42AF9FC3, 0x03983CA8, 0xDE1B372A, 0xDED733E8, 0x9148624F, 0xF7B0B7D2, + 0x72AE2244, 0xEECE328B, + ], + t: 0xB4E1357D4A84EB03, + block_len: 42, + flags: 52, + out: [ + 0xCED9D1FF, 0xC248EEAB, 0xBD109B7F, 0x911B48F6, 0x923D62C0, 0xD804903F, 0x5974223E, + 0xAA4F0C80, 0xAD61007F, 0xB50B8DDB, 0xE7372BE1, 0x33D3D6C3, 0x42AA284B, 0xC5A25F28, + 0x79AC8370, 0xB75F3915, + ], + }, + Vector { + h: [ + 0xC386BBC4, 0x414C343C, 0x7311D8A3, 0xA6CECC1B, 0xC9E9C616, 0x18072E8C, 0xD5F4B3B2, + 0x7204E52D, + ], + m: [ + 0xF1FD42A2, 0xE6C3F339, 0x07D4BEDC, 0x8A9A021E, 0x3BAB6C39, 0x05805975, 0xA46D6753, + 0xDC2574BD, 0xAB99254A, 0x4DA98F1D, 0xE1EA24C4, 0x815A47C5, 0x08D6AF57, 0xCC22AF58, + 0x2C4A3698, 0x5FEC898F, + ], + t: 0xC74803E31BA16215, + block_len: 50, + flags: 94, + out: [ + 0xF2A972E9, 0x81FDB8EC, 0x40C50EBC, 0x4BA1CAF9, 0x9EE9E930, 0x6B1A16B2, 0xE9156F47, + 0xA89FB436, 0xA2F616B3, 0x12874C12, 0x30768035, 0xE01A17D9, 0xBEE5C17C, 0xD61C0BE0, + 0x3041FF46, 0xDFB91125, + ], + }, + Vector { + h: [ + 0x0E7A269F, 0x15BA2BDD, 0xD5E34124, 0x4EE207F8, 0x9B1F282E, 0x9B575BD1, 0xF30B94FA, + 0x0706A045, + ], + m: [ + 0x6148A86F, 0x8697BBD0, 0x8F7D9B78, 0x3C729578, 0x061B9030, 0x533C9135, 0x829E07B0, + 0xE4C11AB2, 0xCBF87544, 0xC34C769F, 0x5A91C89B, 0xF63F23D0, 0xC1066932, 0x87C56473, + 0x7D718D73, 0xECC1CB63, + ], + t: 0x7604E4B4E73695C3, + block_len: 58, + flags: 124, + out: [ + 0x5AA6B114, 0xC9D6740C, 0x8738CAF4, 0xAC5F4B72, 0x9FC6B9DE, 0x3F2EFB8F, 0x8CB7A912, + 0xF497A285, 0x3D062266, 0x7F22380C, 0xAFD468FA, 0x122CBA80, 0x446B156D, 0xB239D8C2, + 0xC3EAB2CF, 0x775F2F92, + ], + }, + Vector { + h: [ + 0x8B529B4A, 0x9A9A80FD, 0xD6645FA9, 0x3BFD1D33, 0x79F248B0, 0x268ECC45, 0xA2863A7F, + 0x85EF3430, + ], + m: [ + 0xBDC2AE99, 0x10645D51, 0x97524D6A, 0xDD933160, 0xE0F9E038, 0xEBCD1F5E, 0xEF829C88, + 0xE0FD67DD, 0x18F2C41C, 0x22CEDAFB, 0x378C74DC, 0x4D100D8F, 0x95C76AB4, 0x95918694, + 0xE779C470, 0xEDCF6109, + ], + t: 0x92D3043AFCF249F3, + block_len: 36, + flags: 31, + out: [ + 0xEED92FAB, 0x138D9358, 0x915BFE3C, 0x13718B01, 0xB506E277, 0xBE4007CD, 0x35847E06, + 0xCE1C6896, 0x52FA01B5, 0x4AA26AF8, 0xB1078A61, 0x2C517AED, 0xA08867A0, 0xEA6ECFEA, + 0x6D33D3B0, 0xDC293166, + ], + }, + Vector { + h: [ + 0x3C6DA5D7, 0x656412A9, 0x27AC435A, 0x11072231, 0xEAFF1A09, 0xC3E1B258, 0x8963DC6E, + 0x1B2ED40E, + ], + m: [ + 0xED6F0B09, 0xCE80C4B0, 0xCCEA2645, 0x3184FF27, 0x4F5253A0, 0xE14B0190, 0x9B191BF4, + 0xABF4A07C, 0x81862FC9, 0x2D83A823, 0x793D0E45, 0x4CDCE7A6, 0xE8ABB93F, 0xE1DF8AF9, + 0x8224B122, 0x69F85E31, + ], + t: 0x49C7B59B995253FD, + block_len: 57, + flags: 41, + out: [ + 0xCA00BDA3, 0x84239A3A, 0xE7C88E6D, 0x33A8A3D6, 0x09DCD1CE, 0xA1B10212, 0xF48E1156, + 0x8F039915, 0x8A055EAA, 0xFF5B11D5, 0xB725085B, 0x2E1AB267, 0x6AE7323D, 0xB2FF6FA8, + 0x7102C8A1, 0x7561EB37, + ], + }, + Vector { + h: [ + 0x9F767C45, 0xBDE5C099, 0xF17FD374, 0xA6233255, 0xE6A16A3B, 0x1CFB10F6, 0x3F1F65A8, + 0x8B33E968, + ], + m: [ + 0x92EDCF45, 0x377B9AA2, 0x478C281D, 0xC4069545, 0xCC11D357, 0x9E115E4B, 0x206F5C66, + 0xDF1461AA, 0xFB7FF337, 0xDF561D80, 0x4A0FE75D, 0xF6236BF2, 0x346C6E2B, 0xB0CDE917, + 0xE4CC4132, 0x4C7D6DF0, + ], + t: 0x6A3753915C76F18A, + block_len: 18, + flags: 67, + out: [ + 0x14A9F66F, 0x101BDFE8, 0x9B0A50DD, 0xEE4BB45B, 0x7A914502, 0x77B3486B, 0x59BFC114, + 0xA1AD2AFD, 0xC194DDE6, 0x894EC54D, 0xAD36C805, 0x9018F3F5, 0x165AF5D8, 0x3E85B598, + 0x78E76653, 0xBB7A485D, + ], + }, + Vector { + h: [ + 0xD26B9496, 0x42F9A039, 0x001D9A88, 0x5F877031, 0xC527E279, 0x45CF8AA4, 0xCD4A5557, + 0xAE9AF169, + ], + m: [ + 0xAF895F5B, 0xD822E2F9, 0x17D7AB26, 0xCCDF540B, 0xCE06294D, 0x4A8B0188, 0xF38D2E64, + 0x5C41D5C5, 0xE8D5B9E3, 0x5C832A51, 0x9A0C1B76, 0x4DE8344E, 0x96D2F9E0, 0x8677A5F2, + 0xA9A967C1, 0x323BBEAF, + ], + t: 0x390567C27BD6AA42, + block_len: 26, + flags: 3, + out: [ + 0x32A6FF70, 0xC30560BC, 0xD1C777C8, 0xF1871821, 0x7207AB54, 0x9F5B83C7, 0xB6561C5D, + 0x991E738F, 0xB38B62B9, 0x0EF6D156, 0x994BECB1, 0x09A85D0E, 0x32221741, 0xADA3CC5F, + 0x5B654ED6, 0x2A7A62B2, + ], + }, + Vector { + h: [ + 0x269E0D37, 0xA6A3A450, 0x892F902B, 0x81E74EF5, 0x099950D8, 0x6F03675A, 0x11E20B8F, + 0x6CAD4A26, + ], + m: [ + 0xF29D0DA9, 0x658CDA14, 0xF9EBDACC, 0xDBC496CB, 0x4A23D596, 0x2E44158B, 0xA38FD547, + 0x5F557203, 0x34B9B5DF, 0x506BF2EF, 0x7403E430, 0x4CBD87AD, 0xCB5C7427, 0x3E7D1BFB, + 0x930D6EAF, 0x86734721, + ], + t: 0x12BD4ACEFAECBD38, + block_len: 53, + flags: 42, + out: [ + 0xA632AD45, 0x12CE41F4, 0xD21B2CBD, 0x76795C62, 0x6BEC36C1, 0xDAFAFCDE, 0x53CA87B7, + 0x92E8465B, 0x7B424F5D, 0xE1E6AD7F, 0x753BA387, 0xCCC50824, 0x69AEDF6D, 0xBBBBF253, + 0x78D04883, 0xF3F33689, + ], + }, + Vector { + h: [ + 0x3A096533, 0xF658F7A7, 0x205738D1, 0xB46EE1DA, 0x15CEB3A1, 0x359B1548, 0xA4517D6C, + 0x7589CA4A, + ], + m: [ + 0x74007CB4, 0xD49D0AC1, 0x16EDC5D4, 0x685CA8AF, 0x4223AA56, 0x10269470, 0x60908405, + 0xA92D04A3, 0x56A3E957, 0xB0F91306, 0xE6C08269, 0xF2306D4A, 0x31A06A7C, 0x9436D6F6, + 0xE18692E2, 0xE0C99F3E, + ], + t: 0x329911DA9FBD8735, + block_len: 19, + flags: 91, + out: [ + 0x913B2AE1, 0xC7F73082, 0x45E1C023, 0x6F1F3F82, 0x20AEE6F5, 0xDAF21D94, 0xF2C1E4AF, + 0xD4F7D4AC, 0x44A45F87, 0xF4C40CE5, 0x613E9B94, 0x08CE53DE, 0x4FF07AA4, 0x456BF2E2, + 0x2066EA7F, 0x3C5A654B, + ], + }, + Vector { + h: [ + 0x5F915EF0, 0x237751AA, 0x01A5BA50, 0x80B65386, 0x14B044D7, 0x61076DC3, 0xB99DE255, + 0x283B73A6, + ], + m: [ + 0x3CEE5E2C, 0x1C670EA9, 0x972651DA, 0x4A8AA593, 0xAC9ABB0C, 0x35BB5C11, 0x47FBB3B4, + 0xCF3C17E5, 0xE2EB17C8, 0xE11E99FB, 0x7DE0D208, 0x0602FE0C, 0x98CAE043, 0x9425B3E2, + 0x33FB4B4F, 0x15607DF9, + ], + t: 0xEAEB999B8A2E547E, + block_len: 64, + flags: 21, + out: [ + 0xF5EE9114, 0x856CABB8, 0x29BE2CF1, 0x603BE91C, 0x94A7DD0E, 0x28FC3E27, 0xB64E2CC8, + 0x2D2C67FF, 0x69FAC1BA, 0x0C949090, 0xD68DE435, 0xCE91A527, 0xE80C1815, 0x6D44EFE6, + 0x87C7B175, 0xD18A8B94, + ], + }, +]; + +/// The same ten inputs as [`CANONICAL_VECTORS`], at **7 rounds** — that is, +/// under standard BLAKE3's compression function. +/// +/// Provenance, and it is a rung stronger than the 6-round table's: these were +/// emitted by the gate-oracle's independently-written Python reference +/// (`thoughts/shared/lfm-real-hash/gate-oracle/blake3_oracle.py`) at +/// `rounds = 7` and cross-checked word-for-word against the second in-repo +/// reference (`thoughts/blake3/blake3-oracle/blake3_ref.py`) — two +/// implementations, agreeing on all ten. Both references' 7-round paths are +/// themselves pinned by the OFFICIAL BLAKE3 test vectors, so unlike +/// [`CANONICAL_VECTORS`] this table has an external anchor rather than one a +/// step removed. The same generation run re-derived the 6-round table and +/// reproduced it 10/10, which is what ties the two together. +/// +/// Only the outputs are stored: the inputs are [`CANONICAL_VECTORS`]'s, and +/// duplicating them would be a second place for them to drift. +pub const CANONICAL_OUT_7ROUND: [[u32; 16]; 10] = [ + [ + 0xEE79E5DC, 0xEA647B8C, 0x964C097E, 0xE2F3383A, 0xFE2E6D00, 0x78EE613A, 0xC33C8572, + 0xCD444391, 0x0C890604, 0xC3209591, 0x45633FF8, 0xCB171C6A, 0x760247AE, 0xF6D0FC1E, + 0xCD550F20, 0xCD54BF83, + ], + [ + 0xD68593D0, 0xDBC8157A, 0xF6E1687C, 0x52A60555, 0xB56D418A, 0x0CCBB863, 0xADBFB51E, + 0x8BF7D125, 0x75C23432, 0xF484D7A6, 0x06E85F4A, 0x2771FE96, 0x00F6E24D, 0x48368A3E, + 0x04EE7E88, 0x501D8539, + ], + [ + 0xBC92D7C4, 0x56542092, 0x3490E2CB, 0x2E3328CD, 0x13E3746F, 0xA5B88E66, 0x2B5FE530, + 0x92C7AD52, 0xFF502AE5, 0x1F088FBF, 0x9163752F, 0x8A0C8B4D, 0xB557B0E8, 0xE76F23CB, + 0xD054C959, 0x74813CFD, + ], + [ + 0xCF4FB929, 0x1DBADE2A, 0x70E63AAF, 0x2E0FFB48, 0x60123045, 0x798AEAE8, 0x5A911D30, + 0x15977C61, 0x6F7C8334, 0x5EB0BCE2, 0xAB240F17, 0x66B7A3CD, 0xA9064E0B, 0x6AC4747B, + 0x1206F62B, 0x9F3E91EC, + ], + [ + 0xFF525F0F, 0xD892E3D2, 0xFB566B40, 0x3BDF4ED0, 0x78B961CD, 0x9CB86B48, 0x6AB54F3D, + 0x3EF5F695, 0xBD896ED8, 0x6265AC08, 0xF6695D78, 0x9F3795EA, 0x943E0342, 0xD1437B3B, + 0x4F6BAF78, 0x85DFD2C9, + ], + [ + 0xD22912BB, 0x627F992C, 0xE883AF5D, 0x50E58A48, 0xF3D071C6, 0xB20D47A4, 0x29011151, + 0xFE50E232, 0x594B76A3, 0x8706296B, 0x2C1D1E31, 0x6A478D0D, 0x64004E61, 0xA072DA1E, + 0xAB3FCA42, 0x09BB269E, + ], + [ + 0xA101CEAB, 0x9232E0EC, 0x2FE4B24E, 0x35F7F4FE, 0x61A5AB42, 0xBE417503, 0xEB740D5E, + 0x8BB2FE96, 0xC6863DA9, 0x1F31FF5D, 0x5763EA12, 0xDC862699, 0x1A60ADE2, 0x9E3E6745, + 0xE3C8F87E, 0xD3EFB0EA, + ], + [ + 0x318604BE, 0x22A35843, 0x6CA63195, 0xA2E7E2F8, 0x48769A04, 0xC462F1E3, 0x5CF053C7, + 0xFD1EE629, 0x69366332, 0x0ACC819B, 0xBBD2456A, 0xF1DA9DB6, 0x4A7B7D68, 0x6DD1A843, + 0x61555466, 0xBDA36F28, + ], + [ + 0x87584719, 0x15C73090, 0x851C1A4A, 0x99D21014, 0x821A82A8, 0xC7307CD5, 0x6797EFE2, + 0xCF38CEDF, 0x777C177D, 0x202BE3EA, 0x19421985, 0x3176132D, 0x7BB8BC22, 0x65C9804B, + 0x22C68EA3, 0x92504162, + ], + [ + 0xDC60D189, 0xE6311F18, 0x9DC3E078, 0x304BB43E, 0x5C616E7D, 0xE168D00F, 0x2E197872, + 0x175B9188, 0x5A99C462, 0xEF311A88, 0xC61836FD, 0x9FFD4DE3, 0x36AE4940, 0x4D813D81, + 0x9B058DA9, 0x9017D38C, + ], +]; diff --git a/crypto/crypto/src/hash/mod.rs b/crypto/crypto/src/hash/mod.rs index 78f89fca3..9099dc26b 100644 --- a/crypto/crypto/src/hash/mod.rs +++ b/crypto/crypto/src/hash/mod.rs @@ -1,3 +1,5 @@ +pub mod blake3; +pub mod platform_blake3; pub mod platform_keccak; pub mod poseidon; pub mod sha3; diff --git a/crypto/crypto/src/hash/platform_blake3.rs b/crypto/crypto/src/hash/platform_blake3.rs new file mode 100644 index 000000000..731f4334e --- /dev/null +++ b/crypto/crypto/src/hash/platform_blake3.rs @@ -0,0 +1,35 @@ +//! [`PlatformBlake3`] — the BLAKE3 byte hash, under the name shape +//! [`PlatformKeccak256`](crate::hash::platform_keccak::PlatformKeccak256) +//! established for a hash that is accelerated on the riscv64 guest and software +//! everywhere else. +//! +//! # This is a re-export, and that is the design +//! +//! `platform_keccak` needs an adapter because the thing it selects between is +//! two different types: `sha3::Keccak256` on the host and a syscall-backed +//! sponge from the syscall crate on the guest. Those two carry their own +//! framing, so something has to give them one `digest` interface. +//! +//! BLAKE3 has no such pair. [`Blake3Chain`](crate::hash::blake3::Blake3Chain) is +//! one type on every target; the accelerator is reached from *inside* it, at the +//! compression function, where `compress_block` selects the syscall on riscv64 +//! and software otherwise. The framing — single chunk, 64-byte blocks, +//! `CHUNK_START` / `CHUNK_END | ROOT`, `t = 0`, the true byte count as the final +//! block's `block_len` (PA-PLAN §1.7) — is above that seam and is therefore the +//! same code on host and guest by construction. +//! +//! INVARIANT (load-bearing): this must remain a **PURE PASSTHROUGH** — a +//! re-export and nothing else. A wrapper type here would be a second place the +//! framing is expressed, which is exactly what PA-PLAN §1.4 forbids and what +//! `executor::vm::instruction::execution`'s duplicate compression already cost +//! us one gating test to contain. It would also break the argument the `TypeId` +//! specializations in `merkle_tree::backends::field_element_vector` rest on: +//! they dispatch on the concrete `PlatformKeccak256` type, so a BLAKE3 `D` +//! reaches the generic `D::new()/update/finalize` path — correct only while +//! `PlatformBlake3` *is* `Blake3Chain` and hashes identically through both. +//! +//! The round count is not selected here either. It is +//! [`BLAKE3_ROUNDS`](crate::hash::blake3::BLAKE3_ROUNDS), one crate-global knob, +//! so a build cannot commit under two hashes. + +pub use crate::hash::blake3::chain::Blake3Chain as PlatformBlake3; diff --git a/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs b/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs index 6d0cc6491..560cb5852 100644 --- a/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs +++ b/crypto/crypto/src/merkle_tree/backends/field_element_vector.rs @@ -1,7 +1,7 @@ use core::marker::PhantomData; use crate::hash::poseidon::Poseidon; -use crate::merkle_tree::traits::IsMerkleTreeBackend; +use crate::merkle_tree::traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}; use alloc::vec::Vec; use digest::{Digest, Output}; use math::{ @@ -202,6 +202,31 @@ where } } +/// Exposes the streaming leaf routes to callers that reach this backend through +/// a commitment configuration rather than by name. Both bodies go through +/// [`hash_streamed`], which is where the absorbed byte layout is defined, so +/// they agree with `hash_data` by construction. +impl IsStreamingLeafBackend + for FieldElementVectorBackend +where + F: IsField, + FieldElement: AsBytes, + [u8; NUM_BYTES]: From>, + Vec>: Sync + Send, +{ + fn hash_bytes(data: &[u8]) -> [u8; NUM_BYTES] { + hash_streamed::(|sink| sink(data)) + } + + fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> [u8; NUM_BYTES] { + hash_streamed::(|sink| { + for element in a.iter().chain(b.iter()) { + element.stream_bytes(sink); + } + }) + } +} + #[derive(Clone, Default)] pub struct BatchPoseidonTree { _poseidon: PhantomData

, diff --git a/crypto/crypto/src/merkle_tree/backends/types.rs b/crypto/crypto/src/merkle_tree/backends/types.rs index 2384fda3a..35ebb207c 100644 --- a/crypto/crypto/src/merkle_tree/backends/types.rs +++ b/crypto/crypto/src/merkle_tree/backends/types.rs @@ -1,15 +1,33 @@ +use crate::hash::blake3::Blake3Chain; use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256; -use super::{ - field_element::FieldElementBackend, - field_element_vector::{FieldElementPairBackend, FieldElementVectorBackend}, -}; - -// Field element backend definitions -pub type Keccak256Backend = FieldElementBackend; +use super::field_element_vector::{FieldElementPairBackend, FieldElementVectorBackend}; // Vector of field elements backend definitions pub type BatchKeccak256Backend = FieldElementVectorBackend; // Fixed-size pair backends (more efficient for FRI layers) pub type PairKeccak256Backend = FieldElementPairBackend; + +/// The BLAKE3 batched-leaf backend, over [`Blake3Chain`] — the single-chunk +/// chain specified in PA-PLAN §1.7. +/// +/// It is the *same* generic backend the keccak alias is, with the digest +/// swapped, and that is load-bearing rather than an economy. The leaf byte +/// layout, both streaming routes and the parent framing then have one definition +/// each (`field_element_vector.rs`), so the batched and paired families cannot +/// encode a two-element leaf differently: the invariant `stark::config::StarkHash` +/// requires holds because they are one function, not because two implementations +/// were shown to coincide. +/// +/// A parent is `Blake3Chain` over the two concatenated 32-byte nodes — 64 bytes, +/// so one compression with `h = IV`, `t = 0`, `block_len = 64`, `flags = +/// CHUNK_START | CHUNK_END | ROOT`. That is bit-for-bit what the device kernel +/// computes (`math-cuda/kernels/blake3.cu`, `blake3_hash_merkle_parent`), which +/// is what will let a GPU tree and a CPU tree be the same tree once the device +/// leaf kernels land. +pub type BatchBlake3Backend = FieldElementVectorBackend; + +/// The FRI-layer twin of [`BatchBlake3Backend`] — one leaf per fixed pair, no +/// `Vec` per leaf. See there. +pub type PairBlake3Backend = FieldElementPairBackend; diff --git a/crypto/crypto/src/merkle_tree/traits.rs b/crypto/crypto/src/merkle_tree/traits.rs index c09cff9d0..ceae91d95 100644 --- a/crypto/crypto/src/merkle_tree/traits.rs +++ b/crypto/crypto/src/merkle_tree/traits.rs @@ -1,4 +1,7 @@ use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::traits::IsField; +use math::traits::AsBytes; #[cfg(feature = "parallel")] use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; @@ -27,3 +30,30 @@ pub trait IsMerkleTreeBackend { /// It will be used in the construction of the Merkle tree. fn hash_new_parent(child_1: &Self::Node, child_2: &Self::Node) -> Self::Node; } + +/// A leaf backend that can hash a leaf without being handed one. +/// +/// [`IsMerkleTreeBackend::hash_data`] takes a `&Self::Data`, which for the +/// batched backends is a `Vec>`. Building one per leaf costs an +/// allocation per leaf — millions on a real trace — so the prover and verifier +/// never do: they serialize into a reused buffer, or hold two slices they want +/// hashed as if concatenated. These are the two shapes they use. +/// +/// Both must agree with `hash_data` on the bytes they absorb, so a leaf hashed +/// through either route is the leaf the tree was built from. That is the whole +/// contract, and it is why these live on a trait rather than staying inherent +/// methods on one concrete backend: a commitment configuration that names its +/// leaf backend generically still has to reach them. +pub trait IsStreamingLeafBackend: IsMerkleTreeBackend +where + F: IsField, + FieldElement: AsBytes, +{ + /// Hash a pre-serialized leaf buffer. Equals `hash_data` applied to the + /// elements `data` encodes, in that order. + fn hash_bytes(data: &[u8]) -> Self::Node; + + /// Hash `a ‖ b` without materializing the concatenation. Equals + /// `hash_data(&[a, b].concat())`. + fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> Self::Node; +} diff --git a/crypto/crypto/src/tests/blake3_transcript_tests.rs b/crypto/crypto/src/tests/blake3_transcript_tests.rs new file mode 100644 index 000000000..78523120d --- /dev/null +++ b/crypto/crypto/src/tests/blake3_transcript_tests.rs @@ -0,0 +1,268 @@ +//! The BLAKE3 Fiat-Shamir configuration, and rider 1's fixed consumption +//! schedule. +//! +//! Two things are under test and they are independent: that the transcript's +//! sponge is `Blake3Chain` in the framing the keccak one has always used, and +//! that a field draw under this configuration consumes a *fixed* number of +//! candidates. The second is what a straight-line machine needs — the LFM +//! transcript replay encodes one consumption schedule, so a draw whose count +//! varies with the bytes it happened to see is unprovable against it. + +use alloc::vec::Vec; +use digest::Digest; +use math::field::{ + element::FieldElement, extensions_goldilocks::Degree3GoldilocksExtensionField, + goldilocks::GoldilocksField, traits::HasDefaultTranscript, +}; + +use crate::fiat_shamir::default_transcript::{ + Blake3Transcript, DefaultTranscript, candidate_under_fixed_schedule, +}; +use crate::fiat_shamir::is_transcript::IsTranscript; +use crate::fiat_shamir::transcript_hash::{ + Blake3TranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; +use crate::hash::blake3::chain::Blake3Chain; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; + +const SEED: &[u8] = b"lambda-vm-blake3-transcript-kat-v1"; + +// ========================================================================= +// The sponge is Blake3Chain, in the framing the keccak transcript uses. +// ========================================================================= + +/// ★ Anchor: the transcript's state is the chain hash of exactly what was +/// absorbed. +/// +/// `state()` is not an internal detail — it is the grinding seed. Pinning it +/// against `Blake3Chain` computed directly is what says the transcript absorbs +/// what it claims to, and `Blake3Chain` is in turn anchored from outside (its +/// 7-round arm reproduces the `blake3` crate at all 1025 lengths ≤ 1 chunk, and +/// its 6-round arm has the committed `CHAIN_KAT_6ROUND` table). +#[test] +fn the_blake3_transcript_state_is_the_chain_of_what_was_absorbed() { + let mut t = Blake3Transcript::::new(SEED); + assert_eq!( + t.state(), + <[u8; 32]>::from(Blake3Chain::digest(SEED)), + "a fresh transcript's state must be the chain hash of its seed" + ); + + t.append_bytes(b"a-merkle-root"); + let mut expected = Vec::from(SEED); + expected.extend_from_slice(b"a-merkle-root"); + assert_eq!( + t.state(), + <[u8; 32]>::from(Blake3Chain::digest(&expected)), + "absorbing must concatenate into the same chain, not reset it" + ); +} + +/// The duplex squeeze is the same construction under the new digest: finalize +/// **and reset**, reverse, absorb the reversed output. +/// +/// Reimplemented here rather than compared against itself — this is the one +/// place where a hash swap could silently drop the reverse-and-reabsorb step, +/// and prover and verifier would still agree with each other while producing a +/// transcript nobody else can reproduce. +/// +/// Note the reset: the squeeze is `finalize_reset`, so squeeze `k+1` hashes the +/// reversed output of squeeze `k` **alone**, not the whole absorbed history. +/// That is what makes the sponge a chain rather than a growing buffer, and it +/// is the detail this test exists to pin. +#[test] +fn the_blake3_squeeze_chain_matches_the_construction() { + let mut t = Blake3Transcript::::new(SEED); + + let mut pending = Vec::from(SEED); + for k in 0..3 { + let mut expected = <[u8; 32]>::from(Blake3Chain::digest(&pending)); + expected.reverse(); + assert_eq!(t.sample(), expected, "squeeze {k}"); + pending = Vec::from(expected); + } +} + +/// CONTROL: the two configurations are actually different transcripts. +/// +/// Without this, every test here would pass just as well if `Blake3Transcript` +/// had been left resolving to keccak. +#[test] +fn the_blake3_and_keccak_transcripts_diverge() { + let mut blake3 = Blake3Transcript::::new(SEED); + let mut keccak = DefaultTranscript::::new(SEED); + assert_ne!(blake3.state(), keccak.state()); + assert_ne!(blake3.sample(), keccak.sample()); +} + +// ========================================================================= +// Rider 1 — the consumption schedule. +// ========================================================================= + +/// The configurations' schedules, as a fact rather than as prose. +/// +/// The keccak arm MUST stay `None`. Rider 1 is adopted for BLAKE3 only, because +/// changing the keccak schedule would move every existing proof's challenges — +/// the one thing P-a's staging holds still until the flip. +#[test] +fn only_the_blake3_configuration_takes_the_fixed_schedule() { + assert!( + KeccakTranscriptHash::CANDIDATES_PER_COORDINATE.is_none(), + "the keccak schedule must not move before the flip" + ); + assert_eq!( + Blake3TranscriptHash::CANDIDATES_PER_COORDINATE.map(|n| n.get()), + Some(2) + ); +} + +/// ★ Rider 1's whole content: the draw consumes exactly `n` candidates, +/// wherever the acceptable one sits — including when there is none. +/// +/// Counting the calls is the only way to see this; the returned value cannot +/// distinguish "took the first and stopped" from "took the first and kept +/// drawing", and it is the *stopping* that a straight-line machine cannot +/// follow. +#[test] +fn a_fixed_schedule_draw_consumes_exactly_n_candidates() { + // `p = 2^64 - 2^32 + 1`, so anything ≥ p is rejected. `u64::MAX` is. + let out_of_range = u64::MAX; + assert!(!F::candidate_in_range(out_of_range)); + let in_range = [7u64, 11, 13, 17]; + for c in in_range { + assert!(F::candidate_in_range(c)); + } + + for n in 1..=4usize { + for hit in 0..n { + // A stream whose only in-range value sits at position `hit`. + let stream: Vec = (0..n) + .map(|i| if i == hit { in_range[0] } else { out_of_range }) + .collect(); + let mut calls = 0usize; + let mut it = stream.iter(); + let got = candidate_under_fixed_schedule::(n, || { + calls += 1; + *it.next().expect("the schedule must not overdraw") + }); + assert_eq!( + calls, n, + "n={n}, acceptable candidate at {hit}: the draw must consume exactly n" + ); + assert_eq!(got, in_range[0], "it must return the acceptable candidate"); + } + + // No acceptable candidate: still exactly `n`, and the value handed back + // is one the field rejects, so its own loop draws another full `n`. + let mut calls = 0usize; + let got = candidate_under_fixed_schedule::(n, || { + calls += 1; + out_of_range + }); + assert_eq!(calls, n, "n={n}, no acceptable candidate: still exactly n"); + assert!( + !F::candidate_in_range(got), + "the fallback must NOT be reduced into range — a modular fallback \ + would bias challenges by ~2^-32, which at this system's security \ + level would dominate the soundness error" + ); + } +} + +/// ★ The schedule as the transcript actually runs it: an extension-field draw +/// takes SIX candidates from the squeeze stream, two per coordinate. +/// +/// Reconstructed from the raw squeezes, so it distinguishes the fixed schedule +/// from the unbounded one: under `None` the coordinates would be candidates +/// 0, 1, 2 of the stream, and under `Some(2)` they are the first acceptable of +/// (0,1), (2,3), (4,5). +#[test] +fn an_extension_draw_consumes_two_candidates_per_coordinate() { + // The raw candidate stream this transcript will hand out, taken from a + // clone so the transcript under test is untouched. + let candidates: Vec = { + let mut probe = Blake3Transcript::::new(SEED); + let mut out = Vec::new(); + for _ in 0..2 { + let squeeze = probe.sample(); + for chunk in squeeze.chunks_exact(8) { + out.push(u64::from_be_bytes(chunk.try_into().unwrap())); + } + } + out + }; + assert_eq!(candidates.len(), 8); + + let pick = |a: u64, b: u64| { + if F::candidate_in_range(a) { a } else { b } + }; + let expected = [ + pick(candidates[0], candidates[1]), + pick(candidates[2], candidates[3]), + pick(candidates[4], candidates[5]), + ]; + + let mut t = Blake3Transcript::::new(SEED); + let drawn = t.sample_field_element(); + let coords: Vec = drawn.value().iter().map(|c| *c.value()).collect(); + assert_eq!( + coords, + expected.to_vec(), + "each coordinate must be the first acceptable of its OWN pair" + ); + + // NEGATIVE CONTROL: it is not the unbounded schedule, which would take one + // candidate per coordinate and so read 0, 1, 2. + let unbounded = [candidates[0], candidates[1], candidates[2]]; + assert_ne!( + coords, + unbounded.to_vec(), + "the fixed schedule must be distinguishable from the unbounded one" + ); +} + +/// The keccak configuration still draws one candidate per coordinate. +/// +/// The honest-path partner of the test above: it says the branch is a branch, +/// and that the default side of it did not move. +#[test] +fn the_keccak_extension_draw_still_takes_one_candidate_per_coordinate() { + let candidates: Vec = { + let mut probe = DefaultTranscript::::new(SEED); + let squeeze = probe.sample(); + squeeze + .chunks_exact(8) + .map(|c| u64::from_be_bytes(c.try_into().unwrap())) + .collect() + }; + + let mut t = DefaultTranscript::::new(SEED); + let coords: Vec = t + .sample_field_element() + .value() + .iter() + .map(|c| *c.value()) + .collect(); + assert_eq!( + coords, + candidates[..3].to_vec(), + "the keccak draw must still be one candidate per coordinate" + ); +} + +/// A transcript is still deterministic under the fixed schedule — the property +/// every replaying verifier depends on. +#[test] +fn the_blake3_transcript_replays_identically() { + let mut a = Blake3Transcript::::new(SEED); + let mut b = Blake3Transcript::::new(SEED); + a.append_bytes(b"round-1"); + b.append_bytes(b"round-1"); + + let draw_a: Vec> = (0..8).map(|_| a.sample_field_element()).collect(); + let draw_b: Vec> = (0..8).map(|_| b.sample_field_element()).collect(); + assert_eq!(draw_a, draw_b); + assert_eq!(a.sample_u64(1 << 20), b.sample_u64(1 << 20)); +} diff --git a/crypto/crypto/src/tests/mod.rs b/crypto/crypto/src/tests/mod.rs index 96bf36e92..8273bfa05 100644 --- a/crypto/crypto/src/tests/mod.rs +++ b/crypto/crypto/src/tests/mod.rs @@ -1,3 +1,4 @@ +pub mod blake3_transcript_tests; pub mod default_transcript_tests; pub mod field_element_tests; pub mod field_element_vector_tests; diff --git a/crypto/math-cuda/Cargo.toml b/crypto/math-cuda/Cargo.toml index 2304af398..4de0fe896 100644 --- a/crypto/math-cuda/Cargo.toml +++ b/crypto/math-cuda/Cargo.toml @@ -41,8 +41,19 @@ test-faults = [] # stark/prover layers). Zero-cost when disabled; when enabled but # libnvToolsExt is absent at runtime, every call is a cheap no-op. nvtx = ["dep:libloading"] +# Compile `kernels/blake3.cu` for the 6-round internal BLAKE3 variant instead of +# the 7-round standard one. Same polarity as the host tree's `blake3-6round` +# (default = 7), and it has to be set in lockstep with it: they are separate +# crates' features and a mismatch is a GPU tree committing under a different hash +# than the CPU one. `blake3::device_rounds` reads the cubin's round count back so +# that is assertable rather than discoverable — see +# `tests/blake3_compress_parity.rs`. +blake3-6round = [] [dev-dependencies] +# 7-round-only, so it can anchor the standard arm of the BLAKE3 kernels and +# nothing else — the same dev-only role it has in `prover/Cargo.toml`. +blake3 = { version = "1.8.5", default-features = false, features = ["std", "pure"] } crypto = { path = "../crypto" } rand = { version = "0.8.5", features = ["std"] } rand_chacha = "0.3.1" diff --git a/crypto/math-cuda/build.rs b/crypto/math-cuda/build.rs index fbd70eb5b..17b82d057 100644 --- a/crypto/math-cuda/build.rs +++ b/crypto/math-cuda/build.rs @@ -72,7 +72,7 @@ fn to_real_arch(arch: &str) -> String { } } -fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool) { +fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool, defines: &[&str]) { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let src_path = manifest_dir.join("kernels").join(src); @@ -118,6 +118,7 @@ fn compile_kernel(src: &str, out_name: &str, have_nvcc: bool) { let mut cmd = Command::new(nvcc_path()); cmd.args(["--cubin", "-O3", "-std=c++17", "-arch", &arch]); + cmd.args(defines); // SASS→source line mapping for Nsight Compute. Unlike -G this does not // change codegen, but keep it opt-in so production cubins stay byte-stable. if env::var("LAMBDA_VM_NVCC_LINEINFO").is_ok_and(|v| v != "0" && !v.is_empty()) { @@ -157,13 +158,31 @@ fn main() { ); } - compile_kernel("arith.cu", "arith.cubin", have_nvcc); - compile_kernel("ntt.cu", "ntt.cubin", have_nvcc); - compile_kernel("keccak.cu", "keccak.cubin", have_nvcc); - compile_kernel("barycentric.cu", "barycentric.cubin", have_nvcc); - compile_kernel("deep.cu", "deep.cubin", have_nvcc); - compile_kernel("fri.cu", "fri.cubin", have_nvcc); - compile_kernel("inverse.cu", "inverse.cubin", have_nvcc); - compile_kernel("logup.cu", "logup.cubin", have_nvcc); - compile_kernel("constraint_interp.cu", "constraint_interp.cubin", have_nvcc); + compile_kernel("arith.cu", "arith.cubin", have_nvcc, &[]); + compile_kernel("ntt.cu", "ntt.cubin", have_nvcc, &[]); + compile_kernel("keccak.cu", "keccak.cubin", have_nvcc, &[]); + compile_kernel("barycentric.cu", "barycentric.cubin", have_nvcc, &[]); + compile_kernel("deep.cu", "deep.cubin", have_nvcc, &[]); + compile_kernel("fri.cu", "fri.cubin", have_nvcc, &[]); + compile_kernel("inverse.cu", "inverse.cubin", have_nvcc, &[]); + compile_kernel("logup.cu", "logup.cubin", have_nvcc, &[]); + compile_kernel( + "constraint_interp.cu", + "constraint_interp.cubin", + have_nvcc, + &[], + ); + // The BLAKE3 kernels' round count is a compile-time knob with the same + // polarity as the host tree's `blake3-6round` feature: 7 rounds (standard + // BLAKE3) unless the feature selects the 6-round variant. The `.cu` defaults + // to 7 on its own, so a stale `-D` can never silently pick 6. + // `CARGO_FEATURE_*`, not `cfg!(feature = ..)`: cargo passes a build script + // the active features as environment variables and does NOT cfg them into + // its compilation, so the `cfg!` form here would silently always be false. + let blake3_defines: &[&str] = if env::var_os("CARGO_FEATURE_BLAKE3_6ROUND").is_some() { + &["-DBLAKE3_ROUNDS=6"] + } else { + &[] + }; + compile_kernel("blake3.cu", "blake3.cubin", have_nvcc, blake3_defines); } diff --git a/crypto/math-cuda/kernels/blake3.cu b/crypto/math-cuda/kernels/blake3.cu new file mode 100644 index 000000000..3b30e25f6 --- /dev/null +++ b/crypto/math-cuda/kernels/blake3.cu @@ -0,0 +1,740 @@ +// BLAKE3 compression on device, round-count parameterized, plus the Merkle +// parent compressors and the field-element byte serialization the leaf kernels +// share with the CPU commit path. +// +// THE PARITY REFERENCE is the host `blake3_compress_rounds(h, m, t, block_len, +// flags, rounds)` in `prover/src/lfm/blake3.rs:125` — one function whose ONLY +// parameter is the round count. `blake3_compress` below is a +// transcription of it and must agree bit-for-bit at both 6 and 7 rounds; at 7 +// rounds both are standard BLAKE3, so the `blake3` crate anchors the pair from +// outside this tree. (P-a Stage 1 moves the host reference down into +// `crypto/crypto`; nothing here changes when it does.) +// +// ROUND COUNT is a compile-time knob with the same polarity as the host's +// `BLAKE3_ROUNDS`: 7 (standard BLAKE3) by default, 6 when build.rs passes +// `-DBLAKE3_ROUNDS=6` under math-cuda's `blake3-6round` feature. The two knobs +// are separate crates' features and nothing forces them equal — see +// `blake3_rounds_probe`, which exports this cubin's round count so a caller can +// assert the match rather than discover it as a wrong commitment. +// +// THE CHAINING CONSTRUCTION is `Blake3Chain`, specified in PA-PLAN §1.7 and +// implemented on host at `crypto/crypto/src/hash/blake3/chain.rs`: standard +// BLAKE3 restricted to a single chunk that never ends. `t = 0` on every block, +// CHUNK_START on the first, CHUNK_END|ROOT and the true byte count as +// `block_len` on the last, digest = the low 8 output words little-endian. The +// device `Blake3Chain` below is a transcription of that host type, and every +// leaf kernel here streams its message through one. +// +// ⚠ The construction is a DRAFT pending ratification of forks F1-F3 +// (PA-PLAN §1.7.3): `t = 0` throughout rather than a block counter, the +// three-flag schedule rather than one constant, and no leaf/parent domain +// separation. It is implemented as the working default by standing decision. If +// a fork is ratified the other way, the change lands in `Blake3Chain::finalize` +// and `compress_pending` — the leaf kernels themselves do not move. + +#include +#include "goldilocks.cuh" + +// 7 = standard BLAKE3. Overridden to 6 by build.rs; see the header comment. +#ifndef BLAKE3_ROUNDS +#define BLAKE3_ROUNDS 7 +#endif + +// The BLAKE3 IV (= SHA-256's initial state). Mirror of `BLAKE3_IV` +// (`blake3.rs:46`). `IV[0..4]` also seeds `v[8..12]` of the working state. +__device__ __constant__ uint32_t BLAKE3_IV[8] = { + 0x6A09E667u, 0xBB67AE85u, 0x3C6EF372u, 0xA54FF53Au, + 0x510E527Fu, 0x9B05688Cu, 0x1F83D9ABu, 0x5BE0CD19u, +}; + +// The three BLAKE3 domain flags this construction uses. Mirrors of `CHUNK_START` +// / `CHUNK_END` / `ROOT` in `crypto/crypto/src/hash/blake3/chain.rs:53-58`. +// `Blake3Chain` sets CHUNK_START on the first block only and CHUNK_END|ROOT on +// the last only; interior blocks carry no flags at all. +#define BLAKE3_FLAG_CHUNK_START 1u +#define BLAKE3_FLAG_CHUNK_END 2u +#define BLAKE3_FLAG_ROOT 8u + +// CHUNK_START | CHUNK_END | ROOT: the flags of a BLAKE3 hash whose whole message +// is one block of one chunk. At 7 rounds a compression under these flags with +// `h = IV` and `t = 0` IS `blake3::hash(message)`, which is what makes the crate +// an anchor for the framing and not just for the round function. Same framing +// the live LFM socket uses (`blake3_socket.rs:258` `FLAGS_LFMC = 0x0B`). +#define BLAKE3_FLAGS_ONE_BLOCK \ + (BLAKE3_FLAG_CHUNK_START | BLAKE3_FLAG_CHUNK_END | BLAKE3_FLAG_ROOT) + +// A Merkle parent's message is two 32-byte child digests = exactly 64 bytes. +#define BLAKE3_PARENT_BLOCK_LEN 64u + +__device__ __forceinline__ uint32_t rotr32(uint32_t x, uint32_t n) { + // Every call site passes 16, 12, 8 or 7, so the 32-n shift is never a + // shift-by-32. Kept as an explicit expression rather than __funnelshift_r + // so the transcription against the host `rotate_right` is readable. + return (x >> n) | (x << (32 - n)); +} + +// Reverse the four bytes of a 32-bit word. nvcc lowers this to a single PRMT. +__device__ __forceinline__ uint32_t bswap32(uint32_t x) { + return (x >> 24) | ((x >> 8) & 0x0000FF00u) | ((x << 8) & 0x00FF0000u) | (x << 24); +} + +// The BLAKE3 quarter-round G (spec §2.1). Mirror of `blake3_g` +// (`blake3.rs:89`); uint32_t arithmetic wraps, matching `wrapping_add`. +__device__ __forceinline__ void blake3_g(uint32_t *v, int a, int b, int c, int d, + uint32_t mx, uint32_t my) { + v[a] = v[a] + v[b] + mx; + v[d] = rotr32(v[d] ^ v[a], 16); + v[c] = v[c] + v[d]; + v[b] = rotr32(v[b] ^ v[c], 12); + v[a] = v[a] + v[b] + my; + v[d] = rotr32(v[d] ^ v[a], 8); + v[c] = v[c] + v[d]; + v[b] = rotr32(v[b] ^ v[c], 7); +} + +// The message-schedule permutation `m'[i] = m[PERM[i]]`, written out. The +// indices are `BLAKE3_MSG_PERMUTATION` (`blake3.rs:52`): +// [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] +// Spelled as literals rather than read from a constant array so it stays a +// register shuffle after unrolling; the parity tests are what check the +// transcription. +__device__ __forceinline__ void blake3_permute(uint32_t *m) { + uint32_t p[16] = {m[2], m[6], m[3], m[10], m[7], m[0], m[4], m[13], + m[1], m[11], m[12], m[5], m[9], m[14], m[15], m[8]}; + #pragma unroll + for (int i = 0; i < 16; ++i) m[i] = p[i]; +} + +// The BLAKE3 compression function `f` at `ROUNDS` rounds, full 16-word output. +// +// State init: `v[0..8] = h`, `v[8..12] = IV[0..4]`, `v[12] = t as u32`, +// `v[13] = (t >> 32) as u32`, `v[14] = block_len`, `v[15] = flags`. Each round +// is 8 G-calls (4 columns then 4 diagonals); the schedule is permuted between +// rounds only (`r < ROUNDS - 1` — the trailing permute is never consumed). +// Feed-forward: `out[i] = v[i] ^ v[i+8]`, `out[i+8] = v[i+8] ^ h[i]`; the +// truncated chaining value is `out[0..8]`. +// +// Callers that need only the chaining value still get the full 16 words: the +// second half is 8 XORs the compiler drops when they are unused, and one +// function is one place to be wrong. +template +__device__ __forceinline__ void blake3_compress(const uint32_t *h, const uint32_t *m_in, + uint64_t t, uint32_t block_len, uint32_t flags, + uint32_t *out) { + uint32_t v[16] = { + h[0], h[1], h[2], h[3], + h[4], h[5], h[6], h[7], + BLAKE3_IV[0], BLAKE3_IV[1], BLAKE3_IV[2], BLAKE3_IV[3], + (uint32_t)t, (uint32_t)(t >> 32), block_len, flags, + }; + + uint32_t m[16]; + #pragma unroll + for (int i = 0; i < 16; ++i) m[i] = m_in[i]; + + #pragma unroll + for (int r = 0; r < ROUNDS; ++r) { + // Mix the columns. + blake3_g(v, 0, 4, 8, 12, m[0], m[1]); + blake3_g(v, 1, 5, 9, 13, m[2], m[3]); + blake3_g(v, 2, 6, 10, 14, m[4], m[5]); + blake3_g(v, 3, 7, 11, 15, m[6], m[7]); + // Mix the diagonals. + blake3_g(v, 0, 5, 10, 15, m[8], m[9]); + blake3_g(v, 1, 6, 11, 12, m[10], m[11]); + blake3_g(v, 2, 7, 8, 13, m[12], m[13]); + blake3_g(v, 3, 4, 9, 14, m[14], m[15]); + if (r < ROUNDS - 1) blake3_permute(m); + } + + #pragma unroll + for (int i = 0; i < 8; ++i) { + out[i] = v[i] ^ v[i + 8]; + out[i + 8] = v[i + 8] ^ h[i]; + } +} + +// --------------------------------------------------------------------------- +// Byte serialization — shared with the leaf kernels and with the CPU commit. +// +// The leaf byte encoding does NOT move under P-a: `leaves_bit_reversed_grouped` +// (`crypto/stark/src/commitment.rs:55`) serializes each field element in +// canonical BIG-endian form and concatenates. BLAKE3 reads a 64-byte block as +// 16 LITTLE-endian u32 words, so one 8-byte element is two words: the +// byte-reverse of its high half, then of its low half. That transposition is +// the whole of the serialization difference from keccak, which absorbs the same +// bytes as one byte-swapped u64 lane. +// --------------------------------------------------------------------------- + +// The two BLAKE3 message words covered by one Goldilocks element's canonical +// big-endian bytes. `raw` may be non-canonical; canonicalising here matches +// `canonical_u64().to_be_bytes()` on host. +__device__ __forceinline__ void blake3_words_of_felt(uint64_t raw, uint32_t &w0, uint32_t &w1) { + uint64_t canon = goldilocks::canonical(raw); + w0 = bswap32((uint32_t)(canon >> 32)); + w1 = bswap32((uint32_t)canon); +} + +// A 64-byte BLAKE3 message block under construction. +// +// The SINK is deliberately the caller's: a leaf kernel compresses each full +// block into a chaining value, and which chaining construction that is (bare +// cv-chain vs standard chunk tree, PA-PLAN §1.6) is still open. Everything this +// struct does — word packing, block boundaries, zero-padding the tail, the byte +// count the final `block_len` comes from — is the same under either. +// +// Usage: `push_word` returns true when the block just filled, at which point the +// caller consumes `m` and calls `reset()`; pushing into a full block is the one +// way to misuse it. Field elements go in two words at a time (via +// `blake3_words_of_felt`) and straddle a block boundary whenever the element +// count is not a multiple of 8 — ext3 elements, at three felts, straddle +// routinely — which is why this works at word granularity and not element +// granularity. +struct Blake3Block { + uint32_t m[16]; + uint32_t nwords; // words filled in the current block, 0..15 between pushes + + __device__ __forceinline__ void init() { + nwords = 0; + #pragma unroll + for (int i = 0; i < 16; ++i) m[i] = 0; + } + + __device__ __forceinline__ bool push_word(uint32_t w) { + m[nwords++] = w; + return nwords == 16; + } + + __device__ __forceinline__ void reset() { init(); } + + // Bytes occupied in the pending (partial) block — the `block_len` a final + // compression over it takes. Zero exactly when no partial block is pending. + __device__ __forceinline__ uint32_t pending_bytes() const { return nwords * 4u; } +}; + +// --------------------------------------------------------------------------- +// `Blake3Chain` — the byte hash every leaf kernel commits with. +// +// Transcription of the host `Blake3Chain` (`crypto/crypto/src/hash/blake3/ +// chain.rs:98`), and the two are checked against each other by the leaf parity +// tests at the build's round count. The construction is PA-PLAN §1.7: +// +// n = max(1, ceil(|M| / 64)) blocks; the empty message is ONE +// L = |M| - 64*(n-1) 0 when |M| = 0, else 1..=64 +// F_i = (CHUNK_START if i = 0) | (CHUNK_END|ROOT if i = n-1) +// cv_0 = IV +// cv_i+1 = compress(cv_i, m_i, 0, 64, F_i)[0..8] for i < n-1 +// digest = compress(cv_n-1, m_n-1, 0, L, F_n-1)[0..8] little-endian +// +// ★ THE ONE SUBTLETY, and the reason this is a state machine rather than a +// loop: a FULL block is *held*, not compressed. The last block's flags and +// `block_len` differ from every other block's, and whether a block is the last +// is not known until the message ends — so a block is only folded into the +// chaining value once a further word proves it was not the last. `push_word` +// therefore compresses on the *next* push, never on filling. This mirrors the +// host `update`, which tests `block_len == BLOCK_LEN` at the top of the loop +// body and so only compresses when there is more input (`chain.rs:186-195`). +// +// Compressing eagerly on fill is the bug this shape exists to prevent: it would +// hash a 64-byte message as two blocks (one flagged CHUNK_START, one empty +// final) instead of one, breaking P2 — the property that a 64-byte message is +// exactly a Merkle parent — and with it the `StarkHash` two-element invariant. +// +// Word granularity, not byte: every message these kernels hash is a whole +// number of 8-byte field elements, so `block_len` is always a multiple of 4 and +// a partial word can never occur. `Blake3Block` is reused for the pending block +// so that the framing (packing, boundaries, zero-padding, the byte count) has +// exactly one implementation shared with `blake3_blocks_of_felts_probe`. +// --------------------------------------------------------------------------- +struct Blake3Chain { + uint32_t cv[8]; + Blake3Block block; + // Whether any block has been compressed — i.e. whether the pending block + // still carries CHUNK_START. Host counterpart: `started` (`chain.rs:109`). + bool started; + + __device__ __forceinline__ void init() { + #pragma unroll + for (int i = 0; i < 8; ++i) cv[i] = BLAKE3_IV[i]; + block.init(); + started = false; + } + + // The pending block's flags. CHUNK_START while nothing has been compressed + // yet; CHUNK_END|ROOT when this is the message's last block. Mirror of the + // host `flags(is_final)` (`chain.rs:160`). + __device__ __forceinline__ uint32_t flags(bool is_final) const { + uint32_t start = started ? 0u : BLAKE3_FLAG_CHUNK_START; + uint32_t end = is_final ? (BLAKE3_FLAG_CHUNK_END | BLAKE3_FLAG_ROOT) : 0u; + return start | end; + } + + // Fold the pending block — known NOT to be the last — into the chaining + // value, and clear it so the next block starts zero-padded. + __device__ __forceinline__ void compress_pending() { + uint32_t out[16]; + blake3_compress(cv, block.m, 0, 64u, flags(false), out); + #pragma unroll + for (int i = 0; i < 8; ++i) cv[i] = out[i]; + block.reset(); + started = true; + } + + // Absorb one message word. The full-block test comes FIRST: reaching here + // with a full block is what proves that block was not the last. + __device__ __forceinline__ void push_word(uint32_t w) { + if (block.nwords == 16) compress_pending(); + block.push_word(w); + } + + // Absorb one Goldilocks field element as its two message words — the + // canonical big-endian element bytes read back as little-endian words. + __device__ __forceinline__ void push_felt(uint64_t raw) { + uint32_t w0, w1; + blake3_words_of_felt(raw, w0, w1); + push_word(w0); + push_word(w1); + } + + // The 32-byte digest: one final compression over the pending block with the + // true byte count as `block_len` and CHUNK_END|ROOT set. The empty message + // takes this path with an all-zero block and `block_len = 0`, which is ONE + // compression, not zero. + // + // `dst` is 32-byte aligned at every call site (node buffers come from + // cuMemAlloc, 256-byte aligned, and every leaf sits at a multiple of 32), so + // the u32 store is safe. A digest's 32 bytes ARE its 8 output words + // little-endian and the device is little-endian, so this is a plain copy + // with no byte swapping — contrast the leaf INPUT path, whose field bytes + // are big-endian. + __device__ __forceinline__ void finalize(uint8_t *dst) { + uint32_t out[16]; + blake3_compress(cv, block.m, 0, block.pending_bytes(), flags(true), out); + uint32_t *w = reinterpret_cast(dst); + #pragma unroll + for (int i = 0; i < 8; ++i) w[i] = out[i]; + } +}; + +// --------------------------------------------------------------------------- +// Leaf kernels. +// +// Twins of the seven keccak leaf kernels, one for one, with the sponge replaced +// by a `Blake3Chain` and the lane byte-swap replaced by the two-word field +// serialization. THE READ PATTERN IS IDENTICAL IN EVERY CASE — same bit +// reversal, same column/component order, same row-pair ordering — because the +// leaf byte layout does not move under P-a: `leaves_bit_reversed_grouped` +// (`crypto/stark/src/commitment.rs:55`) serializes each element in canonical +// big-endian and concatenates, and only the hash over those bytes changes. +// +// So the correctness argument for each kernel below is two independent halves: +// the byte stream (identical to the keccak twin's, and checked against the CPU +// leaf helpers by the parity tests) and the hash over it (`Blake3Chain`, checked +// against the host chain by the same tests). +// --------------------------------------------------------------------------- + +// Goldilocks BASE-FIELD leaf hashing, one leaf per bit-reversed row. +// Twin of `keccak256_leaves_base_batched` (`keccak.cu:152`). +extern "C" __global__ void blake3_leaves_base_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + + // Read columns at the bit-reversed row, write the leaf at `tid` — matching + // the CPU per-row `commit_bit_reversed(.., 1)`. + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < num_cols; ++c) { + h.push_felt(columns_base_ptr[c * col_stride + br]); + } + h.finalize(hashed_leaves_out + tid * 32); +} + +// Goldilocks BASE-FIELD row-pair leaf hashing: leaf `tid` hashes bit-reversed +// rows `2*tid` and `2*tid+1`, each written column-by-column, first row then +// second. `num_leaves = num_rows / 2`. +// Twin of `keccak256_leaves_base_row_pair_batched` (`keccak.cu:196`). +extern "C" __global__ void blake3_leaves_base_row_pair_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < num_cols; ++c) { + h.push_felt(columns_base_ptr[c * col_stride + br_0]); + } + for (uint64_t c = 0; c < num_cols; ++c) { + h.push_felt(columns_base_ptr[c * col_stride + br_1]); + } + h.finalize(hashed_leaves_out + tid * 32); +} + +// Goldilocks EXT3 leaf hashing, one leaf per bit-reversed row. Components live +// in three separate base-field slabs: column `c` component `k` is at +// `columns_base_ptr[(c*3 + k)*col_stride + br]`, and per-element bytes are +// `[comp0, comp1, comp2]` each 8 big-endian bytes (matching +// `FieldElement::::write_bytes_be`). +// Twin of `keccak256_leaves_ext3_batched` (`keccak.cu:237`). +extern "C" __global__ void blake3_leaves_ext3_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, // number of ext3 columns (NOT slabs) + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < num_cols; ++c) { + #pragma unroll + for (int k = 0; k < 3; ++k) { + h.push_felt(columns_base_ptr[(c * 3 + (uint64_t)k) * col_stride + br]); + } + } + h.finalize(hashed_leaves_out + tid * 32); +} + +// R2 composition-polynomial leaf hashing: each leaf hashes `2 * num_parts` ext3 +// values from bit-reversed rows `2*tid` and `2*tid+1`, in (row 0: parts) then +// (row 1: parts) order, three base components per value. +// Twin of `keccak_comp_poly_leaves_ext3` (`keccak.cu:277`). +extern "C" __global__ void blake3_comp_poly_leaves_ext3( + const uint64_t *parts_base_ptr, + uint64_t col_stride, + uint64_t num_parts, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + Blake3Chain h; + h.init(); + for (uint64_t p = 0; p < num_parts; ++p) { + #pragma unroll + for (int k = 0; k < 3; ++k) { + h.push_felt(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_0]); + } + } + for (uint64_t p = 0; p < num_parts; ++p) { + #pragma unroll + for (int k = 0; k < 3; ++k) { + h.push_felt(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_1]); + } + } + h.finalize(leaves_out + tid * 32); +} + +// FRI layer leaf hashing: each leaf hashes two consecutive ext3 values from an +// interleaved eval vector `[a0,a1,a2,b0,b1,b2,...]` = 48 bytes. No bit reversal +// and no slab layout. +// +// Note 48 bytes is under one block, so a FRI leaf is a SINGLE compression with +// `flags = 0x0B` and `block_len = 48` — the chain's degenerate one-block case, +// same shape as a Merkle parent but at a different length. +// Twin of `keccak_fri_leaves_ext3` (`keccak.cu:326`). +extern "C" __global__ void blake3_fri_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_evals u64s (ext3 interleaved) + uint64_t num_leaves, // = num_evals / 2 + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + const uint64_t *left = evals_interleaved + 2 * tid * 3; // 3 u64s + const uint64_t *right = left + 3; + + Blake3Chain h; + h.init(); + #pragma unroll + for (int i = 0; i < 3; ++i) h.push_felt(left[i]); + #pragma unroll + for (int i = 0; i < 3; ++i) h.push_felt(right[i]); + h.finalize(leaves_out + tid * 32); +} + +// Row-major ROW-PAIR leaf hashing: the row-major analog of +// `blake3_leaves_base_row_pair_batched`. Leaf `tid` hashes row +// `reverse_index(2*tid)` then row `reverse_index(2*tid+1)`, each `m` lanes read +// contiguously from `data + br * m`. `m` is the row stride in u64s: base trace = +// column count, ext3 trace = 3 * column count (an ext3 element's components are +// consecutive, matching `write_bytes_be`). +// Twin of `keccak256_leaves_base_row_major_row_pair` (`keccak.cu:473`). +extern "C" __global__ void blake3_leaves_base_row_major_row_pair( + const uint64_t *data, + uint64_t m, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) +{ + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + Blake3Chain h; + h.init(); + for (uint64_t c = 0; c < m; ++c) h.push_felt(row_0[c]); + for (uint64_t c = 0; c < m; ++c) h.push_felt(row_1[c]); + h.finalize(hashed_leaves_out + tid * 32); +} + +// Column-range variant: each leaf hashes only columns `[col_start, col_end)` of +// the two bit-reversed rows, while `m` stays the full row stride. Byte layout +// equals the CPU `commit_rows_bit_reversed_subset` — used for preprocessed +// tables, whose precomputed and multiplicity column ranges commit to separate +// Merkle trees over the same row-major LDE. +// Twin of `keccak256_leaves_base_row_major_row_pair_range` (`keccak.cu:511`). +extern "C" __global__ void blake3_leaves_base_row_major_row_pair_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) +{ + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + Blake3Chain h; + h.init(); + for (uint64_t c = col_start; c < col_end; ++c) h.push_felt(row_0[c]); + for (uint64_t c = col_start; c < col_end; ++c) h.push_felt(row_1[c]); + h.finalize(hashed_leaves_out + tid * 32); +} + +// --------------------------------------------------------------------------- +// Merkle parent / level compressors. +// +// A parent is ONE compression over the 64 bytes of its two child digests: +// `h = IV`, `t = 0`, `block_len = 64`, `flags = CHUNK_START|CHUNK_END|ROOT`, +// digest = `out[0..8]` little-endian. That is `hash_bytes(left ‖ right)`, which +// is what `hash_new_parent` is on host for every existing backend +// (`hash_new_parent_bytes`, `field_element_vector.rs:74`) — so a parent needs no +// chaining and is construction-independent: with a single-block message the +// chunk tree and a bare cv-chain agree bit-for-bit. +// +// The u32 casts are byte-order-free in both directions and that is not an +// accident: a digest's 32 bytes ARE its 8 output words little-endian, and BLAKE3 +// reads message bytes as little-endian words, so on a little-endian device +// (every NVIDIA GPU) reading a child digest as `uint32_t[8]` yields exactly the +// message words, and storing `out[0..8]` as u32 yields exactly the digest bytes. +// No byte swapping anywhere on this path — contrast the leaf path above, whose +// input is big-endian field bytes. +// +// Node buffer layout mirrors `keccak.cu`'s and the CPU +// `crypto/crypto/src/merkle_tree/merkle.rs`: children at +// `nodes[parent_begin + n_pairs .. parent_begin + 3*n_pairs]`, parents at +// `nodes[parent_begin .. parent_begin + n_pairs]`, 32 bytes per node. +// --------------------------------------------------------------------------- +__device__ __forceinline__ void blake3_hash_merkle_parent(uint8_t *nodes, uint64_t parent_begin, + uint64_t n_pairs, uint64_t tid) { + // `nodes` comes from cuMemAlloc (256-byte aligned) and every 32-byte node + // sits at a 32-byte-aligned offset, so the u32 casts are safe. + const uint32_t *left = reinterpret_cast( + nodes + (parent_begin + n_pairs + 2 * tid) * 32); + const uint32_t *right = reinterpret_cast( + nodes + (parent_begin + n_pairs + 2 * tid + 1) * 32); + + uint32_t m[16]; + #pragma unroll + for (int i = 0; i < 8; ++i) { + m[i] = left[i]; + m[i + 8] = right[i]; + } + + uint32_t out[16]; + blake3_compress(BLAKE3_IV, m, 0, BLAKE3_PARENT_BLOCK_LEN, + BLAKE3_FLAGS_ONE_BLOCK, out); + + uint32_t *dst = reinterpret_cast(nodes + (parent_begin + tid) * 32); + #pragma unroll + for (int i = 0; i < 8; ++i) dst[i] = out[i]; +} + +// One level of the inner Merkle tree: each thread hashes one child pair. +extern "C" __global__ void blake3_merkle_level(uint8_t *nodes, + uint64_t parent_begin, // in 32-byte nodes + uint64_t n_pairs) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_pairs) return; + blake3_hash_merkle_parent(nodes, parent_begin, n_pairs, tid); +} + +// Build every remaining level (from `level_begin` up to the root) in ONE +// single-block launch: each level's pairs are grid-strided over the block, with +// a __syncthreads() barrier between levels. Replaces log2 launches of +// `blake3_merkle_level` for the small top levels, whose per-level work is +// dwarfed by launch overhead. Twin of `keccak_merkle_tail`. +extern "C" __global__ void blake3_merkle_tail(uint8_t *nodes, uint64_t level_begin) { + uint64_t lb = level_begin; + while (lb != 0) { + uint64_t nb = lb / 2; + uint64_t n_pairs = lb - nb; + for (uint64_t tid = threadIdx.x; tid < n_pairs; tid += blockDim.x) { + blake3_hash_merkle_parent(nodes, nb, n_pairs, tid); + } + __syncthreads(); + lb = nb; + } +} + +// --------------------------------------------------------------------------- +// Parity-harness entry points. +// +// The device compression function is not otherwise reachable from host code, so +// there would be nothing to check it against the host reference with. These are +// that oracle — the same role `build_fri_layer_tree_from_evals_ext3` plays for +// the keccak tree. Not on any production path. +// --------------------------------------------------------------------------- + +// `n` independent compressions, full 16-word outputs. One thread per vector. +template +__device__ __forceinline__ void compress_probe_body(const uint32_t *h, const uint32_t *m, + const uint64_t *t, const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + blake3_compress(h + tid * 8, m + tid * 16, t[tid], block_len[tid], flags[tid], + out + tid * 16); +} + +extern "C" __global__ void blake3_compress_probe_6r(const uint32_t *h, const uint32_t *m, + const uint64_t *t, const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + compress_probe_body<6>(h, m, t, block_len, flags, n, out); +} + +extern "C" __global__ void blake3_compress_probe_7r(const uint32_t *h, const uint32_t *m, + const uint64_t *t, const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + compress_probe_body<7>(h, m, t, block_len, flags, n, out); +} + +// The same probe at the round count this cubin's PRODUCTION kernels are built +// for. Not redundant with the two above: it is the only way to observe from host +// code which of them `blake3_merkle_level` actually uses. +extern "C" __global__ void blake3_compress_probe_default(const uint32_t *h, const uint32_t *m, + const uint64_t *t, + const uint32_t *block_len, + const uint32_t *flags, uint64_t n, + uint32_t *out) { + compress_probe_body(h, m, t, block_len, flags, n, out); +} + +// `n_words` message words streamed through the device `Blake3Chain`, digest out. +// +// ★ This is the harness that lets the device be checked against the COMMITTED +// KAT TABLE (`CHAIN_KAT_6ROUND`, `chain.rs:304`) rather than only against the +// host implementation. That distinction is the whole of risk R13: a device port +// checked solely against the Rust it was transcribed from is checked against +// nothing. The KAT digests came from a Python oracle, so asserting the device +// against them closes the loop with an artifact this tree did not produce. +// +// Word-granular, because that is all the device ever hashes: every production +// message is a whole number of 8-byte field elements. The KAT lengths that are +// not multiples of 4 are therefore unreachable from device code by construction, +// and the host tests cover them instead. +// +// Single-threaded on purpose — same shape as a leaf kernel, one thread hashing +// one whole message sequentially. +extern "C" __global__ void blake3_chain_probe(const uint32_t *words, uint64_t n_words, + uint8_t *out32) { + if (threadIdx.x != 0 || blockIdx.x != 0) return; + Blake3Chain h; + h.init(); + for (uint64_t i = 0; i < n_words; ++i) h.push_word(words[i]); + h.finalize(out32); +} + +// This cubin's compiled-in round count, so a caller can assert it against the +// host's `BLAKE3_ROUNDS` instead of discovering a mismatch as a wrong root. +extern "C" __global__ void blake3_rounds_probe(uint32_t *out) { + if (threadIdx.x == 0 && blockIdx.x == 0) *out = (uint32_t)BLAKE3_ROUNDS; +} + +// The two message words of each of `n` field elements, in order — the +// serialization contract on its own (canonicalisation, big-endian element bytes, +// little-endian word packing), with no hashing over it. +extern "C" __global__ void blake3_serialize_felts_probe(const uint64_t *vals, uint64_t n, + uint32_t *out_words) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint32_t w0, w1; + blake3_words_of_felt(vals[tid], w0, w1); + out_words[tid * 2] = w0; + out_words[tid * 2 + 1] = w1; +} + +// `n` field elements streamed through `Blake3Block`, with the completed blocks +// written out instead of compressed. Single-threaded on purpose: that is the +// shape a leaf kernel has (one thread hashes one whole leaf, sequentially), so +// this exercises the block framing on the code path the chaining loop will use. +// Writes `ceil(2n/16)` blocks of 16 words; the tail block is zero-padded. +extern "C" __global__ void blake3_blocks_of_felts_probe(const uint64_t *vals, uint64_t n, + uint32_t *out_blocks) { + if (threadIdx.x != 0 || blockIdx.x != 0) return; + + Blake3Block b; + b.init(); + uint64_t nblocks = 0; + for (uint64_t i = 0; i < n; ++i) { + uint32_t w0, w1; + blake3_words_of_felt(vals[i], w0, w1); + if (b.push_word(w0)) { + #pragma unroll + for (int k = 0; k < 16; ++k) out_blocks[nblocks * 16 + k] = b.m[k]; + ++nblocks; + b.reset(); + } + if (b.push_word(w1)) { + #pragma unroll + for (int k = 0; k < 16; ++k) out_blocks[nblocks * 16 + k] = b.m[k]; + ++nblocks; + b.reset(); + } + } + // Flush the partial tail block, zero-padded (`init` zeroed it, and + // `pending_bytes` is what a real final compression would pass as block_len). + if (b.pending_bytes() != 0) { + #pragma unroll + for (int k = 0; k < 16; ++k) out_blocks[nblocks * 16 + k] = b.m[k]; + } +} diff --git a/crypto/math-cuda/src/blake3.rs b/crypto/math-cuda/src/blake3.rs new file mode 100644 index 000000000..f05f46063 --- /dev/null +++ b/crypto/math-cuda/src/blake3.rs @@ -0,0 +1,827 @@ +//! GPU BLAKE3 for Merkle commits — the leaf kernels, the parent/level +//! compressors, and the parity-harness handles on the device compression +//! function, byte serialization and chain construction. +//! +//! Twin of [`crate::merkle`]'s keccak path, kernel for kernel, so the two read +//! against each other. Keccak stays the prover's default hash: nothing in the +//! production dispatch reaches this module yet. +//! +//! # What a parent is +//! +//! `hash_new_parent(left, right)` is one BLAKE3 compression over the 64 bytes of +//! the two child digests: `h = IV`, `t = 0`, `block_len = 64`, `flags = +//! CHUNK_START|CHUNK_END|ROOT`, digest = the low 8 output words little-endian. +//! That is `hash_bytes(left ‖ right)`, which is what `hash_new_parent` already +//! is for every host backend (`hash_new_parent_bytes`, +//! `crypto/crypto/src/merkle_tree/backends/field_element_vector.rs:74`), and at +//! 7 rounds it is literally `blake3::hash(left ‖ right)`. +//! +//! A parent is therefore construction-independent: its message is a single +//! block, and over a single block every candidate chaining construction agrees +//! bit-for-bit. Only multi-block messages — leaves — depend on the construction, +//! which is why the parent compressor could land before it was settled and the +//! leaf kernels could not. +//! +//! # What a leaf is +//! +//! A leaf's bytes are unchanged from the keccak path — `leaves_bit_reversed_grouped` +//! (`crypto/stark/src/commitment.rs:55`) serializes each element in canonical +//! big-endian and concatenates, and only the hash over those bytes moves. The +//! hash is `Blake3Chain` (PA-PLAN §1.7): standard BLAKE3 restricted to a single +//! chunk that never ends, host implementation at +//! `crypto/crypto/src/hash/blake3/chain.rs`. +//! +//! ⚠ That construction is a DRAFT pending ratification of forks F1-F3 +//! (PA-PLAN §1.7.3), implemented here as the working default by standing +//! decision. +//! +//! # What is missing, and why +//! +//! Nothing on the kernel side: all seven leaf kernels, both tree compressors and +//! the six wrapper twins are here. What has NOT happened is production dispatch — +//! `stark::config::StarkHash` still requires `KeccakTreeBackend` under `cuda` +//! (`config.rs:116-122`), so no prover path reaches this module. Retiring that +//! bound is PA-PLAN's Stage 6, not track G. + +use cudarc::driver::{CudaSlice, CudaStream, CudaViewMut, LaunchConfig, PushKernelArg}; +use std::sync::Arc; + +use crate::Result; +use crate::device::{Backend, backend}; +use crate::lde::pack_ext3_to_pinned_slabs; + +/// Threads per block for the BLAKE3 kernels. +/// +/// Wider than [`crate::merkle`]'s 128 because the register footprint is a third +/// of keccak's: 16 working-state words + 16 message words + the output, all u32, +/// against keccak's 25 u64 lanes plus a 25-lane scratch. The 128 there is a +/// Blackwell register-file limit, not a shape this path shares. +const BLAKE3_BLOCK_DIM: u32 = 256; + +pub(crate) fn blake3_launch_cfg(num_threads: u64) -> LaunchConfig { + debug_assert!( + num_threads <= u32::MAX as u64, + "blake3_launch_cfg: num_threads ({num_threads}) exceeds u32 grid range", + ); + let grid = (num_threads as u32).div_ceil(BLAKE3_BLOCK_DIM); + LaunchConfig { + grid_dim: (grid, 1, 1), + block_dim: (BLAKE3_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + } +} + +/// BLAKE3 leaf hashing over a base-field column buffer. Twin of +/// [`crate::merkle::keccak_leaves_base`], argument for argument. +/// +/// `columns` must hold `num_cols * col_stride` u64s with column `c`'s data at +/// `[c*col_stride .. c*col_stride + num_rows]`. `rows_per_leaf` selects the leaf +/// layout: `1` = one leaf per bit-reversed row (`num_rows` leaves), `2` = one +/// leaf per bit-reversed row pair (`num_rows/2` leaves, the trace-commit +/// layout). Returns `(num_rows / rows_per_leaf) * 32` hash bytes. +pub fn leaves_base( + columns: &[u64], + col_stride: usize, + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + assert!(num_rows.is_power_of_two()); + assert!(rows_per_leaf == 1 || rows_per_leaf == 2); + assert!( + num_rows >= 2, + "num_rows must be at least 2 for bit-reversed GPU leaf hashing" + ); + assert!( + col_stride >= num_rows, + "col_stride must be >= num_rows to keep per-column reads in-bounds" + ); + let total = num_cols + .checked_mul(col_stride) + .expect("num_cols * col_stride overflows usize"); + assert!(columns.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(&columns[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + let launch = if rows_per_leaf == 2 { + launch_leaves_base_row_pair + } else { + launch_leaves_base + }; + launch( + stream.as_ref(), + &cols_dev, + col_stride as u64, + num_cols as u64, + num_rows as u64, + &mut out_dev.as_view_mut(), + )?; + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Ext3 variant of [`leaves_base`]: columns arrive as three base slabs per ext3 +/// column, so `columns.len() >= num_cols * 3 * col_stride`. Twin of +/// [`crate::merkle::keccak_leaves_ext3`]. +pub fn leaves_ext3( + columns: &[u64], + col_stride: usize, + num_cols: usize, + num_rows: usize, + rows_per_leaf: usize, +) -> Result> { + assert!(num_rows.is_power_of_two()); + assert!(rows_per_leaf == 1 || rows_per_leaf == 2); + assert!( + num_rows >= 2, + "num_rows must be at least 2 for bit-reversed GPU leaf hashing" + ); + assert!( + col_stride >= num_rows, + "col_stride must be >= num_rows to keep per-column reads in-bounds" + ); + let total = num_cols + .checked_mul(3) + .and_then(|v| v.checked_mul(col_stride)) + .expect("num_cols * 3 * col_stride overflows usize"); + assert!(columns.len() >= total); + let be = backend()?; + let stream = be.next_stream(); + let cols_dev = stream.clone_htod(&columns[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / rows_per_leaf) * 32)?; + // Row-pair ext3 leaves reuse the comp-poly kernel, exactly as the keccak + // path does: hashing all ext3 columns of rows `2i`, `2i+1` is the same + // traversal whether the columns are called "aux trace" or "parts". + let launch = if rows_per_leaf == 2 { + launch_ext3_row_pair + } else { + launch_leaves_ext3 + }; + launch( + stream.as_ref(), + &cols_dev, + col_stride as u64, + num_cols as u64, + num_rows as u64, + &mut out_dev.as_view_mut(), + )?; + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(out) +} + +pub(crate) fn launch_leaves_base( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + // The kernel computes `__brevll(tid) >> (64 - log_num_rows)`, which is UB + // for `log_num_rows == 0` (single-row trees are degenerate anyway). + debug_assert!(num_rows >= 2, "blake3 leaf kernel: num_rows must be >= 2"); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg(num_rows); + unsafe { + stream + .launch_builder(&be.blake3_leaves_base_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_leaves_base_row_pair( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "blake3 row-pair leaf kernel: num_rows must be >= 2" + ); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + // One thread per leaf (= row pair). + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_leaves_base_row_pair_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_leaves_ext3( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!(num_rows >= 2, "blake3 leaf kernel: num_rows must be >= 2"); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg(num_rows); + unsafe { + stream + .launch_builder(&be.blake3_leaves_ext3_batched) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +pub(crate) fn launch_ext3_row_pair( + stream: &CudaStream, + cols_dev: &CudaSlice, + col_stride: u64, + num_cols: u64, + num_rows: u64, + out_dev: &mut CudaViewMut<'_, u8>, +) -> Result<()> { + debug_assert!( + num_rows >= 2, + "blake3 row-pair leaf kernel: num_rows must be >= 2" + ); + let be = backend()?; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg(num_rows >> 1); + unsafe { + stream + .launch_builder(&be.blake3_comp_poly_leaves_ext3) + .arg(cols_dev) + .arg(&col_stride) + .arg(&num_cols) + .arg(&num_rows) + .arg(&log_num_rows) + .arg(out_dev) + .launch(cfg)?; + } + Ok(()) +} + +/// Row-major row-pair leaf hashing: leaf `i` hashes the two consecutive +/// bit-reversed rows `reverse_index(2i)`, `reverse_index(2i+1)`, each `m` lanes +/// read contiguously from the row-major `data`. Matches the CPU +/// `commit_bit_reversed(.., 2)`; twin of the keccak launcher in +/// [`crate::lde`]. +/// +/// Returns `(num_rows / 2) * 32` hash bytes. Public because the blake3 path has +/// no production caller yet — the parity tests are what reach this kernel, and +/// the keccak twin's private launcher is called from the LDE pipeline instead. +pub fn leaves_base_row_major_row_pair(data: &[u64], m: usize, num_rows: usize) -> Result> { + leaves_row_major_row_pair_inner(data, m, 0, m, num_rows, false) +} + +/// Column-range variant of [`leaves_base_row_major_row_pair`]: each leaf hashes +/// only columns `[col_start, col_end)` of the row pair, while `m` stays the full +/// row stride. Matches the CPU `commit_rows_bit_reversed_subset`, which is how +/// preprocessed tables commit their precomputed and multiplicity column ranges +/// to separate Merkle trees over one row-major LDE. +pub fn leaves_base_row_major_row_pair_range( + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, +) -> Result> { + leaves_row_major_row_pair_inner(data, m, col_start, col_end, num_rows, true) +} + +fn leaves_row_major_row_pair_inner( + data: &[u64], + m: usize, + col_start: usize, + col_end: usize, + num_rows: usize, + ranged: bool, +) -> Result> { + assert!(num_rows.is_power_of_two()); + assert!(num_rows >= 2, "num_rows must be at least 2"); + assert!( + col_start < col_end && col_end <= m, + "column range in bounds" + ); + let total = num_rows + .checked_mul(m) + .expect("num_rows * m overflows usize"); + assert!(data.len() >= total); + + let be = backend()?; + let stream = be.next_stream(); + let data_dev = stream.clone_htod(&data[..total])?; + let mut out_dev = stream.alloc_zeros::((num_rows / 2) * 32)?; + + let m_u64 = m as u64; + let num_rows_u64 = num_rows as u64; + let log_num_rows = num_rows.trailing_zeros() as u64; + let cfg = blake3_launch_cfg((num_rows / 2) as u64); + unsafe { + if ranged { + let cs = col_start as u64; + let ce = col_end as u64; + stream + .launch_builder(&be.blake3_leaves_base_row_major_row_pair_range) + .arg(&data_dev) + .arg(&m_u64) + .arg(&cs) + .arg(&ce) + .arg(&num_rows_u64) + .arg(&log_num_rows) + .arg(&mut out_dev.as_view_mut()) + .launch(cfg)?; + } else { + stream + .launch_builder(&be.blake3_leaves_base_row_major_row_pair) + .arg(&data_dev) + .arg(&m_u64) + .arg(&num_rows_u64) + .arg(&log_num_rows) + .arg(&mut out_dev.as_view_mut()) + .launch(cfg)?; + } + } + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Walk the inner Merkle tree on device under BLAKE3. `nodes_dev` already has +/// the `leaves_len` hashed leaves written into the tail; this fills in the inner +/// nodes bottom-up. Twin of [`crate::merkle::build_inner_tree_levels`], and the +/// tail cutover has the same rationale: one single-block launch takes over once a +/// level is no wider than the block, where per-level launch overhead dominates +/// the work and the tail's grid-striding adds no serialization over the launches +/// it replaces. +pub(crate) fn build_inner_tree_levels( + stream: &CudaStream, + be: &Backend, + nodes_dev: &mut CudaSlice, + leaves_len: usize, +) -> Result<()> { + const TAIL_MAX_PAIRS: u64 = BLAKE3_BLOCK_DIM as u64; + let mut level_begin: u64 = (leaves_len - 1) as u64; + while level_begin != 0 { + let new_begin = level_begin / 2; + let n_pairs = level_begin - new_begin; + if n_pairs <= TAIL_MAX_PAIRS { + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (BLAKE3_BLOCK_DIM, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_merkle_tail) + .arg(&mut *nodes_dev) + .arg(&level_begin) + .launch(cfg)?; + } + return Ok(()); + } + let cfg = blake3_launch_cfg(n_pairs); + unsafe { + stream + .launch_builder(&be.blake3_merkle_level) + .arg(&mut *nodes_dev) + .arg(&new_begin) + .arg(&n_pairs) + .launch(cfg)?; + } + level_begin = new_begin; + } + Ok(()) +} + +/// Given `hashed_leaves` of length `leaves_len * 32`, build the full BLAKE3 +/// Merkle tree on device and return the `(2*leaves_len - 1) * 32`-byte node +/// buffer in the standard layout: `nodes[0..leaves_len - 1]` are inner nodes +/// (root at index 0) and `nodes[leaves_len - 1..]` are the leaves themselves. +/// +/// Matches the CPU `crypto/crypto/src/merkle_tree/merkle.rs` construction, so +/// the result plugs into `MerkleTree::from_precomputed_nodes` the same way +/// [`crate::merkle::build_merkle_tree_on_device`]'s does. +/// +/// `leaves_len` must be a power of two and >= 2. +pub fn build_merkle_tree_on_device(hashed_leaves: &[u8]) -> Result> { + assert!(hashed_leaves.len().is_multiple_of(32)); + let leaves_len = hashed_leaves.len() / 32; + assert!(leaves_len >= 2, "tree needs at least two leaves"); + assert!( + leaves_len.is_power_of_two(), + "leaves_len must be a power of two" + ); + + let total_nodes = 2 * leaves_len - 1; + let be = backend()?; + let stream = be.next_stream(); + + // SAFETY: every byte is written before it is read — leaves by the H2D + // below, inner nodes by the level walk that follows. + let mut nodes_dev = unsafe { stream.alloc::(total_nodes * 32) }?; + let leaves_offset_bytes = (leaves_len - 1) * 32; + { + let mut slice = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + hashed_leaves.len()); + stream.memcpy_htod(hashed_leaves, &mut slice)?; + } + + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, leaves_len)?; + + let out = stream.clone_dtoh(&nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Build the composition Merkle tree under BLAKE3 straight from a +/// device-resident slab buffer (`3*m` slabs of `lde_size` u64s, component `k` of +/// part `c` at `(c*3 + k) * lde_size` — the [`crate::lde::GpuLdeExt3`] layout). +/// No host staging and no H2D: the leaf kernel reads `buf` in place on `stream`. +/// +/// Twin of [`crate::merkle::build_comp_poly_tree_from_slabs_dev`]. +pub fn build_comp_poly_tree_from_slabs_dev( + stream: &Arc, + buf: &CudaSlice, + m: usize, + lde_size: usize, +) -> Result { + assert!(m > 0); + assert!(lde_size.is_power_of_two() && lde_size >= 2); + assert_eq!(buf.len(), 3 * m * lde_size, "slab buffer shape"); + let num_leaves = lde_size / 2; + let tight_total_nodes = 2 * num_leaves - 1; + let be = backend()?; + + // SAFETY: every byte is written before it is read — leaves by the kernel + // below, inner nodes by the level walk after it. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + let leaves_offset_bytes = (num_leaves - 1) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + launch_ext3_row_pair( + stream.as_ref(), + buf, + lde_size as u64, + m as u64, + lde_size as u64, + &mut leaves_view, + )?; + } + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + let mut root = [0u8; 32]; + stream.memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; + stream.synchronize()?; + Ok(crate::lde::GpuMerkleTree { + nodes: Arc::new(nodes_dev), + leaves_len: num_leaves, + root, + }) +} + +/// Build the composition Merkle tree under BLAKE3 from host-side interleaved +/// ext3 parts, keeping the nodes device-resident so openings can gather paths on +/// device. `parts_interleaved` is `num_parts` slices, each `[a0,a1,a2,b0,b1,b2,…]` +/// of length `3*lde_size`. Leaves hash row pairs, so `leaves_len = lde_size / 2`. +/// +/// Twin of [`crate::merkle::build_comp_poly_tree_from_evals_ext3_keep`], and it +/// stages through the same pinned de-interleave buffer for the same reason. +pub fn build_comp_poly_tree_from_evals_ext3_keep( + parts_interleaved: &[&[u64]], +) -> Result { + assert!(!parts_interleaved.is_empty()); + let m = parts_interleaved.len(); + let ext3_elems = parts_interleaved[0].len() / 3; + assert_eq!( + parts_interleaved[0].len(), + 3 * ext3_elems, + "ext3 buffer length must be 3 * lde_size" + ); + for p in parts_interleaved.iter() { + assert_eq!(p.len(), 3 * ext3_elems); + } + let lde_size = ext3_elems; + assert!(lde_size.is_power_of_two() && lde_size >= 2); + + let be = backend()?; + let stream = be.next_stream(); + let staging_slot = be.pinned_staging(); + + // Stage: de-interleave each part into 3 base slabs in pinned memory. + let mb = 3 * m; + let mut staging = staging_slot.lock().unwrap(); + staging.ensure_capacity(mb * lde_size, &be.ctx)?; + let pinned = unsafe { staging.as_mut_slice(mb * lde_size) }; + + pack_ext3_to_pinned_slabs(parts_interleaved, pinned, lde_size); + + // H2D the de-interleaved parts, then release the staging lock: the tree + // build reads the device `buf`, not `pinned`. Synchronize first so the async + // H2D has consumed `pinned` before it can be freed or reused. + let mut buf = stream.alloc_zeros::(mb * lde_size)?; + stream.memcpy_htod(&pinned[..mb * lde_size], &mut buf)?; + stream.synchronize()?; + drop(staging); + + build_comp_poly_tree_from_slabs_dev(&stream, &buf, m, lde_size) +} + +/// Build a FRI-layer Merkle tree on device under BLAKE3 from an interleaved ext3 +/// eval vector, returning the full host node buffer so tests can compare it byte +/// for byte against the CPU. Each leaf hashes two consecutive ext3 values; +/// `num_leaves = evals.len() / 6`. Returns `(2*num_leaves - 1) * 32` bytes in +/// standard layout. +/// +/// Twin of [`crate::merkle::build_fri_layer_tree_from_evals_ext3`], and like it a +/// parity harness rather than a production path: production folds and commits +/// through [`crate::fri::FriLayer::fold_and_commit_layer`]. +pub fn build_fri_layer_tree_from_evals_ext3(evals: &[u64]) -> Result> { + assert!( + evals.len().is_multiple_of(6), + "evals must hold whole pair-leaves" + ); + let num_evals = evals.len() / 3; + let num_leaves = num_evals / 2; + assert!(num_leaves.is_power_of_two() && num_leaves >= 2); + let tight_total_nodes = 2 * num_leaves - 1; + + let be = backend()?; + let stream = be.next_stream(); + + let evals_dev = stream.clone_htod(evals)?; + // SAFETY: leaves are written by the kernel below, inner nodes by the level + // walk after it, before either is read. + let mut nodes_dev = unsafe { stream.alloc::(tight_total_nodes * 32) }?; + + let leaves_offset_bytes = (num_leaves - 1) * 32; + { + let mut leaves_view = + nodes_dev.slice_mut(leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32); + let num_leaves_u64 = num_leaves as u64; + let cfg = blake3_launch_cfg(num_leaves as u64); + unsafe { + stream + .launch_builder(&be.blake3_fri_leaves_ext3) + .arg(&evals_dev) + .arg(&num_leaves_u64) + .arg(&mut leaves_view) + .launch(cfg)?; + } + } + + build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + + let out = stream.clone_dtoh(&nodes_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// One compression's inputs, in the argument order of the host reference +/// `blake3_compress_rounds(h, m, t, block_len, flags, rounds)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CompressInput { + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, +} + +/// Which round count [`compress_probe`] should run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProbeRounds { + /// 6 — the internal variant. + Six, + /// 7 — standard BLAKE3, where the `blake3` crate is an external anchor. + Seven, + /// Whatever the cubin's production kernels are compiled for. The only way to + /// observe from host code which of the two `blake3_merkle_level` uses. + CompiledIn, +} + +/// Parity harness: run the device compression function over `inputs` and return +/// each full 16-word output. +/// +/// Not a production path — the device compression is otherwise unreachable from +/// host code, so without this there would be nothing to check it against the +/// host reference with. +pub fn compress_probe(inputs: &[CompressInput], rounds: ProbeRounds) -> Result> { + if inputs.is_empty() { + return Ok(Vec::new()); + } + let n = inputs.len(); + let mut h = Vec::with_capacity(n * 8); + let mut m = Vec::with_capacity(n * 16); + let mut t = Vec::with_capacity(n); + let mut block_len = Vec::with_capacity(n); + let mut flags = Vec::with_capacity(n); + for i in inputs { + h.extend_from_slice(&i.h); + m.extend_from_slice(&i.m); + t.push(i.t); + block_len.push(i.block_len); + flags.push(i.flags); + } + + let be = backend()?; + let stream = be.next_stream(); + let h_dev = stream.clone_htod(&h)?; + let m_dev = stream.clone_htod(&m)?; + let t_dev = stream.clone_htod(&t)?; + let bl_dev = stream.clone_htod(&block_len)?; + let fl_dev = stream.clone_htod(&flags)?; + let mut out_dev = stream.alloc_zeros::(n * 16)?; + + let kernel = match rounds { + ProbeRounds::Six => &be.blake3_compress_probe_6r, + ProbeRounds::Seven => &be.blake3_compress_probe_7r, + ProbeRounds::CompiledIn => &be.blake3_compress_probe_default, + }; + let n_u64 = n as u64; + let cfg = blake3_launch_cfg(n_u64); + unsafe { + stream + .launch_builder(kernel) + .arg(&h_dev) + .arg(&m_dev) + .arg(&t_dev) + .arg(&bl_dev) + .arg(&fl_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let flat = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(flat + .chunks_exact(16) + .map(|c| { + let mut w = [0u32; 16]; + w.copy_from_slice(c); + w + }) + .collect()) +} + +/// The round count `kernels/blake3.cu` was compiled for. +/// +/// The host tree's round count and this one are separate crates' features, so +/// nothing forces them equal; a mismatch would be a GPU tree committing under a +/// different hash than the CPU one, with no symptom short of a failing verify. +/// Reading it back makes that assertable. +pub fn device_rounds() -> Result { + let be = backend()?; + let stream = be.next_stream(); + let mut out_dev = stream.alloc_zeros::(1)?; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_rounds_probe) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(out[0]) +} + +/// Parity harness: the BLAKE3 message words each of `vals` serializes to — two +/// per element, the byte-reverse of its canonical value's high then low half. +/// +/// This is the serialization the leaf kernels share with the CPU commit path +/// (`leaves_bit_reversed_grouped`, `crypto/stark/src/commitment.rs:55`), isolated +/// from any hashing: canonicalisation, big-endian element bytes, little-endian +/// word packing. +pub fn serialize_felts(vals: &[u64]) -> Result> { + if vals.is_empty() { + return Ok(Vec::new()); + } + let be = backend()?; + let stream = be.next_stream(); + let vals_dev = stream.clone_htod(vals)?; + let mut out_dev = stream.alloc_zeros::(vals.len() * 2)?; + let n_u64 = vals.len() as u64; + let cfg = blake3_launch_cfg(n_u64); + unsafe { + stream + .launch_builder(&be.blake3_serialize_felts_probe) + .arg(&vals_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(out) +} + +/// Parity harness: `words` streamed through the device `Blake3Chain`, returning +/// the 32-byte digest. +/// +/// ★ This is what lets the device be asserted against the COMMITTED KAT TABLE +/// (`crypto::hash::blake3::chain::CHAIN_KAT_6ROUND`) rather than only against +/// the host implementation — the difference risk R13 is about. The KAT digests +/// were produced by a Python oracle, so a device digest matching them is checked +/// against an artifact this tree did not compute. +/// +/// Word-granular because that is all the device ever hashes: production messages +/// are whole numbers of 8-byte field elements. KAT lengths that are not +/// multiples of 4 are unreachable from device code by construction and are +/// covered by the host tests instead. +pub fn chain_probe(words: &[u32]) -> Result<[u8; 32]> { + let be = backend()?; + let stream = be.next_stream(); + // The empty message is a legitimate input (one compression, `block_len = 0`), + // so an empty slice must still reach the kernel. `clone_htod` of an empty + // slice is not portable, so allocate a one-word buffer and pass `n = 0`. + let words_dev = if words.is_empty() { + stream.alloc_zeros::(1)? + } else { + stream.clone_htod(words)? + }; + let mut out_dev = stream.alloc_zeros::(32)?; + let n_words = words.len() as u64; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_chain_probe) + .arg(&words_dev) + .arg(&n_words) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + let mut digest = [0u8; 32]; + digest.copy_from_slice(&out); + Ok(digest) +} + +/// Parity harness: `vals` streamed through the device block builder, returning +/// the `ceil(2*len/16)` completed 64-byte blocks as 16 words each, tail block +/// zero-padded. +/// +/// Exercises the block framing on the code path a leaf kernel will use — one +/// thread streaming a whole leaf — with the compression sink replaced by a copy +/// out. Small inputs only; it is single-threaded by design. +pub fn blocks_of_felts(vals: &[u64]) -> Result> { + if vals.is_empty() { + return Ok(Vec::new()); + } + let n_words = vals.len() * 2; + let n_blocks = n_words.div_ceil(16); + let be = backend()?; + let stream = be.next_stream(); + let vals_dev = stream.clone_htod(vals)?; + let mut out_dev = stream.alloc_zeros::(n_blocks * 16)?; + let n_u64 = vals.len() as u64; + let cfg = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + }; + unsafe { + stream + .launch_builder(&be.blake3_blocks_of_felts_probe) + .arg(&vals_dev) + .arg(&n_u64) + .arg(&mut out_dev) + .launch(cfg)?; + } + let out = stream.clone_dtoh(&out_dev)?; + stream.synchronize()?; + Ok(out) +} diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs index a7c129cc8..c603796e4 100644 --- a/crypto/math-cuda/src/device.rs +++ b/crypto/math-cuda/src/device.rs @@ -135,6 +135,7 @@ const INVERSE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/inverse.c const LOGUP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/logup.cubin")); const CONSTRAINT_INTERP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/constraint_interp.cubin")); +const BLAKE3_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/blake3.cubin")); /// Number of CUDA streams in the pool. Larger pools let many rayon-parallel /// callers overlap on the GPU without serializing on stream ownership. The @@ -202,6 +203,32 @@ pub struct Backend { pub keccak_merkle_tail: CudaFunction, pub merkle_gather_paths: CudaFunction, + // blake3.cubin — the leaf kernels, the Merkle level/tail compressors, and + // the parity-harness probes that are the only host-visible handle on the + // device compression function, byte serialization and chain construction + // (see `kernels/blake3.cu`). Twin for twin with the keccak set above, and in + // the same order. `merkle_gather_paths` has no twin: path gathering copies + // nodes and never hashes, so it is hash-agnostic and both trees share it. + // + // Keccak stays the prover's default, so no production dispatch reaches these + // yet — they exist so the GPU can follow the CPU's hash switch (PA-PLAN §6.1). + pub blake3_leaves_base_row_major_row_pair: CudaFunction, + pub blake3_leaves_base_row_major_row_pair_range: CudaFunction, + pub blake3_leaves_base_batched: CudaFunction, + pub blake3_leaves_base_row_pair_batched: CudaFunction, + pub blake3_leaves_ext3_batched: CudaFunction, + pub blake3_comp_poly_leaves_ext3: CudaFunction, + pub blake3_fri_leaves_ext3: CudaFunction, + pub blake3_merkle_level: CudaFunction, + pub blake3_merkle_tail: CudaFunction, + pub blake3_compress_probe_6r: CudaFunction, + pub blake3_compress_probe_7r: CudaFunction, + pub blake3_compress_probe_default: CudaFunction, + pub blake3_rounds_probe: CudaFunction, + pub blake3_serialize_felts_probe: CudaFunction, + pub blake3_blocks_of_felts_probe: CudaFunction, + pub blake3_chain_probe: CudaFunction, + // barycentric.cubin pub barycentric_base_batched: CudaFunction, pub barycentric_ext3_batched: CudaFunction, @@ -347,6 +374,7 @@ impl Backend { let logup = ctx.load_module(Ptx::from_binary(LOGUP_CUBIN.to_vec()))?; let constraint_interp = ctx.load_module(Ptx::from_binary(CONSTRAINT_INTERP_CUBIN.to_vec()))?; + let blake3 = ctx.load_module(Ptx::from_binary(BLAKE3_CUBIN.to_vec()))?; let mut streams = Vec::with_capacity(STREAM_POOL_SIZE); for _ in 0..STREAM_POOL_SIZE { @@ -432,6 +460,25 @@ impl Backend { keccak_merkle_level: keccak.load_function("keccak_merkle_level")?, keccak_merkle_tail: keccak.load_function("keccak_merkle_tail")?, merkle_gather_paths: keccak.load_function("merkle_gather_paths")?, + blake3_leaves_base_row_major_row_pair: blake3 + .load_function("blake3_leaves_base_row_major_row_pair")?, + blake3_leaves_base_row_major_row_pair_range: blake3 + .load_function("blake3_leaves_base_row_major_row_pair_range")?, + blake3_leaves_base_batched: blake3.load_function("blake3_leaves_base_batched")?, + blake3_leaves_base_row_pair_batched: blake3 + .load_function("blake3_leaves_base_row_pair_batched")?, + blake3_leaves_ext3_batched: blake3.load_function("blake3_leaves_ext3_batched")?, + blake3_comp_poly_leaves_ext3: blake3.load_function("blake3_comp_poly_leaves_ext3")?, + blake3_fri_leaves_ext3: blake3.load_function("blake3_fri_leaves_ext3")?, + blake3_merkle_level: blake3.load_function("blake3_merkle_level")?, + blake3_merkle_tail: blake3.load_function("blake3_merkle_tail")?, + blake3_compress_probe_6r: blake3.load_function("blake3_compress_probe_6r")?, + blake3_compress_probe_7r: blake3.load_function("blake3_compress_probe_7r")?, + blake3_compress_probe_default: blake3.load_function("blake3_compress_probe_default")?, + blake3_rounds_probe: blake3.load_function("blake3_rounds_probe")?, + blake3_serialize_felts_probe: blake3.load_function("blake3_serialize_felts_probe")?, + blake3_blocks_of_felts_probe: blake3.load_function("blake3_blocks_of_felts_probe")?, + blake3_chain_probe: blake3.load_function("blake3_chain_probe")?, barycentric_base_batched: bary.load_function("barycentric_base_batched")?, barycentric_ext3_batched: bary.load_function("barycentric_ext3_batched")?, barycentric_base_batched_strided: bary diff --git a/crypto/math-cuda/src/lib.rs b/crypto/math-cuda/src/lib.rs index 6b58d935b..4a2f0c6cc 100644 --- a/crypto/math-cuda/src/lib.rs +++ b/crypto/math-cuda/src/lib.rs @@ -6,6 +6,7 @@ //! pipelines or used by the parity test suite. pub mod barycentric; +pub mod blake3; pub mod constraint_interp; pub mod deep; pub mod device; diff --git a/crypto/math-cuda/tests/blake3_chain_kat.rs b/crypto/math-cuda/tests/blake3_chain_kat.rs new file mode 100644 index 000000000..4b3687d6e --- /dev/null +++ b/crypto/math-cuda/tests/blake3_chain_kat.rs @@ -0,0 +1,243 @@ +//! ★ Known-answer tests for the DEVICE `Blake3Chain`, against references that +//! are not this tree's Rust. +//! +//! # Why this file exists (risk R13) +//! +//! Every other parity test here asserts device == host. That is necessary and +//! not sufficient: the device kernels were transcribed from the host reference, +//! so a shared misreading of the construction passes all of them. R13 is exactly +//! that gap — "track G would then be checking a device port against the same +//! code path it was derived from". +//! +//! Two references close it, and neither is Rust in this repository: +//! +//! - **At 7 rounds, the official `blake3` crate.** `Blake3Chain` over any +//! message of at most one chunk (1024 bytes) IS `blake3::hash`, because +//! standard BLAKE3's first chunk is this chain and a one-chunk message has +//! that chunk's output as its root (PA-PLAN §1.7.2, P1). So for the whole +//! range the prover actually hashes in, the device is checked against a +//! published, externally maintained implementation with nothing in between. +//! - **At 6 rounds, `CHAIN_KAT_6ROUND`.** Those digests came from #903's Python +//! oracle, not from this code (`chain.rs:280-294`). Asserting the device +//! against them is a check against an artifact this tree did not compute. +//! +//! # Why the coverage is at multiples of four bytes +//! +//! The device chain is word-granular, because that is all it ever hashes: every +//! production message is a whole number of 8-byte field elements. The KAT +//! lengths that are not multiples of 4 (1, 31, 63, 127) are unreachable from +//! device code by construction and are covered by the host tests in +//! `crypto/crypto/src/hash/blake3/chain.rs` instead. What remains — 0, 64, 128, +//! 192, 256, 1024, 1088 — still covers every structural case PA-PLAN §1.7.4 +//! names except the partial-tail ones: the empty message is one block (0), a +//! 64-byte message is the parent form (64), an exact multiple of 64 emits no +//! spurious final block (128), interior blocks carry no flags (192, 256, 1024), +//! and 1088 is where this construction leaves standard BLAKE3. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::{expected_device_rounds, merkle_parent}; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::hash::blake3::chain::{ + CHAIN_KAT_6ROUND, CHAIN_KAT_LENS, blake3_chain_rounds, kat_message_byte, +}; +use math_cuda::blake3::chain_probe; + +/// The KAT message of a given length: byte `i` is `37i + 11 (mod 256)`. +fn message(len: usize) -> Vec { + (0..len).map(kat_message_byte).collect() +} + +/// A byte message as the little-endian u32 words the device chain absorbs. +/// Panics on a length that is not a whole number of words — see the module docs +/// for why that case cannot arise on device. +fn words(msg: &[u8]) -> Vec { + assert!(msg.len().is_multiple_of(4), "device chain is word-granular"); + msg.chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect() +} + +fn assert_lockstep() { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep" + ); +} + +/// The KAT lengths the device can hash, paired with their index into +/// `CHAIN_KAT_6ROUND`. +fn device_reachable_lengths() -> Vec<(usize, usize)> { + CHAIN_KAT_LENS + .iter() + .enumerate() + .filter(|&(_, &len)| len.is_multiple_of(4)) + .map(|(i, &len)| (i, len)) + .collect() +} + +/// ★ THE EXTERNAL ANCHOR. At 7 rounds the device chain must be the `blake3` +/// crate's hash, for every reachable length up to one full chunk. +/// +/// This is the strongest statement available about the device port: no oracle, +/// no table, no transcription — a published implementation computes the same +/// bytes. It pins the block splitting, the zero padding, the final block's +/// `block_len`, the CHUNK_START/CHUNK_END/ROOT schedule, `t = 0` throughout, and +/// the little-endian digest read-back, all at once. +/// +/// Only meaningful when the cubin is built for 7 rounds; under `blake3-6round` +/// nothing external recomputes this, which is PA-PLAN §1.6's premise and why the +/// 6-round arm needs the committed table instead. +#[test] +fn device_chain_is_the_blake3_crate_at_seven_rounds() { + assert_lockstep(); + if expected_device_rounds() != 7 { + return; + } + for (_, len) in device_reachable_lengths() { + if len > 1024 { + continue; + } + let msg = message(len); + let device = chain_probe(&words(&msg)).unwrap(); + assert_eq!( + device, + *blake3::hash(&msg).as_bytes(), + "device chain must equal the blake3 crate at length {len}" + ); + } +} + +/// ★ P3, on device. Past one chunk the construction deliberately leaves standard +/// BLAKE3 — the standard would start chunk 1 with `t = 1` and a reset chaining +/// value, this keeps chaining. Without this the test above would pass +/// identically if the kernels had implemented the whole chunk tree, so "the +/// device implements the single-chunk chain" would be unfalsifiable. +#[test] +fn device_chain_leaves_the_blake3_crate_past_one_chunk() { + assert_lockstep(); + if expected_device_rounds() != 7 { + return; + } + // 1024 is the last length where they agree; 1088 the first reachable one + // past it. Asserting both locates the divergence rather than just observing + // one. + let agreeing = message(1024); + assert_eq!( + chain_probe(&words(&agreeing)).unwrap(), + *blake3::hash(&agreeing).as_bytes(), + "1024 bytes is still one chunk and must agree" + ); + let diverging = message(1088); + assert_ne!( + chain_probe(&words(&diverging)).unwrap(), + *blake3::hash(&diverging).as_bytes(), + "past one chunk the device must leave the standard" + ); +} + +/// ★ THE 6-ROUND ANCHOR. The device must reproduce the committed KAT table, +/// whose digests came from a Python oracle rather than from this code. +/// +/// This is the assertion R13 asks for: at the round count the campaign actually +/// ships, the device port is pinned by numbers no Rust in this tree produced. +#[test] +fn device_chain_matches_the_committed_table_at_six_rounds() { + assert_lockstep(); + if expected_device_rounds() != 6 { + return; + } + for (i, len) in device_reachable_lengths() { + let device = chain_probe(&words(&message(len))).unwrap(); + assert_eq!( + device, CHAIN_KAT_6ROUND[i], + "device chain must match the committed 6-round KAT at length {len}" + ); + } +} + +/// The device chain against the host chain at whatever round count this build +/// uses. Weaker than the two anchors above — both sides are ours — but it is the +/// one that runs in every configuration, and it is the property the commitment +/// path actually needs: a GPU tree and a CPU tree over the same leaves must be +/// the same tree. +#[test] +fn device_chain_matches_the_host_chain() { + assert_lockstep(); + let rounds = expected_device_rounds(); + for (_, len) in device_reachable_lengths() { + let msg = message(len); + assert_eq!( + chain_probe(&words(&msg)).unwrap(), + blake3_chain_rounds(&msg, rounds), + "device/host chain mismatch at length {len}, {rounds} rounds" + ); + } + // Lengths off the KAT list, stepping through several block boundaries, so + // the agreement is not an artifact of the seven lengths chosen above. + for len in (0..=520usize).step_by(4) { + let msg = message(len); + assert_eq!( + chain_probe(&words(&msg)).unwrap(), + blake3_chain_rounds(&msg, rounds), + "device/host chain mismatch at length {len}" + ); + } +} + +/// ★ P2, on device: a 64-byte message through the chain is exactly the Merkle +/// parent compression. +/// +/// This is the invariant that lets the leaf and parent layers be one hash — and +/// the reason `blake3_hash_merkle_parent` can be a single compression with no +/// chaining at all. If the chain's flag schedule or `block_len` moved, the two +/// would part here while every leaf test still passed. +#[test] +fn a_sixty_four_byte_chain_is_the_parent_compression() { + assert_lockstep(); + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(7)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(3)); + let mut msg = [0u8; 64]; + msg[..32].copy_from_slice(&left); + msg[32..].copy_from_slice(&right); + + assert_eq!( + chain_probe(&words(&msg)).unwrap(), + merkle_parent(&left, &right, expected_device_rounds()), + "a 64-byte device chain must be the parent form" + ); +} + +/// NEGATIVE CONTROL: distinct lengths must give distinct digests, or the tests +/// above would pass with a probe that ignored its input length. In particular a +/// chain that ignored `block_len` would collide 0 with nothing visible here, but +/// one that dropped the final partial block would collide 64 with 128. +#[test] +fn device_digests_are_distinct_across_lengths() { + assert_lockstep(); + let mut seen: Vec<[u8; 32]> = Vec::new(); + for len in (0..=256usize).step_by(4) { + let d = chain_probe(&words(&message(len))).unwrap(); + assert!( + !seen.contains(&d), + "length {len} collides with a shorter message" + ); + seen.push(d); + } +} + +/// The cubin's compiled-in round count must be the one the Rust side thinks it +/// is. Reading it back is the only way to observe from host code which arm +/// `blake3_merkle_level` and the leaf kernels were built for; a mismatch here is +/// a GPU tree committing under a different hash with no other symptom. +#[test] +fn the_cubin_round_count_is_what_the_feature_selected() { + assert_eq!( + math_cuda::blake3::device_rounds().unwrap() as usize, + expected_device_rounds(), + "cubin round count disagrees with math-cuda's blake3-6round feature" + ); +} diff --git a/crypto/math-cuda/tests/blake3_comp_poly_tree.rs b/crypto/math-cuda/tests/blake3_comp_poly_tree.rs new file mode 100644 index 000000000..71231968b --- /dev/null +++ b/crypto/math-cuda/tests/blake3_comp_poly_tree.rs @@ -0,0 +1,159 @@ +//! Parity: the device BLAKE3 composition-polynomial tree must equal the CPU +//! tree node for node, through BOTH wrappers that build it. +//! +//! `build_comp_poly_tree_from_evals_ext3_keep` takes host-side interleaved parts +//! and stages them through the pinned de-interleave buffer; +//! `build_comp_poly_tree_from_slabs_dev` takes an already-resident slab buffer +//! and never touches the host. They share the leaf kernel and the level walk but +//! not the staging, so a de-interleave bug shows in the first and not the second +//! — which is why both are exercised here rather than only the one the leaf test +//! happens to call. +//! +//! CPU reference is the production leaf function plus the production tree +//! builder, so nothing in the comparison is written for the test. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::expected_device_rounds; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::merkle_tree::backends::types::BatchBlake3Backend; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use stark::commitment::leaves_bit_reversed_grouped; + +type Fp = FieldElement; +type Fp3 = FieldElement; +type Ext3 = Degree3GoldilocksExtensionField; + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +fn assert_lockstep() { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep" + ); +} + +/// The CPU node buffer for these parts: production row-pair leaves, production +/// tree walk. +fn cpu_nodes(parts: &[Vec]) -> Vec<[u8; 32]> { + let leaves = leaves_bit_reversed_grouped::>(parts, 2); + let tree = MerkleTree::>::build_from_hashed_leaves(leaves).unwrap(); + tree.nodes().to_vec() +} + +fn interleave(parts: &[Vec], lde_size: usize) -> Vec> { + parts + .iter() + .map(|p| { + let mut v = vec![0u64; 3 * lde_size]; + for (i, e) in p.iter().enumerate() { + v[i * 3] = *e.value()[0].value(); + v[i * 3 + 1] = *e.value()[1].value(); + v[i * 3 + 2] = *e.value()[2].value(); + } + v + }) + .collect() +} + +/// The de-interleaved slab layout the device wrapper consumes directly: +/// component `k` of part `c` at `(c*3 + k) * lde_size`. +fn slabs(parts: &[Vec], lde_size: usize) -> Vec { + let mut buf = vec![0u64; 3 * parts.len() * lde_size]; + for (c, p) in parts.iter().enumerate() { + for (r, e) in p.iter().enumerate() { + buf[(c * 3) * lde_size + r] = *e.value()[0].value(); + buf[(c * 3 + 1) * lde_size + r] = *e.value()[1].value(); + buf[(c * 3 + 2) * lde_size + r] = *e.value()[2].value(); + } + } + buf +} + +fn assert_nodes_eq(gpu: &[u8], cpu: &[[u8; 32]], what: &str) { + assert_eq!(gpu.len(), cpu.len() * 32, "{what}: node count"); + for (i, expected) in cpu.iter().enumerate() { + assert_eq!( + &gpu[i * 32..(i + 1) * 32], + &expected[..], + "{what}: node {i} mismatch" + ); + } +} + +fn run_parity(log_lde: u32, num_parts: usize, seed: u64) { + assert_lockstep(); + let lde_size = 1usize << log_lde; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let parts: Vec> = (0..num_parts) + .map(|_| (0..lde_size).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + let expected = cpu_nodes(&parts); + let what = format!("log_lde={log_lde} parts={num_parts}"); + + let be = math_cuda::device::backend().unwrap(); + + // Route 1: host-side interleaved parts through the pinned staging path. + let interleaved = interleave(&parts, lde_size); + let slices: Vec<&[u64]> = interleaved.iter().map(|v| v.as_slice()).collect(); + let keep = math_cuda::blake3::build_comp_poly_tree_from_evals_ext3_keep(&slices).unwrap(); + { + let stream = be.next_stream(); + let nodes: Vec = stream.clone_dtoh(&*keep.nodes).unwrap(); + assert_nodes_eq(&nodes, &expected, &format!("keep {what}")); + assert_eq!(&keep.root[..], &expected[0][..], "keep {what}: root"); + assert_eq!(keep.leaves_len, lde_size / 2, "keep {what}: leaf count"); + } + + // Route 2: an already-resident slab buffer, no host staging. + { + let stream = be.next_stream(); + let buf = stream.clone_htod(&slabs(&parts, lde_size)).unwrap(); + stream.synchronize().unwrap(); + let dev = math_cuda::blake3::build_comp_poly_tree_from_slabs_dev( + &stream, &buf, num_parts, lde_size, + ) + .unwrap(); + let nodes: Vec = stream.clone_dtoh(&*dev.nodes).unwrap(); + assert_nodes_eq(&nodes, &expected, &format!("slabs {what}")); + assert_eq!(&dev.root[..], &expected[0][..], "slabs {what}: root"); + } +} + +/// Small trees: the tail kernel builds every level in one launch. +#[test] +fn blake3_comp_poly_tree_small() { + for log_lde in [2u32, 4, 6, 8] { + for num_parts in [1usize, 2, 5] { + run_parity(log_lde, num_parts, 300 + log_lde as u64 + num_parts as u64); + } + } +} + +/// Deep enough to cross from the per-level kernel into the tail. +#[test] +fn blake3_comp_poly_tree_medium() { + for log_lde in [10u32, 12, 14] { + run_parity(log_lde, 17, 700 + log_lde as u64); + } +} + +#[test] +fn blake3_comp_poly_tree_large() { + run_parity(18, 3, 4242); +} diff --git a/crypto/math-cuda/tests/blake3_compress_parity.rs b/crypto/math-cuda/tests/blake3_compress_parity.rs new file mode 100644 index 000000000..9032ba525 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_compress_parity.rs @@ -0,0 +1,217 @@ +//! Parity: the device BLAKE3 compression function must equal the host reference +//! bit-for-bit, at both round counts. +//! +//! The reference is one function whose only parameter is the round count +//! (`crypto::hash::blake3::blake3_compress_rounds`, re-exported through +//! `blake3_reference` — the same function the host commitment backends and the +//! in-circuit chip use, not a copy of it). So the 7-round arm, where the `blake3` crate +//! is an external known-answer test, certifies the whole device code path — the +//! G function, the message schedule, the counter split, the feed-forward — and +//! the 6-round arm differs from it by a loop bound alone. That is why the +//! anchor below is worth more than a table of 6-round vectors would be. +//! +//! Every test here needs a GPU, like the rest of this crate's parity suite. + +mod blake3_reference; + +use blake3_reference::{ + BLAKE3_IV, BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS, FLAGS_ONE_BLOCK, blake3_compress_rounds, + expected_device_rounds, +}; +use math_cuda::blake3::{CompressInput, ProbeRounds, compress_probe, device_rounds}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +/// Inputs that between them move every field of the compression's framing. +/// +/// `block_len` and `flags` are state words, not lengths the kernel loops over, so +/// a port that dropped either would still pass on a single value of it — hence +/// the spread, including the 18..=64 range the host's canonical vectors cover and +/// the 36 the LFM socket uses. `t` carries values whose halves differ, since the +/// counter split is a real way to be wrong and a symmetric `t` cannot see it. +fn vectors(seed: u64) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let block_lens = [0u32, 1, 4, 18, 36, 63, 64]; + let flags = [0u32, 1, 2, 8, FLAGS_ONE_BLOCK, 0x0C, 0xFFFF_FFFF]; + let counters = [ + 0u64, + 1, + 0xFFFF_FFFF, + 0x1_0000_0000, + 0xB4E1_357D_4A84_EB03, + u64::MAX, + ]; + + let mut out = Vec::new(); + // Walk the framing exhaustively over random h/m: 7 × 7 × 6 = 294, plus the + // eight `h = IV` cases below for 302 — not a multiple of the kernel's block + // width, so the `tid >= n` guard is exercised too. + for &block_len in block_lens.iter() { + for &fl in flags.iter() { + for &t in counters.iter() { + out.push(CompressInput { + h: core::array::from_fn(|_| rng.r#gen::()), + m: core::array::from_fn(|_| rng.r#gen::()), + t, + block_len, + flags: fl, + }); + } + } + } + // `h = IV` is the case every real call site uses, and a random h would never + // hit it: the feed-forward `out[i+8] = v[i+8] ^ h[i]` reads h a second time. + for _ in 0..8 { + out.push(CompressInput { + h: BLAKE3_IV, + m: core::array::from_fn(|_| rng.r#gen::()), + t: 0, + block_len: 64, + flags: FLAGS_ONE_BLOCK, + }); + } + out +} + +fn host_outputs(inputs: &[CompressInput], rounds: usize) -> Vec<[u32; 16]> { + inputs + .iter() + .map(|i| blake3_compress_rounds(&i.h, &i.m, i.t, i.block_len, i.flags, rounds)) + .collect() +} + +fn assert_parity(rounds: usize, probe: ProbeRounds, seed: u64) { + let inputs = vectors(seed); + let device = compress_probe(&inputs, probe).unwrap(); + let host = host_outputs(&inputs, rounds); + assert_eq!(device.len(), host.len()); + for (i, (d, h)) in device.iter().zip(host.iter()).enumerate() { + assert_eq!( + d, h, + "vector {i} mismatch at {rounds} rounds: input {:?}", + inputs[i] + ); + } +} + +#[test] +fn device_compression_matches_host_at_six_rounds() { + assert_parity(BLAKE3_SIX_ROUNDS, ProbeRounds::Six, 6001); +} + +#[test] +fn device_compression_matches_host_at_seven_rounds() { + assert_parity(BLAKE3_STANDARD_ROUNDS, ProbeRounds::Seven, 7001); +} + +/// ★ **The external anchor.** At 7 rounds a message of at most 64 bytes is one +/// chunk and one block, so the whole tree hasher collapses to a single `f`: +/// `h = IV`, the block zero-padded and read as 16 little-endian words, `t = 0`, +/// `block_len` the true length, `flags = CHUNK_START|CHUNK_END|ROOT`. The digest +/// is `out[0..8]` little-endian. +/// +/// Mirrors `prover/src/lfm/blake3.rs`'s `seven_rounds_is_the_blake3_crate`, over +/// the same 65 lengths and for the same reason: the length keys both `block_len` +/// and the padding, and a port that ignored `block_len` would pass at one length. +#[test] +fn seven_rounds_on_device_is_the_blake3_crate() { + let inputs: Vec = (0..=64usize) + .map(|len| { + let mut block = [0u8; 64]; + for (i, b) in block.iter_mut().take(len).enumerate() { + *b = (i as u8).wrapping_mul(37).wrapping_add(11); + } + CompressInput { + h: BLAKE3_IV, + m: core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }), + t: 0, + block_len: len as u32, + flags: FLAGS_ONE_BLOCK, + } + }) + .collect(); + + let device = compress_probe(&inputs, ProbeRounds::Seven).unwrap(); + for (len, out) in device.iter().enumerate() { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_eq!( + ours, + *blake3::hash(&msg).as_bytes(), + "device 7-round compression must equal the blake3 crate at length {len}" + ); + } +} + +/// NEGATIVE CONTROL for the anchor above: at 6 rounds the device must NOT match +/// the crate. Without it, the anchor would pass just as well if the round count +/// were being ignored on device — the one bug that makes the whole +/// external-anchor argument vacuous, since the 6-round arm's only defence is +/// "the same code path with the loop bound changed". +#[test] +fn six_rounds_on_device_is_not_the_blake3_crate() { + let msg: [u8; 36] = core::array::from_fn(|i| i as u8); + let mut block = [0u8; 64]; + block[..36].copy_from_slice(&msg); + let input = CompressInput { + h: BLAKE3_IV, + m: core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }), + t: 0, + block_len: 36, + flags: FLAGS_ONE_BLOCK, + }; + let out = compress_probe(&[input], ProbeRounds::Six).unwrap()[0]; + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_ne!(ours, *blake3::hash(&msg).as_bytes()); +} + +/// The two round counts must actually differ on device. Guards the shape of the +/// port itself: `blake3_compress` is one template instantiated twice, and a +/// template that collapsed (or a `#pragma unroll` that outran the bound) would +/// leave both parity tests above passing against a single arm. +#[test] +fn the_two_device_round_counts_differ() { + let inputs = vectors(4242); + let six = compress_probe(&inputs, ProbeRounds::Six).unwrap(); + let seven = compress_probe(&inputs, ProbeRounds::Seven).unwrap(); + for (i, (a, b)) in six.iter().zip(seven.iter()).enumerate() { + assert_ne!(a, b, "6r and 7r agree on vector {i}"); + } +} + +/// The round count the production kernels are compiled for must be the one the +/// `blake3-6round` feature selects. +/// +/// This is the tripwire for a cross-crate feature mismatch: math-cuda's feature +/// and the host tree's are separate, nothing forces them equal, and the symptom +/// of a mismatch is a GPU tree that commits under a different hash than the CPU +/// one — no panic, no log line, just a proof that fails to verify. Asserting the +/// cubin's own round count is what turns that into a test failure. +#[test] +fn the_compiled_in_round_count_is_the_feature() { + assert_eq!( + device_rounds().unwrap() as usize, + expected_device_rounds(), + "kernels/blake3.cu was compiled for a different round count than the \ + math-cuda `blake3-6round` feature selects — check build.rs's -D plumbing" + ); + + // And the default probe must be the corresponding explicit arm, which is what + // ties `blake3_merkle_level`'s hash to the number reported above. + let inputs = vectors(909); + let default = compress_probe(&inputs, ProbeRounds::CompiledIn).unwrap(); + let host = host_outputs(&inputs, expected_device_rounds()); + assert_eq!(default, host); +} diff --git a/crypto/math-cuda/tests/blake3_fri_layer_tree.rs b/crypto/math-cuda/tests/blake3_fri_layer_tree.rs new file mode 100644 index 000000000..9bdaa33ba --- /dev/null +++ b/crypto/math-cuda/tests/blake3_fri_layer_tree.rs @@ -0,0 +1,96 @@ +//! Parity: the device BLAKE3 FRI-layer tree must equal the CPU tree node for +//! node — leaves and inner nodes alike. +//! +//! Mirror of `fri_layer_tree.rs` with the backend swapped. The CPU side is the +//! production `MerkleTree::build` over `PairBlake3Backend`, so what is compared +//! is the kernel pair against the real commitment path, not against a tree +//! builder written for the test. +//! +//! `blake3_leaves.rs` already pins the leaf layer alone; this adds the inner +//! nodes, which is where the level/tail launch split lives. Deep trees cross the +//! threshold where `blake3_merkle_level` hands over to `blake3_merkle_tail`, so +//! both kernels run. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::expected_device_rounds; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::merkle_tree::backends::types::PairBlake3Backend; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math_cuda::blake3::build_fri_layer_tree_from_evals_ext3; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; +type Fp3 = FieldElement; +type Ext3 = Degree3GoldilocksExtensionField; + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +fn run_parity(log_num_leaves: u32, seed: u64) { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep" + ); + + let num_leaves = 1usize << log_num_leaves; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let evals: Vec = (0..num_leaves * 2).map(|_| rand_ext3(&mut rng)).collect(); + + let mut evals_u64 = Vec::with_capacity(evals.len() * 3); + for e in &evals { + evals_u64.push(*e.value()[0].value()); + evals_u64.push(*e.value()[1].value()); + evals_u64.push(*e.value()[2].value()); + } + + let leaves: Vec<[Fp3; 2]> = evals.chunks_exact(2).map(|c| [c[0], c[1]]).collect(); + let cpu_tree = MerkleTree::>::build(&leaves).unwrap(); + let cpu_nodes = cpu_tree.nodes(); + + let gpu_bytes = build_fri_layer_tree_from_evals_ext3(&evals_u64).unwrap(); + + assert_eq!(cpu_nodes.len() * 32, gpu_bytes.len(), "node count"); + for (i, expected) in cpu_nodes.iter().enumerate() { + assert_eq!( + &gpu_bytes[i * 32..(i + 1) * 32], + &expected[..], + "node {i} mismatch at log_num_leaves={log_num_leaves}" + ); + } +} + +/// Small trees: every level fits the block width, so the tail kernel builds the +/// whole tree in one launch. +#[test] +fn blake3_fri_layer_tree_small() { + for log in 1u32..=6 { + run_parity(log, 100 + log as u64); + } +} + +/// Deep enough that the per-level kernel runs first and hands over to the tail +/// partway up — the launch path a real commit takes. +#[test] +fn blake3_fri_layer_tree_medium() { + for log in [10u32, 12, 14] { + run_parity(log, 500 + log as u64); + } +} + +#[test] +fn blake3_fri_layer_tree_large() { + run_parity(18, 9999); +} diff --git a/crypto/math-cuda/tests/blake3_leaves.rs b/crypto/math-cuda/tests/blake3_leaves.rs new file mode 100644 index 000000000..193d149ea --- /dev/null +++ b/crypto/math-cuda/tests/blake3_leaves.rs @@ -0,0 +1,386 @@ +//! Parity: the device BLAKE3 leaf kernels must reproduce the CPU prover's leaf +//! hashes byte for byte. +//! +//! Structural mirror of `keccak_leaves.rs`, and deliberately so: the leaf BYTE +//! layout does not move under P-a. `leaves_bit_reversed_grouped` +//! (`crypto/stark/src/commitment.rs:55`) serializes each element in canonical +//! big-endian, concatenates, and hashes the buffer once — the same bytes for +//! both hashes. What changes is only the hash over them, so the CPU reference +//! here is the *production* leaf function instantiated at the BLAKE3 backend +//! rather than a second implementation written for the test. +//! +//! That makes each assertion below a check of two things at once: that the +//! kernel's read pattern (bit reversal, column order, component order, row-pair +//! ordering) matches the CPU's, and that the device `Blake3Chain` matches the +//! host one over multi-block messages. +//! +//! Needs a GPU. See `RESUME-TRACKG.md` for the run command. + +mod blake3_reference; + +use blake3_reference::expected_device_rounds; +use crypto::hash::blake3::BLAKE3_ROUNDS; +use crypto::merkle_tree::backends::types::{BatchBlake3Backend, PairBlake3Backend}; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; +use math::traits::{AsBytes, ByteConversion}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use stark::commitment::leaves_bit_reversed_grouped; +use stark::config::Commitment; + +type Fp = FieldElement; +type Fp3 = FieldElement; + +/// The CPU leaf hashes for `columns`, through the production leaf function at +/// the BLAKE3 batched backend. +fn cpu_leaves(columns: &[Vec>], rows_per_leaf: usize) -> Vec +where + E: IsField, + FieldElement: AsBytes + Sync + Send + ByteConversion, +{ + leaves_bit_reversed_grouped::>(columns, rows_per_leaf) +} + +/// ★ LOCKSTEP GUARD. `crypto`'s `blake3-6round` and `math-cuda`'s are separate +/// features and nothing forces them equal. Out of lockstep, every assertion in +/// this file compares a 6-round device tree against a 7-round host one (or the +/// reverse) and fails with a wall of unequal bytes that says nothing about the +/// cause. Failing here first names it. +/// +/// This is the same guard `blake3_reference`'s parent test carries, repeated +/// because a leaf-kernel failure has the same ambiguity and a developer running +/// only this file would not see the other one. +fn assert_round_lockstep() { + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep: the GPU \ + kernels would commit under a different hash than the CPU backend. Set \ + both features or neither." + ); +} + +fn rand_base(rng: &mut ChaCha8Rng) -> Fp { + Fp::from_raw(rng.r#gen::()) +} + +fn rand_ext3(rng: &mut ChaCha8Rng) -> Fp3 { + Fp3::new([ + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + Fp::from_raw(rng.r#gen::()), + ]) +} + +/// Base columns into the contiguous `[col * stride + row]` slab the kernels read +/// — the layout `coset_lde_batch_base_into` writes to pinned staging. +fn base_slabs(columns: &[Vec], n: usize) -> Vec { + let mut flat = vec![0u64; columns.len() * n]; + for (c, col) in columns.iter().enumerate() { + for (r, e) in col.iter().enumerate() { + flat[c * n + r] = *e.value(); + } + } + flat +} + +/// Ext3 columns into three base slabs per column: `[col*3 + k]`, each a +/// contiguous slab of `n` u64s. +fn ext3_slabs(columns: &[Vec], n: usize) -> Vec { + let mut flat = vec![0u64; columns.len() * 3 * n]; + for (c, col) in columns.iter().enumerate() { + for (r, e) in col.iter().enumerate() { + flat[(c * 3) * n + r] = *e.value()[0].value(); + flat[(c * 3 + 1) * n + r] = *e.value()[1].value(); + flat[(c * 3 + 2) * n + r] = *e.value()[2].value(); + } + } + flat +} + +fn assert_leaves_eq(gpu: &[u8], cpu: &[Commitment], what: &str) { + assert_eq!(gpu.len(), cpu.len() * 32, "{what}: leaf count"); + for (i, expected) in cpu.iter().enumerate() { + assert_eq!( + &gpu[i * 32..(i + 1) * 32], + &expected[..], + "{what}: leaf {i} mismatch" + ); + } +} + +/// Column counts are chosen to straddle the 64-byte block boundary in both +/// directions: 8 base elements fill a block exactly, so 1/5/17/41 columns give +/// leaves that end mid-block, on a boundary, and several blocks in. That is +/// where a chaining bug lives — a kernel that compressed eagerly on fill, or +/// mis-set `CHUNK_START` on a later block, agrees with the host at one column +/// count and not at the next. +#[test] +fn blake3_leaves_base_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10, 12] { + for num_cols in [1usize, 5, 8, 17, 41] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(100 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_base(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 1); + let flat = base_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_base(&flat, n, num_cols, n, 1).unwrap(); + assert_leaves_eq(&gpu, &cpu, &format!("base log_n={log_n} cols={num_cols}")); + } + } +} + +#[test] +fn blake3_leaves_base_row_pair_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10, 12] { + for num_cols in [1usize, 5, 8, 17, 41] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(500 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_base(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 2); + assert_eq!(cpu.len(), n / 2); + let flat = base_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_base(&flat, n, num_cols, n, 2).unwrap(); + assert_leaves_eq( + &gpu, + &cpu, + &format!("base row-pair log_n={log_n} cols={num_cols}"), + ); + } + } +} + +/// Ext3 elements are three felts = six words, so they straddle block boundaries +/// on most column counts rather than only on a few — the case the word-granular +/// (rather than element-granular) block builder exists for. +#[test] +fn blake3_leaves_ext3_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + for num_cols in [1usize, 3, 11, 20] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(200 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 1); + let flat = ext3_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_ext3(&flat, n, num_cols, n, 1).unwrap(); + assert_leaves_eq(&gpu, &cpu, &format!("ext3 log_n={log_n} cols={num_cols}")); + } + } +} + +#[test] +fn blake3_leaves_ext3_row_pair_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + for num_cols in [1usize, 3, 11, 20] { + let n = 1 << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(600 + log_n as u64 + num_cols as u64); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + + let cpu = cpu_leaves(&columns, 2); + assert_eq!(cpu.len(), n / 2); + let flat = ext3_slabs(&columns, n); + let gpu = math_cuda::blake3::leaves_ext3(&flat, n, num_cols, n, 2).unwrap(); + assert_leaves_eq( + &gpu, + &cpu, + &format!("ext3 row-pair log_n={log_n} cols={num_cols}"), + ); + } + } +} + +/// FRI leaves are 48 bytes — under one block — so this is the chain's degenerate +/// single-compression case: `flags = 0x0B`, `block_len = 48`. Same shape as a +/// Merkle parent at a different length, which is why it is worth pinning +/// separately from the multi-block leaves above. +#[test] +fn blake3_fri_leaves_matches_cpu() { + assert_round_lockstep(); + for log_lde in [2u32, 4, 6, 8, 10, 12] { + let lde_size = 1usize << log_lde; + let mut rng = ChaCha8Rng::seed_from_u64(400 + log_lde as u64); + let evals: Vec = (0..lde_size).map(|_| rand_ext3(&mut rng)).collect(); + + let cpu: Vec<[u8; 32]> = evals + .chunks_exact(2) + .map(|c| PairBlake3Backend::::hash_data(&[c[0], c[1]])) + .collect(); + + let mut evals_interleaved = vec![0u64; 3 * lde_size]; + for (i, e) in evals.iter().enumerate() { + evals_interleaved[i * 3] = *e.value()[0].value(); + evals_interleaved[i * 3 + 1] = *e.value()[1].value(); + evals_interleaved[i * 3 + 2] = *e.value()[2].value(); + } + let nodes = + math_cuda::blake3::build_fri_layer_tree_from_evals_ext3(&evals_interleaved).unwrap(); + let num_leaves = lde_size / 2; + let leaves_offset = (num_leaves - 1) * 32; + assert_leaves_eq( + &nodes[leaves_offset..leaves_offset + num_leaves * 32], + &cpu, + &format!("fri log_lde={log_lde}"), + ); + } +} + +/// The comp-poly kernel through the production keep path, checked at the leaf +/// layer: the resident node buffer's leaf half must be the CPU's row-pair leaves. +#[test] +fn blake3_comp_poly_leaves_matches_cpu() { + assert_round_lockstep(); + for log_lde in [2u32, 4, 6, 8, 10, 12] { + for num_parts in [1usize, 2, 5, 17] { + let lde_size = 1usize << log_lde; + let mut rng = ChaCha8Rng::seed_from_u64(300 + log_lde as u64 + num_parts as u64); + let parts: Vec> = (0..num_parts) + .map(|_| (0..lde_size).map(|_| rand_ext3(&mut rng)).collect()) + .collect(); + let cpu = cpu_leaves(&parts, 2); + + let parts_interleaved: Vec> = parts + .iter() + .map(|p| { + let mut v = vec![0u64; 3 * lde_size]; + for (i, e) in p.iter().enumerate() { + v[i * 3] = *e.value()[0].value(); + v[i * 3 + 1] = *e.value()[1].value(); + v[i * 3 + 2] = *e.value()[2].value(); + } + v + }) + .collect(); + let parts_slices: Vec<&[u64]> = + parts_interleaved.iter().map(|v| v.as_slice()).collect(); + + let tree = math_cuda::blake3::build_comp_poly_tree_from_evals_ext3_keep(&parts_slices) + .unwrap(); + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let nodes: Vec = stream.clone_dtoh(&*tree.nodes).unwrap(); + let num_leaves = lde_size / 2; + let leaves_offset = (num_leaves - 1) * 32; + assert_leaves_eq( + &nodes[leaves_offset..leaves_offset + num_leaves * 32], + &cpu, + &format!("comp-poly log_lde={log_lde} parts={num_parts}"), + ); + } + } +} + +/// Row-major row-pair leaves. The CPU reference is the same +/// `leaves_bit_reversed_grouped(.., 2)` — over the column-major view of the same +/// buffer, which is exactly the equivalence the row-major kernel exists to +/// exploit (`commit_rows_bit_reversed` reads rows contiguously instead of +/// transposing). +#[test] +fn blake3_leaves_row_major_row_pair_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + for m in [1usize, 5, 8, 17] { + let n = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(800 + log_n as u64 + m as u64); + // Row-major: row r occupies `data[r*m .. r*m + m]`. + let data: Vec = (0..n * m).map(|_| rand_base(&mut rng)).collect(); + + let columns: Vec> = (0..m) + .map(|c| (0..n).map(|r| data[r * m + c]).collect()) + .collect(); + let cpu = cpu_leaves(&columns, 2); + + let raw: Vec = data.iter().map(|e| *e.value()).collect(); + let gpu = math_cuda::blake3::leaves_base_row_major_row_pair(&raw, m, n).unwrap(); + assert_leaves_eq(&gpu, &cpu, &format!("row-major log_n={log_n} m={m}")); + } + } +} + +/// The column-range variant, which is how preprocessed tables commit their +/// precomputed and multiplicity ranges to separate trees over one LDE. The +/// reference is the same function over just those columns — so this pins that +/// `m` stays the full row stride while only `[col_start, col_end)` is hashed. +#[test] +fn blake3_leaves_row_major_row_pair_range_matches_cpu() { + assert_round_lockstep(); + for log_n in [4u32, 6, 8, 10] { + let n = 1usize << log_n; + let m = 13usize; + let mut rng = ChaCha8Rng::seed_from_u64(900 + log_n as u64); + let data: Vec = (0..n * m).map(|_| rand_base(&mut rng)).collect(); + + // A split that is not on a block boundary either side of it. + for (col_start, col_end) in [(0usize, 5usize), (5, 13), (0, 13), (3, 4)] { + let columns: Vec> = (col_start..col_end) + .map(|c| (0..n).map(|r| data[r * m + c]).collect()) + .collect(); + let cpu = cpu_leaves(&columns, 2); + + let raw: Vec = data.iter().map(|e| *e.value()).collect(); + let gpu = math_cuda::blake3::leaves_base_row_major_row_pair_range( + &raw, m, col_start, col_end, n, + ) + .unwrap(); + assert_leaves_eq( + &gpu, + &cpu, + &format!("row-major range log_n={log_n} cols=[{col_start},{col_end})"), + ); + } + } +} + +/// ★ NEGATIVE CONTROL for the whole file. +/// +/// Every test above asserts device == host. All of them would pass just as well +/// if both sides were a constant, or if the kernel ignored its input entirely +/// and the CPU reference happened to be compared against itself. This asserts +/// the leaves actually depend on the data: two column sets differing in one +/// element must give different leaves, and distinct rows must give distinct +/// leaves. +#[test] +fn leaves_depend_on_the_data() { + assert_round_lockstep(); + let n = 64usize; + let num_cols = 5usize; + let mut rng = ChaCha8Rng::seed_from_u64(4242); + let columns: Vec> = (0..num_cols) + .map(|_| (0..n).map(|_| rand_base(&mut rng)).collect()) + .collect(); + + let flat = base_slabs(&columns, n); + let a = math_cuda::blake3::leaves_base(&flat, n, num_cols, n, 1).unwrap(); + + // Perturb one element and re-hash. + let mut perturbed = columns.clone(); + perturbed[2][7] += Fp::from(1u64); + let flat2 = base_slabs(&perturbed, n); + let b = math_cuda::blake3::leaves_base(&flat2, n, num_cols, n, 1).unwrap(); + assert_ne!(a, b, "a one-element change must move some leaf"); + + // And the leaves are not all the same digest. + let first = &a[0..32]; + assert!( + a.chunks_exact(32).any(|c| c != first), + "all leaves identical — the kernel is not reading its row index" + ); +} diff --git a/crypto/math-cuda/tests/blake3_merkle_gather.rs b/crypto/math-cuda/tests/blake3_merkle_gather.rs new file mode 100644 index 000000000..5afbed0e3 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_merkle_gather.rs @@ -0,0 +1,102 @@ +//! Parity: authentication paths gathered from a BLAKE3 tree must be the paths +//! the CPU `MerkleTree::get_proof_by_pos` returns. +//! +//! `merkle_gather_paths` is HASH-AGNOSTIC — it copies sibling nodes and never +//! hashes — so PA-PLAN §6.1 correctly says it needs no BLAKE3 twin, and none is +//! written. What is not free is the claim that it walks a BLAKE3 tree correctly: +//! that depends on `blake3::build_merkle_tree_on_device` laying nodes out in the +//! same order the keccak builder does, which is a property of the new code. This +//! file is that check, and it is why the gather is reused rather than twinned +//! *and tested*, rather than reused on the strength of the argument alone. +//! +//! Mirror of `merkle_gather.rs` with the tree builder swapped. +//! +//! Needs a GPU. + +mod blake3_reference; + +use blake3_reference::{expected_device_rounds, merkle_parent}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +/// The host parent hash as a Merkle backend, so the CPU reference is the +/// production tree walk. Leaves arrive already hashed, so `hash_data` is +/// unreachable; it is wired to the same parent function rather than to +/// `unimplemented!()` so the backend stays a total function. +#[derive(Clone, Default)] +struct Blake3ParentBackend; + +impl IsMerkleTreeBackend for Blake3ParentBackend { + type Node = [u8; 32]; + type Data = [u8; 32]; + + fn hash_data(leaf: &Self::Data) -> Self::Node { + merkle_parent(leaf, leaf, expected_device_rounds()) + } + + fn hash_new_parent(a: &Self::Node, b: &Self::Node) -> Self::Node { + merkle_parent(a, b, expected_device_rounds()) + } +} + +fn run_gather_parity(log_n: u32, seed: u64) { + let leaves_len = 1usize << log_n; + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let leaves: Vec<[u8; 32]> = (0..leaves_len) + .map(|_| core::array::from_fn(|_| rng.r#gen::())) + .collect(); + let flat: Vec = leaves.iter().flatten().copied().collect(); + + // Build the BLAKE3 tree on device, then upload its nodes back as the + // resident buffer the gather reads. + let gpu_nodes_bytes = math_cuda::blake3::build_merkle_tree_on_device(&flat).unwrap(); + let cpu_tree = MerkleTree::::build_from_hashed_leaves(leaves).unwrap(); + + // A spread of positions: first, last, and random interior ones. + let mut positions: Vec = vec![0, (leaves_len - 1) as u32]; + let mut r = ChaCha8Rng::seed_from_u64(seed ^ 0xabcd); + for _ in 0..16usize.min(leaves_len) { + positions.push(r.gen_range(0..leaves_len) as u32); + } + + let be = math_cuda::device::backend().unwrap(); + let stream = be.next_stream(); + let nodes_dev = stream.clone_htod(&gpu_nodes_bytes).unwrap(); + stream.synchronize().unwrap(); + + let depth = log_n as usize; + let paths = + math_cuda::merkle::gather_merkle_paths_dev(&nodes_dev, leaves_len, &positions, &stream) + .unwrap(); + assert_eq!(paths.len(), positions.len() * depth * 32); + + for (q, &pos) in positions.iter().enumerate() { + let cpu_proof = cpu_tree.get_proof_by_pos(pos as usize).unwrap(); + assert_eq!( + cpu_proof.merkle_path.len(), + depth, + "depth mismatch at log_n={log_n} pos={pos}" + ); + for (level, cpu_node) in cpu_proof.merkle_path.iter().enumerate() { + assert_eq!( + &paths[(q * depth + level) * 32..(q * depth + level + 1) * 32], + &cpu_node[..], + "path node mismatch: log_n={log_n} pos={pos} level={level}" + ); + } + } +} + +#[test] +fn blake3_merkle_gather_small() { + for log_n in 1u32..=6 { + run_gather_parity(log_n, 200 + log_n as u64); + } +} + +#[test] +fn blake3_merkle_gather_large() { + run_gather_parity(18, 7777); +} diff --git a/crypto/math-cuda/tests/blake3_merkle_tree.rs b/crypto/math-cuda/tests/blake3_merkle_tree.rs new file mode 100644 index 000000000..562d2be15 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_merkle_tree.rs @@ -0,0 +1,118 @@ +//! Parity: the device BLAKE3 Merkle tree must equal the CPU tree node for node. +//! +//! The CPU side is the *production* tree walk — `MerkleTree::build_from_hashed_leaves` +//! over a backend whose only new code is `hash_new_parent` — so what this compares +//! is the parent compression and the node layout, not a second tree builder. +//! Both the per-level kernel and the single-block tail kernel are exercised: the +//! tail takes over once a level is no wider than the block, so a tree deep enough +//! to cross that threshold runs both, and the small trees run the tail alone. +//! +//! Mirrors `merkle_root_parity.rs` / `merkle_tree.rs` in structure. Needs a GPU. + +mod blake3_reference; + +use blake3_reference::{expected_device_rounds, merkle_parent}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math_cuda::blake3::build_merkle_tree_on_device; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +/// The host parent hash under test, wrapped as a Merkle backend so the CPU +/// reference is the production tree walk rather than a hand-rolled one. +/// +/// `hash_data` is unreachable here — leaves are supplied already hashed — and is +/// wired to the same parent function rather than to `unimplemented!()` so the +/// backend stays a total function if a later test does call it. +#[derive(Clone, Default)] +struct Blake3ParentBackend; + +impl IsMerkleTreeBackend for Blake3ParentBackend { + type Node = [u8; 32]; + type Data = [u8; 32]; + + fn hash_data(leaf: &Self::Data) -> Self::Node { + merkle_parent(leaf, leaf, expected_device_rounds()) + } + + fn hash_new_parent(a: &Self::Node, b: &Self::Node) -> Self::Node { + merkle_parent(a, b, expected_device_rounds()) + } +} + +fn random_leaves(count: usize, seed: u64) -> Vec<[u8; 32]> { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + (0..count) + .map(|_| core::array::from_fn(|_| rng.r#gen::())) + .collect() +} + +fn run_parity(log_num_leaves: u32, seed: u64) { + let num_leaves = 1usize << log_num_leaves; + let leaves = random_leaves(num_leaves, seed); + + let cpu = MerkleTree::::build_from_hashed_leaves(leaves.clone()).unwrap(); + let cpu_nodes = cpu.nodes(); + + let flat: Vec = leaves.iter().flatten().copied().collect(); + let gpu = build_merkle_tree_on_device(&flat).unwrap(); + + assert_eq!(cpu_nodes.len() * 32, gpu.len(), "node count"); + for (i, expected) in cpu_nodes.iter().enumerate() { + assert_eq!( + &gpu[i * 32..(i + 1) * 32], + &expected[..], + "node {i} mismatch at log_num_leaves = {log_num_leaves}" + ); + } +} + +/// Small trees: every level fits the block width, so the tail kernel builds the +/// whole tree in one launch. +#[test] +fn blake3_merkle_tree_small() { + for log in 1u32..=8 { + run_parity(log, 300 + log as u64); + } +} + +/// Deep enough that the per-level kernel runs first and hands over to the tail +/// partway up — the launch path a real commit takes. +#[test] +fn blake3_merkle_tree_medium() { + for log in [10u32, 12, 14] { + run_parity(log, 700 + log as u64); + } +} + +#[test] +fn blake3_merkle_tree_large() { + run_parity(18, 4242); +} + +/// The parent is `hash_bytes(left ‖ right)` — a plain library call at 7 rounds, +/// which is the property that makes the framing (`h = IV`, `t = 0`, +/// `block_len = 64`, `flags = CHUNK_START|CHUNK_END|ROOT`) externally anchored +/// rather than merely self-consistent. +/// +/// Only meaningful when the kernels are built for 7 rounds; under +/// `blake3-6round` there is nothing in the world that recomputes the parent, which +/// is exactly PA-PLAN §1.6's premise. +#[test] +fn the_parent_is_the_blake3_crate_at_seven_rounds() { + if expected_device_rounds() != 7 { + return; + } + let leaves = random_leaves(2, 31337); + let flat: Vec = leaves.iter().flatten().copied().collect(); + let gpu = build_merkle_tree_on_device(&flat).unwrap(); + + let mut msg = Vec::with_capacity(64); + msg.extend_from_slice(&leaves[0]); + msg.extend_from_slice(&leaves[1]); + assert_eq!( + &gpu[0..32], + blake3::hash(&msg).as_bytes(), + "a two-leaf root must be blake3::hash(left ‖ right)" + ); +} diff --git a/crypto/math-cuda/tests/blake3_reference/mod.rs b/crypto/math-cuda/tests/blake3_reference/mod.rs new file mode 100644 index 000000000..574b383bb --- /dev/null +++ b/crypto/math-cuda/tests/blake3_reference/mod.rs @@ -0,0 +1,189 @@ +//! The host BLAKE3 compression reference the device kernels are checked against. +//! +//! **Not a copy any more.** The compression function, the IV, the permutation +//! and the round-count constants are re-exported from `crypto::hash::blake3`, +//! which P-a Stage 1 made their single home — so the device kernels, the host +//! commitment backends and the in-circuit chip are now all checked against one +//! function rather than three transcriptions of it. `crypto` is a dev-dependency +//! of this crate, which is what makes the re-export legal: `prover` (the old +//! home) depends on this crate, so it could never have been imported here. +//! +//! What stays local is [`merkle_parent`] — the *framing* a device Merkle parent +//! uses, which is a property of the kernel, not of the primitive. It is checked +//! two ways: against the `blake3` crate at 7 rounds, and against the production +//! host backend at the build's round count, so the reference cannot drift from +//! either the standard or the thing the CPU prover actually commits with. +//! +//! Lives in a subdirectory of `tests/`, so cargo treats it as a shared module +//! the parity tests `mod blake3_reference;` rather than as a test binary of its +//! own. `#![allow(dead_code)]` because not every includer uses every item. + +#![allow(dead_code)] + +pub use crypto::hash::blake3::chain::FLAGS_ONE_BLOCK; +pub use crypto::hash::blake3::{ + BLAKE3_IV, BLAKE3_SIX_ROUNDS, BLAKE3_STANDARD_ROUNDS, blake3_compress_rounds, +}; + +/// A Merkle parent: one compression over the 64 bytes of two child digests, with +/// the digest read back out little-endian. The host `hash_new_parent` for a +/// BLAKE3 backend, and the reference for `blake3_merkle_level`. +pub fn merkle_parent(left: &[u8; 32], right: &[u8; 32], rounds: usize) -> [u8; 32] { + let mut m = [0u32; 16]; + for i in 0..8 { + m[i] = u32::from_le_bytes(left[4 * i..4 * i + 4].try_into().unwrap()); + m[i + 8] = u32::from_le_bytes(right[4 * i..4 * i + 4].try_into().unwrap()); + } + let out = blake3_compress_rounds(&BLAKE3_IV, &m, 0, 64, FLAGS_ONE_BLOCK, rounds); + let mut digest = [0u8; 32]; + for i in 0..8 { + digest[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + digest +} + +/// The round count the kernels are compiled for, as the Rust side of the +/// feature. Mirrors `math-cuda`'s `blake3-6round`, which build.rs turns into +/// `-DBLAKE3_ROUNDS=6`. +pub const fn expected_device_rounds() -> usize { + if cfg!(feature = "blake3-6round") { + BLAKE3_SIX_ROUNDS + } else { + BLAKE3_STANDARD_ROUNDS + } +} + +/// The compression function is shared now, but [`merkle_parent`]'s framing is +/// still written here, so it gets its own checks that it did not drift — +/// otherwise a device-vs-host parity failure would be ambiguous between "the +/// kernel is wrong" and "the reference is wrong". +/// +/// Host-only: no GPU, so these run wherever the suite compiles, including the +/// laptops where the kernels are stubbed out. +#[cfg(test)] +mod tests { + use super::*; + + /// At 7 rounds the reference must be the `blake3` crate, over every length a + /// single block can hold — `block_len` and the zero-padding both key off the + /// length, and one length would not see a port that ignored either. + #[test] + fn the_reference_copy_is_the_blake3_crate_at_seven_rounds() { + for len in 0..=64usize { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + let mut block = [0u8; 64]; + block[..len].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + let out = blake3_compress_rounds( + &BLAKE3_IV, + &words, + 0, + len as u32, + FLAGS_ONE_BLOCK, + BLAKE3_STANDARD_ROUNDS, + ); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_eq!( + ours, + *blake3::hash(&msg).as_bytes(), + "reference copy diverged from the blake3 crate at length {len}" + ); + } + } + + /// And the parent framing on top of it: `hash_new_parent(a, b)` is + /// `blake3::hash(a ‖ b)` at 7 rounds. Pins `block_len = 64`, `t = 0`, + /// `h = IV`, the flag set, and the little-endian digest read-back in one shot. + #[test] + fn the_reference_parent_is_the_blake3_crate_at_seven_rounds() { + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(7)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(31).wrapping_add(3)); + let mut msg = Vec::with_capacity(64); + msg.extend_from_slice(&left); + msg.extend_from_slice(&right); + assert_eq!( + merkle_parent(&left, &right, BLAKE3_STANDARD_ROUNDS), + *blake3::hash(&msg).as_bytes() + ); + } + + /// ★ The reference parent is what the **host commitment backend** computes. + /// + /// The two checks above anchor the framing against the standard; this one + /// anchors it against the thing the CPU prover actually commits with, so a + /// GPU tree and a CPU tree over the same leaves are the same tree. Without + /// it, the device could be faithful to `blake3::hash(a ‖ b)` and still + /// disagree with the backend the proof is verified against. + /// + /// It runs at [`expected_device_rounds`], and so it doubles as the LOCKSTEP + /// alarm for the two crates' `blake3-6round` features: they are separate + /// features and nothing forces them equal, and a mismatch means a GPU tree + /// committing under a different hash than the CPU one. If this fails with + /// the round counts differing, set both features or neither — `make lint` + /// has a combined pass that compiles them together for the same reason. + #[test] + fn the_reference_parent_is_the_host_commitment_backend() { + use crypto::hash::blake3::BLAKE3_ROUNDS; + use crypto::merkle_tree::backends::types::BatchBlake3Backend; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + use math::field::goldilocks::GoldilocksField; + + assert_eq!( + BLAKE3_ROUNDS, + expected_device_rounds(), + "crypto's blake3-6round and math-cuda's are out of lockstep: the GPU \ + kernels would commit under a different hash than the CPU backend" + ); + + let left: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(11).wrapping_add(5)); + let right: [u8; 32] = core::array::from_fn(|i| (i as u8).wrapping_mul(23)); + assert_eq!( + merkle_parent(&left, &right, expected_device_rounds()), + as IsMerkleTreeBackend>::hash_new_parent( + &left, &right + ), + ); + } + + /// NEGATIVE CONTROL: at 6 rounds neither must match, or the two checks above + /// would pass with `rounds` ignored. + #[test] + fn six_rounds_is_not_the_blake3_crate() { + let msg: [u8; 36] = core::array::from_fn(|i| i as u8); + let mut block = [0u8; 64]; + block[..36].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + let out = blake3_compress_rounds( + &BLAKE3_IV, + &words, + 0, + 36, + FLAGS_ONE_BLOCK, + BLAKE3_SIX_ROUNDS, + ); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_ne!(ours, *blake3::hash(&msg).as_bytes()); + + let left = [1u8; 32]; + let right = [2u8; 32]; + let mut pmsg = Vec::with_capacity(64); + pmsg.extend_from_slice(&left); + pmsg.extend_from_slice(&right); + assert_ne!( + merkle_parent(&left, &right, BLAKE3_SIX_ROUNDS), + *blake3::hash(&pmsg).as_bytes() + ); + } +} diff --git a/crypto/math-cuda/tests/blake3_serialize.rs b/crypto/math-cuda/tests/blake3_serialize.rs new file mode 100644 index 000000000..3ecd8ccc7 --- /dev/null +++ b/crypto/math-cuda/tests/blake3_serialize.rs @@ -0,0 +1,125 @@ +//! Parity: the device field-element serialization and 64-byte block framing must +//! reproduce the bytes the CPU commit path hashes. +//! +//! The leaf byte encoding does not move under P-a: `leaves_bit_reversed_grouped` +//! (`crypto/stark/src/commitment.rs:55`) writes each element in canonical +//! big-endian form and concatenates, and `hash_bytes` hashes that buffer. BLAKE3 +//! reads a 64-byte block as 16 little-endian u32 words, so the device has to +//! transpose: one element becomes the byte-reverse of its canonical high half, +//! then of its low half. This pins that transposition, the canonicalisation in +//! front of it, and the block boundaries and zero-padding around it — everything +//! a leaf kernel needs that does not depend on the still-open chaining +//! construction (PA-PLAN §1.6). + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::traits::AsBytes; +use math_cuda::blake3::{blocks_of_felts, serialize_felts}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +type Fp = FieldElement; + +const PRIME: u64 = 0xFFFF_FFFF_0000_0001; + +/// Raw values that include the ones canonicalisation is the only thing standing +/// between: `p` and above are representable in the prover's non-canonical u64 +/// form and serialize as their reduced value, so a kernel that skipped the +/// reduction would differ from the CPU on exactly these. +fn raws(seed: u64, n: usize) -> Vec { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let mut v = vec![ + 0u64, + 1, + PRIME - 1, + PRIME, + PRIME + 1, + PRIME + 12345, + u64::MAX, + ]; + v.truncate(n.min(7)); + while v.len() < n { + // Half in range, half deliberately non-canonical. + let x = rng.r#gen::(); + v.push(if v.len().is_multiple_of(2) { + x % PRIME + } else { + x + }); + } + v +} + +/// The bytes the CPU hashes for these elements, via the same `AsBytes` route +/// `leaves_bit_reversed_grouped` serializes through. +fn cpu_bytes(raws: &[u64]) -> Vec { + let mut out = Vec::with_capacity(raws.len() * 8); + for &r in raws { + out.extend_from_slice(&Fp::from_raw(r).as_bytes()); + } + out +} + +/// Those bytes as BLAKE3 message words, zero-padded to whole 64-byte blocks. +fn cpu_block_words(bytes: &[u8]) -> Vec { + let n_blocks = bytes.len().div_ceil(64); + let mut padded = bytes.to_vec(); + padded.resize(n_blocks * 64, 0); + padded + .chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect() +} + +#[test] +fn device_serialization_is_the_cpu_leaf_bytes() { + for n in [1usize, 2, 7, 8, 9, 64, 1000] { + let vals = raws(11 + n as u64, n); + let device = serialize_felts(&vals).unwrap(); + let expected = cpu_block_words(&cpu_bytes(&vals)); + // `serialize_felts` emits exactly two words per element with no padding, + // so compare against the unpadded prefix of the block view. + assert_eq!(device.len(), 2 * n); + assert_eq!( + device[..], + expected[..2 * n], + "serialization mismatch at n = {n}" + ); + } +} + +/// A field element is 8 bytes = 2 words, and a block is 16 words, so elements +/// straddle a block boundary only when the count is not a multiple of 8 — but +/// ext3 elements are 6 words and straddle routinely, which is why the builder +/// works at word granularity. Both cases are covered by the counts below. +#[test] +fn device_block_framing_matches_the_cpu_byte_stream() { + for n in [1usize, 3, 8, 9, 16, 17, 63, 64, 255] { + let vals = raws(500 + n as u64, n); + let device = blocks_of_felts(&vals).unwrap(); + let expected = cpu_block_words(&cpu_bytes(&vals)); + assert_eq!( + device.len(), + expected.len(), + "block count mismatch at n = {n}" + ); + assert_eq!(device, expected, "block words mismatch at n = {n}"); + } +} + +/// The tail block must be zero-padded, not left holding stale words. The check +/// above would catch that only if the padding happened to differ from whatever +/// was there; asserting the padded region directly is what makes it a test of the +/// padding rather than of the allocator. +#[test] +fn the_tail_block_is_zero_padded() { + // 9 elements = 18 words = one full block plus 2 words, leaving 14 to pad. + let vals = raws(77, 9); + let device = blocks_of_felts(&vals).unwrap(); + assert_eq!(device.len(), 32, "expected exactly two blocks"); + assert!( + device[18..].iter().all(|&w| w == 0), + "tail block not zero-padded: {:?}", + &device[18..] + ); +} diff --git a/crypto/math-cuda/tests/host_kat/blake3_host_kat.cpp b/crypto/math-cuda/tests/host_kat/blake3_host_kat.cpp new file mode 100644 index 000000000..42b0b05f4 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/blake3_host_kat.cpp @@ -0,0 +1,779 @@ +// Known-answer tests for `kernels/blake3.cu`, run on the host. +// +// WHY THIS EXISTS. The GPU parity suite (`tests/blake3_compress_parity.rs` and +// friends) is the authority on these kernels, but it runs only where a GPU does, +// and per-PR CI has none — GPU CI is merge_group-only. Without this the kernels +// have no per-PR gate at all: an edit to `blake3.cu` that broke the hash would +// reach the merge queue before anything noticed. This compiles the real kernel +// source through `cuda_host_shim.h` and runs external known-answer vectors +// through it, in seconds, with no GPU and no cargo. +// +// WHAT IT COVERS: the compression function at both round counts, the field +// element serialization, the 64-byte block framing, the Merkle parent, the +// `Blake3Chain` construction over multi-block messages, and every leaf kernel's +// byte stream (replayed thread by thread through the shim). +// +// WHAT IT DOES NOT COVER, and what the GPU tests are still required for: +// whether nvcc accepts the file, and every property of execution rather than +// arithmetic — grid indexing, `__syncthreads` ordering up the Merkle levels, +// device memory alignment, and register pressure. Passing here is necessary, +// never sufficient. +// +// HOW THE ANCHORING LAYERS. Nothing here is checked against itself: +// 1. The compression function is anchored by the OFFICIAL BLAKE3 vectors at 7 +// rounds (Table 1) and by the canonical vectors at 6 (Table 2). +// 2. `HostChain` below — a byte-level transcription of the construction — is +// anchored by the OFFICIAL multi-block vectors at 7 rounds (Table 3) and +// the committed 6-round chain KAT (Table 4). It is built ON the device +// compression, so layer 1 carries into it. +// 3. The device `Blake3Chain` is checked against `HostChain`, at word +// granularity, which is all the kernels ever need. +// 4. Each leaf kernel is replayed on host and checked against `HostChain` over +// the byte stream `leaves_bit_reversed_grouped` specifies — so the read +// pattern and the hash are anchored separately rather than together. +// +// ★ TABLES 1 AND 2 ARE COMPLEMENTARY, NOT REDUNDANT, and the difference is not +// obvious enough to leave unwritten. The official-vector path hashes whole +// messages, so it only ever exercises `t = 0`. A build with `v[12]` and `v[13]` +// transposed — the counter split inverted — reproduces the official vectors at +// ALL 65 single-block lengths, and is caught only by Table 2, whose ten vectors +// all carry `t >= 2^32`. That was measured, not assumed. Do not retire Table 2 +// as "covered by the standard", and do not describe Table 1 as subsuming it. +// +// ⚠ CONVERSELY, one thing here pins LESS than it appears to. The compression +// loop's `if (r < ROUNDS - 1)` permutation guard is UNOBSERVABLE: always +// permuting produces identical output at both round counts, because the schedule +// permuted after the final round is never read. It is an optimization, not a +// convention any known-answer test can validate — upstream expresses the same +// schedule as an indexed table with no guard at all. Do not cite these vectors +// as evidence the guard is correct; they cannot be. +// +// Build and run with `make test-blake3-host-kat`. + +#include +#include +#include +#include + +#include "cuda_host_shim.h" + +// The kernel under test. Included, not linked: the shim turns its device +// functions into host functions, and there is no other way to call them. +#include "blake3.cu" + +#include "blake3_kat_vectors.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char *what) { + if (!ok) { + printf("FAIL: %s\n", what); + ++failures; + } +} + +// The official vectors' input: the first `len` bytes of the repeating 251-byte +// sequence 0, 1, ..., 250. +void official_input(uint32_t len, uint8_t *out) { + for (uint32_t i = 0; i < len; ++i) out[i] = (uint8_t)(i % 251); +} + +// The 32-byte digest of a message of at most 64 bytes: ONE compression with +// `h = IV`, `t = 0`, the block zero-padded and read as little-endian words, +// `block_len` the true length, and the one-block flag set. The digest is the low +// eight output words, little-endian. +std::string hash_one_block(const uint8_t *msg, uint32_t len, int rounds) { + uint8_t block[64] = {0}; + memcpy(block, msg, len); + uint32_t m[16]; + for (int i = 0; i < 16; ++i) { + m[i] = (uint32_t)block[4 * i] | ((uint32_t)block[4 * i + 1] << 8) | + ((uint32_t)block[4 * i + 2] << 16) | ((uint32_t)block[4 * i + 3] << 24); + } + uint32_t out[16]; + if (rounds == 6) { + blake3_compress<6>(BLAKE3_IV, m, 0, len, BLAKE3_FLAGS_ONE_BLOCK, out); + } else { + blake3_compress<7>(BLAKE3_IV, m, 0, len, BLAKE3_FLAGS_ONE_BLOCK, out); + } + char hex[65]; + for (int i = 0; i < 8; ++i) { + for (int b = 0; b < 4; ++b) { + snprintf(hex + (i * 4 + b) * 2, 3, "%02x", (unsigned)((out[i] >> (8 * b)) & 0xff)); + } + } + return std::string(hex, 64); +} + +// ★ The external anchor. At 7 rounds the kernel must BE standard BLAKE3. +// +// Run over every length a single block can hold rather than one: the length keys +// both `block_len` and the zero-padding, so a port that ignored either would +// still pass at a single length. +void official_vectors_at_seven_rounds() { + check(NUM_OFFICIAL_VECTORS == 11, "official vector table lost entries"); + for (int i = 0; i < NUM_OFFICIAL_VECTORS; ++i) { + const OfficialVector &v = OFFICIAL_VECTORS[i]; + uint8_t msg[64]; + official_input(v.input_len, msg); + std::string got = hash_one_block(msg, v.input_len, 7); + if (got != v.hash_hex) { + printf("FAIL official vector len=%u\n got %s\n want %s\n", v.input_len, got.c_str(), + v.hash_hex); + ++failures; + } + } + printf("official BLAKE3 vectors at 7 rounds: %d checked\n", NUM_OFFICIAL_VECTORS); +} + +// NEGATIVE CONTROL for the anchor above: at 6 rounds nothing must match. +// +// Without this the anchor would pass just as well if the round count were being +// ignored — the one bug that makes the whole external-anchor argument vacuous, +// since the 6-round arm's only defence is "the same code path with the loop +// bound changed". The zero-length case is skipped: an empty message is the one +// input where the rounds have nothing to diffuse and a collision would not be +// evidence of anything. +void six_rounds_is_not_standard_blake3() { + int discriminated = 0; + for (int i = 0; i < NUM_OFFICIAL_VECTORS; ++i) { + const OfficialVector &v = OFFICIAL_VECTORS[i]; + if (v.input_len == 0) continue; + uint8_t msg[64]; + official_input(v.input_len, msg); + check(hash_one_block(msg, v.input_len, 6) != v.hash_hex, + "6 rounds reproduced an official 7-round vector"); + ++discriminated; + } + printf("6-round negative control: %d lengths discriminated\n", discriminated); +} + +// ★ The 6-round known-answer test, and the reason it is worth more than a +// self-comparison: `out6` came from #903's Python oracle, not from any code in +// this tree. All 16 output words are checked, not just the chaining value. +void canonical_vectors_at_both_round_counts() { + check(NUM_CANONICAL_VECTORS == 10, "canonical vector table lost entries"); + for (int i = 0; i < NUM_CANONICAL_VECTORS; ++i) { + const CanonicalVector &v = CANONICAL_VECTORS[i]; + uint32_t out6[16], out7[16]; + blake3_compress<6>(v.h, v.m, v.t, v.block_len, v.flags, out6); + blake3_compress<7>(v.h, v.m, v.t, v.block_len, v.flags, out7); + for (int w = 0; w < 16; ++w) { + if (out6[w] != v.out6[w]) { + printf("FAIL canonical %d word %d at 6 rounds: got %08x want %08x\n", i, w, out6[w], + v.out6[w]); + ++failures; + } + if (out7[w] != v.out7[w]) { + printf("FAIL canonical %d word %d at 7 rounds: got %08x want %08x\n", i, w, out7[w], + v.out7[w]); + ++failures; + } + } + } + printf("canonical vectors at 6 AND 7 rounds: %d checked, all 16 words each\n", + NUM_CANONICAL_VECTORS); +} + +// The serialization: one field element becomes the two message words its +// canonical big-endian bytes are read as, little-endian. The non-canonical raws +// are the cases where the reduction is the only thing that matters. +void serialization_is_the_canonical_big_endian_bytes() { + const uint64_t P = 0xFFFFFFFF00000001ull; + const uint64_t raws[] = {0, 1, P - 1, P, P + 1, P + 12345, ~0ull, 0x0123456789ABCDEFull}; + for (uint64_t raw : raws) { + uint32_t w0, w1; + blake3_words_of_felt(raw, w0, w1); + uint64_t canon = raw >= P ? raw - P : raw; + uint8_t be[8]; + for (int i = 0; i < 8; ++i) be[i] = (uint8_t)(canon >> (56 - 8 * i)); + uint32_t e0 = (uint32_t)be[0] | ((uint32_t)be[1] << 8) | ((uint32_t)be[2] << 16) | + ((uint32_t)be[3] << 24); + uint32_t e1 = (uint32_t)be[4] | ((uint32_t)be[5] << 8) | ((uint32_t)be[6] << 16) | + ((uint32_t)be[7] << 24); + check(w0 == e0 && w1 == e1, "blake3_words_of_felt"); + } + printf("serialization: %zu elements checked, non-canonical raws included\n", + sizeof(raws) / sizeof(raws[0])); +} + +// The block framing: nine elements are eighteen words, so one block completes and +// a two-word tail stays pending with fourteen words of zero padding behind it. +void block_framing_completes_and_pads() { + Blake3Block b; + b.init(); + int completed = 0; + uint32_t blocks[2][16] = {{0}}; + for (int i = 0; i < 9; ++i) { + uint32_t w0, w1; + blake3_words_of_felt((uint64_t)(i + 1) * 0x1111111111111111ull, w0, w1); + if (b.push_word(w0)) { + memcpy(blocks[completed++], b.m, 64); + b.reset(); + } + if (b.push_word(w1)) { + memcpy(blocks[completed++], b.m, 64); + b.reset(); + } + } + check(completed == 1, "exactly one block should have completed"); + check(b.pending_bytes() == 8, "the pending tail should be 8 bytes"); + memcpy(blocks[1], b.m, 64); + bool padded = true; + for (int k = 2; k < 16; ++k) padded = padded && blocks[1][k] == 0; + check(padded, "the tail block must be zero-padded"); + printf("block framing: 1 completed block + an 8-byte zero-padded tail\n"); +} + +// The Merkle parent: one compression over the 64 bytes of two child digests, so +// it must equal the one-block hash of their concatenation — which at 7 rounds is +// a plain `blake3::hash` call, and is what makes the parent framing externally +// anchored rather than merely self-consistent. +void parent_is_the_one_block_hash_of_its_children() { + uint8_t children[64]; + official_input(64, children); + uint8_t nodes[3 * 32]; + memcpy(nodes + 32, children, 32); // node 1 = left child + memcpy(nodes + 64, children + 32, 32); // node 2 = right child + blake3_hash_merkle_parent(nodes, 0, 1, 0); + char hex[65]; + for (int i = 0; i < 32; ++i) snprintf(hex + i * 2, 3, "%02x", (unsigned)nodes[i]); + check(std::string(hex, 64) == hash_one_block(children, 64, BLAKE3_ROUNDS), + "parent must equal the one-block hash of left || right"); + printf("Merkle parent at BLAKE3_ROUNDS=%d: %s\n", BLAKE3_ROUNDS, hex); +} + +// =========================================================================== +// The chain construction. +// =========================================================================== + +// The device compression with the round count as a run-time argument, so the +// reference below can be evaluated at either arm from one code path. +void compress_dyn(const uint32_t *h, const uint32_t *m, uint64_t t, uint32_t block_len, + uint32_t flags, int rounds, uint32_t *out) { + if (rounds == 6) { + blake3_compress<6>(h, m, t, block_len, flags, out); + } else { + blake3_compress<7>(h, m, t, block_len, flags, out); + } +} + +// `Blake3Chain` at BYTE granularity — PA-PLAN §1.7.1 written out directly. +// +// Why this exists when `blake3.cu` already has a `Blake3Chain`: the device one +// is word-granular, because every message the kernels hash is a whole number of +// 8-byte field elements. The official vectors are not — 65, 127, 1023 — and +// those lengths are where a final-block `block_len` bug lives. So the vectors +// anchor THIS, and the device chain is then checked against it at the word +// lengths it can actually reach. +// +// It is a transcription of the same spec as the host Rust `Blake3Chain`, not of +// the device struct, and it holds a full block rather than compressing it for +// the same reason: whether a block is the last is unknown until the message ends. +struct HostChain { + uint32_t cv[8]; + uint8_t block[64]; + uint32_t block_len; + bool started; + int rounds; + + void init(int r) { + memcpy(cv, BLAKE3_IV, sizeof(cv)); + memset(block, 0, sizeof(block)); + block_len = 0; + started = false; + rounds = r; + } + + void block_words(uint32_t *m) const { + for (int i = 0; i < 16; ++i) { + m[i] = (uint32_t)block[4 * i] | ((uint32_t)block[4 * i + 1] << 8) | + ((uint32_t)block[4 * i + 2] << 16) | ((uint32_t)block[4 * i + 3] << 24); + } + } + + uint32_t flags(bool is_final) const { + return (started ? 0u : BLAKE3_FLAG_CHUNK_START) | + (is_final ? (BLAKE3_FLAG_CHUNK_END | BLAKE3_FLAG_ROOT) : 0u); + } + + void compress_pending() { + uint32_t m[16], out[16]; + block_words(m); + compress_dyn(cv, m, 0, 64, flags(false), rounds, out); + memcpy(cv, out, sizeof(cv)); + memset(block, 0, sizeof(block)); + block_len = 0; + started = true; + } + + void update(const uint8_t *in, size_t n) { + while (n != 0) { + // Only now is the pending block known not to be the last. + if (block_len == 64) compress_pending(); + size_t take = 64 - block_len; + if (take > n) take = n; + memcpy(block + block_len, in, take); + block_len += (uint32_t)take; + in += take; + n -= take; + } + } + + void finalize(uint8_t *out32) const { + uint32_t m[16], out[16]; + block_words(m); + compress_dyn(cv, m, 0, block_len, flags(true), rounds, out); + for (int i = 0; i < 8; ++i) { + out32[4 * i] = (uint8_t)(out[i] & 0xff); + out32[4 * i + 1] = (uint8_t)((out[i] >> 8) & 0xff); + out32[4 * i + 2] = (uint8_t)((out[i] >> 16) & 0xff); + out32[4 * i + 3] = (uint8_t)((out[i] >> 24) & 0xff); + } + } +}; + +void host_chain(const uint8_t *msg, size_t len, int rounds, uint8_t *out32) { + HostChain c; + c.init(rounds); + c.update(msg, len); + c.finalize(out32); +} + +std::string to_hex(const uint8_t *b, size_t n) { + std::string s(n * 2, '\0'); + for (size_t i = 0; i < n; ++i) snprintf(&s[i * 2], 3, "%02x", (unsigned)b[i]); + return s; +} + +// The KAT message for Table 4: byte `i` is `37i + 11 (mod 256)`. +void kat_message(size_t len, std::vector &out) { + out.resize(len); + for (size_t i = 0; i < len; ++i) out[i] = (uint8_t)((i * 37 + 11) & 0xff); +} + +// ★ The chain's external anchor: over multi-block messages of at most one +// chunk, at 7 rounds, the construction IS standard BLAKE3, so the official +// vectors are direct known-answer tests for the framing — the flag schedule +// across blocks, the chaining value, and the final block's `block_len`. +// +// The `agrees == false` rows are the P3 negative control and are not decoration: +// without them, every matching row would pass identically if the full chunk tree +// had been implemented instead of the single unbounded chunk. +void chain_against_official_multiblock_vectors() { + check(NUM_CHAIN_VECTORS == 8, "chain vector table lost entries"); + int matched = 0, diverged = 0; + for (int i = 0; i < NUM_CHAIN_VECTORS; ++i) { + const ChainVector &v = CHAIN_VECTORS[i]; + std::vector msg(v.input_len); + for (uint32_t k = 0; k < v.input_len; ++k) msg[k] = (uint8_t)(k % 251); + uint8_t digest[32]; + host_chain(msg.data(), msg.size(), 7, digest); + std::string got = to_hex(digest, 32); + if (v.agrees) { + if (got != v.hash_hex) { + printf("FAIL chain vector len=%u\n got %s\n want %s\n", v.input_len, got.c_str(), + v.hash_hex); + ++failures; + } else { + ++matched; + } + } else { + check(got != v.hash_hex, + "past one chunk the chain must LEAVE standard BLAKE3 (P3 control)"); + ++diverged; + } + } + printf("chain vs official multi-block vectors at 7 rounds: %d must-match, %d P3 controls\n", + matched, diverged); +} + +// ★ The 6-round chain anchor: the committed table, whose digests came from the +// Python oracle rather than from any code in this tree. +void chain_against_the_committed_six_round_table() { + check(NUM_CHAIN_KAT_6ROUND == 12, "6-round chain KAT table lost entries"); + for (int i = 0; i < NUM_CHAIN_KAT_6ROUND; ++i) { + const ChainKat6Round &v = CHAIN_KAT_6ROUND[i]; + std::vector msg; + kat_message(v.input_len, msg); + uint8_t digest[32]; + host_chain(msg.data(), msg.size(), 6, digest); + std::string got = to_hex(digest, 32); + if (got != v.hash_hex) { + printf("FAIL 6-round chain KAT len=%u\n got %s\n want %s\n", v.input_len, got.c_str(), + v.hash_hex); + ++failures; + } + } + printf("chain vs committed 6-round KAT: %d lengths checked\n", NUM_CHAIN_KAT_6ROUND); +} + +// The DEVICE chain against the anchored reference, at every word-multiple length +// through several block boundaries. This is what carries the anchors above onto +// the struct the kernels actually use. +// +// The step of 4 is the device chain's granularity, and the range crosses the +// first, second and eighth block boundaries — the places a mis-set CHUNK_START, +// an eagerly compressed final block, or a wrong `block_len` would show. +void device_chain_matches_the_reference() { + int checked = 0; + for (size_t len = 0; len <= 600; len += 4) { + std::vector msg; + kat_message(len, msg); + + Blake3Chain dev; + dev.init(); + for (size_t i = 0; i < len; i += 4) { + uint32_t w = (uint32_t)msg[i] | ((uint32_t)msg[i + 1] << 8) | + ((uint32_t)msg[i + 2] << 16) | ((uint32_t)msg[i + 3] << 24); + dev.push_word(w); + } + uint8_t got[32]; + dev.finalize(got); + + uint8_t want[32]; + host_chain(msg.data(), msg.size(), BLAKE3_ROUNDS, want); + if (memcmp(got, want, 32) != 0) { + printf("FAIL device chain at len=%zu\n got %s\n want %s\n", len, + to_hex(got, 32).c_str(), to_hex(want, 32).c_str()); + ++failures; + break; + } + ++checked; + } + printf("device chain vs reference at BLAKE3_ROUNDS=%d: %d lengths (0..600 step 4)\n", + BLAKE3_ROUNDS, checked); +} + +// ★ P2 on the device struct: a 64-byte message through the chain must be the +// parent compression. This is the invariant that lets the leaf and parent layers +// be one hash, and it is why `blake3_hash_merkle_parent` needs no chaining. +void device_chain_at_64_bytes_is_the_parent() { + uint8_t children[64]; + official_input(64, children); + uint8_t nodes[3 * 32]; + memcpy(nodes + 32, children, 32); + memcpy(nodes + 64, children + 32, 32); + blake3_hash_merkle_parent(nodes, 0, 1, 0); + + Blake3Chain dev; + dev.init(); + for (int i = 0; i < 16; ++i) { + uint32_t w = (uint32_t)children[4 * i] | ((uint32_t)children[4 * i + 1] << 8) | + ((uint32_t)children[4 * i + 2] << 16) | ((uint32_t)children[4 * i + 3] << 24); + dev.push_word(w); + } + uint8_t got[32]; + dev.finalize(got); + check(memcmp(got, nodes, 32) == 0, "a 64-byte device chain must be the parent compression"); + printf("P2: a 64-byte chain is the Merkle parent compression\n"); +} + +// =========================================================================== +// The leaf kernels, replayed thread by thread. +// =========================================================================== + +const uint64_t GOLDILOCKS_P = 0xFFFFFFFF00000001ull; + +uint64_t canon(uint64_t raw) { return raw >= GOLDILOCKS_P ? raw - GOLDILOCKS_P : raw; } + +// `reverse_index(i, n)` — the CPU commit's row permutation, and what the kernels +// compute as `__brevll(tid) >> (64 - log_num_rows)`. +uint64_t reverse_index(uint64_t i, uint32_t log_n) { return __brevll(i) >> (64 - log_n); } + +// Append a field element's canonical BIG-endian bytes — the serialization +// `leaves_bit_reversed_grouped` writes and every leaf kernel must reproduce. +void push_be(std::vector &buf, uint64_t raw) { + uint64_t c = canon(raw); + for (int i = 0; i < 8; ++i) buf.push_back((uint8_t)(c >> (56 - 8 * i))); +} + +// A deterministic value stream, including deliberately non-canonical raws so the +// reduction is exercised rather than assumed. +uint64_t sample(uint64_t seed, uint64_t i) { + uint64_t x = seed * 0x9E3779B97F4A7C15ull + i * 0xBF58476D1CE4E5B9ull; + x ^= x >> 31; + x *= 0x94D049BB133111EBull; + x ^= x >> 29; + // Every fifth value is left above the modulus. + return (i % 5 == 0) ? x : x % GOLDILOCKS_P; +} + +void check_leaves(const std::vector &got, const std::vector> &want, + const char *what) { + if (got.size() != want.size() * 32) { + printf("FAIL %s: leaf count %zu vs %zu\n", what, got.size() / 32, want.size()); + ++failures; + return; + } + for (size_t i = 0; i < want.size(); ++i) { + uint8_t expect[32]; + host_chain(want[i].data(), want[i].size(), BLAKE3_ROUNDS, expect); + if (memcmp(got.data() + i * 32, expect, 32) != 0) { + printf("FAIL %s: leaf %zu\n got %s\n want %s\n", what, i, + to_hex(got.data() + i * 32, 32).c_str(), to_hex(expect, 32).c_str()); + ++failures; + return; + } + } +} + +// The two column-major base kernels: one leaf per bit-reversed row, and one per +// bit-reversed row pair. The expected byte stream is built from the CPU leaf +// spec — rows in bit-reversed order, each written column by column in canonical +// big-endian — so what is compared is the kernel's READ PATTERN against that +// spec, with the hash anchored separately above. +void base_leaf_kernels_read_the_specified_bytes() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 5ull, 8ull, 17ull}) { + uint64_t n = 1ull << log_n; + std::vector cols(num_cols * n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t r = 0; r < n; ++r) cols[c * n + r] = sample(log_n * 31 + num_cols, c * n + r); + } + + // rows_per_leaf = 1 + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) push_be(want[leaf], cols[c * n + br]); + } + check_leaves(out, want, "blake3_leaves_base_batched"); + } + + // rows_per_leaf = 2 + { + uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_pair_batched(cols.data(), n, num_cols, n, log_n, + out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < num_cols; ++c) push_be(want[leaf], cols[c * n + br]); + } + } + check_leaves(out, want, "blake3_leaves_base_row_pair_batched"); + } + } + } + printf("base leaf kernels: read pattern matches the CPU leaf spec\n"); +} + +// The ext3 kernels, over the de-interleaved three-slab layout. An ext3 element +// is three consecutive components, each 8 big-endian bytes — six words, so +// elements straddle block boundaries routinely, which is the case the +// word-granular block builder exists for. +void ext3_leaf_kernels_read_the_specified_bytes() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 3ull, 11ull}) { + uint64_t n = 1ull << log_n; + std::vector cols(num_cols * 3 * n); + for (uint64_t s = 0; s < num_cols * 3; ++s) { + for (uint64_t r = 0; r < n; ++r) cols[s * n + r] = sample(log_n * 17 + num_cols, s * n + r); + } + + // One leaf per bit-reversed row. + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_ext3_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) push_be(want[leaf], cols[(c * 3 + k) * n + br]); + } + } + check_leaves(out, want, "blake3_leaves_ext3_batched"); + } + + // Row pairs — the comp-poly kernel, which the aux trace also uses. + { + uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_comp_poly_leaves_ext3(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int j = 0; j < 2; ++j) { + uint64_t br = reverse_index(2 * leaf + j, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) + push_be(want[leaf], cols[(c * 3 + k) * n + br]); + } + } + } + check_leaves(out, want, "blake3_comp_poly_leaves_ext3"); + } + } + } + printf("ext3 + comp-poly leaf kernels: read pattern matches the CPU leaf spec\n"); +} + +// FRI leaves: two consecutive ext3 values from an interleaved vector, 48 bytes, +// no bit reversal. Under one block, so this is the chain's single-compression +// case at a length that is neither 64 nor a block multiple. +void fri_leaf_kernel_reads_the_specified_bytes() { + for (uint64_t num_leaves : {1ull, 2ull, 8ull, 33ull}) { + std::vector evals(num_leaves * 2 * 3); + for (size_t i = 0; i < evals.size(); ++i) evals[i] = sample(0xF41, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_fri_leaves_ext3(evals.data(), num_leaves, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int i = 0; i < 6; ++i) push_be(want[leaf], evals[leaf * 6 + i]); + } + check_leaves(out, want, "blake3_fri_leaves_ext3"); + } + printf("FRI leaf kernel: read pattern matches the CPU leaf spec\n"); +} + +// The row-major row-pair kernels, plain and column-ranged. `m` is the row +// stride; the ranged variant hashes only `[col_start, col_end)` while the stride +// stays full, which is how preprocessed tables commit two column ranges to +// separate trees over one LDE. +void row_major_leaf_kernels_read_the_specified_bytes() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t m : {1ull, 5ull, 13ull}) { + uint64_t n = 1ull << log_n; + uint64_t num_leaves = n / 2; + std::vector data(n * m); + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(log_n * 7 + m, i); + + { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair(data.data(), m, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < m; ++c) push_be(want[leaf], data[br * m + c]); + } + } + check_leaves(out, want, "blake3_leaves_base_row_major_row_pair"); + } + + // Every non-empty column range, so the boundary handling is checked + // rather than sampled. + for (uint64_t cs = 0; cs < m; ++cs) { + for (uint64_t ce = cs + 1; ce <= m; ++ce) { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair_range(data.data(), m, cs, ce, n, + log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = cs; c < ce; ++c) push_be(want[leaf], data[br * m + c]); + } + } + check_leaves(out, want, "blake3_leaves_base_row_major_row_pair_range"); + } + } + } + } + printf("row-major leaf kernels: read pattern matches the CPU leaf spec, all column ranges\n"); +} + +// The full-range ranged kernel must be the unranged one — the same bytes by two +// code paths. A cheap check that the range arithmetic has no off-by-one at the +// boundary it is most likely to have one at. +void the_full_range_variant_equals_the_plain_one() { + const uint32_t log_n = 5; + const uint64_t n = 1ull << log_n, m = 7, num_leaves = n / 2; + std::vector data(n * m); + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(0xBEEF, i); + + std::vector plain(num_leaves * 32, 0), ranged(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair(data.data(), m, n, log_n, plain.data()); + } + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + blake3_leaves_base_row_major_row_pair_range(data.data(), m, 0, m, n, log_n, ranged.data()); + } + check(plain == ranged, "the full-range kernel must equal the unranged one"); + printf("row-major range [0, m) equals the plain kernel\n"); +} + +// NEGATIVE CONTROL for the leaf checks: the leaves must depend on the data and +// on the row index. Every check above compares kernel output to an expectation +// built from the same buffer, and all of them would pass if the kernel emitted a +// constant and the expectation happened to be that constant. +void leaves_depend_on_data_and_row() { + const uint32_t log_n = 4; + const uint64_t n = 1ull << log_n, num_cols = 3; + std::vector cols(num_cols * n); + for (size_t i = 0; i < cols.size(); ++i) cols[i] = sample(0xD00D, i); + + std::vector a(n * 32, 0), b(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_batched(cols.data(), n, num_cols, n, log_n, a.data()); + } + cols[n + 3] ^= 1ull; // one element of one column + CUDA_HOST_FOR_EACH_THREAD(t, n) { + blake3_leaves_base_batched(cols.data(), n, num_cols, n, log_n, b.data()); + } + check(a != b, "a one-element change must move some leaf"); + + bool all_same = true; + for (uint64_t i = 1; i < n; ++i) { + if (memcmp(a.data(), a.data() + i * 32, 32) != 0) { + all_same = false; + break; + } + } + check(!all_same, "all leaves identical — the kernel is not reading its row index"); + printf("negative control: leaves depend on the data and on the row index\n"); +} + +} // namespace + +int main() { + printf("BLAKE3 device-kernel known-answer tests, host-compiled from " + "crypto/math-cuda/kernels/blake3.cu\n\n"); + official_vectors_at_seven_rounds(); + six_rounds_is_not_standard_blake3(); + canonical_vectors_at_both_round_counts(); + serialization_is_the_canonical_big_endian_bytes(); + block_framing_completes_and_pads(); + parent_is_the_one_block_hash_of_its_children(); + printf("\n-- chain construction --\n"); + chain_against_official_multiblock_vectors(); + chain_against_the_committed_six_round_table(); + device_chain_matches_the_reference(); + device_chain_at_64_bytes_is_the_parent(); + printf("\n-- leaf kernels --\n"); + base_leaf_kernels_read_the_specified_bytes(); + ext3_leaf_kernels_read_the_specified_bytes(); + fri_leaf_kernel_reads_the_specified_bytes(); + row_major_leaf_kernels_read_the_specified_bytes(); + the_full_range_variant_equals_the_plain_one(); + leaves_depend_on_data_and_row(); + if (failures != 0) { + printf("\n*** %d FAILURE(S) ***\n", failures); + return 1; + } + printf("\nALL HOST KAT CHECKS PASS\n"); + printf("NOTE: arithmetic only. nvcc acceptance and GPU execution are covered " + "by tests/blake3_*.rs, which need a GPU.\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/blake3_kat_vectors.h b/crypto/math-cuda/tests/host_kat/blake3_kat_vectors.h new file mode 100644 index 000000000..ee1127c9b --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/blake3_kat_vectors.h @@ -0,0 +1,360 @@ +// Known-answer vectors for the BLAKE3 device kernels, embedded rather than +// parsed at run time. +// +// Embedded on purpose: a test that reads its vectors from a file passes +// silently when the read finds nothing, which is a failure mode that has +// already happened once on this harness. A table cannot have a zero-vector +// run, and `main` asserts the counts below as well. +// +// This file is DATA. It is transcribed, never computed, and the two tables +// come from outside this crate — see each one's provenance note. +#pragma once +#include + +// --------------------------------------------------------------------------- +// Table 1 — the OFFICIAL BLAKE3 test vectors, standard 7-round hash. +// +// Transcribed from `thoughts/blake3/blake3-oracle/official_test_vectors.json` +// (tracked in this repo, sourced from the BLAKE3 reference implementation). +// Only the cases with `input_len <= 64` appear: a message that fits one block +// of one chunk is a SINGLE compression, which is what the device function +// computes. Longer cases need the chunk tree and are not this kernel's job. +// +// The input for length N is the first N bytes of the repeating 251-byte +// sequence 0, 1, 2, ..., 250 — the generator the vector file specifies. +// --------------------------------------------------------------------------- +struct OfficialVector { + uint32_t input_len; + const char *hash_hex; // the first 32 bytes of the extended output +}; + +inline constexpr int NUM_OFFICIAL_VECTORS = 11; +inline constexpr OfficialVector OFFICIAL_VECTORS[NUM_OFFICIAL_VECTORS] = { + { 0, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"}, + { 1, "2d3adedff11b61f14c886e35afa036736dcd87a74d27b5c1510225d0f592e213"}, + { 2, "7b7015bb92cf0b318037702a6cdd81dee41224f734684c2c122cd6359cb1ee63"}, + { 3, "e1be4d7a8ab5560aa4199eea339849ba8e293d55ca0a81006726d184519e647f"}, + { 4, "f30f5ab28fe047904037f77b6da4fea1e27241c5d132638d8bedce9d40494f32"}, + { 5, "b40b44dfd97e7a84a996a91af8b85188c66c126940ba7aad2e7ae6b385402aa2"}, + { 6, "06c4e8ffb6872fad96f9aaca5eee1553eb62aed0ad7198cef42e87f6a616c844"}, + { 7, "3f8770f387faad08faa9d8414e9f449ac68e6ff0417f673f602a646a891419fe"}, + { 8, "2351207d04fc16ade43ccab08600939c7c1fa70a5c0aaca76063d04c3228eaeb"}, + {63, "e9bc37a594daad83be9470df7f7b3798297c3d834ce80ba85d6e207627b7db7b"}, + {64, "4eed7141ea4a5cd4b788606bd23f46e212af9cacebacdc7d1f4c6dc7f2511b98"}, +}; + +// --------------------------------------------------------------------------- +// Table 2 — the ten canonical vectors, at BOTH round counts. +// +// Inputs and `out6` are transcribed from `CANONICAL_VECTORS` +// (now `crypto/crypto/src/hash/blake3/vectors.rs`), whose 6-round outputs came +// from #903's Python oracle rather than from any Rust code. `out7` is +// `CANONICAL_OUT_7ROUND` from the same file, itself pinned by the official crate. +// +// This is the point of the table: it gives the SIX-round arm a known-answer +// test whose expected values no implementation in this tree produced. +// +// ★ It ALSO does a job Table 1 structurally CANNOT, and the reason is not +// obvious enough to leave unwritten: the official-vector path hashes whole +// messages, so it only ever exercises `t = 0`. A compression with the counter +// split inverted (`v[12]` and `v[13]` transposed) reproduces the official +// vectors at every single-block length, and is caught only here — all ten of +// these vectors carry `t >= 2^32`. Measured, not assumed. This table is not +// redundant with Table 1 and must not be retired as "covered by the standard". +// +// ★ PROVENANCE, strengthened 2026-08-15 — the 6-round column is no longer +// pinned by Python alone. `thoughts/blake3/reference-impl/` holds UPSTREAM +// BLAKE3 1.8.5 with a 2 KB reviewable diff (`PARAMETERISATION.diff`) whose only +// functional edit replaces seven unrolled `round_fn` calls with a loop bounded +// by `BLAKE3_ROUNDS_PARAM`. Built at both round counts and run over these ten +// inputs, it reproduces `out6` AND `out7` 10/10, all 16 words. It is C rather +// than Python, upstream's own code rather than a transcription, and it encodes +// the message schedule as an INDEXED TABLE (`MSG_SCHEDULE[r]`) rather than as an +// in-place permutation between rounds — a structurally different expression of +// the same convention, so its agreement cross-validates the schedule instead of +// restating it. Rebuild with `thoughts/blake3/reference-impl/build.sh`: a ~1 +// second C compile, no cargo, no GPU. +// --------------------------------------------------------------------------- +struct CanonicalVector { + uint32_t h[8]; + uint32_t m[16]; + uint64_t t; + uint32_t block_len; + uint32_t flags; + uint32_t out6[16]; + uint32_t out7[16]; +}; + +inline constexpr int NUM_CANONICAL_VECTORS = 10; +inline constexpr CanonicalVector CANONICAL_VECTORS[NUM_CANONICAL_VECTORS] = { + { + {0xD82C07CDu, 0x6BAA9455u, 0x82E2E662u, 0x7A024204u, + 0xE87A1613u, 0x81332876u, 0x48268673u, 0xC17C6279u}, + {0xE6F4590Bu, 0x4F65D4D9u, 0xBAD640FBu, 0xAF19922Au, + 0x19C78DF4u, 0x6F25E2A2u, 0xE9BB17BCu, 0x7A1D5006u, + 0x42AF9FC3u, 0x03983CA8u, 0xDE1B372Au, 0xDED733E8u, + 0x9148624Fu, 0xF7B0B7D2u, 0x72AE2244u, 0xEECE328Bu}, + 0xB4E1357D4A84EB03ull, 42u, 52u, + {0xCED9D1FFu, 0xC248EEABu, 0xBD109B7Fu, 0x911B48F6u, + 0x923D62C0u, 0xD804903Fu, 0x5974223Eu, 0xAA4F0C80u, + 0xAD61007Fu, 0xB50B8DDBu, 0xE7372BE1u, 0x33D3D6C3u, + 0x42AA284Bu, 0xC5A25F28u, 0x79AC8370u, 0xB75F3915u}, + {0xEE79E5DCu, 0xEA647B8Cu, 0x964C097Eu, 0xE2F3383Au, + 0xFE2E6D00u, 0x78EE613Au, 0xC33C8572u, 0xCD444391u, + 0x0C890604u, 0xC3209591u, 0x45633FF8u, 0xCB171C6Au, + 0x760247AEu, 0xF6D0FC1Eu, 0xCD550F20u, 0xCD54BF83u}, + }, + { + {0xC386BBC4u, 0x414C343Cu, 0x7311D8A3u, 0xA6CECC1Bu, + 0xC9E9C616u, 0x18072E8Cu, 0xD5F4B3B2u, 0x7204E52Du}, + {0xF1FD42A2u, 0xE6C3F339u, 0x07D4BEDCu, 0x8A9A021Eu, + 0x3BAB6C39u, 0x05805975u, 0xA46D6753u, 0xDC2574BDu, + 0xAB99254Au, 0x4DA98F1Du, 0xE1EA24C4u, 0x815A47C5u, + 0x08D6AF57u, 0xCC22AF58u, 0x2C4A3698u, 0x5FEC898Fu}, + 0xC74803E31BA16215ull, 50u, 94u, + {0xF2A972E9u, 0x81FDB8ECu, 0x40C50EBCu, 0x4BA1CAF9u, + 0x9EE9E930u, 0x6B1A16B2u, 0xE9156F47u, 0xA89FB436u, + 0xA2F616B3u, 0x12874C12u, 0x30768035u, 0xE01A17D9u, + 0xBEE5C17Cu, 0xD61C0BE0u, 0x3041FF46u, 0xDFB91125u}, + {0xD68593D0u, 0xDBC8157Au, 0xF6E1687Cu, 0x52A60555u, + 0xB56D418Au, 0x0CCBB863u, 0xADBFB51Eu, 0x8BF7D125u, + 0x75C23432u, 0xF484D7A6u, 0x06E85F4Au, 0x2771FE96u, + 0x00F6E24Du, 0x48368A3Eu, 0x04EE7E88u, 0x501D8539u}, + }, + { + {0x0E7A269Fu, 0x15BA2BDDu, 0xD5E34124u, 0x4EE207F8u, + 0x9B1F282Eu, 0x9B575BD1u, 0xF30B94FAu, 0x0706A045u}, + {0x6148A86Fu, 0x8697BBD0u, 0x8F7D9B78u, 0x3C729578u, + 0x061B9030u, 0x533C9135u, 0x829E07B0u, 0xE4C11AB2u, + 0xCBF87544u, 0xC34C769Fu, 0x5A91C89Bu, 0xF63F23D0u, + 0xC1066932u, 0x87C56473u, 0x7D718D73u, 0xECC1CB63u}, + 0x7604E4B4E73695C3ull, 58u, 124u, + {0x5AA6B114u, 0xC9D6740Cu, 0x8738CAF4u, 0xAC5F4B72u, + 0x9FC6B9DEu, 0x3F2EFB8Fu, 0x8CB7A912u, 0xF497A285u, + 0x3D062266u, 0x7F22380Cu, 0xAFD468FAu, 0x122CBA80u, + 0x446B156Du, 0xB239D8C2u, 0xC3EAB2CFu, 0x775F2F92u}, + {0xBC92D7C4u, 0x56542092u, 0x3490E2CBu, 0x2E3328CDu, + 0x13E3746Fu, 0xA5B88E66u, 0x2B5FE530u, 0x92C7AD52u, + 0xFF502AE5u, 0x1F088FBFu, 0x9163752Fu, 0x8A0C8B4Du, + 0xB557B0E8u, 0xE76F23CBu, 0xD054C959u, 0x74813CFDu}, + }, + { + {0x8B529B4Au, 0x9A9A80FDu, 0xD6645FA9u, 0x3BFD1D33u, + 0x79F248B0u, 0x268ECC45u, 0xA2863A7Fu, 0x85EF3430u}, + {0xBDC2AE99u, 0x10645D51u, 0x97524D6Au, 0xDD933160u, + 0xE0F9E038u, 0xEBCD1F5Eu, 0xEF829C88u, 0xE0FD67DDu, + 0x18F2C41Cu, 0x22CEDAFBu, 0x378C74DCu, 0x4D100D8Fu, + 0x95C76AB4u, 0x95918694u, 0xE779C470u, 0xEDCF6109u}, + 0x92D3043AFCF249F3ull, 36u, 31u, + {0xEED92FABu, 0x138D9358u, 0x915BFE3Cu, 0x13718B01u, + 0xB506E277u, 0xBE4007CDu, 0x35847E06u, 0xCE1C6896u, + 0x52FA01B5u, 0x4AA26AF8u, 0xB1078A61u, 0x2C517AEDu, + 0xA08867A0u, 0xEA6ECFEAu, 0x6D33D3B0u, 0xDC293166u}, + {0xCF4FB929u, 0x1DBADE2Au, 0x70E63AAFu, 0x2E0FFB48u, + 0x60123045u, 0x798AEAE8u, 0x5A911D30u, 0x15977C61u, + 0x6F7C8334u, 0x5EB0BCE2u, 0xAB240F17u, 0x66B7A3CDu, + 0xA9064E0Bu, 0x6AC4747Bu, 0x1206F62Bu, 0x9F3E91ECu}, + }, + { + {0x3C6DA5D7u, 0x656412A9u, 0x27AC435Au, 0x11072231u, + 0xEAFF1A09u, 0xC3E1B258u, 0x8963DC6Eu, 0x1B2ED40Eu}, + {0xED6F0B09u, 0xCE80C4B0u, 0xCCEA2645u, 0x3184FF27u, + 0x4F5253A0u, 0xE14B0190u, 0x9B191BF4u, 0xABF4A07Cu, + 0x81862FC9u, 0x2D83A823u, 0x793D0E45u, 0x4CDCE7A6u, + 0xE8ABB93Fu, 0xE1DF8AF9u, 0x8224B122u, 0x69F85E31u}, + 0x49C7B59B995253FDull, 57u, 41u, + {0xCA00BDA3u, 0x84239A3Au, 0xE7C88E6Du, 0x33A8A3D6u, + 0x09DCD1CEu, 0xA1B10212u, 0xF48E1156u, 0x8F039915u, + 0x8A055EAAu, 0xFF5B11D5u, 0xB725085Bu, 0x2E1AB267u, + 0x6AE7323Du, 0xB2FF6FA8u, 0x7102C8A1u, 0x7561EB37u}, + {0xFF525F0Fu, 0xD892E3D2u, 0xFB566B40u, 0x3BDF4ED0u, + 0x78B961CDu, 0x9CB86B48u, 0x6AB54F3Du, 0x3EF5F695u, + 0xBD896ED8u, 0x6265AC08u, 0xF6695D78u, 0x9F3795EAu, + 0x943E0342u, 0xD1437B3Bu, 0x4F6BAF78u, 0x85DFD2C9u}, + }, + { + {0x9F767C45u, 0xBDE5C099u, 0xF17FD374u, 0xA6233255u, + 0xE6A16A3Bu, 0x1CFB10F6u, 0x3F1F65A8u, 0x8B33E968u}, + {0x92EDCF45u, 0x377B9AA2u, 0x478C281Du, 0xC4069545u, + 0xCC11D357u, 0x9E115E4Bu, 0x206F5C66u, 0xDF1461AAu, + 0xFB7FF337u, 0xDF561D80u, 0x4A0FE75Du, 0xF6236BF2u, + 0x346C6E2Bu, 0xB0CDE917u, 0xE4CC4132u, 0x4C7D6DF0u}, + 0x6A3753915C76F18Aull, 18u, 67u, + {0x14A9F66Fu, 0x101BDFE8u, 0x9B0A50DDu, 0xEE4BB45Bu, + 0x7A914502u, 0x77B3486Bu, 0x59BFC114u, 0xA1AD2AFDu, + 0xC194DDE6u, 0x894EC54Du, 0xAD36C805u, 0x9018F3F5u, + 0x165AF5D8u, 0x3E85B598u, 0x78E76653u, 0xBB7A485Du}, + {0xD22912BBu, 0x627F992Cu, 0xE883AF5Du, 0x50E58A48u, + 0xF3D071C6u, 0xB20D47A4u, 0x29011151u, 0xFE50E232u, + 0x594B76A3u, 0x8706296Bu, 0x2C1D1E31u, 0x6A478D0Du, + 0x64004E61u, 0xA072DA1Eu, 0xAB3FCA42u, 0x09BB269Eu}, + }, + { + {0xD26B9496u, 0x42F9A039u, 0x001D9A88u, 0x5F877031u, + 0xC527E279u, 0x45CF8AA4u, 0xCD4A5557u, 0xAE9AF169u}, + {0xAF895F5Bu, 0xD822E2F9u, 0x17D7AB26u, 0xCCDF540Bu, + 0xCE06294Du, 0x4A8B0188u, 0xF38D2E64u, 0x5C41D5C5u, + 0xE8D5B9E3u, 0x5C832A51u, 0x9A0C1B76u, 0x4DE8344Eu, + 0x96D2F9E0u, 0x8677A5F2u, 0xA9A967C1u, 0x323BBEAFu}, + 0x390567C27BD6AA42ull, 26u, 3u, + {0x32A6FF70u, 0xC30560BCu, 0xD1C777C8u, 0xF1871821u, + 0x7207AB54u, 0x9F5B83C7u, 0xB6561C5Du, 0x991E738Fu, + 0xB38B62B9u, 0x0EF6D156u, 0x994BECB1u, 0x09A85D0Eu, + 0x32221741u, 0xADA3CC5Fu, 0x5B654ED6u, 0x2A7A62B2u}, + {0xA101CEABu, 0x9232E0ECu, 0x2FE4B24Eu, 0x35F7F4FEu, + 0x61A5AB42u, 0xBE417503u, 0xEB740D5Eu, 0x8BB2FE96u, + 0xC6863DA9u, 0x1F31FF5Du, 0x5763EA12u, 0xDC862699u, + 0x1A60ADE2u, 0x9E3E6745u, 0xE3C8F87Eu, 0xD3EFB0EAu}, + }, + { + {0x269E0D37u, 0xA6A3A450u, 0x892F902Bu, 0x81E74EF5u, + 0x099950D8u, 0x6F03675Au, 0x11E20B8Fu, 0x6CAD4A26u}, + {0xF29D0DA9u, 0x658CDA14u, 0xF9EBDACCu, 0xDBC496CBu, + 0x4A23D596u, 0x2E44158Bu, 0xA38FD547u, 0x5F557203u, + 0x34B9B5DFu, 0x506BF2EFu, 0x7403E430u, 0x4CBD87ADu, + 0xCB5C7427u, 0x3E7D1BFBu, 0x930D6EAFu, 0x86734721u}, + 0x12BD4ACEFAECBD38ull, 53u, 42u, + {0xA632AD45u, 0x12CE41F4u, 0xD21B2CBDu, 0x76795C62u, + 0x6BEC36C1u, 0xDAFAFCDEu, 0x53CA87B7u, 0x92E8465Bu, + 0x7B424F5Du, 0xE1E6AD7Fu, 0x753BA387u, 0xCCC50824u, + 0x69AEDF6Du, 0xBBBBF253u, 0x78D04883u, 0xF3F33689u}, + {0x318604BEu, 0x22A35843u, 0x6CA63195u, 0xA2E7E2F8u, + 0x48769A04u, 0xC462F1E3u, 0x5CF053C7u, 0xFD1EE629u, + 0x69366332u, 0x0ACC819Bu, 0xBBD2456Au, 0xF1DA9DB6u, + 0x4A7B7D68u, 0x6DD1A843u, 0x61555466u, 0xBDA36F28u}, + }, + { + {0x3A096533u, 0xF658F7A7u, 0x205738D1u, 0xB46EE1DAu, + 0x15CEB3A1u, 0x359B1548u, 0xA4517D6Cu, 0x7589CA4Au}, + {0x74007CB4u, 0xD49D0AC1u, 0x16EDC5D4u, 0x685CA8AFu, + 0x4223AA56u, 0x10269470u, 0x60908405u, 0xA92D04A3u, + 0x56A3E957u, 0xB0F91306u, 0xE6C08269u, 0xF2306D4Au, + 0x31A06A7Cu, 0x9436D6F6u, 0xE18692E2u, 0xE0C99F3Eu}, + 0x329911DA9FBD8735ull, 19u, 91u, + {0x913B2AE1u, 0xC7F73082u, 0x45E1C023u, 0x6F1F3F82u, + 0x20AEE6F5u, 0xDAF21D94u, 0xF2C1E4AFu, 0xD4F7D4ACu, + 0x44A45F87u, 0xF4C40CE5u, 0x613E9B94u, 0x08CE53DEu, + 0x4FF07AA4u, 0x456BF2E2u, 0x2066EA7Fu, 0x3C5A654Bu}, + {0x87584719u, 0x15C73090u, 0x851C1A4Au, 0x99D21014u, + 0x821A82A8u, 0xC7307CD5u, 0x6797EFE2u, 0xCF38CEDFu, + 0x777C177Du, 0x202BE3EAu, 0x19421985u, 0x3176132Du, + 0x7BB8BC22u, 0x65C9804Bu, 0x22C68EA3u, 0x92504162u}, + }, + { + {0x5F915EF0u, 0x237751AAu, 0x01A5BA50u, 0x80B65386u, + 0x14B044D7u, 0x61076DC3u, 0xB99DE255u, 0x283B73A6u}, + {0x3CEE5E2Cu, 0x1C670EA9u, 0x972651DAu, 0x4A8AA593u, + 0xAC9ABB0Cu, 0x35BB5C11u, 0x47FBB3B4u, 0xCF3C17E5u, + 0xE2EB17C8u, 0xE11E99FBu, 0x7DE0D208u, 0x0602FE0Cu, + 0x98CAE043u, 0x9425B3E2u, 0x33FB4B4Fu, 0x15607DF9u}, + 0xEAEB999B8A2E547Eull, 64u, 21u, + {0xF5EE9114u, 0x856CABB8u, 0x29BE2CF1u, 0x603BE91Cu, + 0x94A7DD0Eu, 0x28FC3E27u, 0xB64E2CC8u, 0x2D2C67FFu, + 0x69FAC1BAu, 0x0C949090u, 0xD68DE435u, 0xCE91A527u, + 0xE80C1815u, 0x6D44EFE6u, 0x87C7B175u, 0xD18A8B94u}, + {0xDC60D189u, 0xE6311F18u, 0x9DC3E078u, 0x304BB43Eu, + 0x5C616E7Du, 0xE168D00Fu, 0x2E197872u, 0x175B9188u, + 0x5A99C462u, 0xEF311A88u, 0xC61836FDu, 0x9FFD4DE3u, + 0x36AE4940u, 0x4D813D81u, 0x9B058DA9u, 0x9017D38Cu}, + }, +}; + +// --------------------------------------------------------------------------- +// Table 3 — MULTI-BLOCK official BLAKE3 vectors, for the `Blake3Chain` +// construction rather than the bare compression function. +// +// Table 1 above stops at 64 bytes because a single compression is all it can +// check. The chain spans many blocks, and its flag schedule, its `block_len` +// handling and its chaining value are only exercised past the first block — so +// it needs vectors Table 1 cannot supply. +// +// ★ These are still the OFFICIAL vectors, not an oracle's. `Blake3Chain` over a +// message of at most one chunk (1024 bytes) IS `blake3::hash` — standard +// BLAKE3's first chunk is exactly this chain, and a one-chunk message has that +// chunk's output as its root (PA-PLAN §1.7.2, P1). So for every length here up +// to 1024 the published hash is a direct known-answer test for the device +// chain, with no oracle and no transcription of anything computed in this repo. +// +// Transcribed from `thoughts/blake3/blake3-oracle/official_test_vectors.json` +// (tracked), first 32 bytes of each case's `hash`. Input for length N is the +// first N bytes of the repeating 251-byte sequence 0, 1, ..., 250 — the same +// generator Table 1 uses. +// +// `agrees` marks whether the chain must MATCH the published hash. The 1025 and +// 2048 rows are the P3 negative control: past one chunk standard BLAKE3 starts +// chunk 1 with a reset chaining value and builds a tree, and this construction +// deliberately does not. Without them "we implement the single-chunk chain" +// would be unfalsifiable — the matching rows alone would pass identically if the +// whole chunk tree had been implemented instead. +// +// ★ THE BOUNDARY IS LOCATED, not sampled. All 35 official cases were swept +// (2026-08-15): agreement holds for every length up to and including 1024, and +// fails for every one of the 18 lengths >= 1025. Max agreeing 1024, min +// differing 1025 — the divergence sits exactly on the one-chunk edge, which is +// what P3 predicts. The 1024/1025 pair below is that boundary; the other rows +// cover the multi-block cases in between. +// +// The input generator `i % 251` was itself verified empirically rather than +// taken from the file's prose: it reproduces the `hash` field for 35/35 cases. +// Note it differs from the `(37i + 11) mod 256` generator Table 4 uses — the two +// tables come from different sources and do NOT share a message. +// --------------------------------------------------------------------------- +struct ChainVector { + uint32_t input_len; + const char *hash_hex; + bool agrees; // false = must DIFFER (past one chunk) +}; + +inline constexpr int NUM_CHAIN_VECTORS = 8; +inline constexpr ChainVector CHAIN_VECTORS[NUM_CHAIN_VECTORS] = { + { 65, "de1e5fa0be70df6d2be8fffd0e99ceaa8eb6e8c93a63f2d8d1c30ecb6b263dee", true}, + { 127, "d81293fda863f008c09e92fc382a81f5a0b4a1251cba1634016a0f86a6bd640d", true}, + { 128, "f17e570564b26578c33bb7f44643f539624b05df1a76c81f30acd548c44b45ef", true}, + { 129, "683aaae9f3c5ba37eaaf072aed0f9e30bac0865137bae68b1fde4ca2aebdcb12", true}, + { 1023, "10108970eeda3eb932baac1428c7a2163b0e924c9a9e25b35bba72b28f70bd11", true}, + { 1024, "42214739f095a406f3fc83deb889744ac00df831c10daa55189b5d121c855af7", true}, + { 1025, "d00278ae47eb27b34faecf67b4fe263f82d5412916c1ffd97c8cb7fb814b8444", false}, + { 2048, "e776b6028c7cd22a4d0ba182a8bf62205d2ef576467e838ed6f2529b85fba24a", false}, +}; + +// --------------------------------------------------------------------------- +// Table 4 — the committed 6-ROUND chain KAT. +// +// A byte-for-byte transcription of `CHAIN_KAT_6ROUND` +// (`crypto/crypto/src/hash/blake3/chain.rs:304`), which is the same table +// PA-PLAN §1.7.5 records. Message of length N is byte `i = 37i + 11 (mod 256)`. +// +// Provenance, and why it is worth more than a self-comparison: those digests +// were produced by #903's Python oracle +// (`thoughts/blake3/blake3-oracle/blake3_ref.py`, tracked), a full +// standard-BLAKE3 implementation with the round count as a parameter, whose +// 7-round arm reproduces the official package bit-for-bit. So at the round count +// the campaign actually ships, this pins the device chain against numbers no +// Rust and no CUDA in this tree computed. +// +// Duplicating the table here rather than sharing one copy is deliberate: this +// harness compiles as standalone C++ with no cargo and no Rust in the build, so +// there is nothing to share it with. A drift between the two copies is caught by +// the Rust-side `device_chain_matches_the_committed_table_at_six_rounds`, which +// reads the Rust constant directly. +// --------------------------------------------------------------------------- +struct ChainKat6Round { + uint32_t input_len; + const char *hash_hex; +}; + +inline constexpr int NUM_CHAIN_KAT_6ROUND = 12; +inline constexpr ChainKat6Round CHAIN_KAT_6ROUND[NUM_CHAIN_KAT_6ROUND] = { + { 0, "3c3bbb1f335a31ea86464b651c0206fc81d33262ae00ea1a65f3d1d04afaefc9"}, + { 1, "2a50e45b8921f9efa008d9f39f7165600cf48a7f0e859c2122e3ccb6b9677ee5"}, + { 31, "c38bf62f506040b2600273778d281b8943621e2b8a9f59e2379f8fd7e5c85125"}, + { 63, "c373f51a5eb8b27ea05bb1f6f4e62e924ff4d8a279f0d05afa5cd519391d6389"}, + { 64, "5900a1e398bb2bf6d3ba7f1a29197b79c86b71ad2c2631f4ac736c82db043cb5"}, + { 65, "53953fcadc39b8623901af7b534f2f6933e312f50299331334e6c0a7c9dbc2be"}, + { 127, "9e0dd8168d199a04590c2cba439b270776e42715d518f68655e56692483e505e"}, + { 128, "5caffc8784e817bbba991b2108c26a3dfdf804245ef63ae1040a3c34f1b362ff"}, + { 192, "399d6b9adeb2f88450775f773e9dec08836c135713c2c5dd09f4ceceb0ed3888"}, + { 256, "fbcab3699a4959fa37190e98ca5142ddbc88330f2e7d12335db9c6c8881a0b87"}, + { 1024, "f395e7e2150363b6d200487515425b0204eea424072183b701176eccbe0ffe1b"}, + { 1088, "b4738ede77a6ec166ee97667118d4793cbf2b08b45aac7c6d52943b5d298c688"}, +}; diff --git a/crypto/math-cuda/tests/host_kat/cuda_host_shim.h b/crypto/math-cuda/tests/host_kat/cuda_host_shim.h new file mode 100644 index 000000000..5a2dc6bff --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/cuda_host_shim.h @@ -0,0 +1,70 @@ +// Enough of the CUDA language to compile a `.cu` kernel file as ordinary host +// C++, so its arithmetic can be checked without a GPU. +// +// This exists because the GPU parity suite (`crypto/math-cuda/tests/blake3_*.rs`) +// runs only where a GPU does, and per-PR CI has none. Including a kernel through +// this shim turns its device functions into plain functions a host program can +// call, which is all a known-answer test needs. +// +// ⚠ What it CANNOT check, and what therefore still belongs to the GPU tests: +// anything about execution rather than arithmetic — thread/block indexing, +// `__syncthreads` ordering, memory alignment on device, register pressure, and +// whether nvcc accepts the file at all. A kernel that passes here can still be +// wrong on a GPU. Treat this as a lower bound on correctness, never a substitute. +#pragma once + +#include + +// The execution-space and inlining qualifiers carry no meaning on host. +#define __device__ +#define __constant__ +#define __forceinline__ inline +#define __global__ + +// Single-threaded host execution: one thread, block 0, and a barrier that has +// nothing to wait for. Kernel thread coordinates are ordinary mutable globals, +// so a caller can drive them (see `CUDA_HOST_FOR_EACH_THREAD`) and replay a +// whole launch's worth of thread slices one at a time. +#define __syncthreads() ((void)0) +struct CudaHostDim3 { + unsigned x = 0, y = 0, z = 0; +}; +static CudaHostDim3 blockIdx; +static CudaHostDim3 threadIdx; +static CudaHostDim3 cuda_host_block_dim; +#define blockDim cuda_host_block_dim + +// `goldilocks.cuh`'s field multiply needs this intrinsic. `blake3.cu` only uses +// `goldilocks::canonical`, but the header compiles as a whole, so supply it. +static inline uint64_t __umul64hi(uint64_t a, uint64_t b) { + return (uint64_t)(((unsigned __int128)a * (unsigned __int128)b) >> 64); +} + +// Bit-reverse a 64-bit word. Every leaf kernel derives its row index as +// `__brevll(tid) >> (64 - log_num_rows)`, so replaying one on host needs it. +// Written out rather than deferring to a compiler builtin so the shim stays +// toolchain-neutral. +static inline uint64_t __brevll(uint64_t x) { + x = ((x & 0x5555555555555555ull) << 1) | ((x >> 1) & 0x5555555555555555ull); + x = ((x & 0x3333333333333333ull) << 2) | ((x >> 2) & 0x3333333333333333ull); + x = ((x & 0x0F0F0F0F0F0F0F0Full) << 4) | ((x >> 4) & 0x0F0F0F0F0F0F0F0Full); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +// Replay a `__global__` kernel once per thread index, sequentially, by driving +// the shim's thread coordinates. A kernel computing +// `tid = blockIdx.x * blockDim.x + threadIdx.x` sees `tid = i` on iteration `i`, +// so a whole launch can be reproduced on host: +// +// CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) some_leaf_kernel(args...); +// +// ⚠ Only valid for kernels whose threads are independent — which the leaf +// kernels are (one thread, one leaf, disjoint output) and the Merkle *tail* is +// not. It says nothing about `__syncthreads` ordering, races or occupancy. +#define CUDA_HOST_FOR_EACH_THREAD(i, n) \ + for (unsigned i = 0; \ + i < (unsigned)(n) && \ + (blockIdx.x = 0, blockDim.x = 0, threadIdx.x = i, true); \ + ++i) diff --git a/crypto/math/src/field/extensions_goldilocks.rs b/crypto/math/src/field/extensions_goldilocks.rs index b4814a2c7..42eb51a59 100644 --- a/crypto/math/src/field/extensions_goldilocks.rs +++ b/crypto/math/src/field/extensions_goldilocks.rs @@ -579,6 +579,12 @@ impl HasDefaultTranscript for Degree3GoldilocksExtensionField { GoldilocksField::sample_field_element_from(&mut next_u64) })) } + + /// The coordinates are Goldilocks and the rejection loop runs per + /// coordinate, so the predicate is the base field's. + fn candidate_in_range(candidate: u64) -> bool { + GoldilocksField::candidate_in_range(candidate) + } } // ===================================================== diff --git a/crypto/math/src/field/goldilocks.rs b/crypto/math/src/field/goldilocks.rs index 39fd707b7..e5fa034bd 100644 --- a/crypto/math/src/field/goldilocks.rs +++ b/crypto/math/src/field/goldilocks.rs @@ -548,9 +548,13 @@ impl HasDefaultTranscript for GoldilocksField { fn sample_field_element_from(mut next_u64: impl FnMut() -> u64) -> FieldElement { loop { let candidate = next_u64(); - if candidate < GOLDILOCKS_PRIME { + if Self::candidate_in_range(candidate) { return FieldElement::from(candidate); } } } + + fn candidate_in_range(candidate: u64) -> bool { + candidate < GOLDILOCKS_PRIME + } } diff --git a/crypto/math/src/field/traits.rs b/crypto/math/src/field/traits.rs index a0e0a7fbc..1adc7992b 100644 --- a/crypto/math/src/field/traits.rs +++ b/crypto/math/src/field/traits.rs @@ -305,4 +305,19 @@ pub trait HasDefaultTranscript: IsField { /// straight from the Fiat-Shamir sponge, so no separate CSPRNG keystream is /// generated (see `DefaultTranscript`). fn sample_field_element_from(next_u64: impl FnMut() -> u64) -> FieldElement; + + /// Would [`Self::sample_field_element_from`] accept `candidate` as one + /// coordinate, or reject it and pull another? + /// + /// This exposes the acceptance predicate so a caller can pre-filter the + /// candidate stream and thereby control *how many* candidates a draw + /// consumes — see `DefaultTranscript`'s constant-consumption mode. It has to + /// agree with what `sample_field_element_from` actually does: a pre-filtering + /// caller that disagreed would hand the sampler values it rejects, and the + /// consumption schedule the caller believes it is enforcing would not be the + /// real one. + /// + /// For an extension field this is the predicate for a single *base* + /// coordinate, because that is the granularity the rejection loop runs at. + fn candidate_in_range(candidate: u64) -> bool; } diff --git a/crypto/stark/examples/examples_cli.rs b/crypto/stark/examples/examples_cli.rs index d8d20528b..0997f7f36 100644 --- a/crypto/stark/examples/examples_cli.rs +++ b/crypto/stark/examples/examples_cli.rs @@ -576,6 +576,7 @@ fn prove_multi_table_lookup() -> Result, String> { &mut DefaultTranscript::::new(&[]), #[cfg(feature = "disk-spill")] stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| format!("prove failed: {e:?}"))?; ser(&multi_proof) diff --git a/crypto/stark/src/commitment.rs b/crypto/stark/src/commitment.rs index d4a6dbdbe..9925dfde6 100644 --- a/crypto/stark/src/commitment.rs +++ b/crypto/stark/src/commitment.rs @@ -33,6 +33,9 @@ use math::traits::{AsBytes, ByteConversion}; #[cfg(feature = "parallel")] use rayon::prelude::{IntoParallelIterator, ParallelIterator}; +use crypto::merkle_tree::merkle::MerkleTree; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; + use crate::config::{BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment}; /// Number of consecutive (bit-reversed) rows packed into one Merkle leaf for the @@ -49,12 +52,13 @@ pub const ROWS_PER_LEAF: usize = 2; /// exact leaf byte layout. This is the single code path behind both the per-row /// ([`keccak_leaves_bit_reversed`]) and per-row-pair /// ([`keccak_leaves_row_pair_bit_reversed`]) commitments. -pub fn keccak_leaves_bit_reversed_grouped( +pub fn leaves_bit_reversed_grouped( columns: &[Vec>], rows_per_leaf: usize, ) -> Vec where E: IsField, + B: IsStreamingLeafBackend, FieldElement: AsBytes + Sync + Send + ByteConversion, { if columns.is_empty() || columns[0].is_empty() { @@ -87,7 +91,7 @@ where offset += byte_len; } } - BatchedMerkleTreeBackend::::hash_bytes(buf) + >::hash_bytes(buf) }; // Per-thread buffer reuse (map_init) avoids millions of small allocations. @@ -106,6 +110,19 @@ where result } +/// [`leaves_bit_reversed_grouped`] at the keccak backend — the production leaf +/// hash, and the one the CUDA kernels and their parity tests mirror. +pub fn keccak_leaves_bit_reversed_grouped( + columns: &[Vec>], + rows_per_leaf: usize, +) -> Vec +where + E: IsField, + FieldElement: AsBytes + Sync + Send + ByteConversion, +{ + leaves_bit_reversed_grouped::>(columns, rows_per_leaf) +} + /// Per-row Keccak-256 leaf hashes (one leaf per bit-reversed row). Thin wrapper /// over [`keccak_leaves_bit_reversed_grouped`] with `rows_per_leaf = 1`. /// @@ -117,7 +134,7 @@ where E: IsField, FieldElement: AsBytes + Sync + Send + ByteConversion, { - keccak_leaves_bit_reversed_grouped(columns, 1) + leaves_bit_reversed_grouped::>(columns, 1) } /// Per-row-pair Keccak-256 leaf hashes (leaf `i` hashes bit-reversed rows `2i`, @@ -128,7 +145,7 @@ where E: IsField, FieldElement: AsBytes + Sync + Send + ByteConversion, { - keccak_leaves_bit_reversed_grouped(parts, 2) + leaves_bit_reversed_grouped::>(parts, 2) } /// Builds the Merkle tree committing to `columns`' bit-reversed, column-major LDE @@ -144,12 +161,30 @@ pub fn commit_bit_reversed( where E: IsField, FieldElement: AsBytes + Sync + Send + ByteConversion, +{ + commit_bit_reversed_with::>(columns, rows_per_leaf) +} + +/// [`commit_bit_reversed`] under an explicit leaf backend. +/// +/// The backend is a parameter rather than the fixed alias because the prover is +/// generic over its commitment configuration; `commit_bit_reversed` is this +/// function at the keccak backend, and is what every caller that commits a +/// fixed production table still uses. +pub fn commit_bit_reversed_with( + columns: &[Vec>], + rows_per_leaf: usize, +) -> Option<(MerkleTree, Commitment)> +where + E: IsField, + B: IsStreamingLeafBackend, + FieldElement: AsBytes + Sync + Send + ByteConversion, { if columns.is_empty() || columns[0].is_empty() { return None; } - let hashed_leaves = keccak_leaves_bit_reversed_grouped(columns, rows_per_leaf); - let tree = BatchedMerkleTree::::build_from_hashed_leaves(hashed_leaves)?; + let hashed_leaves = leaves_bit_reversed_grouped::(columns, rows_per_leaf); + let tree = MerkleTree::::build_from_hashed_leaves(hashed_leaves)?; let root = tree.root; Some((tree, root)) } diff --git a/crypto/stark/src/config.rs b/crypto/stark/src/config.rs index 50650e40a..0f09efa19 100644 --- a/crypto/stark/src/config.rs +++ b/crypto/stark/src/config.rs @@ -1,15 +1,20 @@ +use crypto::fiat_shamir::transcript_hash::{KeccakTranscriptHash, TranscriptHash}; +#[cfg(not(feature = "cuda"))] +use crypto::merkle_tree::backends::types::{BatchBlake3Backend, PairBlake3Backend}; use crypto::merkle_tree::{ - backends::types::{BatchKeccak256Backend, Keccak256Backend, PairKeccak256Backend}, + backends::types::{BatchKeccak256Backend, PairKeccak256Backend}, merkle::MerkleTree, + traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}, }; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; +use math::traits::AsBytes; // Merkle Trees configuration // Security of both hashes should match -pub type FriMerkleTreeBackend = Keccak256Backend; -pub type FriMerkleTree = MerkleTree>; - // If using hashes with 256-bit security, commitment size should be 32 // If using hashes with 512-bit security, commitment size should be 64 // TODO: Commitment type should be obtained from MerkleTrees @@ -22,3 +27,321 @@ pub type BatchedMerkleTree = MerkleTree>; // FRI layer uses fixed-size pairs for efficiency (avoids Vec allocation per pair) pub type FriLayerMerkleTreeBackend = PairKeccak256Backend; pub type FriLayerMerkleTree = MerkleTree>; + +/// A Merkle backend whose leaves and parents are Keccak-256, byte for byte. +/// +/// A marker: no methods, nothing to implement wrongly. It exists because +/// `IsMerkleTreeBackend` is too weak a bound wherever the +/// backend does not actually do the hashing. The GPU tree entry points in +/// `gpu_lde` are exactly that case — they take a backend parameter and then +/// launch the `math-cuda` keccak kernels unconditionally, so `B` is a label on +/// bytes `B` never touched. Any 32-byte-node backend satisfies the weak bound, +/// so a backend over some other hash would compile there and hand back keccak +/// trees wearing its name, with nothing failing. +/// +/// Requiring this marker instead makes that a compile error at the call site, +/// and makes implementing it for a non-keccak backend a deliberate, reviewable +/// false statement rather than an omission nobody had to make. +pub trait KeccakTreeBackend: IsMerkleTreeBackend {} + +impl KeccakTreeBackend for BatchKeccak256Backend where + Self: IsMerkleTreeBackend +{ +} +impl KeccakTreeBackend for PairKeccak256Backend where + Self: IsMerkleTreeBackend +{ +} + +/// The hash every commitment this crate produces is built with. +/// +/// It is the machine-readable form of what the aliases above say in types, and +/// it exists so that code reasoning about *which hash is inside a root* can +/// match on it exhaustively rather than assert it in prose — see +/// `build_artifacts_with_hasher` in `prover/src/lfm/registry.rs`, whose +/// artifacts name a hash. Every such match is a place that has to be revisited +/// before this crate commits under a second hash; adding [`Self::Blake3`] broke +/// them, which is what that mechanism is for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CommitmentHash { + /// Keccak-256 at both the leaf and the parent layer. + Keccak256, + /// `Blake3Chain` at both the leaf and the parent layer — the single-chunk + /// BLAKE3 chain of PA-PLAN §1.7, at the round count `crypto`'s + /// `blake3-6round` feature selects. + /// + /// The round count is deliberately **not** a second variant. It is a + /// crate-global compile-time constant precisely so one build cannot produce + /// two hashes, so within a build there is nothing here to distinguish: a + /// proof's roots are named by this variant plus the build's feature set, + /// exactly as the `LFM_BLAKE3` chip's round count is. + Blake3, +} + +/// The hash behind [`Commitment`], [`BatchedMerkleTree`] and +/// [`FriLayerMerkleTree`]. Pinned to the aliases by the assertion below. +/// +/// ⚠ **This describes the DEFAULT configuration — the aliases — and nothing +/// else.** Now that [`Blake3StarkHash`] exists, a prover can run under a +/// configuration whose [`StarkHash::COMMITMENT_HASH`] differs from this const +/// and this const will not know: it is a global, the configuration is per-type. +/// Code that names the hash inside a *particular* proof's roots must read +/// `H::COMMITMENT_HASH` at the call site; only code that names the hash of the +/// aliases may read this. `prover::lfm::registry`'s guard reads this one because +/// its commit helpers are hard-wired to the aliases — when they become generic +/// over `H`, that guard moves with them (PA-PLAN §4.2). +pub const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Keccak256; + +/// One STARK commitment configuration: the Merkle backend families the +/// prover and verifier build trees with, named together so they cannot be +/// mixed, plus the [`CommitmentHash`] they all are. +/// +/// The three are separate families because they hash different leaf shapes, not +/// The two are separate families because they hash different leaf shapes, not +/// because they are different hashes: [`Self::Batched`] takes a whole row group, +/// [`Self::Pair`] a fixed FRI-layer pair. An implementation is expected to build +/// both on one hash — that is what [`Self::COMMITMENT_HASH`] asserts, and what +/// makes a proof's roots describable by a single name. +/// +/// Every member is generic over the field because the prover commits over both +/// the base field (main trace) and the extension (aux, composition, FRI) within +/// one proof, so the configuration cannot be pinned to one field. +/// +/// `Node` is deliberately **not** an associated type: it is [`Commitment`] for +/// every implementation. Keeping 32 bytes on the wire is what lets a +/// configuration change leave `StarkProof`'s fields and their rkyv derives +/// byte-identical — no format bump, no disturbance to the in-place verify path. +/// +/// # Invariant: the two families must agree on a two-element leaf +/// +/// `>::hash_data(&vec![a, b])` must equal `>::hash_data(&[a, b])`. +/// +/// This is load-bearing, not decorative. The prover builds FRI-layer trees with +/// [`Self::Pair`] (`fri/mod.rs`) and the verifier authenticates those same +/// openings with [`Self::Batched`] (`verify_fri_layer_openings`, which builds a +/// two-element `Vec`). Under keccak the two coincide — both stream the same +/// element bytes into one digest — which is why the split went unremarked while +/// there was only one configuration. A configuration whose families encode a +/// pair differently would reject every honest proof at its first FRI query. +/// +/// An implementation that cannot honour this must make the prover and verifier +/// agree on one family instead of implementing this trait and hoping. +pub trait StarkHash: Send + Sync + 'static { + /// The batched leaf backend: one leaf per row group, streamed. + /// + /// Under `cuda` this additionally has to be [`KeccakTreeBackend`]. That is + /// not a preference: `gpu_lde`'s tree entries hash on the device with the + /// keccak kernels and only *label* the result with this type, so a cuda + /// build has no way to honour any other configuration. The bound says so at + /// compile time instead of letting the label be wrong. It comes off when + /// the device kernels stop being keccak-only. + #[cfg(feature = "cuda")] + type Batched: IsStreamingLeafBackend>> + + KeccakTreeBackend + + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The batched leaf backend: one leaf per row group, streamed. + #[cfg(not(feature = "cuda"))] + type Batched: IsStreamingLeafBackend>> + + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The FRI-layer backend: one leaf per fixed pair, no `Vec` per leaf. + /// + /// Under `cuda` this carries the same [`KeccakTreeBackend`] obligation + /// [`Self::Batched`] does, and for the same reason: `gpu_lde`'s FRI commit + /// drives the whole commit phase on device, hashing every layer tree with + /// the keccak kernels and only *labelling* the result with this type. A cuda + /// build cannot honour any other configuration for FRI layers either, so the + /// bound says so at compile time rather than letting the label be wrong. + #[cfg(feature = "cuda")] + type Pair: IsMerkleTreeBackend; 2]> + + KeccakTreeBackend + + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The FRI-layer backend: one leaf per fixed pair, no `Vec` per leaf. + #[cfg(not(feature = "cuda"))] + type Pair: IsMerkleTreeBackend; 2]> + 'static + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + /// The Fiat-Shamir configuration this commitment configuration is paired + /// with — the hash the transcript sponges on, and the one grinding's + /// proof-of-work computes over. + /// + /// Naming it here is what keeps a proof describable by one configuration. + /// The transcript object is still built by the caller and handed to + /// `multi_prove` / `multi_verify`, so this does not *force* the caller's + /// transcript to match; what it forces is that everything the prover and + /// verifier derive internally from the configuration — grinding — follows + /// this hash instead of a hard-wired one. + type Transcript: TranscriptHash; + + /// What both Merkle families hash with. The name a proof's roots may be + /// called by. + const COMMITMENT_HASH: CommitmentHash; +} + +/// The digest a configuration grinds over: its transcript's hash, because the +/// grinding seed is `transcript.state()`. +pub type GrindingDigest = <::Transcript as TranscriptHash>::Digest; + +/// The keccak-256 configuration — the only one, and the one every `Prover` and +/// `Verifier` alias resolves to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeccakStarkHash; + +impl StarkHash for KeccakStarkHash { + type Batched + = BatchKeccak256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Pair + = PairKeccak256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Transcript = KeccakTranscriptHash; + + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Keccak256; +} + +/// The BLAKE3 configuration — `Blake3Chain` at both the leaf and the parent +/// layer, over the *same* two generic backends the keccak instance uses. +/// +/// Sharing those backends is what makes the two-element invariant above hold by +/// construction: `Batched::hash_data(&vec![a, b])` and `Pair::hash_data(&[a, b])` +/// serialize the same 16 bytes and hand them to the same digest, so there are +/// not two encodings to be shown equal. +/// `blake3_batched_and_pair_agree_on_a_two_element_leaf` pins it anyway, because +/// "holds by construction" is a claim about today's code and the invariant has +/// to survive tomorrow's. +/// +/// Separately, and for the parent layer rather than the leaf: a parent's message +/// is the two 32-byte children, and at 64 bytes `Blake3Chain` is a single BLAKE3 +/// compression in the framing the device kernels implement +/// (`crypto::hash::blake3::chain`, PA-PLAN §1.7 P2). +/// +/// # What selects it +/// +/// Nothing, by default: every `Prover` and `Verifier` alias resolves to +/// [`KeccakStarkHash`], and [`COMMITMENT_HASH`] describes those aliases. It is +/// reachable by naming it — `GenericProver` and `GenericVerifier` at this +/// configuration prove and verify a full STARK, FRI layer trees included. +/// +/// What it does **not** cover yet is the rest of the stack: the transcript and +/// grinding are keccak under both configurations (PA-PLAN Stage 3), and the +/// RV64 guest has no BLAKE3 precompile (Stage 4), so a guest verifying a +/// BLAKE3-committed proof hashes in software. +#[cfg(not(feature = "cuda"))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Blake3StarkHash; + +// Under `cuda` there is deliberately no BLAKE3 configuration to name. +// +// [`StarkHash::Batched`] additionally requires [`KeccakTreeBackend`] there, +// because `gpu_lde`'s tree entry points hash on the device with the keccak +// kernels and only *label* the result with the backend type — so a cuda build +// has no way to honour any other configuration, and the bound says so at compile +// time instead of letting the label be wrong. Implementing `KeccakTreeBackend` +// for a BLAKE3 backend to get past it would be precisely the deliberate false +// statement that marker exists to require, so the configuration does not exist +// under `cuda` at all. +// +// This comes off when the device leaf kernels land. Track G has already built +// the device parent layer against the same framing this host backend uses +// (`math-cuda/kernels/blake3.cu`: `blake3_merkle_level`, `blake3_merkle_tail`, +// checked by `tests/blake3_merkle_tree.rs`); what is missing is the multi-block +// leaf kernel, which needs the chaining construction PA-PLAN §1.7 specifies. +#[cfg(not(feature = "cuda"))] +impl StarkHash for Blake3StarkHash { + type Batched + = BatchBlake3Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Pair + = PairBlake3Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; + + type Transcript = crypto::fiat_shamir::transcript_hash::Blake3TranscriptHash; + + const COMMITMENT_HASH: CommitmentHash = CommitmentHash::Blake3; +} + +/// [`Blake3StarkHash`]'s members are the BLAKE3 backends, not a mix. +/// +/// The two families exist because they hash different leaf *shapes*, not because +/// they are different hashes — that is the whole content of +/// [`StarkHash::COMMITMENT_HASH`] being one constant. Asserting it here means a +/// configuration assembled from one hash's batched backend and another's pair +/// backend fails to compile, rather than producing proofs whose roots no single +/// name describes. +#[cfg(not(feature = "cuda"))] +const _: fn() = || { + fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_same::>( + core::marker::PhantomData::<( + BatchBlake3Backend, + ::Batched, + )>, + ); + assert_same::>( + core::marker::PhantomData::<( + PairBlake3Backend, + ::Pair, + )>, + ); +}; + +/// Ties the aliases, [`COMMITMENT_HASH`] and [`KeccakStarkHash`]'s members +/// to each other, so they cannot drift apart silently. +/// +/// The `KeccakTreeBackend` assertions are the H3 marker's tie-in: it is not a +/// parallel ladder to [`StarkHash`] but a consequence of this instance, since +/// the GPU kernels are keccak-only regardless of which configuration the host +/// prover runs. [`Blake3StarkHash`] is that second configuration and it does +/// **not** satisfy the marker — deliberately, which is why it does not exist at +/// all under `cuda`. Point the aliases at it and this is where you find out. +const _: fn() = || { + fn assert_keccak_backend() {} + fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_keccak_backend::>(); + assert_keccak_backend::>(); + + // The aliases ARE the keccak instance's members, not a second opinion. + assert_same::>( + core::marker::PhantomData::<( + BatchedMerkleTreeBackend, + ::Batched, + )>, + ); + assert_same::>( + core::marker::PhantomData::<( + FriLayerMerkleTreeBackend, + ::Pair, + )>, + ); +}; + +const _: () = assert!(matches!( + ::COMMITMENT_HASH, + COMMITMENT_HASH +)); diff --git a/crypto/stark/src/constraint_ir/artifact.rs b/crypto/stark/src/constraint_ir/artifact.rs new file mode 100644 index 000000000..5a614ab04 --- /dev/null +++ b/crypto/stark/src/constraint_ir/artifact.rs @@ -0,0 +1,771 @@ +//! Build-time serialization of an AIR's transition constraints — "constraints +//! as data". +//! +//! [`DeviceProgram`](super::device::DeviceProgram) already flattens a captured +//! [`ConstraintProgram`] into POD arrays, but a program alone does not describe +//! an AIR's transition constraints: evaluating the roots is only part of the +//! job. A consumer also needs each constraint's ZEROFIER shape (which capture +//! discards) and the AIR's shape scalars. This module bundles all of it into one +//! serializable [`ConstraintArtifact`]. +//! +//! # Why the bundle is four things, not one +//! +//! Derived from what [`crate::verifier`] actually calls on an AIR, not from the +//! trait surface: +//! +//! 1. **The program** — `nodes` / `base_consts` / `ext_consts` / `roots` / +//! `num_base`, as a flat POD projection of the captured [`ConstraintProgram`] +//! ([`ArtifactNode`], node-id operands). Replaces `AIR::compute_transition`. +//! Note this is NOT [`DeviceProgram`]'s form — see [`ArtifactNode`] for why +//! the artifact keeps the liftable node-id form and re-lowers on demand. +//! 2. **Per-constraint metadata** — `{kind, end_exemptions}` per constraint. +//! Capture DISCARDS this: [`ConstraintProgram`] records each constraint's +//! root but not the row domain it applies to, and `end_exemptions` is what +//! picks the constraint's zerofier +//! (`AIR::transition_zerofier_evaluations_grouped` keys its dedup groups on +//! exactly this field). A program without it evaluates the right algebra +//! against the wrong divisor. +//! +//! **Production zerofiers are UNIFORM.** Measured, not assumed: every +//! production constraint across all 28 tables emits through `RowDomain::ALL` +//! — `RowDomain::except_last` appears only in `crate::examples` and in tests. +//! So `end_exemptions` is 0 everywhere and every table has exactly ONE +//! zerofier group. Two things follow. The GPU constraint path already +//! *requires* a uniform zerofier, so that precondition holds in fact rather +//! than by luck. And a consumer evaluating these constraints needs one +//! zerofier per AIR, not one per distinct exemption value — worth knowing +//! before speccing the general case defensively. +//! +//! The field is still carried, and is still load-bearing for anything that +//! is not a production VM table (the example AIRs use exemptions). It is +//! covered by `ExemptConstraints` in `artifact_tests`, deliberately, so that +//! "always zero in production" cannot decay into "never tested". +//! 3. **The AIR shape** — widths, step size, transition offsets, the next-row +//! column set (which decides the pruned `g·z` OOD opening), max bus elements. +//! 4. **The composition degree multiplier** — see +//! [`AirShape::composition_degree_multiplier`]. This one is easy to miss: it +//! lives in neither [`AirContext`](crate::context::AirContext) nor +//! [`ConstraintMeta`], only inside the `ConstraintSet` impl and the LogUp +//! layout, yet the verifier needs it to size the composition polynomial. +//! +//! # What is deliberately NOT in the bundle +//! +//! - **`ProofOptions`.** `AirContext` bundles the proof options in with the +//! shape scalars, but a captured program does not depend on them (pinned by +//! the blowup-invariance test in the prover's artifact suite). Storing them +//! would multiply the artifact count by the number of blowup factors for no +//! information gain, and would wrongly imply the constraints are +//! options-dependent. Options are supplied at AIR construction. +//! - **Trace length / epoch size.** No AIR constructor takes one, so the axis is +//! structurally absent; the only route by which it could reach the artifact is +//! `composition_poly_degree_bound(n)`, which the artifact stores divided +//! through by `n`. That division is sound only if the bound is exactly linear, +//! which `artifacts_are_invariant_across_trace_length` sweeps per table rather +//! than assuming. +//! - **The preprocessed COMMITMENT.** `AIR::precomputed_commitment` is a +//! blowup-dependent Merkle root, delivered by the existing static-commitment +//! mechanism. Only the `is_preprocessed` / `num_precomputed_columns` shape +//! flags are artifact material; putting the root here would reintroduce the +//! options dependence the previous point removes. +//! - **Boundary constraints.** `AIR::boundary_constraints` is a function of the +//! public inputs, not a static property of the AIR, so it is not data in the +//! sense this artifact means. Serializing it is a separate problem. +//! - **Derived scalars.** `has_aux_trace` and `num_auxiliary_rap_columns` are +//! pure functions of `trace_layout`; `num_transition_constraints` is +//! `roots.len()`. Storing a second copy only creates a way for the two to +//! disagree. +//! +//! # Guest safety +//! +//! [`ConstraintArtifact::capture`] CAPTURES — it calls +//! `AIR::constraint_program`, which hash-conses. It is a build-time entry point +//! and must never run in a guest. Everything else here (deserialize, +//! [`ConstraintArtifact::program`], [`ConstraintArtifact::validate_against`]) is +//! pure data handling and is guest-safe: that asymmetry is the whole point of +//! the artifact. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as GoldilocksExtension; +use math::field::goldilocks::GoldilocksField; + +use super::device::DeviceProgram; +use super::ir::{ConstraintProgram, Dim, Op}; +use crate::constraints::builder::{ConstraintMeta, RootKind}; +use crate::traits::AIR; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// Trace lengths used to probe `AIR::composition_poly_degree_bound`. Two of +/// them, so [`ConstraintArtifact::capture`] can check the bound really is linear +/// in the trace length rather than assuming it. +const DEGREE_PROBE_LEN: usize = 1 << 10; +const DEGREE_PROBE_LEN_2: usize = 1 << 11; + +// ============================================================================= +// The wire node +// ============================================================================= + +/// Node result is a base-field value. +pub const DIM_BASE: u32 = 0; +/// Node result is an extension-field value. +pub const DIM_EXT: u32 = 1; + +/// One serialized IR instruction: 16 bytes, `#[repr(C)]`. +/// +/// This is a POD projection of [`Op`] + its [`Dim`], NOT of +/// [`DeviceNode`](super::device::DeviceNode). The distinction is the whole +/// reason this type exists and is worth stating plainly. +/// +/// `DeviceNode` is the *lowered* form: its `a`/`b` are slot-encoded operand +/// words, uniform leaves and dead nodes have been eliminated, and it carries a +/// result slot instead of a dim. That lowering is LOSSY — there is no map back +/// to the [`ConstraintProgram`] it came from. An artifact that stored it could +/// not implement [`ConstraintArtifact::program`], which is the method every +/// consumer of this artifact actually uses. +/// +/// So the artifact stores the high-level form instead: `a`/`b` are **node ids** +/// (id `i` references only `< i`, the same invariant [`ConstraintProgram`] +/// carries), `dim` is [`DIM_BASE`] / [`DIM_EXT`], and the node list is dense — +/// one entry per `ConstraintProgram` node, in the same order. `program()` is its +/// exact inverse; the device blob is re-derived on demand by running main's own +/// [`DeviceProgram::lower`] over that lifted program, so the slot encoding is +/// never duplicated here and cannot drift from the prover's. +/// +/// The `OP_*` tags are shared with `device.rs` — those are stable and mean the +/// same thing in both forms; only the operand encoding differs. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ArtifactNode { + /// `OP_*` tag (shared with [`super::device`]). + pub op: u32, + /// Operand word 0: a node id for arithmetic ops, a table index for + /// constants/uniforms, packed [`Op::Var`] fields for `OP_VAR`. + pub a: u32, + /// Operand word 1: as `a`, where the op takes two operands. + pub b: u32, + /// [`DIM_BASE`] or [`DIM_EXT`] — this node's result dim. + pub dim: u32, +} + +// ============================================================================= +// Metadata +// ============================================================================= + +/// [`ConstraintMeta`] as plain serializable data. +/// +/// `kind` is encoded as a `u8` (see [`ArtifactMeta::KIND_BASE`] / +/// [`ArtifactMeta::KIND_EXT`]) rather than reusing [`RootKind`] so the wire +/// encoding is pinned independently of the in-memory enum. +#[derive(Clone, Copy, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ArtifactMeta { + /// Constraint index. Stored rather than implied by position so + /// [`ConstraintArtifact::validate_self`] can CHECK the dense-and-ordered + /// invariant instead of silently depending on it. + pub constraint_idx: u32, + /// [`ArtifactMeta::KIND_BASE`] or [`ArtifactMeta::KIND_EXT`]. + pub kind: u8, + /// Exempted rows at the end of the trace — the constraint's zerofier shape. + pub end_exemptions: u32, +} + +impl ArtifactMeta { + /// Base-field rooted constraint. + pub const KIND_BASE: u8 = 0; + /// Extension-field (LogUp) rooted constraint. + pub const KIND_EXT: u8 = 1; + + fn from_meta(m: &ConstraintMeta) -> Self { + // An exhaustive match, so adding a RootKind variant is a build error + // here rather than a silently wrong wire byte. + let kind = match m.kind { + RootKind::Base => Self::KIND_BASE, + RootKind::Ext => Self::KIND_EXT, + }; + Self { + constraint_idx: m.constraint_idx as u32, + kind, + end_exemptions: m.end_exemptions as u32, + } + } + + /// Back to a [`ConstraintMeta`]. Panics on an unknown `kind` byte — a + /// corrupt artifact must not silently become a base-field constraint. + pub fn to_meta(self) -> ConstraintMeta { + let kind = match self.kind { + Self::KIND_BASE => RootKind::Base, + Self::KIND_EXT => RootKind::Ext, + other => panic!("unknown ArtifactMeta kind byte {other}"), + }; + ConstraintMeta { + constraint_idx: self.constraint_idx as usize, + kind, + end_exemptions: self.end_exemptions as usize, + } + } +} + +// ============================================================================= +// Shape +// ============================================================================= + +/// An AIR's transition-constraint shape: everything the verifier reads off the +/// AIR that is neither the program nor per-constraint metadata. +#[derive(Clone, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct AirShape { + /// `AIR::step_size`. + pub step_size: u32, + /// `AIR::trace_layout().0` — main trace width. + pub main_width: u32, + /// `AIR::trace_layout().1` — aux trace width. + pub aux_width: u32, + /// `AirContext::transition_offsets` — the frame's row offsets. + pub transition_offsets: Vec, + /// `AIR::trace_ood_next_row_columns`, sorted and deduplicated: the + /// full-width `[main | aux]` columns opened at `g·z`. Every other column is + /// reconstructed as ZERO at the next row, so this set is soundness-critical. + pub next_row_columns: Vec, + /// `AIR::max_bus_elements` — decides the LogUp alpha-power count. + pub max_bus_elements: u32, + /// `AIR::has_trace_interaction`. + pub has_trace_interaction: bool, + /// `AIR::is_preprocessed`. + pub is_preprocessed: bool, + /// `AIR::num_precomputed_columns`. + pub num_precomputed_columns: u32, + /// `composition_poly_degree_bound(n) / n` — the trace-length-INDEPENDENT + /// part of the composition degree bound. + /// + /// Stored as this observable rather than as the underlying `max_degree` + /// because `max_degree` is not exposed on the `AIR` trait at all: it is + /// `max(ConstraintSet::max_degree(), logup_max_degree(layout))`, private to + /// the AIR's construction. The multiplier is what the verifier consumes, it + /// is directly measurable through the public trait, and it needs no new + /// trait method. + pub composition_degree_multiplier: u32, +} + +// ============================================================================= +// The artifact +// ============================================================================= + +/// A build-time-serializable bundle of one AIR's transition constraints. +/// +/// Produced by [`ConstraintArtifact::capture`] (build time, captures) and +/// consumed by [`ConstraintArtifact::program`] (guest-safe, pure data). +/// +/// # A trap for anyone optimizing a consumer of this program +/// +/// The node list is HASH-CONSED: structurally identical subexpressions share one +/// node, which is why the program is compact. That same sharing makes the +/// obvious peepholes UNSOUND if applied naively. +/// +/// Concretely, fusing `Add(Mul(a,b), c)` into a fused multiply-add is only valid +/// when the `Mul` has exactly ONE consumer. A shared `Mul` feeds several +/// parents, and fusing it into each would recompute it per parent — turning a +/// saving into a loss. A node named by [`Self::roots`] counts as a consumer too: +/// fusing it away deletes the value the quotient recombination reads. +/// +/// The rule generalizes to any rewrite that moves work into a consumer: compute +/// the fanout over `nodes` (plus `roots`) first and require it to be 1. The +/// property that makes this IR small is the property that makes rewriting it +/// hazardous, and the two are easy to reason about separately and get wrong +/// together. +#[derive(Clone, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ConstraintArtifact { + /// Topologically ordered flat instruction list (id `i` references only + /// `< i`) — see [`ArtifactNode`]. + pub nodes: Vec, + /// Base-field constant table, raw canonical limbs. + pub base_consts: Vec, + /// Extension-field constant table, raw canonical limbs. + pub ext_consts: Vec<[u64; 3]>, + /// Per-constraint root node ids. + pub roots: Vec, + /// Number of leading base-field-rooted constraints. + pub num_base: u32, + /// Idx-ordered, dense per-constraint metadata. + pub meta: Vec, + /// The AIR's shape scalars. + pub shape: AirShape, +} + +/// Why an artifact was rejected. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ArtifactError { + /// The artifact is internally inconsistent. + #[error("malformed constraint artifact: {0}")] + Malformed(String), + /// The artifact does not describe the AIR it was checked against. + #[error( + "constraint artifact does not match this AIR: {field} is {found} in the artifact but {expected} on the AIR" + )] + ShapeMismatch { + /// The disagreeing field. + field: &'static str, + /// The artifact's value. + found: String, + /// The AIR's value. + expected: String, + }, + /// Serialization or deserialization failed. + #[error("constraint artifact codec error: {0}")] + Codec(String), +} + +/// Compare one shape field, producing a [`ArtifactError::ShapeMismatch`]. +fn check_field(field: &'static str, found: T, expected: T) -> Result<(), ArtifactError> +where + T: PartialEq + core::fmt::Debug, +{ + if found == expected { + Ok(()) + } else { + Err(ArtifactError::ShapeMismatch { + field, + found: format!("{found:?}"), + expected: format!("{expected:?}"), + }) + } +} + +impl ConstraintArtifact { + /// Capture an AIR's constraints into a serializable artifact. + /// + /// BUILD TIME ONLY: this calls `AIR::constraint_program`, which hash-conses + /// the whole constraint body. Never call it from a verifier or a guest — + /// that is precisely what the artifact exists to avoid. + /// + /// # Panics + /// + /// If `composition_poly_degree_bound` is not exactly linear in the trace + /// length, since the artifact stores only the linear coefficient. Better a + /// loud failure at build time than an artifact that silently misstates the + /// composition bound. + pub fn capture(air: &A) -> Self + where + A: AIR + ?Sized, + { + use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, pack_var, + }; + + let prog = air.constraint_program(); + + // A 1:1 projection of the captured program — same node count, same + // order, operands left as node ids. Deliberately NOT + // `DeviceProgram::lower`: that is the slot-allocating lowering, and its + // output cannot be lifted back (see `ArtifactNode`). + let nodes: Vec = prog + .nodes + .iter() + .zip(prog.dims.iter()) + .map(|(op, dim)| { + let dim = match dim { + Dim::Base => DIM_BASE, + Dim::Ext => DIM_EXT, + }; + let (op, a, b) = match *op { + Op::ConstBase(idx) => (OP_CONST_BASE, idx, 0), + Op::ConstExt(idx) => (OP_CONST_EXT, idx, 0), + Op::Var { + main, + offset, + row, + col, + } => { + let (a, b) = pack_var(main, offset, row, col); + (OP_VAR, a, b) + } + Op::RapChallenge { idx } => (OP_RAP_CHALLENGE, idx as u32, 0), + Op::AlphaPow { idx } => (OP_ALPHA_POW, idx as u32, 0), + Op::TableOffset => (OP_TABLE_OFFSET, 0, 0), + Op::Add(a, b) => (OP_ADD, a, b), + Op::Sub(a, b) => (OP_SUB, a, b), + Op::Mul(a, b) => (OP_MUL, a, b), + Op::Neg(a) => (OP_NEG, a, 0), + Op::Embed(a) => (OP_EMBED, a, 0), + }; + ArtifactNode { op, a, b, dim } + }) + .collect(); + + let base_consts: Vec = prog.base_consts.iter().map(|c| *c.value()).collect(); + let ext_consts: Vec<[u64; 3]> = prog + .ext_consts + .iter() + .map(|x| { + let limbs = x.value(); + [*limbs[0].value(), *limbs[1].value(), *limbs[2].value()] + }) + .collect(); + + let (main_width, aux_width) = air.trace_layout(); + + let mut next_row_columns: Vec = air + .trace_ood_next_row_columns() + .into_iter() + .map(|c| c as u32) + .collect(); + next_row_columns.sort_unstable(); + next_row_columns.dedup(); + + // The composition bound is `n * k`; recover `k` and check linearity + // across two probe lengths rather than trusting one sample. + let b1 = air.composition_poly_degree_bound(DEGREE_PROBE_LEN); + let b2 = air.composition_poly_degree_bound(DEGREE_PROBE_LEN_2); + assert_eq!( + b1 % DEGREE_PROBE_LEN, + 0, + "composition_poly_degree_bound({DEGREE_PROBE_LEN}) = {b1} is not a multiple of the \ + trace length; the artifact cannot store it as a linear multiplier" + ); + let multiplier = b1 / DEGREE_PROBE_LEN; + assert_eq!( + b2, + multiplier * DEGREE_PROBE_LEN_2, + "composition_poly_degree_bound is not linear in the trace length ({b1} at \ + {DEGREE_PROBE_LEN}, {b2} at {DEGREE_PROBE_LEN_2}); the artifact's single \ + multiplier cannot represent it" + ); + + Self { + nodes, + base_consts, + ext_consts, + roots: prog.roots.clone(), + num_base: prog.num_base as u32, + meta: air + .constraints_meta() + .iter() + .map(ArtifactMeta::from_meta) + .collect(), + shape: AirShape { + step_size: air.step_size() as u32, + main_width: main_width as u32, + aux_width: aux_width as u32, + transition_offsets: air + .context() + .transition_offsets + .iter() + .map(|o| *o as u32) + .collect(), + next_row_columns, + max_bus_elements: air.max_bus_elements() as u32, + has_trace_interaction: air.has_trace_interaction(), + is_preprocessed: air.is_preprocessed(), + num_precomputed_columns: air.num_precomputed_columns() as u32, + composition_degree_multiplier: multiplier as u32, + }, + } + } + + /// The flat device form, produced by re-running the prover's own + /// [`DeviceProgram::lower`] over [`Self::program`]. + /// + /// Not a field copy: the artifact stores node ids, the device form stores + /// slots (see [`ArtifactNode`]). Re-lowering rather than storing the lowered + /// arrays is what keeps this blob identical to the one the prover and the + /// GPU path build from the same AIR — there is one lowering, not two. + /// + /// Guest-safe: `program()` is a POD walk and `lower()` is a liveness scan; + /// neither captures nor hashes. + pub fn device_program(&self) -> DeviceProgram { + DeviceProgram::lower(&self.program()) + } + + /// Lift back to a [`ConstraintProgram`] — the exact inverse of + /// [`DeviceProgram::lower`], so the generic CPU interpreters + /// ([`eval_program`](super::interp::eval_program) / + /// [`eval_program_verifier`](super::interp::eval_program_verifier)) can run + /// a deserialized artifact. + /// + /// Guest-safe: a linear walk over POD arrays, no capture and no hashing. + /// + /// # Panics + /// + /// On an unknown op tag or dim tag — a corrupt program must not evaluate to + /// something plausible. + pub fn program(&self) -> ConstraintProgram { + use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, unpack_var, + }; + + let mut nodes = Vec::with_capacity(self.nodes.len()); + let mut dims = Vec::with_capacity(self.nodes.len()); + + for (i, n) in self.nodes.iter().enumerate() { + let op = match n.op { + OP_CONST_BASE => Op::ConstBase(n.a), + OP_CONST_EXT => Op::ConstExt(n.a), + OP_VAR => { + let (main, offset, row, col) = unpack_var(n.a, n.b); + Op::Var { + main, + offset, + row, + col, + } + } + OP_RAP_CHALLENGE => Op::RapChallenge { idx: n.a as u16 }, + OP_ALPHA_POW => Op::AlphaPow { idx: n.a as u16 }, + OP_TABLE_OFFSET => Op::TableOffset, + OP_ADD => Op::Add(n.a, n.b), + OP_SUB => Op::Sub(n.a, n.b), + OP_MUL => Op::Mul(n.a, n.b), + OP_NEG => Op::Neg(n.a), + OP_EMBED => Op::Embed(n.a), + other => panic!("unknown op tag {other} at node {i}"), + }; + let dim = match n.dim { + DIM_BASE => Dim::Base, + DIM_EXT => Dim::Ext, + other => panic!("unknown dim tag {other} at node {i}"), + }; + nodes.push(op); + dims.push(dim); + } + + ConstraintProgram { + nodes, + dims, + base_consts: self + .base_consts + .iter() + .map(|c| FieldElement::::from_raw(*c)) + .collect(), + ext_consts: self + .ext_consts + .iter() + .map(|limbs| { + FieldElement::::from_raw([ + FieldElement::::from_raw(limbs[0]), + FieldElement::::from_raw(limbs[1]), + FieldElement::::from_raw(limbs[2]), + ]) + }) + .collect(), + roots: self.roots.clone(), + num_base: self.num_base as usize, + } + } + + /// The per-constraint metadata as the engine's own type. + pub fn constraints_meta(&self) -> Vec { + self.meta.iter().map(|m| m.to_meta()).collect() + } + + /// Internal consistency: the invariants a consumer would otherwise assume. + /// + /// Checks that node operands are topologically ordered and in range, that + /// constant/root indices are in range, and that the metadata list is dense, + /// idx-ordered and has its `Base` entries as a prefix of length `num_base`. + pub fn validate_self(&self) -> Result<(), ArtifactError> { + use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, + }; + let bad = |m: String| Err(ArtifactError::Malformed(m)); + + for (i, n) in self.nodes.iter().enumerate() { + if n.dim != DIM_BASE && n.dim != DIM_EXT { + return bad(format!("node {i} has unknown dim tag {}", n.dim)); + } + // Operand ids must reference strictly earlier nodes; constant and + // uniform indices must be in range for their tables. + let check_id = |x: u32| -> Result<(), ArtifactError> { + if (x as usize) < i { + Ok(()) + } else { + Err(ArtifactError::Malformed(format!( + "node {i} references node {x}, which is not strictly earlier" + ))) + } + }; + match n.op { + OP_CONST_BASE => { + if n.a as usize >= self.base_consts.len() { + return bad(format!( + "node {i} reads base_consts[{}] of {}", + n.a, + self.base_consts.len() + )); + } + } + OP_CONST_EXT => { + if n.a as usize >= self.ext_consts.len() { + return bad(format!( + "node {i} reads ext_consts[{}] of {}", + n.a, + self.ext_consts.len() + )); + } + } + // Var/challenge/alpha/table-offset index per-proof inputs whose + // lengths are not part of the artifact; range-checking them is + // the caller's job at evaluation time. + OP_VAR | OP_RAP_CHALLENGE | OP_ALPHA_POW | OP_TABLE_OFFSET => {} + OP_ADD | OP_SUB | OP_MUL => { + check_id(n.a)?; + check_id(n.b)?; + } + OP_NEG | OP_EMBED => check_id(n.a)?, + other => return bad(format!("node {i} has unknown op tag {other}")), + } + } + + if self.roots.len() != self.meta.len() { + return bad(format!( + "{} roots but {} metadata entries", + self.roots.len(), + self.meta.len() + )); + } + for (c, &root) in self.roots.iter().enumerate() { + if root as usize >= self.nodes.len() { + return bad(format!( + "constraint {c} roots at node {root} of {}", + self.nodes.len() + )); + } + } + + let num_base = self.num_base as usize; + if num_base > self.meta.len() { + return bad(format!( + "num_base {num_base} exceeds the {} constraints", + self.meta.len() + )); + } + for (i, m) in self.meta.iter().enumerate() { + if m.constraint_idx as usize != i { + return bad(format!( + "metadata entry {i} claims constraint_idx {}; the list must be dense and \ + idx-ordered", + m.constraint_idx + )); + } + let expected = if i < num_base { + ArtifactMeta::KIND_BASE + } else { + ArtifactMeta::KIND_EXT + }; + if m.kind != expected { + return bad(format!( + "constraint {i} has kind {} but num_base is {num_base}; Base entries must \ + form a prefix of exactly that length", + m.kind + )); + } + } + + Ok(()) + } + + /// Check that this artifact actually describes `air`. + /// + /// # What this proves, and what it does not + /// + /// It compares the SHAPE scalars and the per-constraint metadata — enough to + /// reject an artifact captured from a different AIR, or a stale artifact + /// from before a column was added or a constraint's exemptions changed. + /// + /// It does NOT prove the serialized program computes the same algebra as the + /// AIR's compiled folder: verifying that requires evaluating both, which + /// requires capture. An AIR edit that changes a constraint's arithmetic + /// without changing any width or exemption passes this check. The build-time + /// drift test is what covers that case, and it is not optional. + pub fn validate_against(&self, air: &A) -> Result<(), ArtifactError> + where + A: AIR + ?Sized, + { + self.validate_self()?; + + let (main_width, aux_width) = air.trace_layout(); + check_field("main_width", self.shape.main_width as usize, main_width)?; + check_field("aux_width", self.shape.aux_width as usize, aux_width)?; + check_field("step_size", self.shape.step_size as usize, air.step_size())?; + check_field( + "num_transition_constraints", + self.roots.len(), + air.context().num_transition_constraints, + )?; + check_field( + "num_base", + self.num_base as usize, + air.num_base_transition_constraints(), + )?; + check_field( + "max_bus_elements", + self.shape.max_bus_elements as usize, + air.max_bus_elements(), + )?; + check_field( + "has_trace_interaction", + self.shape.has_trace_interaction, + air.has_trace_interaction(), + )?; + check_field( + "is_preprocessed", + self.shape.is_preprocessed, + air.is_preprocessed(), + )?; + check_field( + "num_precomputed_columns", + self.shape.num_precomputed_columns as usize, + air.num_precomputed_columns(), + )?; + + let offsets: Vec = self + .shape + .transition_offsets + .iter() + .map(|o| *o as usize) + .collect(); + check_field( + "transition_offsets", + offsets, + air.context().transition_offsets.clone(), + )?; + + let mut declared = air.trace_ood_next_row_columns(); + declared.sort_unstable(); + declared.dedup(); + let stored: Vec = self + .shape + .next_row_columns + .iter() + .map(|c| *c as usize) + .collect(); + check_field("next_row_columns", stored, declared)?; + + check_field( + "composition_degree_multiplier", + self.shape.composition_degree_multiplier as usize, + air.composition_poly_degree_bound(DEGREE_PROBE_LEN) / DEGREE_PROBE_LEN, + )?; + + let air_meta: Vec = air.constraints_meta().to_vec(); + check_field("constraints_meta", self.constraints_meta(), air_meta)?; + + Ok(()) + } + + /// Serialize to the on-disk / in-guest byte form (rkyv, matching the + /// proof format's own encoding). + pub fn to_bytes(&self) -> Result, ArtifactError> { + rkyv::to_bytes::(self) + .map(|b| b.to_vec()) + .map_err(|e| ArtifactError::Codec(e.to_string())) + } + + /// Deserialize, then check internal consistency. Guest-safe. + pub fn from_bytes(bytes: &[u8]) -> Result { + let artifact = rkyv::from_bytes::(bytes) + .map_err(|e| ArtifactError::Codec(e.to_string()))?; + artifact.validate_self()?; + Ok(artifact) + } +} diff --git a/crypto/stark/src/constraint_ir/artifact_tests.rs b/crypto/stark/src/constraint_ir/artifact_tests.rs new file mode 100644 index 000000000..4df89aa9d --- /dev/null +++ b/crypto/stark/src/constraint_ir/artifact_tests.rs @@ -0,0 +1,394 @@ +//! Unit tests for [`ConstraintArtifact`]: the codec, the lift back to a +//! [`ConstraintProgram`], the self-consistency and shape checks, and the +//! pre-captured supply path. +//! +//! The per-table bit-exactness sweep over all 25 production AIRs lives in the +//! prover crate (`prover/src/tests/constraint_artifact_tests.rs`) — it needs the +//! production tables. What is here is what the production tables CANNOT cover: +//! +//! - **Nonzero `end_exemptions`.** Every production constraint applies to every +//! row (`RowDomain::ALL`); nothing under `prover/src` uses +//! `RowDomain::except_last`. So the production sweep would exercise the +//! artifact's zerofier metadata only in its all-zero case, which proves +//! nothing about the field that capture actually discards. The AIR below has +//! exemptions on purpose. +//! - **Rejection.** A suite of AIRs that all validate proves the checks accept; +//! it does not prove they can reject. The falsification tests here corrupt an +//! artifact in each way `validate_self` claims to catch and assert it does. + +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as GoldilocksExtension; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::IsField; + +use super::artifact::{ArtifactError, ArtifactMeta, ConstraintArtifact}; +use crate::constraints::builder::{ConstraintBuilder, ConstraintSet, RowDomain}; +use crate::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use crate::proof::options::GoldilocksCubicProofOptions; +use crate::traits::AIR; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +const NUM_COLS: usize = 4; + +/// A constraint set whose three constraints have DIFFERENT row domains, so the +/// artifact's `end_exemptions` is non-uniform and a bug that dropped, zeroed, or +/// permuted it would be visible. No production table does this today. +struct ExemptConstraints; + +impl ConstraintSet for ExemptConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + let m2 = b.main(0, 2); + let n0 = b.main(1, 0); + let n1 = b.main(1, 1); + + // c0: every row — a degree-2 product. + b.emit_base(0, m0.clone() * m1.clone() - m2.clone()); + // c1: skips the last row (reads the next row). + b.emit_base_rows(1, RowDomain::except_last(1), n0.clone() - m0.clone()); + // c2: skips the last two rows. + b.emit_base_rows(2, RowDomain::except_last(2), n1 - m1); + } +} + +fn options() -> crate::proof::options::ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The exemption-bearing AIR, with no bus interactions so its constraints are +/// exactly the three above (no LogUp suffix). +fn exempt_air() -> AirWithBuses { + AirWithBuses::new( + NUM_COLS, + AuxiliaryTraceBuildData { + interactions: vec![], + }, + &options(), + 1, + ExemptConstraints, + ) + .with_name("EXEMPT") +} + +// ============================================================================= +// The zerofier metadata capture discards +// ============================================================================= + +#[test] +fn artifact_carries_the_end_exemptions_capture_discards() { + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + + // The three constraints' row domains, which the ConstraintProgram alone has + // no field for. + let exemptions: Vec = artifact.meta.iter().map(|m| m.end_exemptions).collect(); + assert_eq!( + exemptions, + vec![0, 1, 2], + "the artifact must preserve each constraint's row domain" + ); + assert_eq!(artifact.constraints_meta(), air.constraints_meta().to_vec()); + + // ... and it must survive the wire. + let bytes = artifact.to_bytes().expect("serialize"); + let back = ConstraintArtifact::from_bytes(&bytes).expect("deserialize"); + assert_eq!(back, artifact, "artifact must round-trip exactly"); + assert_eq!( + back.meta + .iter() + .map(|m| m.end_exemptions) + .collect::>(), + vec![0, 1, 2] + ); +} + +#[test] +fn zeroed_exemptions_are_rejected_against_the_air() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + assert!(artifact.validate_against(&air).is_ok()); + + // Drop the zerofier shapes — the exact damage that makes a serialized + // program evaluate the right algebra against the wrong divisor. + for m in &mut artifact.meta { + m.end_exemptions = 0; + } + let err = artifact + .validate_against(&air) + .expect_err("an artifact with the row domains flattened must be rejected"); + assert!( + matches!( + err, + ArtifactError::ShapeMismatch { + field: "constraints_meta", + .. + } + ), + "expected a constraints_meta mismatch, got {err:?}" + ); +} + +// ============================================================================= +// Lift / codec +// ============================================================================= + +#[test] +fn lift_is_the_inverse_of_lower() { + let air = exempt_air(); + let captured = air.constraint_program(); + let artifact = ConstraintArtifact::capture(&air); + let lifted = artifact.program(); + + assert_eq!( + lifted.nodes, captured.nodes, + "nodes must survive the round trip" + ); + assert_eq!( + lifted.dims, captured.dims, + "dims must survive the round trip" + ); + assert_eq!(lifted.roots, captured.roots); + assert_eq!(lifted.num_base, captured.num_base); + assert_eq!(lifted.base_consts, captured.base_consts); + assert_eq!(lifted.ext_consts, captured.ext_consts); + + // And through the wire, not just in memory. + let bytes = artifact.to_bytes().expect("serialize"); + let lifted2 = ConstraintArtifact::from_bytes(&bytes) + .expect("deserialize") + .program(); + assert_eq!(lifted2.nodes, captured.nodes); + assert_eq!(lifted2.dims, captured.dims); + assert_eq!(lifted2.base_consts, captured.base_consts); +} + +#[test] +fn constants_survive_as_exact_field_values() { + // Constants go out as raw limbs and come back through `from_raw`; a + // canonicalization slip there would change a constraint's arithmetic + // silently, so pin the values rather than only their count. + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + let lifted = artifact.program(); + for (i, (a, b)) in lifted + .base_consts + .iter() + .zip(air.constraint_program().base_consts.iter()) + .enumerate() + { + assert_eq!(a, b, "base_consts[{i}] changed value across the round trip"); + assert_eq!( + a.value(), + b.value(), + "base_consts[{i}] changed representation across the round trip" + ); + } +} + +// ============================================================================= +// Falsification: every check must be able to reject +// ============================================================================= + +#[test] +fn validate_self_rejects_a_forward_reference() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + // Point the last node's operand at itself: no longer topologically ordered, + // which an interpreter would read as an uninitialized value. + let last = artifact.nodes.len() - 1; + artifact.nodes[last].a = last as u32; + let err = artifact + .validate_self() + .expect_err("a self-referential node must be rejected"); + assert!(matches!(err, ArtifactError::Malformed(_)), "got {err:?}"); +} + +#[test] +fn validate_self_rejects_an_out_of_range_root() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + artifact.roots[0] = artifact.nodes.len() as u32; + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_self_rejects_an_out_of_range_constant() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + let n_base = artifact.base_consts.len() as u32; + let node = artifact + .nodes + .iter_mut() + .find(|n| n.op == super::device::OP_CONST_BASE) + .expect("the program reads at least one base constant"); + node.a = n_base; + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_self_rejects_a_non_prefix_base_kind() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + // All three constraints are base-rooted here, so flipping one to Ext breaks + // the "Base entries form a prefix of length num_base" invariant that + // `num_base_from_meta` relies on. + artifact.meta[0].kind = ArtifactMeta::KIND_EXT; + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_self_rejects_permuted_metadata() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + artifact.meta.swap(0, 2); + assert!(matches!( + artifact.validate_self(), + Err(ArtifactError::Malformed(_)) + )); +} + +#[test] +fn validate_against_rejects_a_shape_change() { + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + assert!(artifact.validate_against(&air).is_ok()); + + artifact.shape.main_width += 1; + let err = artifact + .validate_against(&air) + .expect_err("a width change must be rejected"); + assert!( + matches!( + err, + ArtifactError::ShapeMismatch { + field: "main_width", + .. + } + ), + "got {err:?}" + ); +} + +#[test] +fn from_bytes_rejects_a_corrupt_artifact() { + // The codec must not hand back a structurally invalid artifact just because + // the bytes deserialized: `from_bytes` runs `validate_self`. + let air = exempt_air(); + let mut artifact = ConstraintArtifact::capture(&air); + artifact.roots[0] = 9999; + let bytes = artifact.to_bytes().expect("serialize"); + assert!(matches!( + ConstraintArtifact::from_bytes(&bytes), + Err(ArtifactError::Malformed(_)) + )); +} + +// ============================================================================= +// The pre-captured supply path (the scoped verify-path unban) +// ============================================================================= + +#[test] +fn precaptured_is_none_even_after_a_capture() { + let air = exempt_air(); + assert!( + air.precaptured_constraint_program().is_none(), + "a freshly built AIR has no build-time program" + ); + + // Force a capture. This fills the AIR's OnceLock — but a captured program is + // NOT a build-time artifact, and the guest-safe accessor must keep saying so. + let _ = air.constraint_program(); + assert!( + air.precaptured_constraint_program().is_none(), + "a program the AIR captured at runtime must never be reported as pre-captured; \ + conflating the two would let a guest path believe capture had been avoided" + ); +} + +#[test] +fn supplying_a_program_short_circuits_capture() { + let program = ConstraintArtifact::capture(&exempt_air()).program(); + let air = exempt_air().with_precaptured(program); + + let supplied = air + .precaptured_constraint_program() + .expect("the supplied program must be visible"); + + // Pointer identity is the actual proof that no capture ran: a capture would + // have built a fresh program in the OnceLock and returned that instead. + assert!( + std::ptr::eq(air.constraint_program(), supplied), + "constraint_program() must hand back the supplied program itself, not a fresh capture" + ); +} + +#[test] +fn a_supplied_program_still_evaluates_correctly() { + // Supplying a program must not change what the AIR computes. + let program = ConstraintArtifact::capture(&exempt_air()).program(); + let air = exempt_air().with_precaptured(program); + let artifact = ConstraintArtifact::capture(&air); + assert!(artifact.validate_against(&air).is_ok()); + assert_eq!( + artifact.program().nodes, + exempt_air().constraint_program().nodes, + "a supplied program must be the same program the AIR would have captured" + ); +} + +#[test] +#[should_panic(expected = "roots")] +fn supplying_a_mismatched_program_panics() { + // A program for a different constraint count must not be installable. + let mut program = ConstraintArtifact::capture(&exempt_air()).program(); + program.roots.pop(); + let _ = exempt_air().with_precaptured(program); +} + +// ============================================================================= +// Degree bound +// ============================================================================= + +#[test] +fn composition_degree_multiplier_reproduces_the_bound() { + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + let k = artifact.shape.composition_degree_multiplier as usize; + assert!(k >= 1, "the multiplier must be positive"); + for log_n in [8usize, 12, 20] { + let n = 1usize << log_n; + assert_eq!( + air.composition_poly_degree_bound(n), + k * n, + "the stored multiplier must reproduce the AIR's own bound at n=2^{log_n}" + ); + } +} + +/// A sanity floor on the field type used for constants, so a field swap does not +/// silently reinterpret the artifact's raw limbs. +#[test] +fn base_constants_are_goldilocks_limbs() { + let air = exempt_air(); + let artifact = ConstraintArtifact::capture(&air); + for &c in &artifact.base_consts { + let fe = FieldElement::::from_raw(c); + assert_eq!( + *fe.value(), + c, + "constant {c} is not a canonical Goldilocks limb" + ); + } +} diff --git a/crypto/stark/src/constraint_ir/device.rs b/crypto/stark/src/constraint_ir/device.rs index e4e170bfc..f86a1f311 100644 --- a/crypto/stark/src/constraint_ir/device.rs +++ b/crypto/stark/src/constraint_ir/device.rs @@ -114,7 +114,7 @@ pub const RES_EXT_BIT: u32 = 1 << 31; /// for root-pinned uniform leaves); `res` is the result slot with [`RES_EXT_BIT`] /// selecting the slot class. #[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct DeviceNode { pub op: u32, pub a: u32, diff --git a/crypto/stark/src/constraint_ir/mod.rs b/crypto/stark/src/constraint_ir/mod.rs index 380f32d86..dfc1c13cb 100644 --- a/crypto/stark/src/constraint_ir/mod.rs +++ b/crypto/stark/src/constraint_ir/mod.rs @@ -19,12 +19,17 @@ //! - [`device`]: the concrete-Goldilocks flat lowering ([`DeviceProgram`]) for //! the GPU kernel, plus a CPU walker over that flat blob (the pre-GPU parity //! oracle). +//! - [`artifact`]: the build-time serializable bundle ([`ConstraintArtifact`]) +//! — the flat program PLUS the zerofier metadata capture discards and the +//! AIR's shape scalars, which is what "constraints as data" actually needs. //! +//! [`ConstraintArtifact`]: artifact::ConstraintArtifact //! [`ConstraintProgram`]: ir::ConstraintProgram //! [`Op`]: ir::Op //! [`Dim`]: ir::Dim //! [`DeviceProgram`]: device::DeviceProgram +pub mod artifact; pub mod builder; pub mod device; #[cfg(feature = "cuda")] @@ -32,9 +37,12 @@ pub mod gpu_interp; pub mod interp; pub mod ir; +#[cfg(test)] +mod artifact_tests; #[cfg(test)] mod tests; +pub use artifact::{AirShape, ArtifactError, ArtifactMeta, ArtifactNode, ConstraintArtifact}; pub use builder::{Expr, IrBuilder}; pub use device::{DeviceNode, DeviceProgram, eval_device_program}; pub use interp::{eval_program, eval_program_base, eval_program_verifier}; diff --git a/crypto/stark/src/constraints/builder.rs b/crypto/stark/src/constraints/builder.rs index 4554dd4ee..983c8ef3d 100644 --- a/crypto/stark/src/constraints/builder.rs +++ b/crypto/stark/src/constraints/builder.rs @@ -199,7 +199,7 @@ impl RowDomain { /// [`num_base_from_meta`]. Degree is intentionally absent: only the per-table /// max is consumed (by `composition_poly_degree_bound`), declared once via /// [`ConstraintSet::max_degree`]. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ConstraintMeta { pub constraint_idx: usize, /// Base | Ext; Base entries MUST be a prefix. @@ -287,6 +287,65 @@ pub trait ConstraintSet: Send + Sync { } } +/// Why an emitted index set can be wrong in a way nothing else catches. +/// +/// [`EmitTracker`]'s duplicate assert is `#[cfg(debug_assertions)]`, and this +/// workspace declares no `[profile.release]` override — so under the house +/// convention of `cargo test --release` it is a no-op and a second +/// `emit_base(idx, …)` silently overwrites the first. Constraint *counts* do not +/// notice: a body that emits one index twice and another never still fills +/// `0..N` slots, so `NUM_CONSTRAINTS`, any hand-written predicted-count test, +/// and `assert_complete` all still pass while a constraint has been deleted. +/// +/// This is the check that does notice, and it runs wherever it is called +/// from — no `cfg`. It returns rather than panics so a caller on a proving or +/// verifying path can decide; the tests call it and assert. +/// +/// `meta` is what [`ConstraintSet::meta`] returns: one entry per `emit_*` call, +/// idx-sorted, duplicates included. +pub fn check_dense_index_set( + meta: &[ConstraintMeta], + num_constraints: usize, +) -> Result<(), String> { + if meta.len() != num_constraints { + return Err(format!( + "emitted {} constraints, declared {num_constraints}", + meta.len() + )); + } + // `meta` arrives idx-sorted, so a repeat is an equal neighbour and a gap is + // a jump. Reporting both by name beats reporting "not dense". + let mut duplicates = Vec::new(); + let mut missing = Vec::new(); + let mut expected = 0usize; + let mut prev: Option = None; + for m in meta { + if prev == Some(m.constraint_idx) { + duplicates.push(m.constraint_idx); + continue; + } + prev = Some(m.constraint_idx); + while expected < m.constraint_idx { + missing.push(expected); + expected += 1; + } + if m.constraint_idx == expected { + expected += 1; + } + } + while expected < num_constraints { + missing.push(expected); + expected += 1; + } + if duplicates.is_empty() && missing.is_empty() { + return Ok(()); + } + Err(format!( + "emitted index set is not exactly 0..{num_constraints}: \ + emitted twice {duplicates:?}, never emitted {missing:?}" + )) +} + /// A [`ConstraintSet`] with no transition constraints — for tables whose /// soundness rests entirely on their bus (LogUp) interactions (e.g. BITWISE, /// PAGE, REGISTER, the continuation GLOBAL_MEMORY / global L2G sub-tables). diff --git a/crypto/stark/src/fri/batched.rs b/crypto/stark/src/fri/batched.rs new file mode 100644 index 000000000..01797613d --- /dev/null +++ b/crypto/stark/src/fri/batched.rs @@ -0,0 +1,950 @@ +//! Batched FRI: one FRI instance over an epoch's DEEP codewords instead of one +//! per table. +//! +//! Codewords are bucketed by height, mixed within a bucket with powers of a +//! single `alpha`, and then folded from the tallest bucket downward, each +//! shorter bucket being *injected* into the running codeword at the layer whose +//! length matches it. One set of query indices, drawn from the tallest domain, +//! tests the whole chain. +//! +//! # Termination +//! +//! Folding stops at the same terminal the unbatched +//! [`crate::fri::commit_phase_from_evaluations`] stops at — the codeword that +//! encodes a polynomial of degree `< 2^fri_final_poly_log_degree` — and sends +//! that polynomial's coefficients, rather than folding all the way down to a +//! scalar. [`BatchedFriLayout`] derives the fold count through the shared +//! [`FriFoldLayout`], with one batched-only floor: the terminal may not sit +//! above the SHORTEST injected codeword, or that codeword would never reach the +//! running word. So the early stop is `min(blowup_log + k, h_min)`. + +use crypto::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; +use math::traits::AsBytes; + +use crate::config::StarkHash; +use crate::fri::fri_commitment::FriLayer; +use crate::fri::fri_functions::{ + compute_coset_twiddles_inv, fold_evaluations_in_place, update_twiddles_in_place, +}; +use crate::fri::terminal::{FriFoldLayout, coeffs_from_terminal_codeword}; + +/// Accumulates DEEP codewords into per-height buckets as they are produced, +/// mixing the `i`-th absorbed codeword with `alpha^i`. +/// +/// The point of absorbing one codeword at a time is memory: a caller that +/// produces a table's quotient, absorbs it and drops it retains only one bucket +/// per distinct height (`O(2^h_max)` in total), where handing +/// [`combine_by_height`] a fully-materialized `Vec` of every table's codeword +/// retains `O(N_tables · 2^h)`. The result is identical either way — absorption +/// order defines the `alpha` powers, so the caller must absorb in the same +/// canonical per-epoch order the verifier assumes. +pub struct HeightCombiner { + buckets: Vec>>>, + alpha: FieldElement, + /// `alpha^i` for the next codeword to be absorbed. + next_power: FieldElement, +} + +impl HeightCombiner { + pub fn new(alpha: FieldElement) -> Self { + Self { + buckets: Vec::new(), + alpha, + next_power: FieldElement::one(), + } + } + + /// Absorb one codeword of length `2^height`, scaled by the next power of + /// `alpha`. + pub fn absorb(&mut self, codeword: &[FieldElement], height: usize) { + let expected_len = 1usize << height; + assert_eq!( + codeword.len(), + expected_len, + "codeword has length {} but height {height} expects {expected_len}", + codeword.len() + ); + + if self.buckets.len() <= height { + self.buckets.resize_with(height + 1, || None); + } + let scaled = &self.next_power; + match &mut self.buckets[height] { + None => { + self.buckets[height] = Some(codeword.iter().map(|x| scaled * x).collect()); + } + Some(acc) => { + for (a, x) in acc.iter_mut().zip(codeword.iter()) { + *a = &*a + &(scaled * x); + } + } + } + self.next_power = &self.next_power * &self.alpha; + } + + /// The per-height buckets. Index `h` is `Some(combined)` when at least one + /// codeword of height `h` was absorbed, `None` otherwise; the `Vec` is + /// `max_absorbed_height + 1` long, or empty if nothing was absorbed. + pub fn finish(self) -> Vec>>> { + self.buckets + } +} + +/// Combine DEEP polynomial codewords by their FRI height for batched FRI. +/// +/// Each element of `inputs` is a pair `(codeword, height)` where `height` is +/// the log₂ of the codeword length (i.e. `codeword.len() == 2^height`). +/// The global index `i` into `inputs` is used to derive the mixing power +/// `alpha^i` (index 0 → alpha^0 = 1, index 1 → alpha^1, …). +/// +/// Returns a `Vec` of length `max_height + 1`. Index `h` contains +/// `Some(combined)` where `combined[j] = Σ_{i : height_i == h} alpha^i * codeword_i[j]`, +/// or `None` when no input has height `h`. +/// +/// This is [`HeightCombiner`] with every codeword already materialized. Prefer +/// the combiner in the prover, where holding all of them at once is the whole +/// memory cost the batching is meant to remove. +pub fn combine_by_height( + inputs: &[(Vec>, usize)], + alpha: &FieldElement, +) -> Vec>>> +where + E: IsField, +{ + let mut combiner = HeightCombiner::new(alpha.clone()); + for (codeword, height) in inputs { + combiner.absorb(codeword, *height); + } + combiner.finish() +} + +/// How far a batched FRI instance folds, and what it sends at the end. +/// +/// Mirrors [`FriFoldLayout`] — same early stop, same terminal codeword, same +/// coefficient count — with the one difference batching forces: the terminal is +/// additionally floored at the SHORTEST injected codeword's height, since a +/// bucket below the terminal would never be folded into the running word. In a +/// real epoch the shortest table is normally well above `blowup_log + k`, so the +/// floor is inert and the layout is exactly the unbatched one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BatchedFriLayout { + /// Folds from the tallest bucket down to the terminal codeword. + pub total_folds: u32, + /// Committed (Merkle-rooted) FRI layers. + pub num_committed: usize, + /// Terminal codeword length. + pub terminal_len: usize, + /// `log2` of the terminal polynomial's degree bound — the number of + /// coefficients sent is `2^effective_k`. + pub effective_k: u32, +} + +impl BatchedFriLayout { + /// Derive the layout from the epoch's codeword heights. + /// + /// * `h_max` / `h_min` — the tallest and shortest codeword heights present. + /// * `blowup_log` — log2 of the LDE blowup factor. + /// * `final_poly_log_degree` — the requested `fri_final_poly_log_degree`. + /// + /// Panics if `h_min < blowup_log` (a codeword shorter than the blowup is not + /// a Reed-Solomon word of any positive rate) or if `h_min > h_max`. + pub fn new(h_max: usize, h_min: usize, blowup_log: u32, final_poly_log_degree: u32) -> Self { + assert!(h_min <= h_max, "h_min {h_min} exceeds h_max {h_max}"); + assert!( + h_min as u32 >= blowup_log, + "codeword height {h_min} is below the blowup {blowup_log}" + ); + // Deriving at `h_min` is what applies the floor: `FriFoldLayout` clamps + // the terminal to its `lde_log` argument, so the terminal comes out at + // `min(blowup_log + k, h_min)`. Its terminal_len / effective_k are then + // exactly what the unbatched prover would send for that codeword. + let shortest = FriFoldLayout::new(h_min as u32, blowup_log, final_poly_log_degree); + let terminal_log = shortest.terminal_len.trailing_zeros(); + // The running codeword starts at h_max, not h_min, so the fold count is + // re-derived from where folding actually begins. + let total_folds = h_max as u32 - terminal_log; + Self { + total_folds, + num_committed: total_folds.saturating_sub(1) as usize, + terminal_len: shortest.terminal_len, + effective_k: shortest.effective_k, + } + } +} + +/// FRI commit phase over the bucketed output of [`combine_by_height`] / +/// [`HeightCombiner::finish`]. +/// +/// `combined[h]` is `Some(codeword)` when there are DEEP contributions at height +/// `h` (codeword length `2^h`), or `None` otherwise. +/// +/// Folding starts from the tallest bucket. After each fold to height `h`, the +/// bucket at `combined[h]` is injected into the running codeword with +/// coefficient `β²` (β being the fold challenge just used), before the layer is +/// committed. Termination follows [`BatchedFriLayout`]: the running codeword is +/// folded to the terminal length and the terminal polynomial's coefficients are +/// appended to the transcript, exactly as +/// [`crate::fri::commit_phase_from_evaluations`] does — not folded down to a +/// single scalar. +/// +/// Layer trees are built with `H::Pair`, the same commitment configuration the +/// unbatched [`crate::fri::commit_phase_from_evaluations`] uses — so a batched +/// prover and the verifier that authenticates its openings through `H::Batched` +/// agree on the hash by naming one configuration, not by two call sites +/// coinciding. +#[allow(clippy::type_complexity)] +pub fn batched_commit_phase( + mut combined: Vec>>>, + transcript: &mut T, + coset_offset: &FieldElement, + blowup_log: u32, + final_poly_log_degree: u32, +) -> (Vec>, Vec>>) +where + F: IsFFTField + IsSubFieldOf + 'static, + E: IsField + 'static + Send + Sync, + T: IsStarkTranscript + Clone, + H: StarkHash, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + let (h_min, h_max) = bucket_height_range(&combined) + .expect("batched_commit_phase: combined must have at least one Some entry"); + + // Take the starting codeword — NOT committed; it plays the role of layer 0. + let mut running = combined[h_max] + .take() + .expect("combined[h_max] is Some by construction"); + + let domain_size = 1usize << h_max; + debug_assert_eq!( + running.len(), + domain_size, + "starting codeword length must equal 2^h_max" + ); + + let layout = BatchedFriLayout::new(h_max, h_min, blowup_log, final_poly_log_degree); + + // Inverse twiddle factors for the initial domain size. + let mut inv_twiddles = compute_coset_twiddles_inv(coset_offset, domain_size); + + let mut fri_layer_list = Vec::with_capacity(layout.num_committed); + + for _ in 0..layout.num_committed { + // <<<< Receive challenge β + let beta = transcript.sample_field_element(); + + // Fold evaluations in-place; running halves in length. + fold_evaluations_in_place(&mut running, &beta, &inv_twiddles); + inject_bucket(&mut running, &mut combined, &beta); + + // Build the row-pair Merkle tree over the current running codeword. + let leaves: Vec<[FieldElement; 2]> = running + .chunks_exact(2) + .map(|chunk| [chunk[0].clone(), chunk[1].clone()]) + .collect(); + let merkle_tree = MerkleTree::>::build(&leaves) + .expect("FRI batched commit: Merkle tree construction must succeed"); + let root = merkle_tree.root; + fri_layer_list.push(FriLayer::new(&running, merkle_tree)); + + // >>>> Send commitment: append root to transcript. + transcript.append_bytes(&root); + + // Update twiddles for the next (halved) level. + update_twiddles_in_place(&mut inv_twiddles); + } + + // One final fold to reach the terminal codeword, unless already there. The + // bucket AT the terminal height is injected here: it is the last one that can + // still enter the running word, which is why the layout floors the terminal + // at the shortest height rather than at `blowup_log + k` alone. + if layout.total_folds > 0 { + let beta = transcript.sample_field_element(); + fold_evaluations_in_place(&mut running, &beta, &inv_twiddles); + inject_bucket(&mut running, &mut combined, &beta); + } + debug_assert_eq!( + running.len(), + layout.terminal_len, + "terminal codeword size mismatch" + ); + debug_assert!( + combined.iter().all(Option::is_none), + "every bucket must have been injected before the terminal" + ); + + // Recover the terminal polynomial's coefficients and send them, mirroring + // `commit_phase_from_evaluations`: the coefficient count follows + // `layout.effective_k` (the actual terminal), and the terminal coset offset + // is `coset_offset^(2^total_folds)`. + let terminal_offset = coset_offset.pow(1u64 << layout.total_folds); + let final_poly_coeffs = + coeffs_from_terminal_codeword::(&running, &terminal_offset, layout.effective_k); + for c in &final_poly_coeffs { + transcript.append_field_element(c); + } + + (final_poly_coeffs, fri_layer_list) +} + +/// The `(h_min, h_max)` of the occupied buckets, or `None` when none are. +fn bucket_height_range( + combined: &[Option>>], +) -> Option<(usize, usize)> { + let mut occupied = combined + .iter() + .enumerate() + .filter_map(|(h, slot)| slot.as_ref().map(|_| h)); + let first = occupied.next()?; + Some((first, occupied.next_back().unwrap_or(first))) +} + +/// `running += β² · combined[h]` for the running codeword's current height `h`, +/// consuming that bucket. A no-op when the bucket is empty. +fn inject_bucket( + running: &mut [FieldElement], + combined: &mut [Option>>], + beta: &FieldElement, +) { + let h = running.len().trailing_zeros() as usize; + let Some(bucket) = combined.get_mut(h).and_then(Option::take) else { + return; + }; + debug_assert_eq!( + bucket.len(), + running.len(), + "a bucket at height {h} must match the running codeword's length" + ); + let beta_sq = beta.square(); + for (val, contribution) in running.iter_mut().zip(bucket.iter()) { + *val = &*val + &(&beta_sq * contribution); + } +} + +/// Canonical, order-deterministic absorption of an epoch's table-SHAPE histogram +/// into the transcript. Single source of truth for the structural binding. +/// +/// The multiset of `lde_log_height`s across an epoch's tables fully determines +/// the fold order and injection points of the batched FRI (arity is uniformly +/// 2), so binding the heights binds the whole injection schedule. The widths are +/// bound alongside them because they are what makes the mixed-height MMCS leaf +/// parse unambiguous (see [`crate::fri::mmcs`]'s width-binding section) — the +/// verifier derives widths from the AIR set rather than the proof, so this is +/// defence in depth rather than the primary binding, and it costs one field per +/// table. +/// +/// Encoding (fixed-width, length-prefixed, order-preserving): +/// `u64::to_le_bytes(len)` followed by `u64::to_le_bytes(h)`, `u64::to_le_bytes(w)` +/// for each `(h, w)` pair, in the exact order given. Caller (prover and verifier +/// alike) must pass the shape in the same canonical per-epoch table order — this +/// function does not sort or deduplicate. +/// +/// Panics if `heights` and `widths` differ in length; both sides construct them +/// from the same table list. +pub fn absorb_shape_histogram(transcript: &mut T, heights: &[usize], widths: &[usize]) +where + E: IsField, + T: IsTranscript, +{ + assert_eq!( + heights.len(), + widths.len(), + "the shape histogram needs one width per height" + ); + transcript.append_bytes(&(heights.len() as u64).to_le_bytes()); + for (h, w) in heights.iter().zip(widths.iter()) { + transcript.append_bytes(&(*h as u64).to_le_bytes()); + transcript.append_bytes(&(*w as u64).to_le_bytes()); + } +} + +/// Challenges derived from replaying the shared batched round-4 transcript +/// sequence. See [`derive_batched_fri_challenges`]. +#[derive(Debug, Clone)] +pub struct BatchedFriChallenges { + /// Sampled once after the shape histogram (and, at the call site, after all + /// per-table OOD evaluations have been absorbed). + pub alpha: FieldElement, + /// One per committed layer, plus one for the final fold when there is one: + /// `betas.len() == layout.num_committed + (layout.total_folds > 0) as usize`. + pub betas: Vec>, + /// The layout the betas and the terminal were derived under. + pub layout: BatchedFriLayout, + /// Transcript state right before the grinding nonce bytes are appended. + /// All-zero when `grinding_factor == 0` or `nonce` is `None`. + pub grinding_seed: [u8; 32], + /// One `sample_u64(2^(h_max - 1))` draw per query — a row-PAIR index in the + /// tallest domain. A round whose own `h_max` is lower must reduce these; see + /// [`crate::fri::mmcs`]'s index-convention section. + pub iotas: Vec, +} + +/// Replays the shared batched round-4 transcript sequence (shape histogram, +/// alpha, per-layer beta/root, final beta, terminal coefficients, grinding, query +/// iotas) and returns the derived challenges. The one routine the prover and the +/// verifier both call, so they provably derive identical challenges. +/// +/// Returns `None` when the proof's layer-root count disagrees with the layout the +/// epoch's shape implies, or when the terminal coefficient count is wrong — both +/// are prover-supplied and both are rejections, not panics. +#[allow(clippy::too_many_arguments)] +pub fn derive_batched_fri_challenges( + transcript: &mut T, + heights: &[usize], + widths: &[usize], + layer_roots: &[[u8; 32]], + final_poly_coeffs: &[FieldElement], + blowup_log: u32, + final_poly_log_degree: u32, + grinding_factor: u8, + nonce: Option, + num_queries: usize, +) -> Option> +where + E: IsField, + T: IsTranscript, +{ + let &h_max = heights.iter().max()?; + let &h_min = heights.iter().min()?; + // `heights` is derived from proof-supplied trace lengths, so bound it before + // it reaches a shift or `BatchedFriLayout`'s asserts: a bogus height is a + // rejection, never a panic on the verifier's path. + if h_max == 0 || h_max >= u32::BITS as usize || h_min < blowup_log as usize { + return None; + } + let layout = BatchedFriLayout::new(h_max, h_min, blowup_log, final_poly_log_degree); + if layer_roots.len() != layout.num_committed + || final_poly_coeffs.len() != 1usize << layout.effective_k + { + return None; + } + + absorb_shape_histogram(transcript, heights, widths); + + let alpha = transcript.sample_field_element(); + + let mut betas = Vec::with_capacity(layout.num_committed + 1); + for root in layer_roots { + let beta = transcript.sample_field_element(); + transcript.append_bytes(root); + betas.push(beta); + } + + if layout.total_folds > 0 { + betas.push(transcript.sample_field_element()); + } + for c in final_poly_coeffs { + transcript.append_field_element(c); + } + + let mut grinding_seed = [0u8; 32]; + if grinding_factor > 0 + && let Some(nonce_value) = nonce + { + grinding_seed = transcript.state(); + transcript.append_bytes(&nonce_value.to_be_bytes()); + } + + let iotas = (0..num_queries) + .map(|_| transcript.sample_u64(1u64 << (h_max - 1)) as usize) + .collect(); + + Some(BatchedFriChallenges { + alpha, + betas, + layout, + grinding_seed, + iotas, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::KeccakStarkHash; + use crate::fri::commit_phase_from_evaluations; + use crate::fri::fri_functions::{compute_coset_twiddles_inv, fold_evaluations_in_place}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use math::field::element::FieldElement; + use math::field::goldilocks::GoldilocksField; + + type FE = FieldElement; + type Transcript = DefaultTranscript; + + #[test] + fn combine_by_height_two_height3_one_height2() { + // Three codewords: indices 0, 1 have height 3 (length 8); + // index 2 has height 2 (length 4). + let cw0: Vec = (1u64..=8).map(FE::from).collect(); + let cw1: Vec = (10u64..=17).map(FE::from).collect(); + let cw2: Vec = (100u64..=103).map(FE::from).collect(); + + let alpha = FE::from(7u64); + + let inputs: Vec<(Vec, usize)> = + vec![(cw0.clone(), 3), (cw1.clone(), 3), (cw2.clone(), 2)]; + + let out = combine_by_height(&inputs, &alpha); + + // Output vec length = max_height + 1 = 4 (indices 0..=3 only). + assert_eq!(out.len(), 4, "output length should be max_height+1 = 4"); + + // Heights 0 and 1 have no inputs. + assert!(out[0].is_none(), "height 0 should be None"); + assert!(out[1].is_none(), "height 1 should be None"); + + // Height 3: combined[j] = alpha^0 * cw0[j] + alpha^1 * cw1[j] + let alpha0 = FE::one(); + let alpha1 = alpha; + let expected3: Vec = cw0 + .iter() + .zip(cw1.iter()) + .map(|(a, b)| &(&alpha0 * a) + &(&alpha1 * b)) + .collect(); + + let got3 = out[3].as_ref().expect("height 3 should be Some"); + assert_eq!( + got3.len(), + 8, + "height-3 combined codeword should have length 8" + ); + assert_eq!(got3, &expected3, "height-3 combined values mismatch"); + + // Height 2: combined[j] = alpha^2 * cw2[j] + let alpha2 = &alpha * α + let expected2: Vec = cw2.iter().map(|x| &alpha2 * x).collect(); + + let got2 = out[2].as_ref().expect("height 2 should be Some"); + assert_eq!( + got2.len(), + 4, + "height-2 combined codeword should have length 4" + ); + assert_eq!(got2, &expected2, "height-2 combined values mismatch"); + } + + /// Absorbing codewords one at a time — the shape a prover uses so it never + /// holds every table's quotient at once — must land on the same buckets as + /// handing them all over materialized. + #[test] + fn streaming_absorption_matches_materialized_combine() { + let inputs: Vec<(Vec, usize)> = vec![ + ((1u64..=16).map(FE::from).collect(), 4), + ((50u64..=57).map(FE::from).collect(), 3), + ((90u64..=105).map(FE::from).collect(), 4), + ((200u64..=203).map(FE::from).collect(), 2), + ((300u64..=307).map(FE::from).collect(), 3), + ]; + let alpha = FE::from(11u64); + + let eager = combine_by_height(&inputs, &alpha); + + let mut combiner = HeightCombiner::new(alpha); + for (codeword, height) in &inputs { + combiner.absorb(codeword, *height); + } + assert_eq!( + combiner.finish(), + eager, + "streaming absorption must equal the materialized combine" + ); + } + + /// After the first fold in `batched_commit_phase`, the committed layer[0] + /// evaluation must equal `fold(combined[4], β₀) + β₀² · combined[3]`. + #[test] + fn batched_commit_phase_first_layer_matches_manual_fold_and_inject() { + // Build synthetic codewords for h=4 (len 16) and h=3 (len 8). + let data_h4: Vec = (1u64..=16).map(FE::from).collect(); + let data_h3: Vec = (101u64..=108).map(FE::from).collect(); + + // combined = [None, None, None, Some(data_h3), Some(data_h4)] + let combined: Vec>> = vec![ + None, + None, + None, + Some(data_h3.clone()), + Some(data_h4.clone()), + ]; + + let coset_offset = FE::from(3u64); + let (blowup_log, k) = (1u32, 1u32); + + // Create transcript; clone before mutating so we can replay independently. + let mut transcript = Transcript::new(b"batched_fri_test"); + let mut transcript_check = transcript.clone(); + + let (_coeffs, layers) = batched_commit_phase::<_, _, _, KeccakStarkHash>( + combined, + &mut transcript, + &coset_offset, + blowup_log, + k, + ); + + // Terminal at min(blowup_log + k, h_min) = min(2, 3) = 2, so folds run + // 4 -> 2: two folds, one committed layer. + let layout = BatchedFriLayout::new(4, 3, blowup_log, k); + assert_eq!(layout.total_folds, 2); + assert_eq!( + layers.len(), + layout.num_committed, + "committed layers must follow the layout" + ); + + // --- Independent recomputation of layer[0] --- + let beta_0 = transcript_check.sample_field_element(); + + let inv_twiddles_h4 = compute_coset_twiddles_inv::(&coset_offset, 16); + let mut expected = data_h4.clone(); + fold_evaluations_in_place(&mut expected, &beta_0, &inv_twiddles_h4); + // expected now has length 8 (height 3) + + // Inject combined[3]: expected[j] += beta_0² · data_h3[j] + let beta_0_sq = beta_0.square(); + for (j, val) in data_h3.iter().enumerate() { + expected[j] = &expected[j] + &(&beta_0_sq * val); + } + + assert_eq!( + layers[0].evaluation, expected, + "layer[0] evaluation does not match manual fold+inject" + ); + } + + /// ★ M-12: the batched commit phase must terminate where the unbatched one + /// does. With a single bucket the two are the same protocol, so they must + /// agree on the committed-layer count, the terminal coefficients, and the + /// resulting transcript state — pinning that batching did not silently switch + /// to folding all the way to a scalar (which for this input would commit + /// `h_max - 1 = 9` layers instead of 4). + #[test] + fn single_bucket_terminal_matches_the_unbatched_commit_phase() { + let h = 10usize; + let (blowup_log, k) = (1u32, 5u32); + let coset_offset = FE::from(3u64); + let evals: Vec = (0..(1u64 << h)).map(|i| FE::from(i * 7 + 1)).collect(); + let inv_twiddles = compute_coset_twiddles_inv::(&coset_offset, 1 << h); + + let mut t_unbatched = Transcript::new(b"terminal_parity"); + let (unbatched_coeffs, unbatched_layers) = commit_phase_from_evaluations::< + GoldilocksField, + GoldilocksField, + Transcript, + KeccakStarkHash, + >( + evals.clone(), + &mut t_unbatched, + &coset_offset, + 1 << h, + blowup_log, + k, + &inv_twiddles, + ); + + let mut combined: Vec>> = vec![None; h + 1]; + combined[h] = Some(evals); + let mut t_batched = Transcript::new(b"terminal_parity"); + let (batched_coeffs, batched_layers) = batched_commit_phase::<_, _, _, KeccakStarkHash>( + combined, + &mut t_batched, + &coset_offset, + blowup_log, + k, + ); + + // total_folds = 10 - (1 + 5) = 4, so 3 committed layers — not h_max-1 = 9. + assert_eq!(unbatched_layers.len(), 3); + assert_eq!( + batched_layers.len(), + unbatched_layers.len(), + "batched and unbatched must commit the same number of layers" + ); + assert_eq!( + batched_coeffs.len(), + 1usize << k, + "the terminal polynomial must carry 2^k coefficients" + ); + assert_eq!( + batched_coeffs, unbatched_coeffs, + "batched and unbatched must send the same terminal polynomial" + ); + for (b, u) in batched_layers.iter().zip(unbatched_layers.iter()) { + assert_eq!(b.merkle_tree.root, u.merkle_tree.root); + } + assert_eq!( + t_batched.state(), + t_unbatched.state(), + "the two commit phases must leave the transcript in the same state" + ); + } + + /// The batched-only floor: the terminal may not sit above the shortest + /// injected codeword, or that bucket would never enter the running word. + #[test] + fn terminal_is_floored_at_the_shortest_codeword() { + let (blowup_log, k) = (1u32, 5u32); + + // Shortest codeword above blowup_log + k = 6: the floor is inert and the + // layout is the unbatched one for h_max. + let inert = BatchedFriLayout::new(10, 8, blowup_log, k); + assert_eq!(inert.total_folds, 4, "10 -> 6"); + assert_eq!(inert.effective_k, k); + + // Shortest codeword BELOW blowup_log + k: folding must continue down to + // it, and the terminal polynomial shrinks accordingly. + let floored = BatchedFriLayout::new(10, 4, blowup_log, k); + assert_eq!(floored.total_folds, 6, "10 -> 4"); + assert_eq!(floored.effective_k, 3, "terminal_log 4 - blowup_log 1"); + + // And the commit phase really does consume that low bucket. + let coset_offset = FE::from(3u64); + let mut combined: Vec>> = vec![None; 8]; + combined[7] = Some((0..128u64).map(|i| FE::from(i + 1)).collect()); + combined[4] = Some((0..16u64).map(|i| FE::from(i * 3 + 5)).collect()); + let mut transcript = Transcript::new(b"floor_test"); + let (coeffs, layers) = batched_commit_phase::<_, _, _, KeccakStarkHash>( + combined, + &mut transcript, + &coset_offset, + blowup_log, + k, + ); + let layout = BatchedFriLayout::new(7, 4, blowup_log, k); + assert_eq!(layers.len(), layout.num_committed); + assert_eq!(coeffs.len(), 1usize << layout.effective_k); + } + + /// The prover, by hand, runs exactly the round-4 sequence; the shared replay + /// routine must reproduce byte-identical outputs from the same start state. + #[test] + fn batched_round4_prover_inline_matches_verifier_replay() { + let heights: Vec = vec![10, 10, 8, 8, 8, 7]; + let widths: Vec = vec![3, 5, 2, 2, 9, 1]; + let (blowup_log, k) = (1u32, 5u32); + // total_folds = 10 - 6 = 4 -> 3 committed layers, 4 betas. + let layout = BatchedFriLayout::new(10, 7, blowup_log, k); + assert_eq!((layout.num_committed, layout.total_folds), (3, 4)); + + let layer_roots: Vec<[u8; 32]> = (0u8..3).map(|i| [i; 32]).collect(); + let final_poly_coeffs: Vec = (0..(1u64 << layout.effective_k)).map(FE::from).collect(); + + let grinding_factor: u8 = 4; + let num_queries = 3; + + let seed_transcript = Transcript::new(b"batched_round4_test"); + let mut transcript_a = seed_transcript.clone(); + let mut transcript_b = seed_transcript.clone(); + + // --- Clone A: prover-inline sequence, by hand --- + absorb_shape_histogram(&mut transcript_a, &heights, &widths); + let alpha_a = transcript_a.sample_field_element(); + + let mut betas_a = Vec::with_capacity(layer_roots.len() + 1); + for root in &layer_roots { + let beta = transcript_a.sample_field_element(); + transcript_a.append_bytes(root); + betas_a.push(beta); + } + betas_a.push(transcript_a.sample_field_element()); + for c in &final_poly_coeffs { + transcript_a.append_field_element(c); + } + assert_eq!( + betas_a.len(), + layout.total_folds as usize, + "one beta per fold, matching batched_commit_phase" + ); + + let grinding_seed_a = transcript_a.state(); + // Test-only: derive a real PoW nonce so the grinding step is exercised + // identically by both sides (the nonce search itself is not under test). + let nonce = + crate::grinding::generate_nonce::>( + &grinding_seed_a, + grinding_factor, + ) + .expect("a valid grinding nonce exists for this small grinding_factor"); + transcript_a.append_bytes(&nonce.to_be_bytes()); + + let iotas_a: Vec = (0..num_queries) + .map(|_| transcript_a.sample_u64(1u64 << 9) as usize) + .collect(); + + // --- Clone B: shared replay routine --- + let result = derive_batched_fri_challenges( + &mut transcript_b, + &heights, + &widths, + &layer_roots, + &final_poly_coeffs, + blowup_log, + k, + grinding_factor, + Some(nonce), + num_queries, + ) + .expect("a well-formed layer-root and coefficient count"); + + assert_eq!(result.alpha, alpha_a, "alpha mismatch"); + assert_eq!(result.betas, betas_a, "beta vector mismatch"); + assert_eq!(result.layout, layout, "layout mismatch"); + assert_eq!( + result.grinding_seed, grinding_seed_a, + "grinding seed mismatch" + ); + assert_eq!(result.iotas, iotas_a, "iotas mismatch"); + assert!( + result.iotas.iter().all(|&i| i < 1usize << 9), + "iotas must be row-pair indices in the tallest domain" + ); + } + + /// A layer-root or coefficient count that disagrees with the shape's layout is + /// prover-supplied, so it is a rejection rather than a panic. + #[test] + fn derive_rejects_a_layer_count_that_contradicts_the_shape() { + let heights: Vec = vec![10, 8]; + let widths: Vec = vec![2, 3]; + let (blowup_log, k) = (1u32, 5u32); + let layout = BatchedFriLayout::new(10, 8, blowup_log, k); + let coeffs: Vec = vec![FE::one(); 1usize << layout.effective_k]; + let roots: Vec<[u8; 32]> = vec![[0u8; 32]; layout.num_committed]; + + let mut ok = Transcript::new(b"reject"); + assert!( + derive_batched_fri_challenges( + &mut ok, &heights, &widths, &roots, &coeffs, blowup_log, k, 0, None, 1 + ) + .is_some() + ); + + let mut too_few = Transcript::new(b"reject"); + assert!( + derive_batched_fri_challenges( + &mut too_few, + &heights, + &widths, + &roots[..roots.len() - 1], + &coeffs, + blowup_log, + k, + 0, + None, + 1 + ) + .is_none(), + "one fewer layer root than the shape implies must be rejected" + ); + + let mut bad_coeffs = Transcript::new(b"reject"); + assert!( + derive_batched_fri_challenges( + &mut bad_coeffs, + &heights, + &widths, + &roots, + &coeffs[..coeffs.len() - 1], + blowup_log, + k, + 0, + None, + 1 + ) + .is_none(), + "a short terminal polynomial must be rejected" + ); + } + + /// `heights` comes from proof-supplied trace lengths, so every out-of-range + /// value is a rejection rather than a shift overflow or a layout assert. + #[test] + fn derive_rejects_out_of_range_heights_without_panicking() { + let widths = vec![2usize, 3]; + let (blowup_log, k) = (1u32, 5u32); + let coeffs: Vec = vec![FE::one(); 1usize << k]; + let roots: Vec<[u8; 32]> = vec![[0u8; 32]; 3]; + + let derive = |heights: &[usize]| { + derive_batched_fri_challenges( + &mut Transcript::new(b"range"), + heights, + &widths, + &roots, + &coeffs, + blowup_log, + k, + 0, + None, + 1, + ) + .is_some() + }; + + assert!(derive(&[10, 8]), "a well-formed shape is accepted"); + assert!(!derive(&[0, 0]), "a zero height must be rejected"); + assert!( + !derive(&[10, 0]), + "a height below the blowup must be rejected" + ); + assert!( + !derive(&[u32::BITS as usize, 8]), + "a height at the shift width must be rejected" + ); + assert!( + !derive(&[usize::MAX, 8]), + "an absurd height must be rejected, not wrapped by the u32 cast" + ); + let empty: [usize; 0] = []; + assert!(!derive(&empty), "an empty epoch must be rejected"); + } + + /// Tampering the shape histogram (without changing anything else) must change + /// the derived batching challenge α — the structural binding that protects the + /// fold/injection schedule. Heights and widths are both bound (M-13a), so a + /// change to either alone must move α. + #[test] + fn absorb_shape_histogram_binds_heights_and_widths_into_alpha() { + let heights: Vec = vec![10, 10, 8, 8, 8, 5]; + let widths: Vec = vec![4, 4, 2, 2, 2, 1]; + + let alpha_of = |h: &[usize], w: &[usize]| { + let mut t = Transcript::new(b"histogram_binding_test"); + absorb_shape_histogram(&mut t, h, w); + t.sample_field_element() + }; + + let base = alpha_of(&heights, &widths); + + let mut other_height = heights.clone(); + other_height[5] = 6; + assert_ne!( + base, + alpha_of(&other_height, &widths), + "different height histograms must yield different alpha" + ); + + let mut other_width = widths.clone(); + other_width[5] = 2; + assert_ne!( + base, + alpha_of(&heights, &other_width), + "different width histograms must yield different alpha" + ); + + // The length prefix plus fixed-width fields make the encoding injective: + // swapping a (height, width) pair between tables also moves alpha. + let swapped_h = vec![10, 10, 8, 8, 5, 8]; + let swapped_w = vec![4, 4, 2, 2, 1, 2]; + assert_ne!( + base, + alpha_of(&swapped_h, &swapped_w), + "table order must be bound, not just the multiset" + ); + } +} diff --git a/crypto/stark/src/fri/mmcs.rs b/crypto/stark/src/fri/mmcs.rs new file mode 100644 index 000000000..7b4ac7b1e --- /dev/null +++ b/crypto/stark/src/fri/mmcs.rs @@ -0,0 +1,1334 @@ +//! Mixed-height, row-pair MMCS (Merkle Mixed Commitment Scheme). +//! +//! Commits ALL of an epoch's matrices (one per table, of possibly different +//! heights) into ONE mixed-height Merkle tree, so a single query opens ONE +//! authentication path that covers every table's row at that query — the +//! proof-size / opening-path win of the unified-shard design (SP1 / OpenVM / +//! Plonky3). Mirrors Plonky3's `MerkleTreeMmcs`, adapted to the [`StarkHash`] +//! commitment configuration and to the row-pair `(x, -x)` leaf layout (#735). +//! +//! This is a standalone primitive: the prover and verifier do not build epoch +//! commitments with it yet. The leaf and injection layout documented below is +//! the single source of truth for whoever wires it in. +//! +//! # Inputs +//! +//! [`MixedMmcs::commit`] reads matrices through a [`LeafSource`], which reports +//! each matrix's `(log_height, width)` and serves its rows on demand: +//! - `log_height`: `log2` of the row count; the matrix has `2^log_height` rows. +//! - `width`: number of committed columns. +//! - rows are addressed by **bit-reversed** LDE position (the same layout the +//! per-table trace commit produces internally). +//! +//! # Row-pair leaves +//! +//! Leaf `k` of a matrix groups LDE positions `2k` and `2k+1` (the FRI fold pair +//! `x` and `-x`), all `width` columns batched. A matrix of `log_height h` has +//! `2^(h-1)` leaves. In [`MixedMmcs::open_batch`] / [`PolynomialOpenings`]: +//! `evaluations` = row `2k`, `evaluations_sym` = row `2k+1`. +//! +//! # Tree layout (the soundness-relevant contract) +//! +//! Let `h_max = max(log_height)`. The base digest layer (layer 0) has +//! `N0 = 2^(h_max-1)` nodes. Layer `i` has `N0 >> i` nodes; the root is the sole +//! node of layer `h_max-1`. A matrix of `log_height h` is *injected* at layer +//! index `i = h_max - h` (so the tallest matrices, `h == h_max`, populate the +//! base layer; shorter matrices enter where the layer width matches their leaf +//! count `2^(h-1)`). +//! +//! Hashing (`H = >::hash_data` over a `Vec` of field elements; +//! `C = >::hash_new_parent`, the 2-input compression — the same +//! two functions, on the same backend, that the existing per-table tree uses): +//! +//! - **Base layer** node `k` (`k in [0, N0)`): +//! `layer0[k] = H( CONCAT_{m : h_m == h_max} (row_m(2k) || row_m(2k+1)) )` +//! where matrices of height `h_max` are concatenated in INPUT order. +//! - **Climb** from layer `i` to layer `i+1` (`j in [0, N_{i+1})`): +//! `parent = C(layer_i[2j], layer_i[2j+1])`. Let `inject_h = h_max - 1 - i`. If +//! any matrix has `h_m == inject_h`, then +//! `layer_{i+1}[j] = C( parent, H( CONCAT_{m : h_m == inject_h} (row_m(2j) || row_m(2j+1)) ) )` +//! (injecting matrices concatenated in INPUT order); otherwise +//! `layer_{i+1}[j] = parent`. +//! - `root = layer_{h_max-1}[0]`. +//! +//! Because the leaf and parent hashes come from `H::Batched` — the backend the +//! per-table row-pair tree already commits with — a single-matrix `MixedMmcs` is +//! byte-identical to that tree by construction, not by coincidence. There is no +//! second encoding of a leaf to keep in step. +//! +//! # Query opening +//! +//! For query `iota in [0, N0)`, matrix `m` is opened at leaf +//! `k_m = iota >> (h_max - h_m)` (`= iota >> i_m`). The shared authentication +//! path holds, for each level `level in [0, h_max-1)`, the sibling +//! `layer_level[(iota >> level) ^ 1]`. ONE path authenticates all matrices. +//! The per-matrix [`PolynomialOpenings::proof`] fields are empty; the single +//! [`MixedOpening::proof`] is the authenticator. +//! +//! # ★ Index convention — a HARD PRECONDITION on the caller +//! +//! `iota` is a leaf index **in THIS tree**: it must be drawn from +//! `[0, 2^(h_max-1))` where `h_max` is *this MMCS's* tallest matrix. +//! [`MixedMmcs::verify_batch`] walks the path with `(iota >> level) & 1`, i.e. it +//! consumes the **low** `h_max - 1` bits, while a shorter matrix inside the tree +//! is located by `iota >> (h_max - h_m)`, i.e. by the **high** bits. Both are +//! consistent only when the two `h_max` agree. +//! +//! A caller that batches several rounds under one shared FRI query index must +//! therefore reduce a global index before calling in: +//! +//! ```text +//! iota_round = iota_fri >> (h_max_fri - h_max_round) +//! ``` +//! +//! Passing the un-reduced `iota_fri` to a round whose `h_max` is below the FRI's +//! is not a loud error — prover and verifier share this routine, so a wrong +//! convention is self-consistent: honest proofs still verify and the failure is +//! that short matrices end up authenticated at positions the FRI join never +//! checks. [`MixedMmcs::verify_batch`] rejects an `iota` outside `[0, 2^(h_max-1))` +//! to turn most of that class of misuse into a rejection rather than a silent +//! mis-binding, but the reduction remains the caller's obligation: an index that +//! happens to land in range is accepted at the wrong leaf. +//! `short_round_low_bit_convention_is_exercised` is the control on this. +//! +//! # Width binding (soundness) +//! +//! [`MixedMmcs::verify_batch`] takes per-matrix `widths` alongside `heights`. +//! Within a height group the leaf hash is over the FLAT concatenation of every +//! matrix's opened row pair (`A.eval ‖ A.eval_sym ‖ B.eval ‖ B.eval_sym ‖ …`), +//! which does NOT by itself record where each matrix's columns end. Fixing +//! `widths[m]` (matrix `m`'s column count) makes those boundaries unambiguous: +//! without it a prover could shift a boundary — e.g. lengthen one matrix's +//! `evaluations` by one element and shorten its `evaluations_sym` by one — +//! leaving the flat bytes (and therefore the group hash) identical while feeding +//! a corrupted row downstream. Consumers MUST pass the committed public +//! per-table column counts, in the same INPUT order as `heights`, derived from +//! the AIR set rather than read out of the proof. +//! +//! `heights` and `widths` must ALSO be bound into the Fiat-Shamir transcript by +//! the consumer, before any challenge that depends on the epoch's shape — see +//! [`crate::fri::batched::absorb_shape_histogram`], which is the canonical +//! encoding of that binding. +//! +//! # Determinism +//! +//! The tree is a pure function of `(matrices, input order)`. Grouping within a +//! height (base batching and injection) follows INPUT order; the prover and +//! verifier MUST pass matrices and `heights` in the same per-epoch order. +//! +//! # Memory: what the caller may drop, and when +//! +//! The MMCS owns no evaluations. It stores the digest layers +//! (`O(2^(h_max-1))` nodes) plus each matrix's `(log_height, width)`; rows are +//! pulled through [`LeafSource`] both at commit and at open time. Two properties +//! follow, and `commit_reads_each_height_group_in_one_contiguous_phase` is the +//! control on the second: +//! +//! - `commit` reads matrix `m`'s rows **only while building level +//! `h_max - h_m`**, and levels are built in descending height order. A caller +//! may therefore produce a height group's LDEs, commit, and drop them before +//! the next group is needed. +//! - Within one height group the leaf is a single `hash_data` over the group's +//! concatenated rows, so every matrix of that height must be *readable* +//! simultaneously. That does not require them all to be resident — a +//! `LeafSource` may serve rows from disk, from device memory, or by +//! recomputation — but a caller that serves them from full in-RAM LDE buffers +//! holds the whole group at once. Streaming *within* a height group would need +//! an incremental leaf hasher (absorb matrix by matrix into one sponge per +//! leaf), which the backend trait does not currently expose. + +use core::marker::PhantomData; + +use crypto::merkle_tree::proof::Proof; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::fft::bit_reversing::reverse_index; +use math::field::element::FieldElement; +use math::field::traits::IsField; +use math::traits::AsBytes; + +use crate::config::{Commitment, StarkHash}; +use crate::proof::stark::PolynomialOpenings; + +/// On-demand supplier of committed matrix rows, so [`MixedMmcs`] builds its +/// digests and serves openings WITHOUT owning a copy of the (large) LDE buffers. +/// Both [`MixedMmcs::commit`] and [`MixedMmcs::open_batch`] read every leaf +/// through this trait, so the root and opened rows are byte-identical to those a +/// matrix-owning MMCS would produce — the prover keeps only the LDE buffers it +/// already retains for DEEP, and each MMCS stores just digests. +/// +/// Rows are addressed in each matrix's committed row-pair layout: `append_row(m, +/// r, out)` appends matrix `m`'s row at **bit-reversed** LDE position `r` (its +/// `width(m)` committed columns, in column order). This is the same `r`-indexing +/// the module's "Tree layout" section uses; an implementor holding the +/// natural-order LDE maps `r` to `reverse_index(r, 2^log_height(m))`. +pub trait LeafSource { + /// Number of committed matrices, in canonical input order. + fn num_matrices(&self) -> usize; + /// `log2` of matrix `m`'s row count. Row-pair leaves require `>= 1`. + fn log_height(&self, m: usize) -> usize; + /// Matrix `m`'s committed column count. + fn width(&self, m: usize) -> usize; + /// Append matrix `m`'s bit-reversed LDE row `bitrev_row` (its `width(m)` + /// committed columns) to `out`. `bitrev_row in [0, 2^log_height(m))`. + fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>); +} + +/// One committed matrix borrowed from a retained LDE buffer. Resolves each +/// bit-reversed row on demand (mapping through `reverse_index`) so the MMCS owns +/// no copy of the evaluations. See [`LeafSource`]. +pub enum BorrowedMatrix<'a, E: IsField> { + /// A `stride`-wide, row-major, NATURAL-order LDE buffer (the main / aux LDE + /// retained in `Round1::lde_trace`). This matrix occupies columns + /// `[col_start, col_start + width)`; its bit-reversed row `r` lives at + /// natural-order row `reverse_index(r, 2^log_height)`. + RowMajorNatural { + data: &'a [FieldElement], + stride: usize, + col_start: usize, + width: usize, + log_height: usize, + }, + /// Column-major NATURAL-order columns (the composition-poly LDE retained in + /// `Round2::lde_composition_poly_evaluations`): `cols[c][nat]` is column `c` + /// at natural-order row `nat`. Every committed column is used. + ColMajorNatural { + cols: &'a [Vec>], + log_height: usize, + }, +} + +impl BorrowedMatrix<'_, E> { + fn log_height(&self) -> usize { + match self { + BorrowedMatrix::RowMajorNatural { log_height, .. } + | BorrowedMatrix::ColMajorNatural { log_height, .. } => *log_height, + } + } + + fn width(&self) -> usize { + match self { + BorrowedMatrix::RowMajorNatural { width, .. } => *width, + BorrowedMatrix::ColMajorNatural { cols, .. } => cols.len(), + } + } + + fn append_row(&self, bitrev_row: usize, out: &mut Vec>) { + match self { + BorrowedMatrix::RowMajorNatural { + data, + stride, + col_start, + width, + log_height, + } => { + let nat = reverse_index(bitrev_row, 1u64 << log_height); + let base = nat * stride + col_start; + out.extend_from_slice(&data[base..base + width]); + } + BorrowedMatrix::ColMajorNatural { cols, log_height } => { + let nat = reverse_index(bitrev_row, 1u64 << log_height); + for col in cols.iter() { + out.push(col[nat].clone()); + } + } + } + } +} + +impl LeafSource for Vec> { + fn num_matrices(&self) -> usize { + self.len() + } + fn log_height(&self, m: usize) -> usize { + self[m].log_height() + } + fn width(&self, m: usize) -> usize { + self[m].width() + } + fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>) { + self[m].append_row(bitrev_row, out); + } +} + +/// A committed mixed-height, row-pair MMCS under the commitment configuration +/// `H`. Stores ONLY the digest layers (to serve the shared authentication path) +/// plus each matrix's `(log_height, width)` (to locate leaves). The row DATA is +/// served on demand by the caller's [`LeafSource`] — the MMCS never owns a copy +/// of the LDE. +pub struct MixedMmcs { + root: Commitment, + /// `layers[0]` is the base digest layer; `layers[h_max-1] == [root]`. + layers: Vec>, + /// Per committed matrix, in input order: `(log_height, width)`. + dims: Vec<(usize, usize)>, + h_max: usize, + _marker: PhantomData<(E, H)>, +} + +/// The opening of ALL matrices at one query index, authenticated by a single +/// shared Merkle path. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct MixedOpening { + /// The one authentication path covering every matrix's row at the query. + pub proof: Proof, + /// Per-matrix row pair (in the same INPUT order as `commit`). Each entry's + /// own `proof` is empty — [`MixedOpening::proof`] is the authenticator. + pub per_matrix: Vec>, +} + +/// Hash the row pair `(row(2*leaf), row(2*leaf+1))` of every matrix whose index +/// is in `group` (in the given order), all columns batched, into one digest. +/// Rows are pulled from `source` — the MMCS owns no copy. +fn hash_group_leaf(source: &S, group: &[usize], leaf: usize) -> Commitment +where + E: IsField + 'static, + H: StarkHash, + S: LeafSource, + FieldElement: AsBytes + Sync + Send, +{ + let mut buf: Vec> = Vec::new(); + for &m in group { + source.append_row(m, 2 * leaf, &mut buf); + source.append_row(m, 2 * leaf + 1, &mut buf); + } + as IsMerkleTreeBackend>::hash_data(&buf) +} + +/// Verifier-side analogue of [`hash_group_leaf`]: hash the opened row pairs of a +/// group of openings (in the given order) into one digest. +fn hash_group_openings(group: &[&PolynomialOpenings]) -> Commitment +where + E: IsField + 'static, + H: StarkHash, + FieldElement: AsBytes + Sync + Send, +{ + let mut buf: Vec> = Vec::new(); + for o in group { + buf.extend_from_slice(&o.evaluations); + buf.extend_from_slice(&o.evaluations_sym); + } + as IsMerkleTreeBackend>::hash_data(&buf) +} + +#[inline] +fn compress(left: &Commitment, right: &Commitment) -> Commitment +where + E: IsField + 'static, + H: StarkHash, + FieldElement: AsBytes + Sync + Send, +{ + as IsMerkleTreeBackend>::hash_new_parent(left, right) +} + +impl MixedMmcs +where + E: IsField + 'static, + H: StarkHash, + FieldElement: AsBytes + Sync + Send, +{ + /// Commit the matrices supplied by `source` into one mixed-height row-pair + /// tree, storing only the digest layers. See the module docs for the exact + /// leaf/injection layout. `source` provides each matrix's dimensions and its + /// bit-reversed rows on demand; no copy of the evaluations is retained. + /// + /// Leaf hashing (the base layer and each injected climb layer) is parallel + /// across leaves via [`crate::par::par_map_collect`]; the per-level output is + /// index-ordered, so the root and layers are byte-identical to a sequential + /// build. `S: Sync` lets leaf closures read `source` from worker threads. + /// + /// Levels are built in descending height order and matrix `m` is read only + /// while its own level is built, so the caller may release a height group's + /// buffers once the next level starts — see the module's memory section. + pub fn commit + Sync>(source: &S) -> Self { + let num_matrices = source.num_matrices(); + assert!( + num_matrices > 0, + "MixedMmcs::commit requires at least one matrix" + ); + + let dims: Vec<(usize, usize)> = (0..num_matrices) + .map(|m| { + let log_height = source.log_height(m); + assert!( + log_height >= 1, + "log_height must be >= 1 (row-pair leaves need at least 2 rows)" + ); + (log_height, source.width(m)) + }) + .collect(); + + let h_max = dims + .iter() + .map(|(log_height, _)| *log_height) + .max() + .expect("dims is non-empty"); + let n0 = 1usize << (h_max - 1); + + // Base digest layer: batch all tallest matrices' row pairs (input order). + let base_group: Vec = (0..num_matrices).filter(|&m| dims[m].0 == h_max).collect(); + + let mut layers: Vec> = Vec::with_capacity(h_max); + // Base layer: 2^(h_max-1) independent group-leaf hashes — the bulk of the + // tree's hashing (half of all nodes). Parallel across leaves. + let base: Vec = crate::par::par_map_collect(0..n0, |k| { + hash_group_leaf::(source, &base_group, k) + }); + layers.push(base); + + // Climb, compressing pairs and injecting shorter matrices where the layer + // width matches their leaf count. Each level's nodes are independent + // (they read only the previous, already-materialized layer), so parallel + // across nodes; levels stay sequential. + let mut i = 0usize; + while layers[i].len() > 1 { + let next_len = layers[i].len() / 2; + let inject_h = h_max - 1 - i; + let inject_group: Vec = (0..num_matrices) + .filter(|&m| dims[m].0 == inject_h) + .collect(); + + let cur = &layers[i]; + let next: Vec = crate::par::par_map_collect(0..next_len, |j| { + let mut parent = compress::(&cur[2 * j], &cur[2 * j + 1]); + if !inject_group.is_empty() { + let inj = hash_group_leaf::(source, &inject_group, j); + parent = compress::(&parent, &inj); + } + parent + }); + layers.push(next); + i += 1; + } + + let root = layers.last().expect("at least the base layer exists")[0]; + + MixedMmcs { + root, + layers, + dims, + h_max, + _marker: PhantomData, + } + } + + /// The committed root. + pub fn root(&self) -> Commitment { + self.root + } + + /// `log2` of the tallest committed matrix. The query index this MMCS accepts + /// lives in `[0, 2^(h_max-1))` — see the module's index-convention section. + pub fn h_max(&self) -> usize { + self.h_max + } + + /// Per committed matrix, in input order: `(log_height, width)`. The verifier + /// is expected to rebuild these from the AIR set rather than read them here; + /// this accessor exists so a prover can bind the shape it actually committed. + pub fn dims(&self) -> &[(usize, usize)] { + &self.dims + } + + /// Open all matrices at query `iota in [0, 2^(h_max-1))`, returning each + /// matrix's row pair plus one shared authentication path. Row data is served + /// by `source`, which MUST describe the same matrices (same order and + /// dimensions) as the one passed to [`Self::commit`]. + pub fn open_batch>(&self, iota: usize, source: &S) -> MixedOpening { + let n0 = 1usize << (self.h_max - 1); + assert!(iota < n0, "iota {iota} out of range (n0 = {n0})"); + debug_assert_eq!( + source.num_matrices(), + self.dims.len(), + "leaf source matrix count must match the committed tree" + ); + + let per_matrix: Vec> = (0..self.dims.len()) + .map(|m| { + let (log_height, width) = self.dims[m]; + debug_assert_eq!(source.log_height(m), log_height); + debug_assert_eq!(source.width(m), width); + let k = iota >> (self.h_max - log_height); + let mut evaluations = Vec::with_capacity(width); + source.append_row(m, 2 * k, &mut evaluations); + let mut evaluations_sym = Vec::with_capacity(width); + source.append_row(m, 2 * k + 1, &mut evaluations_sym); + PolynomialOpenings { + proof: Proof { + merkle_path: Vec::new(), + }, + evaluations, + evaluations_sym, + } + }) + .collect(); + + let mut merkle_path = Vec::with_capacity(self.h_max - 1); + for level in 0..(self.h_max - 1) { + let sibling = (iota >> level) ^ 1; + merkle_path.push(self.layers[level][sibling]); + } + + MixedOpening { + proof: Proof { merkle_path }, + per_matrix, + } + } + + /// Verify a batched opening at `iota` against `root`. `heights[m]` is the + /// `log_height` of matrix `m` and `widths[m]` its column count, both in the + /// SAME order as `opening.per_matrix`, and both supplied by the verifier from + /// the AIR set rather than read out of the proof. + /// + /// `widths` binds each matrix's boundary inside the per-height-group leaf + /// hash (see the module `# Width binding` section): the group leaf hashes the + /// FLAT concatenation of every matrix's `evaluations ‖ evaluations_sym`, so + /// without fixed widths a prover could shift a matrix boundary while keeping + /// the flat bytes — and thus the hash — identical. Pinning `widths` makes the + /// boundaries unambiguous and closes that forgery. + /// + /// `iota` must already be reduced to this tree's index space — see the + /// module's index-convention section. Out-of-range indices are rejected here, + /// but that check is a backstop, not a substitute for the reduction. + /// + /// Returns `false` on every malformed input; it never panics, so a verifier + /// can call it on adversarial data. + pub fn verify_batch( + root: &Commitment, + iota: usize, + opening: &MixedOpening, + heights: &[usize], + widths: &[usize], + ) -> bool { + if opening.per_matrix.len() != heights.len() + || heights.len() != widths.len() + || heights.is_empty() + { + return false; + } + // Bind per-matrix boundaries: every opened matrix must present exactly + // `widths[m]` columns in BOTH rows of its pair. A boundary shift keeps the + // flat per-group concatenation identical but changes these lengths. + for (o, w) in opening.per_matrix.iter().zip(widths.iter()) { + if o.evaluations.len() != *w || o.evaluations_sym.len() != *w { + return false; + } + } + let Some(&h_max) = heights.iter().max() else { + return false; + }; + // Honest heights are >= 1 (row-pair leaves need >= 2 rows) and far below + // the shift width; guard both ends rather than trust the proof's shape. + if h_max == 0 || h_max >= usize::BITS as usize { + return false; + } + // Only the low `h_max - 1` bits of `iota` are consumed (one per level), so + // an index from a taller domain would authenticate the short matrices at a + // position nothing else checks. Reject it instead. + if iota >= 1usize << (h_max - 1) { + return false; + } + if opening.proof.merkle_path.len() != h_max - 1 { + return false; + } + + // Base node: batch all tallest matrices' opened row pairs (input order). + let base_group: Vec<&PolynomialOpenings> = opening + .per_matrix + .iter() + .zip(heights.iter()) + .filter(|(_, h)| **h == h_max) + .map(|(o, _)| o) + .collect(); + let mut acc = hash_group_openings::(&base_group); + + for level in 0..(h_max - 1) { + let sibling = &opening.proof.merkle_path[level]; + let bit = (iota >> level) & 1; + let mut parent = if bit == 0 { + compress::(&acc, sibling) + } else { + compress::(sibling, &acc) + }; + + // Inject matrices whose leaf count matches this (halved) layer, in + // INPUT order — mirroring `commit`'s climb exactly. + let inject_h = h_max - 1 - level; + let inject_group: Vec<&PolynomialOpenings> = opening + .per_matrix + .iter() + .zip(heights.iter()) + .filter(|(_, h)| **h == inject_h) + .map(|(o, _)| o) + .collect(); + if !inject_group.is_empty() { + let inj = hash_group_openings::(&inject_group); + parent = compress::(&parent, &inj); + } + acc = parent; + } + + &acc == root + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commitment::commit_bit_reversed; + use crate::config::KeccakStarkHash; + use math::field::element::FieldElement; + use math::field::goldilocks::GoldilocksField; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + type FE = FieldElement; + type Mmcs = MixedMmcs; + + /// Reference [`LeafSource`] owning bit-reversed row-major matrices. Every + /// test commits/opens through this, so the byte-parity assertion against + /// `commit_bit_reversed` pins the tree contract; `borrowed_sources_match_ + /// owned_reference` cross-checks it against the borrowed (natural-order) + /// sources a prover would use. + struct OwnedMatrices { + /// Each entry: `(bit-reversed row-major data, log_height, width)`. + mats: Vec<(Vec>, usize, usize)>, + } + + impl LeafSource for OwnedMatrices { + fn num_matrices(&self) -> usize { + self.mats.len() + } + fn log_height(&self, m: usize) -> usize { + self.mats[m].1 + } + fn width(&self, m: usize) -> usize { + self.mats[m].2 + } + fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>) { + let (data, _log_height, width) = &self.mats[m]; + out.extend_from_slice(&data[bitrev_row * width..(bitrev_row + 1) * width]); + } + } + + fn owned(mats: Vec<(Vec, usize, usize)>) -> OwnedMatrices { + OwnedMatrices { mats } + } + + /// Build a row-major, bit-reversed flat vec from column-major natural-order + /// `columns`, matching the layout the existing trace commit consumes: row `j` + /// of the output = `[col_0[br(j)], ..., col_{w-1}[br(j)]]` with + /// `br = reverse_index(., num_rows)`. + fn row_major_bit_reversed(columns: &[Vec], num_rows: usize) -> Vec { + let width = columns.len(); + let mut out = vec![FE::from(0u64); num_rows * width]; + for (r, chunk) in out.chunks_exact_mut(width).enumerate() { + let br = reverse_index(r, num_rows as u64); + for (c, col) in columns.iter().enumerate() { + chunk[c] = col[br]; + } + } + out + } + + /// Build a row-major flat vec in NATURAL order (no bit reversal): row `r` = + /// `[col_0[r], ..., col_{w-1}[r]]`. This is the layout the prover's + /// `BorrowedMatrix::RowMajorNatural` reads (the retained main/aux LDE buffer). + fn row_major_natural(columns: &[Vec], num_rows: usize) -> Vec { + let width = columns.len(); + let mut out = vec![FE::from(0u64); num_rows * width]; + for (r, chunk) in out.chunks_exact_mut(width).enumerate() { + for (c, col) in columns.iter().enumerate() { + chunk[c] = col[r]; + } + } + out + } + + fn make_columns(width: usize, num_rows: usize, seed: u64) -> Vec> { + (0..width) + .map(|c| { + (0..num_rows) + .map(|r| { + FE::from(seed.wrapping_mul(31) + (c as u64) * 1009 + (r as u64) * 7 + 1) + }) + .collect() + }) + .collect() + } + + #[test] + fn single_matrix_commit_open_verify_and_tamper() { + let log_height = 2usize; + let num_rows = 1usize << log_height; + let width = 3usize; + let columns = make_columns(width, num_rows, 5); + let data = row_major_bit_reversed(&columns, num_rows); + + let src = owned(vec![(data.clone(), log_height, width)]); + let mmcs = Mmcs::commit(&src); + let heights = [log_height]; + let widths = [width]; + let n0 = 1usize << (log_height - 1); + + for iota in 0..n0 { + let opening = mmcs.open_batch(iota, &src); + assert_eq!(opening.per_matrix.len(), 1); + let k = iota; + let row_2k = data[(2 * k) * width..(2 * k + 1) * width].to_vec(); + let row_2k1 = data[(2 * k + 1) * width..(2 * k + 2) * width].to_vec(); + assert_eq!(opening.per_matrix[0].evaluations, row_2k); + assert_eq!(opening.per_matrix[0].evaluations_sym, row_2k1); + assert!(Mmcs::verify_batch( + &mmcs.root(), + iota, + &opening, + &heights, + &widths + )); + } + + let mut opening = mmcs.open_batch(0, &src); + opening.per_matrix[0].evaluations[0] = + &opening.per_matrix[0].evaluations[0] + &FE::from(1u64); + assert!(!Mmcs::verify_batch( + &mmcs.root(), + 0, + &opening, + &heights, + &widths + )); + } + + /// ★ The [`StarkHash`] backward-compatibility statement: a single-matrix MMCS + /// IS the existing per-table row-pair tree. It holds by construction — both + /// go through `H::Batched`'s `hash_data` / `hash_new_parent` — and this + /// pins that no second leaf encoding crept in. + #[test] + fn single_matrix_root_matches_existing_row_pair_tree() { + let log_height = 3usize; + let num_rows = 1usize << log_height; + let width = 4usize; + let columns = make_columns(width, num_rows, 9); + + let (_, existing_root) = + commit_bit_reversed(&columns, 2).expect("non-empty columns build a tree"); + + let data = row_major_bit_reversed(&columns, num_rows); + let mmcs = Mmcs::commit(&owned(vec![(data, log_height, width)])); + + assert_eq!(mmcs.root(), existing_root); + } + + #[test] + fn mixed_height_open_positions_verify_and_tamper() { + // Three matrices, log_heights {5, 5, 3}, widths {2, 1, 4}. + let (ha, hb, hc) = (5usize, 5usize, 3usize); + let (wa, wb, wc) = (2usize, 1usize, 4usize); + let a = row_major_bit_reversed(&make_columns(wa, 1 << ha, 1), 1 << ha); + let b = row_major_bit_reversed(&make_columns(wb, 1 << hb, 2), 1 << hb); + let c = row_major_bit_reversed(&make_columns(wc, 1 << hc, 3), 1 << hc); + + let src = owned(vec![ + (a.clone(), ha, wa), + (b.clone(), hb, wb), + (c.clone(), hc, wc), + ]); + let mmcs = Mmcs::commit(&src); + let heights = [ha, hb, hc]; + let widths = [wa, wb, wc]; + let h_max = 5usize; + let n0 = 1usize << (h_max - 1); // 16 + + let row = |data: &[FE], w: usize, r: usize| data[r * w..(r + 1) * w].to_vec(); + + for iota in [0usize, 1, 2, 3, 7, 8, 13, n0 - 1] { + let opening = mmcs.open_batch(iota, &src); + assert_eq!(opening.per_matrix.len(), 3); + + // Tall matrices open at k = iota >> 0 = iota. + assert_eq!(opening.per_matrix[0].evaluations, row(&a, wa, 2 * iota)); + assert_eq!( + opening.per_matrix[0].evaluations_sym, + row(&a, wa, 2 * iota + 1) + ); + assert_eq!(opening.per_matrix[1].evaluations, row(&b, wb, 2 * iota)); + + // Height-3 matrix opens at k = iota >> (5 - 3) = iota >> 2. + let kc = iota >> (h_max - hc); + assert_eq!(opening.per_matrix[2].evaluations, row(&c, wc, 2 * kc)); + assert_eq!( + opening.per_matrix[2].evaluations_sym, + row(&c, wc, 2 * kc + 1) + ); + + assert!( + Mmcs::verify_batch(&mmcs.root(), iota, &opening, &heights, &widths), + "honest opening at iota={iota} must verify" + ); + } + + // Tamper the height-3 matrix's opened row -> rejection (proves the short + // matrix is bound by the shared path via injection). + let iota = 6usize; + let mut opening = mmcs.open_batch(iota, &src); + opening.per_matrix[2].evaluations[0] = + &opening.per_matrix[2].evaluations[0] + &FE::from(1u64); + assert!( + !Mmcs::verify_batch(&mmcs.root(), iota, &opening, &heights, &widths), + "tampered height-3 row must be rejected" + ); + + // Tamper a tall-matrix row too -> rejection. + let mut opening2 = mmcs.open_batch(iota, &src); + opening2.per_matrix[0].evaluations[0] = + &opening2.per_matrix[0].evaluations[0] + &FE::from(1u64); + assert!( + !Mmcs::verify_batch(&mmcs.root(), iota, &opening2, &heights, &widths), + "tampered tall-matrix row must be rejected" + ); + } + + /// Vector test: hand-compute the root for `{log_height 2, log_height 1}` + /// matrices per the documented layout and assert equality. Pins the + /// leaf/injection contract, plus determinism. + #[test] + fn vector_root_layout_contract_and_determinism() { + // A: log_height 2 (4 rows), width 2 ; B: log_height 1 (2 rows), width 3. + let a_data = row_major_bit_reversed(&make_columns(2, 4, 3), 4); + let b_data = row_major_bit_reversed(&make_columns(3, 2, 8), 2); + + let src = owned(vec![(a_data.clone(), 2, 2), (b_data.clone(), 1, 3)]); + let mmcs = Mmcs::commit(&src); + + // Hand recomputation via the backend primitives, in the documented order. + let arow = |r: usize| a_data[r * 2..(r + 1) * 2].to_vec(); + let brow = |r: usize| b_data[r * 3..(r + 1) * 3].to_vec(); + let h = |v: Vec| { + <::Batched as IsMerkleTreeBackend>::hash_data(&v) + }; + + // Base layer (matrix A only): leaf k = H(A.row(2k) || A.row(2k+1)). + let mut leaf0 = arow(0); + leaf0.extend(arow(1)); + let mut leaf1 = arow(2); + leaf1.extend(arow(3)); + let l00 = h(leaf0); + let l01 = h(leaf1); + + // Climb to layer 1 (root): compress the base pair, then inject B (h=1). + let parent = compress::(&l00, &l01); + let mut binj = brow(0); + binj.extend(brow(1)); + let inj = h(binj); + let expected_root = compress::(&parent, &inj); + + assert_eq!( + mmcs.root(), + expected_root, + "root must match the hand-computed mixed-height layout" + ); + + // Determinism: a second commit over the same inputs yields the same root. + let mmcs2 = Mmcs::commit(&owned(vec![(a_data, 2, 2), (b_data, 1, 3)])); + assert_eq!(mmcs.root(), mmcs2.root(), "commit must be deterministic"); + + for iota in 0..2usize { + let opening = mmcs.open_batch(iota, &src); + // heights {2, 1}, widths {2, 3}. + assert!(Mmcs::verify_batch( + &mmcs.root(), + iota, + &opening, + &[2, 1], + &[2, 3] + )); + } + } + + /// Two SAME-HEIGHT matrices share one base-group leaf, whose hash is over the + /// FLAT concatenation `A.eval ‖ A.eval_sym ‖ B.eval ‖ B.eval_sym`. A malicious + /// prover can shift the A|A_sym boundary (move one element from A's + /// `evaluations_sym` into A's `evaluations`) leaving that flat concatenation — + /// and hence the leaf hash — byte-identical, so a width-blind `verify_batch` + /// would accept it. The per-matrix width binding rejects the shift. + #[test] + fn boundary_shift_forgery_rejected() { + let h = 2usize; + let num_rows = 1usize << h; + let (wa, wb) = (2usize, 1usize); // wA >= 2 so we can steal one column. + let a = row_major_bit_reversed(&make_columns(wa, num_rows, 11), num_rows); + let b = row_major_bit_reversed(&make_columns(wb, num_rows, 22), num_rows); + + let src = owned(vec![(a, h, wa), (b, h, wb)]); + let mmcs = Mmcs::commit(&src); + let heights = [h, h]; + let widths = [wa, wb]; + + let iota = 0usize; + let opening = mmcs.open_batch(iota, &src); + assert!( + Mmcs::verify_batch(&mmcs.root(), iota, &opening, &heights, &widths), + "honest opening must verify" + ); + + // Forge: lengthen A.evaluations by one element taken from A.evaluations_sym. + let mut forged = mmcs.open_batch(iota, &src); + let moved = forged.per_matrix[0].evaluations_sym.remove(0); + forged.per_matrix[0].evaluations.push(moved); + + // The FLAT per-group concatenation is byte-identical to the honest one, so + // the group leaf hash is UNCHANGED — the rejection must come from the width + // check, not from a differing hash. + let flat = |o: &MixedOpening| -> Vec { + let mut v = Vec::new(); + for m in &o.per_matrix { + v.extend_from_slice(&m.evaluations); + v.extend_from_slice(&m.evaluations_sym); + } + v + }; + assert_eq!( + flat(&opening), + flat(&forged), + "the flat concatenation must be byte-identical (boundary-only shift)" + ); + + assert!( + !Mmcs::verify_batch(&mmcs.root(), iota, &forged, &heights, &widths), + "boundary-shift forgery must be rejected by the width binding" + ); + } + + /// Extension-field (Fp3) coverage: the aux and composition matrices an epoch + /// batches are cubic-extension. Byte-parity cross-check of a single Fp3 matrix + /// against the existing per-table row-pair tree, plus an open/verify/tamper + /// roundtrip over the extension path. + #[test] + fn single_matrix_fp3_root_matches_existing_row_pair_tree() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Fp3; + type F3 = FieldElement; + + let log_height = 3usize; + let num_rows = 1usize << log_height; + let width = 3usize; + + // Populate ALL three components so the 24-byte extension serialization is + // exercised (not just the embedded-base subset). + let columns: Vec> = (0..width) + .map(|c| { + (0..num_rows) + .map(|r| { + F3::new([ + FE::from((c as u64) * 7 + r as u64 + 1), + FE::from((r as u64) * 13 + 2), + FE::from((c as u64) * 5 + (r as u64) * 3 + 4), + ]) + }) + .collect() + }) + .collect(); + + let (_, existing_root) = + commit_bit_reversed(&columns, 2).expect("non-empty columns build a tree"); + + // Row-major bit-reversed equivalent of the same column-major data. + let mut data = vec![F3::zero(); num_rows * width]; + for (r, chunk) in data.chunks_exact_mut(width).enumerate() { + let br = reverse_index(r, num_rows as u64); + for (c, col) in columns.iter().enumerate() { + chunk[c] = col[br]; + } + } + + let src = OwnedMatrices { + mats: vec![(data, log_height, width)], + }; + let mmcs = MixedMmcs::::commit(&src); + assert_eq!( + mmcs.root(), + existing_root, + "Fp3 single-matrix root must match the existing row-pair tree" + ); + + let heights = [log_height]; + let widths = [width]; + for iota in 0..(1usize << (log_height - 1)) { + let opening = mmcs.open_batch(iota, &src); + assert!(MixedMmcs::::verify_batch( + &mmcs.root(), + iota, + &opening, + &heights, + &widths + )); + } + + let mut opening = mmcs.open_batch(0, &src); + opening.per_matrix[0].evaluations[0] = &opening.per_matrix[0].evaluations[0] + &F3::one(); + assert!(!MixedMmcs::::verify_batch( + &mmcs.root(), + 0, + &opening, + &heights, + &widths + )); + } + + /// Equivalence (the soundness contract a batched prover relies on): the + /// digest-only MMCS built from borrowed, NATURAL-order leaf sources yields the + /// SAME root and the SAME opened rows as the reference owning source over the + /// bit-reversed data — for the row-major (main / aux) layout, the column-major + /// (composition) layout, AND a main-split column sub-range (`col_start > 0`). + /// Only the leaf-byte source changes; nothing the verifier sees does. + #[test] + fn borrowed_sources_match_owned_reference() { + // Mixed heights {5, 5, 3}; the height-3 matrix exercises injection. + let specs = [(5usize, 3usize, 100u64), (5, 1, 200), (3, 4, 300)]; + + // Column-major natural-order columns per matrix. + let cols: Vec>> = specs + .iter() + .map(|&(lh, w, seed)| make_columns(w, 1 << lh, seed)) + .collect(); + + // Reference: owned, bit-reversed row-major. + let owned_src = owned( + specs + .iter() + .zip(cols.iter()) + .map(|(&(lh, w, _), c)| (row_major_bit_reversed(c, 1 << lh), lh, w)) + .collect(), + ); + + // Borrowed row-major NATURAL (the retained main / aux LDE buffer). + let rm_natural: Vec> = specs + .iter() + .zip(cols.iter()) + .map(|(&(lh, _, _), c)| row_major_natural(c, 1 << lh)) + .collect(); + let rm_src: Vec> = specs + .iter() + .zip(rm_natural.iter()) + .map(|(&(lh, w, _), data)| BorrowedMatrix::RowMajorNatural { + data: data.as_slice(), + stride: w, + col_start: 0, + width: w, + log_height: lh, + }) + .collect(); + + // Borrowed column-major NATURAL (the retained composition-poly LDE). + let cm_src: Vec> = specs + .iter() + .zip(cols.iter()) + .map(|(&(lh, _, _), c)| BorrowedMatrix::ColMajorNatural { + cols: c.as_slice(), + log_height: lh, + }) + .collect(); + + let owned_mmcs = Mmcs::commit(&owned_src); + let rm_mmcs = Mmcs::commit(&rm_src); + let cm_mmcs = Mmcs::commit(&cm_src); + assert_eq!( + owned_mmcs.root(), + rm_mmcs.root(), + "row-major natural root must match the owned reference" + ); + assert_eq!( + owned_mmcs.root(), + cm_mmcs.root(), + "column-major natural root must match the owned reference" + ); + + let n0 = 1usize << (5 - 1); + for iota in 0..n0 { + let o = owned_mmcs.open_batch(iota, &owned_src); + let rm = rm_mmcs.open_batch(iota, &rm_src); + let cm = cm_mmcs.open_batch(iota, &cm_src); + assert_eq!(o.proof.merkle_path, rm.proof.merkle_path); + assert_eq!(o.proof.merkle_path, cm.proof.merkle_path); + for i in 0..specs.len() { + assert_eq!(o.per_matrix[i].evaluations, rm.per_matrix[i].evaluations); + assert_eq!( + o.per_matrix[i].evaluations_sym, + rm.per_matrix[i].evaluations_sym + ); + assert_eq!(o.per_matrix[i].evaluations, cm.per_matrix[i].evaluations); + assert_eq!( + o.per_matrix[i].evaluations_sym, + cm.per_matrix[i].evaluations_sym + ); + } + } + + // Main-split sub-range: a RowMajorNatural over a wider buffer with a + // leading prefix (`col_start = prefix`) must match an owned matrix built + // over ONLY the committed trailing columns. + let (lh, prefix, w) = (4usize, 2usize, 3usize); + let num_rows = 1usize << lh; + let full = make_columns(prefix + w, num_rows, 42); + let full_natural = row_major_natural(&full, num_rows); + let sub_cols: Vec> = full[prefix..].to_vec(); + let sub_owned = owned(vec![(row_major_bit_reversed(&sub_cols, num_rows), lh, w)]); + let split_src: Vec> = + vec![BorrowedMatrix::RowMajorNatural { + data: full_natural.as_slice(), + stride: prefix + w, + col_start: prefix, + width: w, + log_height: lh, + }]; + let sub_owned_mmcs = Mmcs::commit(&sub_owned); + let split_mmcs = Mmcs::commit(&split_src); + assert_eq!( + sub_owned_mmcs.root(), + split_mmcs.root(), + "main-split (col_start>0) root must match the owned sub-range" + ); + for iota in 0..(1usize << (lh - 1)) { + let a = sub_owned_mmcs.open_batch(iota, &sub_owned); + let b = split_mmcs.open_batch(iota, &split_src); + assert_eq!(a.per_matrix[0].evaluations, b.per_matrix[0].evaluations); + assert_eq!( + a.per_matrix[0].evaluations_sym, + b.per_matrix[0].evaluations_sym + ); + } + } + + /// ★ The index-convention control (the module's "HARD PRECONDITION" section). + /// + /// A round whose tallest matrix is SHORTER than the FRI's tallest is the case + /// where the two index conventions disagree: `verify_batch` consumes the LOW + /// `h_max_round - 1` bits of whatever index it is handed, while a matrix + /// inside the tree is located by the HIGH bits of the FRI index. This asserts + /// three things about that case: + /// + /// 1. honest-path control — the correctly reduced index verifies; + /// 2. a tampered row of a SHORT (injected) matrix is rejected, so the low-bits + /// walk really does authenticate the short matrices at the reduced index; + /// 3. handing the un-reduced FRI index straight in is rejected — the misuse is + /// detectable, not silently accepted at some other leaf. + /// + /// A tamper control on the tallest matrix alone would pass under either + /// convention and catch none of this. + #[test] + fn short_round_low_bit_convention_is_exercised() { + // A hypothetical FRI over a 2^6 domain: iota_fri in [0, 2^5). + let h_max_fri = 6usize; + // This round's matrices are shorter: heights {4, 2}. + let (h_tall, h_short) = (4usize, 2usize); + let (w_tall, w_short) = (3usize, 2usize); + let tall = row_major_bit_reversed(&make_columns(w_tall, 1 << h_tall, 77), 1 << h_tall); + let short = row_major_bit_reversed(&make_columns(w_short, 1 << h_short, 88), 1 << h_short); + + let src = owned(vec![(tall, h_tall, w_tall), (short, h_short, w_short)]); + let mmcs = Mmcs::commit(&src); + let heights = [h_tall, h_short]; + let widths = [w_tall, w_short]; + assert_eq!(mmcs.h_max(), h_tall, "the round's h_max is below the FRI's"); + + // The reduction the caller owes: iota_round = iota_fri >> (h_fri - h_round). + let shift = h_max_fri - h_tall; + // Pick a FRI index whose low bits differ from the reduced index's, so the + // two conventions genuinely disagree here. + let iota_fri = 0b10110usize; + let iota_round = iota_fri >> shift; + assert_ne!( + iota_fri & ((1 << (h_tall - 1)) - 1), + iota_round, + "the test index must distinguish the low-bit and high-bit conventions" + ); + + // (1) Honest-path control at the reduced index. + let opening = mmcs.open_batch(iota_round, &src); + assert!( + Mmcs::verify_batch(&mmcs.root(), iota_round, &opening, &heights, &widths), + "the correctly reduced index must verify" + ); + + // (2) Tamper the SHORT (injected) matrix — the matrix a tall-only control + // would never touch, and the one the disagreeing conventions move. + let mut tampered = mmcs.open_batch(iota_round, &src); + tampered.per_matrix[1].evaluations[0] = + &tampered.per_matrix[1].evaluations[0] + &FE::from(1u64); + assert!( + !Mmcs::verify_batch(&mmcs.root(), iota_round, &tampered, &heights, &widths), + "a tampered SHORT-matrix row must be rejected at the reduced index" + ); + + // (3) The misuse: hand the un-reduced FRI index in. It is out of this + // tree's range, so the range guard rejects it rather than walking to some + // unrelated leaf. + assert!( + iota_fri >= 1usize << (h_tall - 1), + "the un-reduced index is outside this round's leaf range" + ); + assert!( + !Mmcs::verify_batch(&mmcs.root(), iota_fri, &opening, &heights, &widths), + "an un-reduced FRI index must be rejected, not accepted at another leaf" + ); + + // And an in-range index that is simply the wrong leaf is rejected too, so + // the guard is not the only thing standing between the two conventions. + let wrong_but_in_range = iota_fri & ((1 << (h_tall - 1)) - 1); + assert!( + !Mmcs::verify_batch( + &mmcs.root(), + wrong_but_in_range, + &opening, + &heights, + &widths + ), + "an opening replayed at the wrong in-range leaf must be rejected" + ); + } + + /// The malformed-input surface of `verify_batch`: every shape error returns + /// `false` rather than panicking, since a verifier calls this on proof data. + #[test] + fn verify_batch_rejects_malformed_shapes_without_panicking() { + let h = 3usize; + let w = 2usize; + let data = row_major_bit_reversed(&make_columns(w, 1 << h, 4), 1 << h); + let src = owned(vec![(data, h, w)]); + let mmcs = Mmcs::commit(&src); + let root = mmcs.root(); + let opening = mmcs.open_batch(1, &src); + + assert!(Mmcs::verify_batch(&root, 1, &opening, &[h], &[w])); + // Mismatched metadata lengths. + assert!(!Mmcs::verify_batch(&root, 1, &opening, &[h, h], &[w])); + assert!(!Mmcs::verify_batch(&root, 1, &opening, &[h], &[w, w])); + // Empty metadata. + assert!(!Mmcs::verify_batch(&root, 1, &opening, &[], &[])); + // A height that would overflow the level shift. + assert!(!Mmcs::verify_batch( + &root, + 1, + &opening, + &[usize::BITS as usize], + &[w] + )); + // An index past this tree's leaf count. + assert!(!Mmcs::verify_batch( + &root, + 1usize << (h - 1), + &opening, + &[h], + &[w] + )); + // A path of the wrong length. + let mut short_path = opening.clone(); + short_path.proof.merkle_path.pop(); + assert!(!Mmcs::verify_batch(&root, 1, &short_path, &[h], &[w])); + } + + /// The memory contract from the module's "what the caller may drop" section, + /// made falsifiable: `commit` reads each height group's rows inside ONE + /// contiguous window of the build, and the windows run in descending height + /// order. A rewrite that materialized every matrix up front, or that revisited + /// a group after moving on, would fail here. + #[test] + fn commit_reads_each_height_group_in_one_contiguous_phase() { + /// Wraps a source and records, per matrix, the first and last global + /// access sequence number. `Mutex` (not `Cell`) because `commit` reads the + /// source from rayon workers. + struct Tracing<'a, E: IsField> { + inner: &'a OwnedMatrices, + clock: AtomicUsize, + window: Mutex>, + } + + impl LeafSource for Tracing<'_, E> { + fn num_matrices(&self) -> usize { + self.inner.num_matrices() + } + fn log_height(&self, m: usize) -> usize { + self.inner.log_height(m) + } + fn width(&self, m: usize) -> usize { + self.inner.width(m) + } + fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec>) { + let t = self.clock.fetch_add(1, Ordering::SeqCst); + let mut w = self.window.lock().expect("no test thread panics here"); + w[m].0 = w[m].0.min(t); + w[m].1 = w[m].1.max(t); + drop(w); + self.inner.append_row(m, bitrev_row, out); + } + } + + // Heights {5, 5, 3, 2}: two groups sharing the base layer, two injected. + let specs = [(5usize, 2usize, 1u64), (5, 3, 2), (3, 1, 3), (2, 4, 4)]; + let inner = owned( + specs + .iter() + .map(|&(lh, w, seed)| { + ( + row_major_bit_reversed(&make_columns(w, 1 << lh, seed), 1 << lh), + lh, + w, + ) + }) + .collect(), + ); + let tracing = Tracing { + inner: &inner, + clock: AtomicUsize::new(0), + window: Mutex::new(vec![(usize::MAX, 0); specs.len()]), + }; + + let traced_root = Mmcs::commit(&tracing).root(); + assert_eq!( + traced_root, + Mmcs::commit(&inner).root(), + "tracing must not change what is committed" + ); + + let windows = tracing + .window + .into_inner() + .expect("uncontended after commit"); + for (m, (first, last)) in windows.iter().enumerate() { + assert!(*first <= *last, "matrix {m} was never read"); + } + + // Same-height matrices share a window; different heights must not overlap, + // and taller groups must come first. + for (m, &(fm, lm)) in windows.iter().enumerate() { + for (n, &(fn_, ln)) in windows.iter().enumerate() { + if specs[m].0 <= specs[n].0 { + continue; + } + assert!( + lm < fn_ || ln < fm, + "matrices {m} (h={}) and {n} (h={}) were read in overlapping \ + windows [{fm},{lm}] / [{fn_},{ln}] — a height group must be \ + readable and then droppable", + specs[m].0, + specs[n].0 + ); + assert!( + lm < fn_, + "the taller matrix {m} (h={}) must be read before the shorter \ + {n} (h={})", + specs[m].0, + specs[n].0 + ); + } + } + } +} diff --git a/crypto/stark/src/fri/mod.rs b/crypto/stark/src/fri/mod.rs index 1f53b51cf..ea125cd95 100644 --- a/crypto/stark/src/fri/mod.rs +++ b/crypto/stark/src/fri/mod.rs @@ -1,14 +1,17 @@ +pub mod batched; pub mod fri_commitment; pub mod fri_decommit; pub(crate) mod fri_functions; +pub mod mmcs; pub(crate) mod terminal; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; +use crypto::merkle_tree::merkle::MerkleTree; use math::field::element::FieldElement; use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; use math::traits::AsBytes; -use crate::config::{FriLayerMerkleTree, FriLayerMerkleTreeBackend}; +use crate::config::StarkHash; use self::fri_commitment::FriLayer; use self::fri_decommit::FriDecommitment; @@ -19,6 +22,14 @@ use self::fri_functions::{fold_evaluations_in_place, update_twiddles_in_place}; /// of degree < 2^`final_poly_log_degree` with blowup 2^`blowup_log`, and /// returns the coefficient vector of that terminal polynomial. /// +/// Layer trees are built with `H::Pair` — the commitment configuration's +/// FRI-layer backend, the same `H` the caller's prover and verifier are +/// instantiated at. That is what makes the layer roots this returns +/// authenticable by [`crate::verifier::IsStarkVerifier::verify`], which +/// re-hashes each opened pair through `H::Batched`: the two are one hash by +/// [`StarkHash`]'s two-element invariant, so agreement is a property of naming +/// one configuration rather than of two call sites happening to match. +/// /// The `T: Clone` and `F/E: 'static` bounds are required by the cuda GPU /// fast path (`try_fri_commit_gpu` snapshots the transcript and TypeId- /// checks the field types). They are present unconditionally (including @@ -28,6 +39,7 @@ pub fn commit_phase_from_evaluations< F: IsFFTField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, T: IsStarkTranscript + Clone, + H: StarkHash, >( mut evals: Vec>, transcript: &mut T, @@ -36,10 +48,7 @@ pub fn commit_phase_from_evaluations< blowup_log: u32, final_poly_log_degree: u32, inv_twiddles: &[FieldElement], -) -> ( - Vec>, - Vec>>, -) +) -> (Vec>, Vec>>) where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, @@ -59,7 +68,7 @@ where // `Some` with the final-polynomial coefficients. It returns `None` on any // precondition miss or cudarc error — restoring the transcript first — so // the CPU path below then runs as if the GPU had never been tried. - if let Some(result) = crate::gpu_lde::try_fri_commit_gpu::( + if let Some(result) = crate::gpu_lde::try_fri_commit_gpu::>( &evals, transcript, coset_offset, @@ -102,7 +111,7 @@ where .chunks_exact(2) .map(|chunk| [chunk[0].clone(), chunk[1].clone()]) .collect(); - let merkle_tree = FriLayerMerkleTree::build(&leaves) + let merkle_tree = MerkleTree::>::build(&leaves) .expect("FRI commit: Merkle tree construction must succeed"); let root = merkle_tree.root; fri_layer_list.push(FriLayer::new(&evals, merkle_tree)); @@ -150,8 +159,14 @@ where (final_poly_coeffs, fri_layer_list) } -pub fn query_phase( - fri_layers: &[FriLayer>], +/// Open every committed layer at each query index, producing one +/// [`FriDecommitment`] per query. +/// +/// Takes the layers [`commit_phase_from_evaluations`] built, so it is generic +/// over the same configuration `H`: the authentication paths it walks are only +/// meaningful against roots the verifier re-derives through `H`. +pub fn query_phase( + fri_layers: &[FriLayer>], iotas: &[usize], ) -> Vec> where @@ -161,7 +176,9 @@ where // layer trees stay resident from the GPU commit). Falls back to the host // walk below if any layer lacks a device tree. #[cfg(feature = "cuda")] - if let Some(decommits) = crate::gpu_lde::try_fri_query_phase_gpu::(fri_layers, iotas) { + if let Some(decommits) = + crate::gpu_lde::try_fri_query_phase_gpu::>(fri_layers, iotas) + { return decommits; } diff --git a/crypto/stark/src/gpu_lde.rs b/crypto/stark/src/gpu_lde.rs index 98830fcc7..5b4c06b0d 100644 --- a/crypto/stark/src/gpu_lde.rs +++ b/crypto/stark/src/gpu_lde.rs @@ -4,6 +4,13 @@ //! back to CPU for extension-field columns and small columns where kernel //! launch overhead dominates. Produces the same natural-order, non-canonical //! LDE evaluations as the CPU path. +//! +//! The tree-building entries here are generic over a Merkle backend `B` that +//! they never call: the leaf and parent hashing happens in the `math-cuda` +//! keccak kernels, and `B` only types the host `MerkleTree` the root is wrapped +//! in. `B` is therefore bound to [`KeccakTreeBackend`] rather than +//! `IsMerkleTreeBackend`, so the label cannot disagree with the kernel that +//! produced the bytes. use core::mem::transmute_copy; use std::any::TypeId; @@ -20,14 +27,13 @@ use math_cuda::{CudaSlice, CudaStream}; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; use crypto::merkle_tree::merkle::MerkleTree; use crypto::merkle_tree::proof::Proof; -use crypto::merkle_tree::traits::IsMerkleTreeBackend; use math::field::element::FieldElement; use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; use math::field::goldilocks::GoldilocksField; use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; use math::traits::AsBytes; -use crate::config::{Commitment, FriLayerMerkleTreeBackend}; +use crate::config::{Commitment, KeccakTreeBackend}; use crate::domain::Domain; use crate::fri::fri_commitment::FriLayer; use crate::fri::fri_decommit::FriDecommitment; @@ -692,7 +698,7 @@ pub(crate) fn try_expand_leaf_and_tree_row_major_keep( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { let lde_size = n.saturating_mul(blowup_factor); if lde_size < gpu_lde_threshold() { @@ -749,7 +755,7 @@ where /// [`MerkleTree`], the exact layout `from_precomputed_nodes` expects. fn tree_from_node_bytes(nodes: Vec) -> Option> where - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { debug_assert_eq!(nodes.len() % 32, 0); let nodes: Vec<[u8; 32]> = nodes @@ -793,7 +799,7 @@ pub(crate) fn try_expand_split_trees_row_major_keep( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { let lde_size = n.saturating_mul(blowup_factor); if lde_size < gpu_lde_threshold() { @@ -875,7 +881,7 @@ pub(crate) fn try_expand_leaf_and_tree_ext3_row_major_keep( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { let lde_size = n.saturating_mul(blowup_factor); if lde_size < gpu_lde_threshold() { @@ -1111,7 +1117,7 @@ pub(crate) fn try_build_comp_poly_tree_gpu( ) -> Option<(MerkleTree, math_cuda::lde::GpuMerkleTree)> where E: IsField + 'static, - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { if lde_parts.is_empty() { return None; @@ -1162,7 +1168,7 @@ pub(crate) fn try_build_comp_poly_tree_gpu_from_dev( ) -> Option<(MerkleTree, math_cuda::lde::GpuMerkleTree)> where E: IsField + 'static, - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { if TypeId::of::() != TypeId::of::() { return None; @@ -1564,7 +1570,7 @@ pub(crate) fn try_expand_leaf_and_tree_ext3_row_major_keep_dev( where F: IsField + 'static, E: IsField + 'static, - B: IsMerkleTreeBackend, + B: KeccakTreeBackend, { if TypeId::of::() != TypeId::of::() || TypeId::of::() != TypeId::of::() @@ -2165,7 +2171,7 @@ where /// it would have produced had the GPU never been tried. This requires the /// concrete transcript type to support snapshot semantics via `Clone`. #[allow(clippy::type_complexity)] -pub(crate) fn try_fri_commit_gpu( +pub(crate) fn try_fri_commit_gpu( evals: &[FieldElement], transcript: &mut T, coset_offset: &FieldElement, @@ -2173,16 +2179,14 @@ pub(crate) fn try_fri_commit_gpu( blowup_log: u32, final_poly_log_degree: u32, inv_twiddles: &[FieldElement], -) -> Option<( - Vec>, - Vec>>, -)> +) -> Option<(Vec>, Vec>)> where F: IsFFTField + IsField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, FieldElement: AsBytes, FieldElement: AsBytes, T: IsStarkTranscript + Clone, + B: KeccakTreeBackend, { // GPU drives the early-termination FRI commit phase, mirroring // `commit_phase_from_evaluations`: for each committed layer (sample zeta, @@ -2227,7 +2231,7 @@ where Err(_) => return None, }; // Host-evals entry: the caller works with host copies, keep draining them. - fri_commit_gpu_drive( + fri_commit_gpu_drive::( state, transcript, coset_offset, @@ -2241,7 +2245,7 @@ where /// [`try_fri_commit_gpu`] entered from a device-resident DEEP codeword /// (already in FRI order): no evals H2D at all. #[allow(clippy::type_complexity)] -pub(crate) fn try_fri_commit_gpu_from_dev( +pub(crate) fn try_fri_commit_gpu_from_dev( codeword: math_cuda::deep::GpuDeepCodeword, transcript: &mut T, coset_offset: &FieldElement, @@ -2249,16 +2253,14 @@ pub(crate) fn try_fri_commit_gpu_from_dev( final_poly_log_degree: u32, inv_twiddles: &[FieldElement], want_host: bool, -) -> Option<( - Vec>, - Vec>>, -)> +) -> Option<(Vec>, Vec>)> where F: IsFFTField + IsField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, FieldElement: AsBytes, FieldElement: AsBytes, T: IsStarkTranscript + Clone, + B: KeccakTreeBackend, { if TypeId::of::() != TypeId::of::() { return None; @@ -2285,7 +2287,7 @@ where Ok(s) => s, Err(_) => return None, }; - fri_commit_gpu_drive( + fri_commit_gpu_drive::( state, transcript, coset_offset, @@ -2301,7 +2303,7 @@ where /// fold and CPU coefficient extraction. Restores the transcript and returns /// `None` on any mid-loop cudarc failure so the CPU path reruns cleanly. #[allow(clippy::type_complexity)] -fn fri_commit_gpu_drive( +fn fri_commit_gpu_drive( mut state: math_cuda::fri::FriCommitState, transcript: &mut T, coset_offset: &FieldElement, @@ -2309,16 +2311,14 @@ fn fri_commit_gpu_drive( blowup_log: u32, final_poly_log_degree: u32, want_host: bool, -) -> Option<( - Vec>, - Vec>>, -)> +) -> Option<(Vec>, Vec>)> where F: IsFFTField + IsField + IsSubFieldOf + 'static, E: IsField + 'static + Send + Sync, FieldElement: AsBytes, FieldElement: AsBytes, T: IsStarkTranscript + Clone, + B: KeccakTreeBackend, { // The unsafe zeta reads below reinterpret `FieldElement` as 3 u64: // every caller gates the tower, but assert here so a future caller with @@ -2350,8 +2350,7 @@ where return None; } let num_committed = layout.num_committed; - let mut fri_layer_list: Vec>> = - Vec::with_capacity(num_committed); + let mut fri_layer_list: Vec> = Vec::with_capacity(num_committed); for _layer_idx in 0..num_committed { // <<<< Receive challenge zeta_k @@ -2376,7 +2375,7 @@ where .map(|v| u64_to_ext3_vec::(&v)) .unwrap_or_default(); let root = dev_tree.root; - let merkle_tree = MerkleTree::>::from_root(root); + let merkle_tree = MerkleTree::::from_root(root); // Retain the device evals only when no host copy exists (device-only): // with a host copy the query phase reads it, and the retained buffer // would be ~24 bytes/LDE-row of dead VRAM per table. @@ -2436,13 +2435,14 @@ where /// /// Returns None when there are no layers or the layers are host trees (CPU /// commit), so the caller falls back to the host walk. -pub(crate) fn try_fri_query_phase_gpu( - fri_layers: &[FriLayer>], +pub(crate) fn try_fri_query_phase_gpu( + fri_layers: &[FriLayer], iotas: &[usize], ) -> Option>> where E: IsField + 'static, FieldElement: AsBytes + Sync + Send, + B: KeccakTreeBackend, { if fri_layers.is_empty() { return None; diff --git a/crypto/stark/src/grinding.rs b/crypto/stark/src/grinding.rs index 4666b7946..c36b29169 100644 --- a/crypto/stark/src/grinding.rs +++ b/crypto/stark/src/grinding.rs @@ -1,5 +1,17 @@ -use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; -use digest::Digest; +//! Proof-of-work grinding, over whichever hash the proof's configuration +//! transcripts with. +//! +//! The construction is two hashes of one block each — 41 bytes inner, 40 bytes +//! outer — so it costs two compressions whichever hash `D` is, and the seed and +//! digest are `[u8; 32]` on both sides. Swapping the hash is therefore a type +//! substitution with no change to the shape of anything: the seed is +//! `transcript.state()`, which is 32 bytes for every transcript configuration. +//! +//! `D` is deliberately a parameter rather than a default: the PoW hash has to +//! be the proof's hash, and a defaulted one would silently keep grinding on +//! keccak for a configuration that had moved everything else. + +use digest::{Digest, OutputSizeUser, typenum::U32}; #[cfg(feature = "parallel")] use rayon::prelude::{IntoParallelIterator, ParallelIterator}; @@ -18,14 +30,17 @@ const PREFIX: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]; /// # Returns /// /// `true` if the number of leading zeros is at least `grinding_factor`, and `false` otherwise. -pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool { +pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool +where + D: Digest + OutputSizeUser, +{ debug_assert!( (1..=64).contains(&grinding_factor), "grinding_factor must be in 1..=64, got {grinding_factor}" ); - let inner_hash = get_inner_hash(seed, grinding_factor); + let inner_hash = get_inner_hash::(seed, grinding_factor); let limit = 1 << (64 - grinding_factor); - is_valid_nonce_for_inner_hash(&inner_hash, nonce, limit) + is_valid_nonce_for_inner_hash::(&inner_hash, nonce, limit) } /// Performs grinding, returning a new nonce for the proof. @@ -42,34 +57,40 @@ pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool /// # Returns /// /// A `nonce` satisfying the required condition. -pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option { +pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser, +{ debug_assert!( (1..=64).contains(&grinding_factor), "grinding_factor must be in 1..=64, got {grinding_factor}" ); - let inner_hash = get_inner_hash(seed, grinding_factor); + let inner_hash = get_inner_hash::(seed, grinding_factor); let limit = 1 << (64 - grinding_factor); #[cfg(not(feature = "parallel"))] return (0..u64::MAX).find(|&candidate_nonce| { - is_valid_nonce_for_inner_hash(&inner_hash, candidate_nonce, limit) + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) }); #[cfg(feature = "parallel")] return (0..u64::MAX).into_par_iter().find_any(|&candidate_nonce| { - is_valid_nonce_for_inner_hash(&inner_hash, candidate_nonce, limit) + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) }); } /// Checks if the leftmost 8 bytes of `Hash(inner_hash || candidate_nonce)` are less than `limit` /// when interpreted as `u64`. #[inline(always)] -fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool { +fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool +where + D: Digest + OutputSizeUser, +{ let mut data = [0; 40]; data[..32].copy_from_slice(inner_hash); data[32..].copy_from_slice(&candidate_nonce.to_be_bytes()); - let digest = Keccak256::digest(data); + let digest = D::digest(data); let seed_head = u64::from_be_bytes(digest[..8].try_into().unwrap()); seed_head < limit @@ -78,12 +99,15 @@ fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, li /// Returns the bit-string constructed as /// Hash(prefix || seed || grinding_factor) /// `prefix` is the bit-string `0x123456789abcded` -fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] { +fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] +where + D: Digest + OutputSizeUser, +{ let mut inner_data = [0u8; 41]; inner_data[0..8].copy_from_slice(&PREFIX); inner_data[8..40].copy_from_slice(seed); inner_data[40] = grinding_factor; - let digest = Keccak256::digest(inner_data); + let digest = D::digest(inner_data); digest[..32].try_into().unwrap() } diff --git a/crypto/stark/src/lib.rs b/crypto/stark/src/lib.rs index 6f8e7c82e..64a836704 100644 --- a/crypto/stark/src/lib.rs +++ b/crypto/stark/src/lib.rs @@ -29,6 +29,7 @@ pub mod profile_markers; pub mod proof; pub mod prover; pub mod r4_denoms; +pub mod residency_mode; #[cfg(feature = "disk-spill")] pub mod storage_mode; pub mod table; diff --git a/crypto/stark/src/lookup.rs b/crypto/stark/src/lookup.rs index ceda5417a..41d7ade83 100644 --- a/crypto/stark/src/lookup.rs +++ b/crypto/stark/src/lookup.rs @@ -838,6 +838,14 @@ pub struct AirWithBuses< /// program (16-25K nodes on the big tables) per epoch/shard instance. constraint_program: std::sync::OnceLock>>, + /// A build-time program supplied via [`Self::with_precaptured`], if any. + /// + /// Kept separate from `constraint_program` rather than pre-filling that + /// `OnceLock`: `precaptured_constraint_program()` must answer "was one + /// SUPPLIED", not "has one been materialized by any means". Sharing the + /// cell would make a capture triggered by an earlier prover call look like + /// a build-time artifact. + precaptured_program: Option>, auxiliary_trace_build_data: AuxiliaryTraceBuildData, boundary_constraint_builder: PhantomData<(B, PI)>, /// Commitment to precomputed columns (if this is a preprocessed table) @@ -874,6 +882,7 @@ impl< meta: self.meta.clone(), num_base: self.num_base, constraint_program: self.constraint_program.clone(), + precaptured_program: self.precaptured_program.clone(), auxiliary_trace_build_data: self.auxiliary_trace_build_data.clone(), boundary_constraint_builder: PhantomData, preprocessed_commitment: self.preprocessed_commitment, @@ -960,6 +969,7 @@ impl< meta, num_base, constraint_program: std::sync::OnceLock::new(), + precaptured_program: None, auxiliary_trace_build_data, boundary_constraint_builder: PhantomData, preprocessed_commitment: None, @@ -995,6 +1005,43 @@ impl< self } + /// Supply a constraint program captured at BUILD time, so this AIR never + /// has to capture one. + /// + /// This is the guest-safe half of the constraint-program story: with a + /// program supplied, both [`AIR::constraint_program`] and + /// [`AIR::precaptured_constraint_program`] hand it back without running the + /// hash-consing capture, which is what makes a constraint program usable on + /// a verify/recursion path at all. + /// + /// The caller is responsible for the program actually being this AIR's. + /// [`ConstraintArtifact::validate_against`] rejects the shape-level + /// mismatches (wrong table, stale widths, changed exemptions); it cannot + /// detect an edit that changes a constraint's arithmetic without changing + /// any shape, which is what the build-time drift test is for. + /// + /// [`ConstraintArtifact::validate_against`]: + /// crate::constraint_ir::ConstraintArtifact::validate_against + pub fn with_precaptured( + mut self, + program: crate::constraint_ir::ConstraintProgram, + ) -> Self { + assert_eq!( + program.roots.len(), + self.meta.len(), + "pre-captured program has {} roots but this AIR has {} transition constraints", + program.roots.len(), + self.meta.len() + ); + assert_eq!( + program.num_base, self.num_base, + "pre-captured program declares num_base {} but this AIR has {}", + program.num_base, self.num_base + ); + self.precaptured_program = Some(program); + self + } + /// Set a debug name for this AIR (for per-table bus sum tracking). /// /// When set, debug output will show bus sums prefixed with this name, @@ -1124,9 +1171,15 @@ where fn constraint_program( &self, ) -> &crate::constraint_ir::ConstraintProgram { - // Lazily captured once (prover/GPU/tests only — the verify path never - // calls this). Runs the table set AND the LogUp emission through one - // CaptureBuilder, matching the folder emission order/indexing exactly. + // A build-time program, if one was supplied, short-circuits capture + // entirely. + if let Some(prog) = &self.precaptured_program { + return prog; + } + // Otherwise lazily captured once (prover/GPU/tests only — the verify + // path never calls this). Runs the table set AND the LogUp emission + // through one CaptureBuilder, matching the folder emission + // order/indexing exactly. self.constraint_program .get_or_init(|| { let mut cb = crate::constraints::builder::CaptureBuilder::::new(); @@ -1138,6 +1191,14 @@ where .as_ref() } + fn precaptured_constraint_program( + &self, + ) -> Option<&crate::constraint_ir::ConstraintProgram> { + // Deliberately NOT `constraint_program.get()`: only a program supplied + // at build time counts, never one a prover run happened to capture. + self.precaptured_program.as_ref() + } + fn build_auxiliary_trace( &self, trace: &mut TraceTable, diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index 4047458bc..c306e9e4b 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -27,12 +27,13 @@ use crate::debug::validate_trace; use crate::fri; use crate::lookup::LOGUP_NUM_CHALLENGES; use crate::proof::stark::{DeepPolynomialOpenings, PolynomialOpenings}; +use crate::residency_mode::ResidencyMode; #[cfg(feature = "disk-spill")] use crate::storage_mode::StorageMode; use crate::table::Table; use crate::trace::LDETraceTable; -use super::config::{BatchedMerkleTree, BatchedMerkleTreeBackend, Commitment}; +use super::config::{Commitment, KeccakStarkHash, StarkHash}; use super::constraints::evaluator::ConstraintEvaluator; use super::domain::Domain; use super::fri::fri_decommit::FriDecommitment; @@ -41,8 +42,10 @@ use super::lookup::BusPublicInputs; use super::proof::stark::{DeepPolynomialOpening, MultiProof, StarkProof}; use super::trace::TraceTable; use super::traits::AIR; +use crypto::merkle_tree::merkle::MerkleTree; #[cfg(feature = "cuda")] use crypto::merkle_tree::proof::Proof; +use crypto::merkle_tree::traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}; pub use crate::commitment::{keccak_leaves_bit_reversed, keccak_leaves_row_pair_bit_reversed}; @@ -53,20 +56,32 @@ type AirTracePair<'a, Field, FieldExtension, PI> = ( &'a PI, ); -/// A default STARK prover implementing `IsStarkProver`. -pub struct Prover< +/// A default STARK prover implementing `IsStarkProver`, generic over the +/// commitment configuration `H`. +/// +/// `H` rides on the concrete type rather than defaulting on the trait: a +/// defaulted trait parameter would be uninferable at a bare +/// `Prover::multi_prove(..)` call, whereas an alias pins it. That is what keeps +/// every existing call site resolving unchanged — see [`Prover`]. +pub struct GenericProver< Field: IsSubFieldOf + IsFFTField + Send + Sync, FieldExtension: Send + Sync + IsField, PI, + H, > { - p: PhantomData<(Field, FieldExtension, PI)>, + p: PhantomData<(Field, FieldExtension, PI, H)>, } +/// The production prover: [`GenericProver`] at the keccak configuration. +pub type Prover = + GenericProver; + impl< Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, FieldExtension: Send + Sync + IsField + 'static, PI, -> IsStarkProver for Prover + H: StarkHash, +> IsStarkProver for GenericProver where FieldElement: math::traits::ByteConversion, FieldElement: math::traits::ByteConversion, @@ -106,28 +121,28 @@ impl From for ProvingError { /// separate Merkle tree over their precomputed columns, hence the optional /// `precomputed_tree`/`precomputed_root` pair and the `num_precomputed_cols` /// index used when opening positions. -pub(crate) struct TableCommit +pub(crate) struct TableCommit where - FieldElement: AsBytes, + FieldElement: AsBytes + Sync + Send, { /// Merkle tree over the trace columns (multiplicities only for preprocessed tables). - pub(crate) tree: Arc>, + pub(crate) tree: Arc>>, /// Root of `tree`. pub(crate) root: Commitment, /// Preprocessed tables only: Merkle tree over precomputed columns. - pub(crate) precomputed_tree: Option>>, + pub(crate) precomputed_tree: Option>>>, /// Preprocessed tables only: root of `precomputed_tree`. pub(crate) precomputed_root: Option, /// Preprocessed tables only: number of precomputed columns. Zero otherwise. pub(crate) num_precomputed_cols: usize, } -impl TableCommit +impl TableCommit where - FieldElement: AsBytes, + FieldElement: AsBytes + Sync + Send, { /// Build a `TableCommit` for a plain (non-preprocessed) table. - fn plain(tree: BatchedMerkleTree, root: Commitment) -> Self { + fn plain(tree: MerkleTree>, root: Commitment) -> Self { Self { tree: Arc::new(tree), root, @@ -141,9 +156,9 @@ where /// arrives as an `Arc` because it may be shared from the process-wide /// cache (see [`precomputed_tree_cache_get`]). fn preprocessed( - tree: BatchedMerkleTree, + tree: MerkleTree>, root: Commitment, - precomputed_tree: Arc>, + precomputed_tree: Arc>>, precomputed_root: Commitment, num_precomputed_cols: usize, ) -> Self { @@ -188,25 +203,20 @@ fn precomputed_tree_cache() CACHE.get_or_init(|| Mutex::new(std::collections::HashMap::new())) } -fn precomputed_tree_cache_get( +fn precomputed_tree_cache_get( root: &Commitment, -) -> Option>> -where - FieldElement: AsBytes, -{ +) -> Option>> { let cache = precomputed_tree_cache().lock().unwrap(); cache .get(root) .cloned() - .and_then(|any| any.downcast::>().ok()) + .and_then(|any| any.downcast::>().ok()) } -fn precomputed_tree_cache_put( +fn precomputed_tree_cache_put( root: Commitment, - tree: Arc>, -) where - FieldElement: AsBytes, -{ + tree: Arc>, +) { precomputed_tree_cache() .lock() .unwrap() @@ -214,19 +224,20 @@ fn precomputed_tree_cache_put( } /// A container for the results of the first round of the STARK Prove protocol. -pub(crate) struct Round1 +pub(crate) struct Round1 where - Field: IsSubFieldOf + IsFFTField, - FieldExtension: IsField, - FieldElement: AsBytes, - FieldElement: AsBytes, + Field: IsSubFieldOf + IsFFTField + 'static, + FieldExtension: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { /// The table of evaluations over the LDE of the main and auxiliary trace tables. pub(crate) lde_trace: LDETraceTable, /// Commitment to the main trace. - pub(crate) main: TableCommit, + pub(crate) main: TableCommit, /// Commitment to the auxiliary (RAP) trace, if any. - pub(crate) aux: Option>, + pub(crate) aux: Option>, /// The challenges of the RAP round. pub(crate) rap_challenges: Vec>, /// Bus interaction public inputs (initial and final aux column values). @@ -237,25 +248,26 @@ where /// and (under cuda) the optional device LDE buffer kept alive for downstream /// rounds when the R1 fused GPU pipeline ran. #[cfg(feature = "cuda")] -type MainCommitTuple = ( - TableCommit, +type MainCommitTuple = ( + TableCommit, (Vec>, usize), Option, ); #[cfg(not(feature = "cuda"))] -type MainCommitTuple = (TableCommit, (Vec>, usize)); +type MainCommitTuple = (TableCommit, (Vec>, usize)); /// Round 1 commitment artifacts — Merkle trees, roots, challenges, and bus inputs. /// Borrowed (not consumed) when building `Round1`. -pub(crate) struct Round1Commitments +pub(crate) struct Round1Commitments where - Field: IsFFTField + IsSubFieldOf, - FieldExtension: IsField, - FieldElement: AsBytes, - FieldElement: AsBytes, + Field: IsFFTField + IsSubFieldOf + 'static, + FieldExtension: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { - main: TableCommit, - aux: Option>, + main: TableCommit, + aux: Option>, rap_challenges: Vec>, bus_public_inputs: Option>, } @@ -286,12 +298,26 @@ struct Lde { gpu_aux: Option, } -impl Round1Commitments +/// A table's Round-1 main LDE, held between the main commit and the table's +/// fused task. +/// +/// `Dropped` is the `ResidencyMode::RecomputeLde` state. It carries no buffer +/// at all, so a consumer added between Round 1 and the fused task cannot read +/// empty data believing it is an LDE — it has to handle the recompute arm or +/// fail to compile. That is the loud guard for the one real risk in dropping +/// the buffer: a retention point the audit missed. +enum MainLdeSlot { + Retained((Vec>, usize)), + Dropped { num_cols: usize }, +} + +impl Round1Commitments where - Field: IsFFTField + IsSubFieldOf + Send + Sync, - FieldExtension: IsField + Send + Sync, - FieldElement: AsBytes, - FieldElement: AsBytes, + Field: IsFFTField + IsSubFieldOf + Send + Sync + 'static, + FieldExtension: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { /// Build a `Round1` by consuming a `Lde` and borrowing commitment data. /// The `TableCommit::share` calls are cheap — only bump Arc refcounts. @@ -300,7 +326,7 @@ where lde: Lde, step_size: usize, blowup_factor: usize, - ) -> Round1 { + ) -> Round1 { let (main_data, num_main_cols) = lde.main; let (aux_data, num_aux_cols) = lde.aux; @@ -729,15 +755,16 @@ fn heaviest_first(estimates: &[u64]) -> Vec { } /// A container for the results of the second round of the STARK Prove protocol. -pub(crate) struct Round2 +pub(crate) struct Round2 where - F: IsField, - FieldElement: AsBytes, + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send, + H: StarkHash, { /// Evaluations of the composition polynomial parts over the LDE domain. pub(crate) lde_composition_poly_evaluations: Vec>>, /// The Merkle tree built to compute the commitment to the composition polynomial parts. - pub(crate) composition_poly_merkle_tree: BatchedMerkleTree, + pub(crate) composition_poly_merkle_tree: MerkleTree>, /// The commitment to the composition polynomial parts. pub(crate) composition_poly_root: Commitment, /// The composition Merkle tree kept resident on device (when the R2 GPU tree @@ -808,6 +835,7 @@ pub trait IsStarkProver< Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, FieldExtension: Send + Sync + IsField + 'static, PI, + H: StarkHash, > where FieldElement: math::traits::ByteConversion, FieldElement: math::traits::ByteConversion, @@ -820,7 +848,7 @@ pub trait IsStarkProver< fn commit_rows_bit_reversed( data: &[FieldElement], num_cols: usize, - ) -> Option<(BatchedMerkleTree, Commitment)> + ) -> Option<(MerkleTree>, Commitment)> where FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, E: IsField, @@ -837,7 +865,7 @@ pub trait IsStarkProver< num_cols: usize, col_start: usize, col_end: usize, - ) -> Option<(BatchedMerkleTree, Commitment)> + ) -> Option<(MerkleTree>, Commitment)> where FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, E: IsField, @@ -876,7 +904,7 @@ pub trait IsStarkProver< offset += byte_len; } } - BatchedMerkleTreeBackend::::hash_bytes(buf) + as IsStreamingLeafBackend>::hash_bytes(buf) }; #[cfg(feature = "parallel")] @@ -895,7 +923,7 @@ pub trait IsStarkProver< .collect() }; - let tree = BatchedMerkleTree::::build_from_hashed_leaves(hashed_leaves)?; + let tree = MerkleTree::>::build_from_hashed_leaves(hashed_leaves)?; let root = tree.root; Some((tree, root)) } @@ -922,8 +950,10 @@ pub trait IsStarkProver< let twiddles = LdeTwiddles::new(&domain); let evals = Self::compute_lde_from_columns_cached::(&precomputed, &domain, &twiddles); - let (_, commitment) = - crate::commitment::commit_bit_reversed(&evals, crate::commitment::ROWS_PER_LEAF)?; + let (_, commitment) = crate::commitment::commit_bit_reversed_with::< + Field, + H::Batched, + >(&evals, crate::commitment::ROWS_PER_LEAF)?; Some(commitment) } @@ -1061,18 +1091,22 @@ pub trait IsStarkProver< precomputed: Option<(Commitment, usize)>, #[cfg(feature = "cuda")] device_only: bool, #[cfg(feature = "disk-spill")] storage_mode: StorageMode, - ) -> Result, ProvingError> + #[cfg_attr(not(feature = "cuda"), allow(unused_variables))] residency: ResidencyMode, + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, { - let lde_size = domain.interpolation_domain_size * domain.blowup_factor; - // Fused GPU path (cuda only): row-major NTT — single H2D from the // already-row-major trace, no column extraction, no transpose. // Falls back to CPU if GPU path returns None. + // + // `RecomputeLde` skips both device paths: the LDE it drops after this + // commit is recomputed on the host, so the buffer the tree was built + // from must be the host one. Same posture as disk-spill — the mode is + // for CPU proving and forces the host path per table. #[cfg(feature = "cuda")] - if precomputed.is_none() { + if precomputed.is_none() && !residency.recomputes_main_lde() { let (trace_slice, num_cols) = trace.main_data_row_major(); let n = if num_cols > 0 { trace_slice.len() / num_cols @@ -1085,7 +1119,7 @@ pub trait IsStarkProver< crate::gpu_lde::try_expand_leaf_and_tree_row_major_keep::< Field, Field, - BatchedMerkleTreeBackend, + H::Batched, >( trace_slice, n, @@ -1125,7 +1159,9 @@ pub trait IsStarkProver< // are gathered on device. The handle keeps the LDE device-resident for // the downstream GPU rounds. #[cfg(feature = "cuda")] - if let Some((expected_precomputed_root, num_precomputed)) = precomputed { + if let Some((expected_precomputed_root, num_precomputed)) = precomputed + && !residency.recomputes_main_lde() + { let (trace_slice, num_cols) = trace.main_data_row_major(); let n = if num_cols > 0 { trace_slice.len() / num_cols @@ -1137,7 +1173,9 @@ pub trait IsStarkProver< #[cfg(not(feature = "disk-spill"))] let cache_ok = true; let cached_pre = cache_ok - .then(|| precomputed_tree_cache_get::(&expected_precomputed_root)) + .then(|| { + precomputed_tree_cache_get::>(&expected_precomputed_root) + }) .flatten(); #[cfg(feature = "instruments")] let t_sub = Instant::now(); @@ -1145,7 +1183,7 @@ pub trait IsStarkProver< crate::gpu_lde::try_expand_split_trees_row_major_keep::< Field, Field, - BatchedMerkleTreeBackend, + H::Batched, >( trace_slice, n, @@ -1172,7 +1210,7 @@ pub trait IsStarkProver< Self::spill_tree(&mut tree, storage_mode, "precomputed Merkle tree")?; let tree = Arc::new(tree); if cache_ok { - precomputed_tree_cache_put::( + precomputed_tree_cache_put::>( expected_precomputed_root, Arc::clone(&tree), ); @@ -1201,28 +1239,16 @@ pub trait IsStarkProver< // (one memcpy — no transpose) and expand in place with the cache-blocked // batched two-half FFT. Row-major end-to-end: no LDE-size transpose, // contiguous Merkle leaves. - let (trace_data, total_cols) = trace.main_data_row_major(); - #[cfg(feature = "instruments")] let t_sub = Instant::now(); - let mut main_data: Vec> = Vec::with_capacity(lde_size * total_cols); - main_data.extend_from_slice(trace_data); - - #[cfg(feature = "disk-spill")] - if storage_mode == StorageMode::Disk { - trace.main_table.advise_drop_cache(); - } - - Polynomial::>::coset_lde_full_expand_row_major::( - &mut main_data, - total_cols, - domain.blowup_factor, - &twiddles.coset_weights, - &twiddles.two_half_inv, - &twiddles.two_half_fwd, - ) - .expect("row-major coset LDE expansion"); + let (main_data, total_cols) = Self::expand_main_lde_row_major( + trace, + domain, + twiddles, + #[cfg(feature = "disk-spill")] + storage_mode, + ); #[cfg(feature = "instruments")] let main_lde_dur = t_sub.elapsed(); @@ -1251,7 +1277,9 @@ pub trait IsStarkProver< #[cfg(not(feature = "disk-spill"))] let cache_ok = true; let precomputed_tree = match cache_ok - .then(|| precomputed_tree_cache_get::(&expected_precomputed_root)) + .then(|| { + precomputed_tree_cache_get::>(&expected_precomputed_root) + }) .flatten() { // Cache key == the root a rebuild would be verified @@ -1273,7 +1301,7 @@ pub trait IsStarkProver< Self::spill_tree(&mut tree, storage_mode, "precomputed Merkle tree")?; let tree = Arc::new(tree); if cache_ok { - precomputed_tree_cache_put::( + precomputed_tree_cache_put::>( expected_precomputed_root, Arc::clone(&tree), ); @@ -1310,12 +1338,50 @@ pub trait IsStarkProver< Ok((commit, (main_data, total_cols))) } + /// Expand a table's main trace to its coset LDE, row-major, without + /// building any Merkle tree. + /// + /// The Round-1 CPU commit and the `ResidencyMode::RecomputeLde` recompute + /// both go through here, which is what makes the recomputed buffer + /// bit-identical to the one the tree was built from — identical by + /// construction rather than by argument. The twiddles are process-cached, + /// so the second call re-runs the NTT over the same inputs. + fn expand_main_lde_row_major( + trace: &TraceTable, + domain: &Domain, + twiddles: &LdeTwiddles, + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + ) -> (Vec>, usize) { + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + let (trace_data, total_cols) = trace.main_data_row_major(); + + let mut main_data: Vec> = Vec::with_capacity(lde_size * total_cols); + main_data.extend_from_slice(trace_data); + + #[cfg(feature = "disk-spill")] + if storage_mode == StorageMode::Disk { + trace.main_table.advise_drop_cache(); + } + + Polynomial::>::coset_lde_full_expand_row_major::( + &mut main_data, + total_cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .expect("row-major coset LDE expansion"); + + (main_data, total_cols) + } + /// Spill a committed Merkle tree to disk when `storage_mode` is `Disk`, /// tagging any I/O error with `label`. No-op otherwise. Shared by every commit /// site (main / preprocessed split / aux). #[cfg(feature = "disk-spill")] fn spill_tree( - tree: &mut BatchedMerkleTree, + tree: &mut MerkleTree>, storage_mode: StorageMode, label: &str, ) -> Result<(), ProvingError> @@ -1339,9 +1405,9 @@ pub trait IsStarkProver< air: &dyn AIR, trace: &TraceTable, domain: &Domain, - commitment: &Round1Commitments, + commitment: &Round1Commitments, twiddles: &LdeTwiddles, - ) -> Result, ProvingError> + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, @@ -1415,7 +1481,7 @@ pub trait IsStarkProver< #[cfg(feature = "debug-checks")] fn run_debug_checks( pair_cells: &[std::sync::Mutex>], - commitments: &[Round1Commitments], + commitments: &[Round1Commitments], domains: &[Arc>], twiddle_caches: &[Arc>], ) where @@ -1423,7 +1489,7 @@ pub trait IsStarkProver< FieldElement: AsBytes, PI: Send + Sync + Clone, { - let mut temp_results: Vec> = + let mut temp_results: Vec> = Vec::with_capacity(pair_cells.len()); for ((cell, commitment), (domain, twiddles)) in pair_cells .iter() @@ -1555,10 +1621,10 @@ pub trait IsStarkProver< pub_inputs: &PI, domain: &Domain, twiddles: &LdeTwiddles, - round_1_result: &mut Round1, + round_1_result: &mut Round1, transition_coefficients: &[FieldElement], boundary_coefficients: &[FieldElement], - ) -> Result, ProvingError> + ) -> Result, ProvingError> where FieldElement: AsBytes, FieldElement: AsBytes, @@ -1741,13 +1807,13 @@ pub trait IsStarkProver< .and_then(|h| { crate::gpu_lde::try_build_comp_poly_tree_gpu_from_dev::< FieldExtension, - BatchedMerkleTreeBackend, + H::Batched, >(h) }) .or_else(|| { crate::gpu_lde::try_build_comp_poly_tree_gpu::< FieldExtension, - BatchedMerkleTreeBackend, + H::Batched, >(&lde_composition_poly_parts_evaluations) }) { Some((host_tree, dev_tree)) => { @@ -1770,7 +1836,10 @@ pub trait IsStarkProver< "R2 composition commit fell back to the host part evals, \ but they are device-only (empty)" ); - let (tree, root) = crate::commitment::commit_bit_reversed( + let (tree, root) = crate::commitment::commit_bit_reversed_with::< + FieldExtension, + H::Batched, + >( &lde_composition_poly_parts_evaluations, crate::commitment::ROWS_PER_LEAF, ) @@ -1780,7 +1849,7 @@ pub trait IsStarkProver< }; #[cfg(not(feature = "cuda"))] let (composition_poly_merkle_tree, composition_poly_root) = - crate::commitment::commit_bit_reversed( + crate::commitment::commit_bit_reversed_with::>( &lde_composition_poly_parts_evaluations, crate::commitment::ROWS_PER_LEAF, ) @@ -1811,8 +1880,8 @@ pub trait IsStarkProver< fn round_3_evaluate_polynomials_in_out_of_domain_element( air: &dyn AIR, domain: &Domain, - round_1_result: &Round1, - round_2_result: &Round2, + round_1_result: &Round1, + round_2_result: &Round2, z: &FieldElement, ) -> Round3 where @@ -1947,8 +2016,8 @@ pub trait IsStarkProver< fn round_4_compute_and_run_fri_on_the_deep_composition_polynomial( air: &dyn AIR, domain: &Domain, - round_1_result: &Round1, - round_2_result: &Round2, + round_1_result: &Round1, + round_2_result: &Round2, round_3_result: &Round3, z: &FieldElement, transcript: &mut (impl IsStarkTranscript + Clone), @@ -2005,7 +2074,12 @@ pub trait IsStarkProver< &trace_term_coeffs, ) .and_then(|dw| { - crate::gpu_lde::try_fri_commit_gpu_from_dev( + crate::gpu_lde::try_fri_commit_gpu_from_dev::< + Field, + FieldExtension, + _, + H::Pair, + >( dw, transcript, &coset_offset, @@ -2019,12 +2093,7 @@ pub trait IsStarkProver< #[allow(clippy::type_complexity)] let precomputed_fri: Option<( Vec>, - Vec< - crate::fri::fri_commitment::FriLayer< - FieldExtension, - crate::config::FriLayerMerkleTreeBackend, - >, - >, + Vec>>, )> = None; #[cfg(feature = "instruments")] let mut other_dur_1 = t_sub.elapsed(); @@ -2068,7 +2137,7 @@ pub trait IsStarkProver< // FRI commit phase from pre-computed evaluations #[cfg(feature = "instruments")] let t_sub = Instant::now(); - let res = fri::commit_phase_from_evaluations( + let res = fri::commit_phase_from_evaluations::( lde_evals, transcript, &coset_offset, @@ -2090,8 +2159,11 @@ pub trait IsStarkProver< let security_bits = air.context().proof_options.grinding_factor; let mut nonce = None; if security_bits > 0 { - let nonce_value = grinding::generate_nonce(&transcript.state(), security_bits) - .expect("nonce not found"); + let nonce_value = grinding::generate_nonce::>( + &transcript.state(), + security_bits, + ) + .expect("nonce not found"); transcript.append_bytes(&nonce_value.to_be_bytes()); nonce = Some(nonce_value); } @@ -2099,7 +2171,7 @@ pub trait IsStarkProver< let number_of_queries = air.options().fri_number_of_queries; let iotas = Self::sample_query_indexes(number_of_queries, domain, transcript); - let query_list = fri::query_phase(&fri_layers, &iotas); + let query_list = fri::query_phase::(&fri_layers, &iotas); let fri_layers_merkle_roots: Vec<_> = fri_layers .iter() @@ -2152,7 +2224,7 @@ pub trait IsStarkProver< #[allow(clippy::too_many_arguments)] fn try_compute_deep_dev( lde_trace: &LDETraceTable, - round_2_result: &Round2, + round_2_result: &Round2, round_3_result: &Round3, z: &FieldElement, domain: &Domain, @@ -2202,7 +2274,7 @@ pub trait IsStarkProver< #[allow(clippy::too_many_arguments)] fn compute_deep_composition_poly_evaluations( lde_trace: &LDETraceTable, - round_2_result: &Round2, + round_2_result: &Round2, round_3_result: &Round3, z: &FieldElement, domain: &Domain, @@ -2392,7 +2464,7 @@ pub trait IsStarkProver< /// at the domain value corresponding to the FRI query challenge `index` and its symmetric /// element. fn open_composition_poly( - composition_poly_merkle_tree: &BatchedMerkleTree, + composition_poly_merkle_tree: &MerkleTree>, lde_composition_poly_evaluations: &[Vec>], index: usize, ) -> PolynomialOpenings @@ -2474,7 +2546,7 @@ pub trait IsStarkProver< /// storage (full main row, ranged main row, or aux row). fn open_polys_with( domain: &Domain, - tree: &BatchedMerkleTree, + tree: &MerkleTree>, challenge: usize, gather: G, ) -> PolynomialOpenings @@ -2601,7 +2673,7 @@ pub trait IsStarkProver< lde_trace: &LDETraceTable, dev_proofs: Option<&Vec>>, dev_values: Option<&Vec>>, - tree: &BatchedMerkleTree, + tree: &MerkleTree>, qi: usize, challenge: usize, ncols: usize, @@ -2658,8 +2730,8 @@ pub trait IsStarkProver< /// Open the deep composition polynomial on a list of indexes and their symmetric elements. fn open_deep_composition_poly( domain: &Domain, - round_1_result: &Round1, - round_2_result: &Round2, + round_1_result: &Round1, + round_2_result: &Round2, indexes_to_open: &[usize], ) -> DeepPolynomialOpenings where @@ -3033,6 +3105,7 @@ pub trait IsStarkProver< #[allow(unused_mut)] mut air_trace_pairs: Vec>, transcript: &mut (impl IsStarkTranscript + Clone + Send), #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + residency: ResidencyMode, ) -> Result, ProvingError> where FieldElement: AsBytes, @@ -3045,6 +3118,16 @@ pub trait IsStarkProver< { info!("Started proof generation..."); + // `debug-checks` reconstructs every table's Round 1 from retained state + // between the aux and rounds stages, so the recompute mode's dropped + // buffers have no meaning there. Forcing `Retain` keeps the debug build + // checking what it always checked. + #[cfg(feature = "debug-checks")] + let residency = { + let _ = residency; + ResidencyMode::Retain + }; + #[cfg(feature = "instruments")] crate::instruments::reset_all(); #[cfg(feature = "instruments")] @@ -3141,8 +3224,8 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_main_commit"); - let mut main_commits: Vec> = Vec::with_capacity(num_airs); - let mut main_ldes: Vec<(Vec>, usize)> = Vec::with_capacity(num_airs); + let mut main_commits: Vec> = Vec::with_capacity(num_airs); + let mut main_ldes: Vec> = Vec::with_capacity(num_airs); // Optional device-side LDE handle per table, populated only when the // R1 fused GPU pipeline produced one. Pairing is by index: this vector // is moved into the per-table `gpu_main_cells` mutex slots below, and @@ -3184,6 +3267,7 @@ pub trait IsStarkProver< device_only, #[cfg(feature = "disk-spill")] storage_mode, + residency, ) }, ); @@ -3198,7 +3282,15 @@ pub trait IsStarkProver< } transcript.append_bytes(&commit.root); main_commits.push(commit); - main_ldes.push(cached_main); + // The root is in the transcript; that is all Fiat-Shamir asks of + // this phase. Under `RecomputeLde` the buffer it was built from + // dies here and the table's fused task rebuilds it from the trace. + main_ldes.push(match residency { + ResidencyMode::Retain => MainLdeSlot::Retained(cached_main), + ResidencyMode::RecomputeLde => MainLdeSlot::Dropped { + num_cols: cached_main.1, + }, + }); #[cfg(feature = "cuda")] main_gpu_handles.push(gpu_main); } @@ -3245,6 +3337,16 @@ pub trait IsStarkProver< } } + // `RecomputeLde` already forced the main commit onto the host path; + // keeping the aux build there too makes the mode wholly host-side, which + // is what its aux release at the end of each fused task acts on. + #[cfg(feature = "cuda")] + if residency.recomputes_main_lde() { + for (_, trace, _) in air_trace_pairs.iter_mut() { + trace.set_resident_aux_ok(false); + } + } + // Thread each table's device-resident trace-domain main columns (kept by // the R1 main LDE) onto its trace so the LogUp aux fingerprint kernel // reads them in place instead of re-uploading ~3 GB. Preprocessed tables @@ -3275,13 +3377,13 @@ pub trait IsStarkProver< // so the handle stays inside its own table's task and never needs a // separate handle vector. #[cfg(feature = "cuda")] - type AuxResult = ( - Option>, + type AuxResult = ( + Option>, (Vec>, usize), Option, ); #[cfg(not(feature = "cuda"))] - type AuxResult = (Option>, (Vec>, usize)); + type AuxResult = (Option>, (Vec>, usize)); // R1 aux commit and rounds 2 to 4 share the peak working set: the main // and aux LDEs are co-resident, plus the composition and Merkle // transients (in the scratch factor). The aux width comes from the AIR @@ -3303,14 +3405,11 @@ pub trait IsStarkProver< .into_iter() .map(std::sync::Mutex::new) .collect(); - let main_commit_cells: Vec>>> = main_commits + let main_commit_cells: Vec>>> = main_commits .into_iter() .map(|c| std::sync::Mutex::new(Some(c))) .collect(); - #[allow(clippy::type_complexity)] - let main_lde_cells: Vec< - std::sync::Mutex>, usize)>>, - > = main_ldes + let main_lde_cells: Vec>>> = main_ldes .into_iter() .map(|l| std::sync::Mutex::new(Some(l))) .collect(); @@ -3335,7 +3434,7 @@ pub trait IsStarkProver< #[allow(clippy::type_complexity)] let aux_stage = |idx: usize| -> Result< ( - Round1Commitments, + Round1Commitments, Lde, ), ProvingError, @@ -3374,8 +3473,8 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_aux_commit_table"); - let aux_full: AuxResult = - (|| -> Result, ProvingError> { + let aux_full: AuxResult = + (|| -> Result, ProvingError> { if air.has_aux_trace() { let lde_size = domain.interpolation_domain_size * domain.blowup_factor; @@ -3399,7 +3498,7 @@ pub trait IsStarkProver< crate::gpu_lde::try_expand_leaf_and_tree_ext3_row_major_keep_dev::< Field, FieldExtension, - BatchedMerkleTreeBackend, + H::Batched, >( ra, domain.blowup_factor, @@ -3439,7 +3538,7 @@ pub trait IsStarkProver< crate::gpu_lde::try_expand_leaf_and_tree_ext3_row_major_keep::< Field, FieldExtension, - BatchedMerkleTreeBackend, + H::Batched, >( trace_slice, n, @@ -3532,11 +3631,33 @@ pub trait IsStarkProver< .unwrap() .take() .expect("main commit consumed once per table"); - let main_lde = main_lde_cells[idx] + let main_lde = match main_lde_cells[idx] .lock() .unwrap() .take() - .expect("main lde consumed once per table"); + .expect("main lde consumed once per table") + { + MainLdeSlot::Retained(lde) => lde, + // The Merkle tree was kept, so this is one forward NTT and no + // re-hashing: the root openings are checked against is still + // the root Round 1 absorbed. The buffer dies with this task. + MainLdeSlot::Dropped { num_cols } => { + #[cfg(feature = "instruments")] + let __sp_recompute = crate::instruments::span("r1_main_lde_recompute_table"); + let recomputed = Self::expand_main_lde_row_major( + &**trace, + domain, + twiddles, + #[cfg(feature = "disk-spill")] + storage_mode, + ); + assert_eq!( + recomputed.1, num_cols, + "recomputed main LDE width must match the committed one" + ); + recomputed + } + }; #[cfg(feature = "cuda")] let gpu_main = gpu_main_cells[idx].lock().unwrap().take(); let commitment = Round1Commitments { @@ -3563,10 +3684,10 @@ pub trait IsStarkProver< // Fused chain, stage 2: Round1 from the cached LDE (consumed by value, // no recomputation) → rounds 2-4 against the table's transcript fork. let rounds_stage = |idx: usize, - commitment: Round1Commitments, + commitment: Round1Commitments, lde: Lde| -> Result, ProvingError> { - let pair = pair_cells[idx].lock().unwrap(); + let mut pair = pair_cells[idx].lock().unwrap(); let (air, trace, pub_inputs) = &*pair; let _ = trace; // used by instruments let domain = &domains[idx]; @@ -3603,6 +3724,15 @@ pub trait IsStarkProver< sub_ops, )); } + + // Phase B of the recompute contract: this table's proof exists, so + // its aux columns are dead weight for the rest of the prove. They + // live in the caller's trace and would otherwise survive to the end + // of `multi_prove` — the second-largest per-table retention after + // the main LDE. + if residency.recomputes_main_lde() { + pair.1.release_aux_columns(); + } Ok(proof) }; @@ -3648,7 +3778,7 @@ pub trait IsStarkProver< let staged: Vec< std::sync::Mutex< Option<( - Round1Commitments, + Round1Commitments, Lde, )>, >, @@ -3711,6 +3841,7 @@ pub trait IsStarkProver< transcript, #[cfg(feature = "disk-spill")] StorageMode::Ram, + ResidencyMode::Retain, ) .map(|mut multi_proof| multi_proof.proofs.remove(0)) } @@ -3721,7 +3852,7 @@ pub trait IsStarkProver< fn prove_rounds_2_to_4( air: &dyn AIR, pub_inputs: &PI, - round_1_result: &mut Round1, + round_1_result: &mut Round1, transcript: &mut (impl IsStarkTranscript + Clone), domain: &Domain, twiddles: &LdeTwiddles, diff --git a/crypto/stark/src/residency_mode.rs b/crypto/stark/src/residency_mode.rs new file mode 100644 index 000000000..52885e703 --- /dev/null +++ b/crypto/stark/src/residency_mode.rs @@ -0,0 +1,36 @@ +/// Whether Round 1 keeps every table's main LDE resident until its fused task +/// runs, or drops it after the commit and recomputes it inside the task. +/// +/// Fiat-Shamir requires the main *roots* to be absorbed before the shared LogUp +/// challenges are sampled; it says nothing about the LDE buffers, so keeping +/// them is a performance choice. `Retain` makes it; `RecomputeLde` trades one +/// extra forward NTT per table for turning an `O(N)` retention into an +/// `O(table_parallelism)` transient. The Merkle tree is kept either way, so a +/// recompute never re-hashes and the root that entered the transcript stays the +/// root openings are checked against. +/// +/// The choice is invisible to the proof: same roots, same transcript order, +/// same proof bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ResidencyMode { + /// Keep every main LDE from its Round-1 commit until its table's fused + /// task consumes it. + #[default] + Retain, + /// Drop each main LDE once its root is absorbed and recompute it from the + /// still-resident trace at the top of the table's fused task. + /// + /// Also releases each table's aux columns from the caller-owned + /// `TraceTable` when that table's proof is complete — a documented part of + /// this mode's contract, since it mutates caller-visible state. Callers + /// that read a trace's aux columns after `multi_prove` returns must use + /// `Retain`. + RecomputeLde, +} + +impl ResidencyMode { + /// True when main LDEs are dropped after Round 1 and recomputed on demand. + pub fn recomputes_main_lde(self) -> bool { + matches!(self, Self::RecomputeLde) + } +} diff --git a/crypto/stark/src/test_utils.rs b/crypto/stark/src/test_utils.rs index f5cd19f80..48b720639 100644 --- a/crypto/stark/src/test_utils.rs +++ b/crypto/stark/src/test_utils.rs @@ -34,5 +34,6 @@ where transcript, #[cfg(feature = "disk-spill")] crate::storage_mode::StorageMode::Ram, + crate::residency_mode::ResidencyMode::Retain, ) } diff --git a/crypto/stark/src/tests/batched_mmcs_soundness_tests.rs b/crypto/stark/src/tests/batched_mmcs_soundness_tests.rs new file mode 100644 index 000000000..ac5178fa7 --- /dev/null +++ b/crypto/stark/src/tests/batched_mmcs_soundness_tests.rs @@ -0,0 +1,339 @@ +//! Soundness negatives for the batched-commitment primitives — the mixed-height +//! MMCS ([`crate::fri::mmcs`]) and the batched-FRI transcript +//! ([`crate::fri::batched`]). +//! +//! Each test builds one honest commitment over a small mixed-height epoch, then +//! tampers a single component and asserts rejection. The honest opening is +//! re-asserted in every test, so a false-reject regression cannot make the +//! negatives pass vacuously. +//! +//! Scope: these reach only what the primitives decide. The forgeries that a +//! batched *proof* must also resist — a tampered per-query FRI layer evaluation, +//! an OOD value, the bus balance, the query count, the grinding nonce — need the +//! prover/verifier integration and belong with it. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::fft::bit_reversing::reverse_index; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use crate::config::KeccakStarkHash; +use crate::fri::batched::{ + BatchedFriLayout, absorb_shape_histogram, derive_batched_fri_challenges, +}; +use crate::fri::mmcs::{LeafSource, MixedMmcs, MixedOpening}; + +type F = GoldilocksField; +type FE = FieldElement; +type Mmcs = MixedMmcs; +type Transcript = DefaultTranscript; + +/// Bit-reversed row-major matrices, in the layout the MMCS commits. +struct Matrices { + /// `(bit-reversed row-major data, log_height, width)`. + mats: Vec<(Vec, usize, usize)>, +} + +impl LeafSource for Matrices { + fn num_matrices(&self) -> usize { + self.mats.len() + } + fn log_height(&self, m: usize) -> usize { + self.mats[m].1 + } + fn width(&self, m: usize) -> usize { + self.mats[m].2 + } + fn append_row(&self, m: usize, bitrev_row: usize, out: &mut Vec) { + let (data, _, width) = &self.mats[m]; + out.extend_from_slice(&data[bitrev_row * width..(bitrev_row + 1) * width]); + } +} + +fn matrix(log_height: usize, width: usize, seed: u64) -> (Vec, usize, usize) { + let num_rows = 1usize << log_height; + let mut data = vec![FE::from(0u64); num_rows * width]; + for (r, chunk) in data.chunks_exact_mut(width).enumerate() { + let br = reverse_index(r, num_rows as u64); + for (c, slot) in chunk.iter_mut().enumerate() { + *slot = FE::from(seed.wrapping_mul(31) + (c as u64) * 1009 + (br as u64) * 7 + 1); + } + } + (data, log_height, width) +} + +/// A four-matrix epoch: two tall (base group), one injected, one injected lower. +/// Heights {5, 5, 4, 2}, widths {3, 3, 2, 4}. Two of the tall matrices share a +/// width so the "swap two openings" forgery below is a pure reordering. +fn epoch() -> (Matrices, Vec, Vec) { + let mats = Matrices { + mats: vec![ + matrix(5, 3, 11), + matrix(5, 3, 22), + matrix(4, 2, 33), + matrix(2, 4, 44), + ], + }; + let heights = vec![5, 5, 4, 2]; + let widths = vec![3, 3, 2, 4]; + (mats, heights, widths) +} + +const IOTA: usize = 9; + +fn honest() -> ([u8; 32], MixedOpening, Vec, Vec) { + let (mats, heights, widths) = epoch(); + let mmcs = Mmcs::commit(&mats); + let opening = mmcs.open_batch(IOTA, &mats); + (mmcs.root(), opening, heights, widths) +} + +/// Sanity anchor: the untampered opening verifies. +#[test] +fn honest_batched_opening_verifies() { + let (root, opening, heights, widths) = honest(); + assert!( + Mmcs::verify_batch(&root, IOTA, &opening, &heights, &widths), + "an honest mixed-height opening must verify" + ); +} + +/// Tampering any matrix's opened row breaks the one shared authentication path — +/// including the SHORT matrices, which are bound through injection rather than +/// through the base leaf. +#[test] +fn rejects_a_tampered_row_in_every_height_group() { + let (root, opening, heights, widths) = honest(); + for m in 0..opening.per_matrix.len() { + let mut tampered = opening.clone(); + tampered.per_matrix[m].evaluations[0] = + &tampered.per_matrix[m].evaluations[0] + &FE::from(1u64); + assert!( + !Mmcs::verify_batch(&root, IOTA, &tampered, &heights, &widths), + "a tampered row of matrix {m} (height {}) must be rejected", + heights[m] + ); + + let mut tampered_sym = opening.clone(); + tampered_sym.per_matrix[m].evaluations_sym[0] = + &tampered_sym.per_matrix[m].evaluations_sym[0] + &FE::from(1u64); + assert!( + !Mmcs::verify_batch(&root, IOTA, &tampered_sym, &heights, &widths), + "a tampered symmetric row of matrix {m} must be rejected" + ); + } +} + +/// Tampering the shared authentication path itself. +#[test] +fn rejects_a_tampered_authentication_path() { + let (root, opening, heights, widths) = honest(); + for level in 0..opening.proof.merkle_path.len() { + let mut tampered = opening.clone(); + tampered.proof.merkle_path[level][0] ^= 1; + assert!( + !Mmcs::verify_batch(&root, IOTA, &tampered, &heights, &widths), + "a tampered sibling at level {level} must be rejected" + ); + } + // Truncating or padding the path is a shape error, not a hash mismatch. + let mut short = opening.clone(); + short.proof.merkle_path.pop(); + assert!(!Mmcs::verify_batch(&root, IOTA, &short, &heights, &widths)); + let mut long = opening.clone(); + long.proof.merkle_path.push([0u8; 32]); + assert!(!Mmcs::verify_batch(&root, IOTA, &long, &heights, &widths)); +} + +/// An honest opening replayed at a different query index must be rejected: the +/// path is position-dependent, so one opening does not authenticate every leaf. +#[test] +fn rejects_an_opening_replayed_at_another_index() { + let (root, opening, heights, widths) = honest(); + let n0 = 1usize << (5 - 1); + for iota in 0..n0 { + let accepted = Mmcs::verify_batch(&root, iota, &opening, &heights, &widths); + assert_eq!( + accepted, + iota == IOTA, + "the opening at {IOTA} must verify at {IOTA} and nowhere else (index {iota})" + ); + } + // And past the tree's leaf range — the index-convention guard. + assert!(!Mmcs::verify_batch(&root, n0, &opening, &heights, &widths)); +} + +/// INPUT ORDER is part of the commitment: swapping two same-height, same-width +/// matrices' openings changes the flat concatenation the group leaf hashes, so +/// the tree no longer reproduces. Without order-dependence a prover could serve +/// one table's rows in another's slot. +#[test] +fn rejects_swapped_openings_within_a_height_group() { + let (root, opening, heights, widths) = honest(); + assert_eq!( + (heights[0], widths[0]), + (heights[1], widths[1]), + "matrices 0 and 1 must share a shape for this to be a pure reordering" + ); + let mut swapped = opening.clone(); + swapped.per_matrix.swap(0, 1); + assert_ne!( + swapped.per_matrix[0].evaluations, opening.per_matrix[0].evaluations, + "the two matrices must carry different data" + ); + assert!( + !Mmcs::verify_batch(&root, IOTA, &swapped, &heights, &widths), + "reordering two same-shape matrices must be rejected" + ); +} + +/// The verifier's `heights` fix the injection schedule. Relabelling a matrix's +/// height — claiming the height-4 matrix is height 3, so it is injected a level +/// later — must not reproduce the root, or a prover could move a table to a +/// layer where its rows are checked against a different query position. +#[test] +fn rejects_a_relabelled_injection_height() { + let (root, opening, heights, widths) = honest(); + let mut relabelled = heights.clone(); + relabelled[2] = 3; + assert!( + !Mmcs::verify_batch(&root, IOTA, &opening, &relabelled, &widths), + "moving a matrix to another injection level must be rejected" + ); + + // Promoting a short matrix into the base group is likewise rejected. + let mut promoted = heights.clone(); + promoted[3] = 5; + assert!(!Mmcs::verify_batch( + &root, IOTA, &opening, &promoted, &widths + )); +} + +/// Widths are verifier-supplied and length-checked, so a width that does not +/// match the opening is rejected before any hashing — the guard that closes the +/// leaf-boundary shift. +#[test] +fn rejects_widths_that_disagree_with_the_opening() { + let (root, opening, heights, widths) = honest(); + for m in 0..widths.len() { + let mut wrong = widths.clone(); + wrong[m] += 1; + assert!( + !Mmcs::verify_batch(&root, IOTA, &opening, &heights, &wrong), + "a width disagreeing with matrix {m}'s opening must be rejected" + ); + } +} + +/// A root committed over a different epoch shape does not authenticate this +/// opening, even where the tree depth coincides. +#[test] +fn rejects_a_root_from_another_epoch_shape() { + let (_, opening, heights, widths) = honest(); + let other = Matrices { + mats: vec![ + matrix(5, 3, 11), + matrix(5, 3, 22), + matrix(4, 2, 33), + // Same height and width, different data. + matrix(2, 4, 99), + ], + }; + let other_root = Mmcs::commit(&other).root(); + assert!( + !Mmcs::verify_batch(&other_root, IOTA, &opening, &heights, &widths), + "an opening must not verify against another epoch's root" + ); +} + +/// The round-4 transcript binds the shape and every committed FRI layer, so +/// tampering a layer root or a terminal coefficient moves the query indices the +/// prover must answer at. This is what stops a prover from choosing its FRI +/// commitments after seeing the queries. +#[test] +fn tampering_the_fri_transcript_moves_the_query_indices() { + let heights = vec![10usize, 10, 8, 7]; + let widths = vec![4usize, 2, 3, 1]; + let (blowup_log, k) = (1u32, 5u32); + let layout = BatchedFriLayout::new(10, 7, blowup_log, k); + let roots: Vec<[u8; 32]> = (0u8..layout.num_committed as u8).map(|i| [i; 32]).collect(); + let coeffs: Vec = (0..(1u64 << layout.effective_k)).map(FE::from).collect(); + + let derive = |roots: &[[u8; 32]], coeffs: &[FE], heights: &[usize], widths: &[usize]| { + derive_batched_fri_challenges( + &mut Transcript::new(b"batched_soundness"), + heights, + widths, + roots, + coeffs, + blowup_log, + k, + 0, + None, + 16, + ) + .expect("a well-formed layer-root and coefficient count") + .iotas + }; + + let base = derive(&roots, &coeffs, &heights, &widths); + assert!(!base.is_empty()); + + let mut other_root = roots.clone(); + other_root[0][0] ^= 1; + assert_ne!( + base, + derive(&other_root, &coeffs, &heights, &widths), + "a tampered FRI layer root must move the query indices" + ); + + let mut other_coeffs = coeffs.clone(); + other_coeffs[0] = &other_coeffs[0] + &FE::from(1u64); + assert_ne!( + base, + derive(&roots, &other_coeffs, &heights, &widths), + "a tampered terminal coefficient must move the query indices" + ); + + let mut other_heights = heights.clone(); + other_heights[2] = 9; + assert_ne!( + base, + derive(&roots, &coeffs, &other_heights, &widths), + "a tampered height must move the query indices" + ); + + let mut other_widths = widths.clone(); + other_widths[2] = 4; + assert_ne!( + base, + derive(&roots, &coeffs, &heights, &other_widths), + "a tampered width must move the query indices" + ); +} + +/// The shape histogram's encoding is injective: no two distinct epoch shapes +/// absorb the same bytes. A collision would let a prover present one shape to +/// the transcript and another to the opening parse. +#[test] +fn the_shape_encoding_separates_distinct_epochs() { + let absorbed = |heights: &[usize], widths: &[usize]| { + let mut t = Transcript::new(b"shape"); + absorb_shape_histogram(&mut t, heights, widths); + t.state() + }; + + // The classic ambiguity a length prefix and fixed-width fields must close: + // one table of shape (h, w) against two tables whose fields interleave to the + // same sequence. + let one = absorbed(&[3, 4], &[4, 5]); + let two = absorbed(&[3], &[4]); + let three = absorbed(&[3, 4, 5], &[4, 5, 6]); + assert_ne!(one, two); + assert_ne!(one, three); + assert_ne!(two, three); + + // Swapping height and width within a table is a different epoch. + assert_ne!(absorbed(&[3, 4], &[4, 3]), absorbed(&[4, 3], &[3, 4])); +} diff --git a/crypto/stark/src/tests/blake3_stark_roundtrip_tests.rs b/crypto/stark/src/tests/blake3_stark_roundtrip_tests.rs new file mode 100644 index 000000000..9013c51c5 --- /dev/null +++ b/crypto/stark/src/tests/blake3_stark_roundtrip_tests.rs @@ -0,0 +1,322 @@ +//! A full STARK prove → verify under [`Blake3StarkHash`], and the FRI-layer +//! evidence that makes it work. +//! +//! The whole commitment path is named by one configuration: the prover builds +//! FRI layer trees with `H::Pair` and the verifier authenticates those openings +//! with `H::Batched`, which are one hash by [`StarkHash`]'s two-element +//! invariant. A configuration that broke that agreement would reject every +//! honest proof at its first FRI query, so these tests are what says it holds +//! for a hash other than the default. +//! +//! Everything here is `cfg(not(cuda))`, because [`Blake3StarkHash`] is: under +//! `cuda` the tree entries hash on the device with the keccak kernels and only +//! label the result, so there is no second configuration to name. +#![cfg(not(feature = "cuda"))] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::merkle_tree::merkle::MerkleTree; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; + +use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; +use crate::examples::simple_addition::{ + SimpleAdditionAIR, SimpleAdditionPublicInputs, simple_addition_trace, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::StarkProof; +use crate::prover::{GenericProver, IsStarkProver}; +use crate::traits::AIR; +use crate::verifier::{GenericVerifier, IsStarkVerifier}; + +type F = GoldilocksField; +type FE = FieldElement; +type PI = SimpleAdditionPublicInputs; + +type Prove = GenericProver; +type Verify = GenericVerifier; + +/// 1024 rows: `trace_bits = 10` against the default `k = 7`, so FRI actually +/// folds and commits layers. A trace that terminates immediately would make +/// every assertion below vacuous, which is why the layer count is asserted. +const TRACE_ROWS: usize = 1024; + +fn air_and_inputs() -> (SimpleAdditionAIR, PI) { + let proof_options = ProofOptions::default_test_options(); + let air = SimpleAdditionAIR::::new(&proof_options); + let pub_inputs = SimpleAdditionPublicInputs { + a: FE::from(1u64), + b: FE::from(2u64), + }; + (air, pub_inputs) +} + +fn prove_with(air: &SimpleAdditionAIR, pub_inputs: &PI) -> StarkProof { + let mut trace = simple_addition_trace::(TRACE_ROWS); + Prove::::prove( + air, + &mut trace, + pub_inputs, + &mut DefaultTranscript::::new(&[]), + ) + .expect("proving must succeed") +} + +/// ★ The Stage-2 oracle: a real STARK proves and verifies end to end under the +/// BLAKE3 configuration. +/// +/// This is a **same-reference** claim — this build's prover and this build's +/// verifier agree — and that is exactly what it is for. It makes no +/// cross-version claim; `scripts/cross_verify_vm.sh` is what covers the default +/// keccak path across refs. +#[test] +fn a_blake3_stark_proof_verifies() { + let (air, pub_inputs) = air_and_inputs(); + let proof = prove_with::(&air, &pub_inputs); + + // Non-vacuity: the proof must actually contain committed FRI layers, or it + // would verify without ever exercising the trees this stage threads `H` + // through. + assert!( + !proof.fri_layers_merkle_roots.is_empty(), + "the test trace must fold; otherwise this proves nothing about fri/" + ); + + assert!( + Verify::::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + "an honest BLAKE3-committed proof must verify under the BLAKE3 verifier" + ); +} + +/// HONEST-PATH CONTROL: the keccak configuration still round-trips. +/// +/// The refactor rewrote the code path the default prover runs through; this +/// says it still proves and verifies. Its stronger sibling is the cross-ref +/// gate, which checks the actual proof BYTES did not move. +#[test] +fn the_keccak_stark_proof_still_verifies() { + let (air, pub_inputs) = air_and_inputs(); + let proof = prove_with::(&air, &pub_inputs); + + assert!(!proof.fri_layers_merkle_roots.is_empty()); + assert!( + Verify::::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + "an honest keccak-committed proof must verify under the keccak verifier" + ); +} + +/// FALSIFICATION: tampering a FRI layer root must break verification. +/// +/// A positive round trip alone cannot distinguish "the verifier checks the FRI +/// layer openings" from "the verifier reached the end without looking". This is +/// aimed at the precise bytes this stage changed the producer of. +#[test] +fn a_tampered_blake3_fri_layer_root_is_rejected() { + let (air, pub_inputs) = air_and_inputs(); + let honest = prove_with::(&air, &pub_inputs); + + for layer in 0..honest.fri_layers_merkle_roots.len() { + let mut tampered = honest.clone(); + tampered.fri_layers_merkle_roots[layer][0] ^= 1; + assert!( + !Verify::::verify( + &tampered, + &air, + &mut DefaultTranscript::::new(&[]) + ), + "a proof with FRI layer root {layer} flipped must be rejected" + ); + } +} + +/// FALSIFICATION: a tampered FRI layer *opening* must break verification. +/// +/// The root tamper above also moves every challenge drawn after it, so it would +/// be caught by a verifier that only replayed Fiat-Shamir. This one leaves the +/// transcript untouched and corrupts the authenticated value instead, so only +/// the Merkle check can catch it — and that check is `H::Batched` against a tree +/// the prover built with `H::Pair`. +#[test] +fn a_tampered_blake3_fri_layer_opening_is_rejected() { + let (air, pub_inputs) = air_and_inputs(); + let honest = prove_with::(&air, &pub_inputs); + + assert!( + !honest.query_list.is_empty() && !honest.query_list[0].layers_evaluations_sym.is_empty(), + "the test proof must carry FRI query openings" + ); + + let mut tampered = honest.clone(); + tampered.query_list[0].layers_evaluations_sym[0] += FE::one(); + assert!( + !Verify::::verify(&tampered, &air, &mut DefaultTranscript::::new(&[])), + "a tampered FRI symmetric evaluation must fail its Merkle authentication" + ); +} + +/// ★ CONTROL — the stark-proof-level analog of +/// `the_blake3_and_keccak_configurations_commit_differently`. +/// +/// Without this, every test above would pass just as well if `Blake3StarkHash` +/// still resolved to the keccak backends, or if `fri/` had kept building keccak +/// layer trees under a BLAKE3 `H`. Both proofs are over the same trace with the +/// same transcript seed, so nothing but the commitment hash can move these +/// bytes — and each verifier must reject the other configuration's proof. +#[test] +fn the_two_configurations_produce_mutually_unverifiable_proofs() { + let (air, pub_inputs) = air_and_inputs(); + let blake3_proof = prove_with::(&air, &pub_inputs); + let keccak_proof = prove_with::(&air, &pub_inputs); + + assert_ne!( + blake3_proof.lde_trace_main_merkle_root, keccak_proof.lde_trace_main_merkle_root, + "the two configurations must commit the same trace to different roots" + ); + assert_ne!( + blake3_proof.fri_layers_merkle_roots, keccak_proof.fri_layers_merkle_roots, + "the two configurations must commit FRI layers to different roots" + ); + + assert!( + !Verify::::verify( + &blake3_proof, + &air, + &mut DefaultTranscript::::new(&[]) + ), + "the keccak verifier must reject a BLAKE3-committed proof" + ); + assert!( + !Verify::::verify( + &keccak_proof, + &air, + &mut DefaultTranscript::::new(&[]) + ), + "the BLAKE3 verifier must reject a keccak-committed proof" + ); +} + +/// ★ The FRI layer tree IS the configuration's tree over that layer's +/// evaluations — checked directly, at both configurations. +/// +/// The round trip above says prover and verifier agree; it does not say *which* +/// hash they agree on, and a `commit_phase_from_evaluations` that ignored `H` +/// and used keccak for both would still round-trip under a keccak verifier. +/// This rebuilds each committed layer's tree from the layer's own evaluations +/// with `H::Pair` and demands the roots match, so the threading is pinned at the +/// producer rather than inferred from the consumer. +#[test] +fn fri_layer_trees_are_built_with_the_configurations_pair_backend() { + use crate::fri::commit_phase_from_evaluations; + use crate::fri::fri_functions::compute_coset_twiddles_inv; + + /// Returns each committed layer's root, and layer 0's folded codeword. + fn check( + offset: &FE, + len: usize, + blowup_log: u32, + k: u32, + ) -> (Vec<[u8; 32]>, Vec) { + let codeword: Vec = (0..len as u64).map(|i| FE::from(i * 7 + 1)).collect(); + let inv_twiddles = compute_coset_twiddles_inv::(offset, len); + let mut transcript = DefaultTranscript::::new(&[]); + let (_coeffs, layers) = commit_phase_from_evaluations::( + codeword, + &mut transcript, + offset, + len, + blowup_log, + k, + &inv_twiddles, + ); + assert!(!layers.is_empty(), "the input must fold"); + + for (i, layer) in layers.iter().enumerate() { + let leaves: Vec<[FE; 2]> = layer + .evaluation + .chunks_exact(2) + .map(|c| [c[0], c[1]]) + .collect(); + let rebuilt = MerkleTree::>::build(&leaves).expect("rebuild layer tree"); + assert_eq!( + rebuilt.root, layer.merkle_tree.root, + "layer {i}'s committed root must be the H::Pair tree over its own evaluations" + ); + } + ( + layers.iter().map(|l| l.merkle_tree.root).collect(), + layers[0].evaluation.clone(), + ) + } + + let offset = FE::from(3u64); + let (len, blowup_log, k) = (1usize << 10, 1u32, 5u32); + + let (keccak_roots, keccak_layer0) = check::(&offset, len, blowup_log, k); + let (blake3_roots, blake3_layer0) = check::(&offset, len, blowup_log, k); + + // ζ₀ is drawn before anything is appended, so both configurations fold the + // same input with the same challenge and layer 0's codeword is identical. + // Checked rather than argued, because it is what makes the root comparison + // below mean "the hash differs" instead of "the input differs". + assert_eq!( + keccak_layer0, blake3_layer0, + "layer 0 must fold identically under both configurations" + ); + assert_ne!( + keccak_roots[0], blake3_roots[0], + "over one identical codeword, the layer root must differ only because \ + the hash does" + ); +} + +/// ★ The whole BLAKE3 configuration at once: BLAKE3 commitments, a BLAKE3 +/// Fiat-Shamir transcript, and BLAKE3 grinding, proving and verifying. +/// +/// The tests above run the BLAKE3 commitment configuration against a *keccak* +/// transcript, which is a legitimate configuration but not the destination. +/// This is the destination minus the guest leg: nothing keccak is left on +/// either side except what the AIR itself does. +/// +/// `default_test_options` sets `grinding_factor: 1`, so the proof carries a +/// nonce and the verifier re-checks it — through `GrindingDigest`, which for +/// this configuration is `Blake3Chain`. A grinding port that had been left +/// hard-wired to keccak would fail here, because prover and verifier would +/// disagree about which work the nonce satisfies. +#[test] +fn the_full_blake3_configuration_proves_and_verifies_with_grinding() { + use crypto::fiat_shamir::default_transcript::Blake3Transcript; + + let (air, pub_inputs) = air_and_inputs(); + assert_eq!( + air.options().grinding_factor, + 1, + "this test is about the grinding path; it must be on" + ); + + let mut trace = simple_addition_trace::(TRACE_ROWS); + let proof = Prove::::prove( + &air, + &mut trace, + &pub_inputs, + &mut Blake3Transcript::::new(&[]), + ) + .expect("proving under the full BLAKE3 configuration must succeed"); + + assert!( + proof.nonce.is_some(), + "grinding is on, so the proof must carry a nonce" + ); + assert!(!proof.fri_layers_merkle_roots.is_empty()); + + assert!( + Verify::::verify(&proof, &air, &mut Blake3Transcript::::new(&[])), + "an honest all-BLAKE3 proof must verify" + ); + + // FALSIFICATION: the transcript is part of the configuration. A keccak + // transcript replaying a BLAKE3-transcripted proof derives different + // challenges and must reject. + assert!( + !Verify::::verify(&proof, &air, &mut DefaultTranscript::::new(&[])), + "verifying with the wrong transcript hash must reject" + ); +} diff --git a/crypto/stark/src/tests/commitment_tests.rs b/crypto/stark/src/tests/commitment_tests.rs index f1684112b..52ce0ef73 100644 --- a/crypto/stark/src/tests/commitment_tests.rs +++ b/crypto/stark/src/tests/commitment_tests.rs @@ -94,3 +94,163 @@ fn empty_and_zero_row_inputs_short_circuit() { assert!(keccak_leaves_bit_reversed_grouped(&zero_rows, ROWS_PER_LEAF).is_empty()); assert!(commit_bit_reversed(&zero_rows, ROWS_PER_LEAF).is_none()); } + +/// ★ The [`StarkHash`] two-element-leaf invariant, for the keccak configuration. +/// +/// The prover commits FRI layers with `Pair` and the verifier authenticates +/// those openings with `Batched` (`verify_fri_layer_openings` builds a +/// two-element `Vec`). Nothing in the type system makes those agree — this is +/// what says they do, so a second configuration that breaks it fails here +/// rather than by rejecting every honest proof at its first FRI query. +#[test] +fn batched_and_pair_agree_on_a_two_element_leaf() { + use crate::config::{KeccakStarkHash, StarkHash}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + type Batched = ::Batched; + type Pair = ::Pair; + + for (a, b) in [(0u64, 1u64), (7, 7), (u64::MAX - 1, 12345)] { + let (x, y) = (Felt::from(a), Felt::from(b)); + assert_eq!( + ::hash_data(&vec![x, y]), + ::hash_data(&[x, y]), + "Batched and Pair must hash the pair ({a}, {b}) identically" + ); + } +} + +/// ★ The same invariant for the BLAKE3 configuration. +/// +/// Under `Blake3StarkHash` it holds by construction rather than by coincidence: +/// both families are the same generic backend over `Blake3Chain`, and a +/// two-element leaf is 16 bytes, one block, one compression. This is here +/// because "holds by construction" is an argument about today's code, and the +/// invariant has to survive tomorrow's. +#[cfg(not(feature = "cuda"))] +#[test] +fn blake3_batched_and_pair_agree_on_a_two_element_leaf() { + use crate::config::{Blake3StarkHash, StarkHash}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + type Batched = ::Batched; + type Pair = ::Pair; + + for (a, b) in [(0u64, 1u64), (7, 7), (u64::MAX - 1, 12345)] { + let (x, y) = (Felt::from(a), Felt::from(b)); + assert_eq!( + ::hash_data(&vec![x, y]), + ::hash_data(&[x, y]), + "Batched and Pair must hash the pair ({a}, {b}) identically" + ); + } +} + +/// The two configurations are actually different hashes. +/// +/// Without this, every BLAKE3 test in this file would pass just as well if the +/// blake3 aliases had been left pointing at the keccak backends — which is a +/// realistic way for a type alias change to be wrong, and one that nothing else +/// here would notice. +#[cfg(not(feature = "cuda"))] +#[test] +fn the_blake3_and_keccak_configurations_commit_differently() { + use crate::config::{Blake3StarkHash, KeccakStarkHash, StarkHash}; + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + + let leaf: Vec = (0..5u64).map(Felt::from).collect(); + assert_ne!( + <::Batched as IsMerkleTreeBackend>::hash_data(&leaf), + <::Batched as IsMerkleTreeBackend>::hash_data(&leaf), + ); +} + +/// A commit → open → verify round trip over the BLAKE3 configuration, through +/// the production commitment path. +/// +/// `commit_bit_reversed_with` is the entry point the prover's main trace commit +/// uses, and it is already backend-generic, so this exercises the real leaf +/// serialization (bit-reversed, row-grouped, big-endian) and the real tree +/// construction — not a hand-rolled tree over the backend. +/// +/// # What this does and does not establish +/// +/// It is a **same-reference** check: this build's prover and this build's +/// verifier agree, over the commitment layer. It makes no cross-version claim. +/// It is also deliberately narrow — the commitment layer alone, isolated from +/// FRI, the transcript and the AIR. The full prove→verify under this +/// configuration lives in `tests::blake3_stark_roundtrip_tests`; when both fail +/// together the fault is here, and this test says where. +#[cfg(not(feature = "cuda"))] +#[test] +fn blake3_commitments_open_and_verify() { + use crate::commitment::{commit_bit_reversed_with, leaves_bit_reversed_grouped}; + use crate::config::{Blake3StarkHash, StarkHash}; + use crypto::merkle_tree::merkle::MerkleTree; + use crypto::merkle_tree::proof::verify_merkle_path_from_leaf_hash; + + type Batched = ::Batched; + + let columns: Vec> = (0..4u64) + .map(|c| (0..16u64).map(|r| Felt::from(1000 * c + r + 1)).collect()) + .collect(); + + let (tree, root) = + commit_bit_reversed_with::(&columns, ROWS_PER_LEAF).expect("commit"); + let leaves = leaves_bit_reversed_grouped::(&columns, ROWS_PER_LEAF); + assert_eq!(root, tree.root); + assert_eq!(leaves.len(), 16 / ROWS_PER_LEAF); + + for (i, leaf) in leaves.iter().enumerate() { + let proof = tree + .get_proof_by_pos(i) + .expect("proof for an in-range leaf"); + assert!( + verify_merkle_path_from_leaf_hash::(&proof.merkle_path, &root, i, *leaf), + "the honest opening of leaf {i} must verify" + ); + + // NEGATIVE CONTROL: the same path must not authenticate a different + // leaf, or "verify" above would be measuring nothing. + let other = leaves[(i + 1) % leaves.len()]; + assert!( + !verify_merkle_path_from_leaf_hash::(&proof.merkle_path, &root, i, other), + "a path must not authenticate a leaf it is not for, at {i}" + ); + } + + // And the root is the tree the leaves make, built independently. + let rebuilt = MerkleTree::::build_from_hashed_leaves(leaves).expect("rebuild"); + assert_eq!(rebuilt.root, root); +} + +/// The streaming routes and the owned-`Data` route are the same leaf. +#[test] +fn streaming_leaf_routes_match_hash_data() { + use crate::config::{KeccakStarkHash, StarkHash}; + use crypto::merkle_tree::traits::{IsMerkleTreeBackend, IsStreamingLeafBackend}; + + type Batched = ::Batched; + + let row: Vec = (0..5u64).map(Felt::from).collect(); + let (left, right) = row.split_at(2); + + let owned = ::hash_data(&row); + assert_eq!( + owned, + >::hash_data_from_slices(left, right), + "hash_data_from_slices must equal hash_data on the concatenation" + ); + + let mut buf = Vec::new(); + for e in &row { + let mut b = [0u8; 8]; + e.write_bytes_be(&mut b); + buf.extend_from_slice(&b); + } + assert_eq!( + owned, + >::hash_bytes(&buf), + "hash_bytes must equal hash_data on the elements those bytes encode" + ); +} diff --git a/crypto/stark/src/tests/constraint_index_tests.rs b/crypto/stark/src/tests/constraint_index_tests.rs new file mode 100644 index 000000000..d2140ec2b --- /dev/null +++ b/crypto/stark/src/tests/constraint_index_tests.rs @@ -0,0 +1,126 @@ +//! `check_dense_index_set` — the release-visible guard against a constraint +//! body emitting one index twice and another never. +//! +//! These tests are the guard's own honest control. A checker that never fires +//! would pass every "the real chip is fine" assertion in the workspace, so the +//! first thing established here is that it fires — on the exact shape that +//! motivated it (`COMMIT.md` §1.4.4 H1: a widened lane loop overrunning into +//! the pins that follow it) and on the degenerate cases either side. + +use crate::constraints::builder::{ + ConstraintBuilder, ConstraintMeta, ConstraintSet, RootKind, check_dense_index_set, +}; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField; +use math::field::goldilocks::GoldilocksField; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; + +fn base(constraint_idx: usize) -> ConstraintMeta { + ConstraintMeta { + constraint_idx, + kind: RootKind::Base, + end_exemptions: 0, + } +} + +/// A body shaped like the hazard: `lanes` lane identities starting at 6, +/// then 8 unused-output pins starting at `pin_base`, then one tail constraint. +/// At `lanes = 8, pin_base = 14` the blocks abut exactly; widening `lanes` to +/// 12 without moving `pin_base` makes 14..17 collide. +struct LaneBody { + lanes: usize, + pin_base: usize, + tail: usize, +} + +impl ConstraintSet for LaneBody { + fn eval>(&self, b: &mut B) { + for lane in 0..self.lanes { + b.emit_base(6 + lane, b.zero()); + } + for j in 0..8 { + b.emit_base(self.pin_base + j, b.zero()); + } + for i in 0..6 { + b.emit_base(i, b.zero()); + } + b.emit_base(self.tail, b.zero()); + } +} + +/// ★ The H1 shape: widening the lane block over the pins that follow it. +/// +/// The count is unchanged — the body still emits 8+8+6+1 = 23 constraints into +/// 23 declared slots — which is exactly why `NUM_CONSTRAINTS`, a predicted-count +/// test, and `assert_complete` all miss it. Four lane identities are silently +/// overwritten and nothing else notices. +#[test] +fn the_widened_lane_block_collides_and_the_checker_says_so() { + let healthy = LaneBody { + lanes: 8, + pin_base: 14, + tail: 22, + }; + check_dense_index_set(&healthy.meta(), 23).expect("the un-widened body is dense"); + + let widened = LaneBody { + lanes: 12, + pin_base: 14, + tail: 22, + }; + // Same declared count — the collision is invisible to counting. + assert_eq!(widened.meta().len(), 27); + let err = check_dense_index_set(&widened.meta(), 27) + .expect_err("a lane block overrunning the pins must be caught"); + assert!( + err.contains("emitted twice [14, 15, 16, 17]"), + "the four colliding indices must be named, got: {err}" + ); + assert!( + err.contains("never emitted [23, 24, 25, 26]"), + "the slots left unwritten must be named, got: {err}" + ); +} + +/// A repeat with no compensating gap is still a repeat. +#[test] +fn a_plain_duplicate_is_caught() { + let meta = vec![base(0), base(1), base(1), base(2)]; + let err = check_dense_index_set(&meta, 4).expect_err("1 emitted twice"); + assert!(err.contains("emitted twice [1]"), "got: {err}"); + assert!(err.contains("never emitted [3]"), "got: {err}"); +} + +/// A hole with no compensating duplicate cannot keep the count, so it surfaces +/// as a count mismatch — the half `assert_complete` used to cover. Worth +/// pinning: it is the reason a gap alone is the *easy* failure, and why H1 +/// (which pairs a gap with a duplicate and so keeps the count) is the hard one. +#[test] +fn a_gap_without_a_duplicate_shows_up_as_a_count_mismatch() { + let meta = vec![base(0), base(2), base(3), base(4)]; + let err = check_dense_index_set(&meta, 5).expect_err("1 never emitted"); + assert!( + err.contains("emitted 4 constraints, declared 5"), + "got: {err}" + ); +} + +/// Wrong total is reported as wrong total, not as a confusing index list. +#[test] +fn a_count_mismatch_is_reported_plainly() { + let meta = vec![base(0), base(1)]; + let err = check_dense_index_set(&meta, 3).expect_err("2 != 3"); + assert!( + err.contains("emitted 2 constraints, declared 3"), + "got: {err}" + ); +} + +/// The honest control: a dense set passes, including the empty one. +#[test] +fn a_dense_set_passes() { + check_dense_index_set(&[], 0).expect("the empty body is dense"); + let meta: Vec<_> = (0..64).map(base).collect(); + check_dense_index_set(&meta, 64).expect("0..64 with no repeats is dense"); +} diff --git a/crypto/stark/src/tests/fri_tests.rs b/crypto/stark/src/tests/fri_tests.rs index 5b599886b..b9f8de565 100644 --- a/crypto/stark/src/tests/fri_tests.rs +++ b/crypto/stark/src/tests/fri_tests.rs @@ -142,6 +142,7 @@ fn test_eval_fold_matches_coeff_fold() { /// reconstructed terminal codeword at the query's terminal-layer position. #[test] fn test_commit_phase_early_termination_roundtrip() { + use crate::config::KeccakStarkHash; use crate::fri::fri_functions::update_twiddles_in_place; use crate::fri::terminal::terminal_codeword_from_coeffs; use crate::fri::{commit_phase_from_evaluations, query_phase}; @@ -176,7 +177,7 @@ fn test_commit_phase_early_termination_roundtrip() { let mut transcript = DefaultTranscript::::new(&[]); let inv_twiddles = crate::fri::fri_functions::compute_coset_twiddles_inv::(&offset, initial_len); - let (final_poly_coeffs, fri_layers) = commit_phase_from_evaluations::( + let (final_poly_coeffs, fri_layers) = commit_phase_from_evaluations::( codeword.clone(), &mut transcript, &offset, @@ -199,7 +200,7 @@ fn test_commit_phase_early_termination_roundtrip() { // query_phase must still work against the committed layers. let iotas = vec![0usize, 1, 5, 17, 30]; - let _decommitments = query_phase(&fri_layers, &iotas); + let _decommitments = query_phase::(&fri_layers, &iotas); // ---- Reconstruct terminal codeword from the emitted coefficients ---- let terminal_len = (1usize << blowup_log) << final_poly_log_degree; // 8 diff --git a/crypto/stark/src/tests/grinding_tests.rs b/crypto/stark/src/tests/grinding_tests.rs index 49c47e81f..53f79fd6d 100644 --- a/crypto/stark/src/tests/grinding_tests.rs +++ b/crypto/stark/src/tests/grinding_tests.rs @@ -1,5 +1,10 @@ +use crate::config::{GrindingDigest, KeccakStarkHash}; use crate::grinding::is_valid_nonce; +/// The default configuration's grinding hash. These vectors were computed +/// against keccak-256, so they name it rather than following any alias. +type Keccak = GrindingDigest; + #[test] fn test_invalid_nonce_grinding_factor_6() { // This setting produces a hash with 5 leading zeros, therefore not enough for grinding @@ -10,7 +15,7 @@ fn test_invalid_nonce_grinding_factor_6() { ]; let nonce = 4; let grinding_factor = 6; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -23,7 +28,7 @@ fn test_invalid_nonce_grinding_factor_9() { ]; let nonce = 287; let grinding_factor = 9; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -34,7 +39,7 @@ fn test_is_valid_nonce_grinding_factor_10() { ]; let nonce = 0x5ba; let grinding_factor = 10; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -45,7 +50,7 @@ fn test_is_valid_nonce_grinding_factor_20() { ]; let nonce = 0x2c5db8; let grinding_factor = 20; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -59,7 +64,7 @@ fn test_invalid_nonce_grinding_factor_19() { ]; let nonce = 0x2c5db8; let grinding_factor = 19; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -70,7 +75,7 @@ fn test_is_valid_nonce_grinding_factor_30() { ]; let nonce = 0x1ae839e1; let grinding_factor = 30; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -81,5 +86,60 @@ fn test_is_valid_nonce_grinding_factor_33() { ]; let nonce = 0x4cc3123f; let grinding_factor = 33; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); +} + +// ========================================================================= +// The BLAKE3 configuration's proof of work. +// +// Grinding is a type substitution — same two one-block hashes, same 32-byte +// seed and digest — so what needs pinning is that the substitution actually +// happened and that the two configurations do not accept each other's work. +// ========================================================================= + +/// The digest the BLAKE3 configuration grinds over. Gated with the tests below +/// because `Blake3StarkHash` does not exist under `cuda` — the device kernels +/// are keccak-only, so there is no second configuration to name there. +#[cfg(not(feature = "cuda"))] +type Blake3 = GrindingDigest; + +/// ★ Honest path: a nonce ground under BLAKE3 satisfies the BLAKE3 check. +/// +/// At `grinding_factor = 20` a nonce passes by chance with probability 2⁻²⁰, so +/// the cross-hash rejection below is a real control rather than a coin flip. +#[cfg(not(feature = "cuda"))] +#[test] +fn a_blake3_ground_nonce_satisfies_the_blake3_check() { + let seed = [0x5au8; 32]; + let factor = 20; + + let nonce = crate::grinding::generate_nonce::(&seed, factor) + .expect("a nonce exists at this factor"); + assert!( + is_valid_nonce::(&seed, nonce, factor), + "the nonce grinding found must satisfy the check it was ground against" + ); + + // FALSIFICATION: work done against one hash is not work against the other. + // Without this, `generate_nonce::` could still be computing keccak + // and every assertion above would hold. + assert!( + !is_valid_nonce::(&seed, nonce, factor), + "a BLAKE3-ground nonce must not satisfy the keccak check" + ); +} + +/// The two configurations compute different work on identical inputs. +#[cfg(not(feature = "cuda"))] +#[test] +fn the_two_configurations_grind_different_work() { + let seed = [0x11u8; 32]; + let factor = 16; + + let blake3 = crate::grinding::generate_nonce::(&seed, factor).expect("blake3 nonce"); + let keccak = crate::grinding::generate_nonce::(&seed, factor).expect("keccak nonce"); + assert_ne!( + blake3, keccak, + "the same seed under two hashes must not grind to the same nonce" + ); } diff --git a/crypto/stark/src/tests/mod.rs b/crypto/stark/src/tests/mod.rs index 468a4cd3c..ffdfd7485 100644 --- a/crypto/stark/src/tests/mod.rs +++ b/crypto/stark/src/tests/mod.rs @@ -1,9 +1,12 @@ pub mod air_tests; pub mod aux_opening_width_tests; +pub mod batched_mmcs_soundness_tests; +pub mod blake3_stark_roundtrip_tests; #[cfg(feature = "debug-checks")] pub mod bus_debug_tests; pub mod bus_tests; pub mod commitment_tests; +pub mod constraint_index_tests; pub mod domain_cache_stats; pub mod fri_tests; pub mod grinding_tests; @@ -11,6 +14,7 @@ pub mod opening_width_tests; pub mod proof_options_tests; pub mod prove_verify_roundtrip_tests; pub mod prover_tests; +pub mod residency_mode_tests; pub mod row_pair_opening_tests; pub mod small_trace_tests; #[cfg(feature = "disk-spill")] diff --git a/crypto/stark/src/tests/residency_mode_tests.rs b/crypto/stark/src/tests/residency_mode_tests.rs new file mode 100644 index 000000000..3d728b325 --- /dev/null +++ b/crypto/stark/src/tests/residency_mode_tests.rs @@ -0,0 +1,253 @@ +//! `ResidencyMode` equivalence: dropping each main LDE after Round 1 and +//! recomputing it inside the table's fused task changes nothing a verifier can +//! see. +//! +//! The oracle is the whole proof, not just the roots. Comparing serialized +//! proof bytes is normally avoided — a committed golden blob turns every +//! legitimate format change into a test failure — but there is no golden blob +//! here: both sides are produced in this process from the same traces, and the +//! only difference between them is the mode. That makes byte equality the +//! sharpest available statement of "invisible to the proof", and it is the same +//! oracle the closed streaming-prover PR (#647) used for the same change. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField, goldilocks::GoldilocksField, +}; + +use crate::examples::multi_table_lookup::{ + new_add_air_with_lookup, new_cpu_air_with_lookup, new_mul_air_with_lookup, +}; +use crate::proof::options::ProofOptions; +use crate::proof::stark::MultiProof; +use crate::prover::{IsStarkProver, Prover}; +use crate::residency_mode::ResidencyMode; +use crate::trace::TraceTable; +use crate::traits::AIR; +use crate::verifier::{IsStarkVerifier, Verifier}; + +type F = GoldilocksField; +type E = Degree3GoldilocksExtensionField; +type FE = FieldElement; + +/// The bus-balanced CPU/ADD/MUL instance from the completeness tests. Rebuilt +/// per prove because `multi_prove` writes the LogUp aux columns into the caller's +/// traces — and under `RecomputeLde` frees them again. +fn traces() -> (TraceTable, TraceTable, TraceTable) { + let cpu = TraceTable::from_columns_main( + vec![ + vec![ + FE::one(), + FE::zero(), + FE::one(), + FE::zero(), + FE::one(), + FE::one(), + FE::zero(), + FE::zero(), + ], + vec![ + FE::zero(), + FE::one(), + FE::zero(), + FE::one(), + FE::zero(), + FE::zero(), + FE::one(), + FE::one(), + ], + (1..=8).map(FE::from).collect(), + (1..=8).map(|i| FE::from(i * 10)).collect(), + vec![ + FE::from(11), + FE::from(40), + FE::from(33), + FE::from(160), + FE::from(55), + FE::from(66), + FE::from(490), + FE::from(640), + ], + ], + 1, + ); + let add = TraceTable::from_columns_main( + vec![ + vec![FE::from(1), FE::from(3), FE::from(5), FE::from(6)], + vec![FE::from(10), FE::from(30), FE::from(50), FE::from(60)], + vec![FE::from(11), FE::from(33), FE::from(55), FE::from(66)], + vec![FE::one(); 4], + ], + 1, + ); + let mul = TraceTable::from_columns_main( + vec![ + vec![FE::from(2), FE::from(4), FE::from(7), FE::from(8)], + vec![FE::from(20), FE::from(40), FE::from(70), FE::from(80)], + vec![FE::from(40), FE::from(160), FE::from(490), FE::from(640)], + vec![FE::one(); 4], + ], + 1, + ); + (cpu, add, mul) +} + +fn prove_under(residency: ResidencyMode) -> MultiProof { + let (mut cpu_trace, mut add_trace, mut mul_trace) = traces(); + let proof_options = ProofOptions::default_test_options(); + let cpu_air = new_cpu_air_with_lookup(&proof_options); + let add_air = new_add_air_with_lookup(&proof_options); + let mul_air = new_mul_air_with_lookup(&proof_options); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + + Prover::multi_prove( + pairs, + &mut DefaultTranscript::::new(&[]), + #[cfg(feature = "disk-spill")] + crate::storage_mode::StorageMode::Ram, + residency, + ) + .unwrap() +} + +fn verifies(proof: &MultiProof) -> bool { + let proof_options = ProofOptions::default_test_options(); + let cpu_air = new_cpu_air_with_lookup(&proof_options); + let add_air = new_add_air_with_lookup(&proof_options); + let mul_air = new_mul_air_with_lookup(&proof_options); + let airs: Vec<&dyn AIR> = + vec![&cpu_air, &add_air, &mul_air]; + Verifier::multi_verify( + &airs, + proof, + &mut DefaultTranscript::::new(&[]), + &FieldElement::zero(), + ) +} + +/// Every commitment root is unchanged. This is the load-bearing half: the root +/// that entered the transcript at Round 1 is the root the recomputed LDE's +/// openings are checked against, so if the recompute produced different values +/// the openings would be answered against a tree that no longer matches them. +#[test_log::test] +fn recompute_lde_preserves_every_commitment_root() { + let retained = prove_under(ResidencyMode::Retain); + let recomputed = prove_under(ResidencyMode::RecomputeLde); + + assert_eq!(retained.proofs.len(), recomputed.proofs.len()); + for (idx, (a, b)) in retained + .proofs + .iter() + .zip(recomputed.proofs.iter()) + .enumerate() + { + assert_eq!( + a.lde_trace_main_merkle_root, b.lde_trace_main_merkle_root, + "table {idx}: main root moved" + ); + assert_eq!( + a.lde_trace_aux_merkle_root, b.lde_trace_aux_merkle_root, + "table {idx}: aux root moved" + ); + assert_eq!( + a.lde_trace_precomputed_merkle_root, b.lde_trace_precomputed_merkle_root, + "table {idx}: precomputed root moved" + ); + assert_eq!( + a.composition_poly_root, b.composition_poly_root, + "table {idx}: composition root moved" + ); + assert_eq!( + a.fri_layers_merkle_roots, b.fri_layers_merkle_roots, + "table {idx}: FRI roots moved" + ); + } +} + +/// The whole proof, byte for byte — openings, FRI decommitments, grinding nonce +/// and all. +#[test_log::test] +fn recompute_lde_produces_byte_identical_proofs() { + let retained = bincode::serialize(&prove_under(ResidencyMode::Retain)).unwrap(); + let recomputed = bincode::serialize(&prove_under(ResidencyMode::RecomputeLde)).unwrap(); + assert_eq!( + retained.len(), + recomputed.len(), + "proof size moved between residency modes" + ); + assert!( + retained == recomputed, + "proof bytes moved between residency modes" + ); +} + +/// A proof made under `RecomputeLde` verifies with the standard verifier. Near +/// tautological given the byte equality above — and that is the point: the mode +/// has no wire presence for a verifier to know about. +#[test_log::test] +fn recompute_lde_proofs_verify() { + assert!(verifies(&prove_under(ResidencyMode::RecomputeLde))); + assert!(verifies(&prove_under(ResidencyMode::Retain))); +} + +/// The caller-visible half of the `RecomputeLde` contract: the aux columns +/// `multi_prove` wrote into the caller's traces are gone when it returns, and +/// under `Retain` they are still there. Pins the documented difference so a +/// caller that needs the aux columns after proving finds out here. +#[test_log::test] +fn recompute_lde_releases_aux_columns_and_retain_keeps_them() { + for (residency, expect_aux_rows) in [ + (ResidencyMode::Retain, true), + (ResidencyMode::RecomputeLde, false), + ] { + let (mut cpu_trace, mut add_trace, mut mul_trace) = traces(); + let proof_options = ProofOptions::default_test_options(); + let cpu_air = new_cpu_air_with_lookup(&proof_options); + let add_air = new_add_air_with_lookup(&proof_options); + let mul_air = new_mul_air_with_lookup(&proof_options); + + { + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + Prover::multi_prove( + pairs, + &mut DefaultTranscript::::new(&[]), + #[cfg(feature = "disk-spill")] + crate::storage_mode::StorageMode::Ram, + residency, + ) + .unwrap(); + } + + for (name, trace) in [ + ("cpu", &cpu_trace), + ("add", &add_trace), + ("mul", &mul_trace), + ] { + let has_rows = trace.aux_table.height > 0; + assert_eq!( + has_rows, expect_aux_rows, + "{name} trace aux residency wrong under {residency:?}" + ); + // The declared width survives either way — only the data is freed. + assert!(trace.aux_table.width > 0, "{name} lost its aux width"); + } + } +} diff --git a/crypto/stark/src/trace.rs b/crypto/stark/src/trace.rs index b34023ac3..0f461a0bf 100644 --- a/crypto/stark/src/trace.rs +++ b/crypto/stark/src/trace.rs @@ -213,6 +213,22 @@ where self.main_trace_dev = None; } + /// Free the auxiliary columns, keeping the declared aux width. + /// + /// Called by `multi_prove` under `ResidencyMode::RecomputeLde` once a + /// table's proof exists: `allocate_aux_table` writes the LogUp columns into + /// this caller-owned trace and nothing reads them afterwards, so under that + /// mode they are released rather than carried to the end of the prove. + /// Callers that do read a trace's aux columns after proving must use + /// `ResidencyMode::Retain`. + pub fn release_aux_columns(&mut self) { + self.aux_table = Table::new(Vec::new(), self.aux_table.width); + #[cfg(feature = "cuda")] + { + self.aux_resident = None; + } + } + pub fn num_steps(&self) -> usize { debug_assert!(self.main_table.height.is_multiple_of(self.step_size)); self.main_table.height / self.step_size diff --git a/crypto/stark/src/traits.rs b/crypto/stark/src/traits.rs index 0aec97a2a..7b9875a4d 100644 --- a/crypto/stark/src/traits.rs +++ b/crypto/stark/src/traits.rs @@ -270,17 +270,48 @@ pub trait AIR: Send + Sync { /// prefix (its length is `num_base_transition_constraints()`). fn constraints_meta(&self) -> &[ConstraintMeta]; - /// The lazily captured flat IR ([`ConstraintProgram`]) of every transition - /// constraint, for the CPU interpreter and the GPU kernel. + /// The flat IR ([`ConstraintProgram`]) of every transition constraint, for + /// the CPU interpreter and the GPU kernel — captured on demand unless a + /// pre-captured program was supplied (see + /// [`Self::precaptured_constraint_program`]). /// - /// GUEST-SAFETY: capture hash-conses, so the verify/recursion path must - /// NEVER call this — only the prover, GPU lowering, and tests do. The - /// default panics precisely so any accidental verify-path use is caught; - /// AIRs that support capture override it with a cached (`OnceLock`) build. + /// GUEST-SAFETY: this MAY capture, and capture hash-conses, so the + /// verify/recursion path must never call it — only the prover, GPU + /// lowering, and tests do. The default panics precisely so any accidental + /// verify-path use is caught; AIRs that support capture override it with a + /// cached (`OnceLock`) build. + /// + /// The prohibition is on CAPTURE, not on constraint programs as such: a + /// program serialized at build time is ordinary data, and consuming one is + /// allowed anywhere. That path is + /// [`Self::precaptured_constraint_program`]. fn constraint_program(&self) -> &ConstraintProgram { unimplemented!("constraint_program is not available for this AIR") } + /// A pre-captured constraint program supplied at build time, if this AIR was + /// given one; `None` otherwise. + /// + /// GUEST-SAFETY: unlike [`Self::constraint_program`], this NEVER captures + /// under any circumstance — it is a borrow of data handed to the AIR at + /// construction, so it is safe on the verify/recursion path. That is the + /// entire distinction between the two methods, and it is why they are + /// separate rather than one method with a flag: an accidental verify-path + /// call to the capturing one still hits the panic above. + /// + /// `None` is not an error — it means nobody supplied an artifact, and the + /// caller must fall back to the compiled folder. A caller that needs a + /// program on a guest path must treat `None` as fatal itself, because + /// falling back to `constraint_program()` there would reintroduce capture. + /// + /// See [`ConstraintArtifact`](crate::constraint_ir::ConstraintArtifact) for + /// the serialized form and for what validating one does and does not prove. + fn precaptured_constraint_program( + &self, + ) -> Option<&ConstraintProgram> { + None + } + fn boundary_constraints( &self, pub_inputs: &Self::PublicInputs, diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index ca6f15152..6cf5178cc 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -1,5 +1,5 @@ use super::{ - config::BatchedMerkleTreeBackend, + config::{KeccakStarkHash, StarkHash}, domain::VerifierDomain, grinding, proof::stark::StarkProof, @@ -19,6 +19,7 @@ use crate::{ }; use crypto::fiat_shamir::is_transcript::IsStarkTranscript; use crypto::merkle_tree::proof::{verify_merkle_path, verify_merkle_path_from_leaf_hash}; +use crypto::merkle_tree::traits::IsStreamingLeafBackend; #[cfg(not(feature = "test_fiat_shamir"))] use log::error; #[cfg(feature = "debug-checks")] @@ -36,20 +37,29 @@ use std::marker::PhantomData; #[cfg(feature = "instruments")] use std::time::Instant; -/// A default STARK verifier implementing `IsStarkVerifier`. -pub struct Verifier< +/// A default STARK verifier implementing `IsStarkVerifier`, generic over the +/// commitment configuration `H`. Mirrors `GenericProver`: `H` rides on the +/// concrete type so [`Verifier`] pins it and existing call sites resolve +/// unchanged. +pub struct GenericVerifier< Field: IsSubFieldOf + IsFFTField + Send + Sync, FieldExtension: Send + Sync + IsField, PI, + H, > { - phantom: PhantomData<(Field, FieldExtension, PI)>, + phantom: PhantomData<(Field, FieldExtension, PI, H)>, } +/// The production verifier: [`GenericVerifier`] at the keccak configuration. +pub type Verifier = + GenericVerifier; + impl< - Field: IsSubFieldOf + IsFFTField + Send + Sync, - FieldExtension: IsField + Send + Sync, + Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, + FieldExtension: IsField + Send + Sync + 'static, PI, -> IsStarkVerifier for Verifier + H: StarkHash, +> IsStarkVerifier for GenericVerifier where Field::BaseType: math::field::element::NativeArchived, FieldExtension::BaseType: math::field::element::NativeArchived, @@ -121,9 +131,10 @@ compile_error!("the zero-copy STARK verifier requires a little-endian target"); /// are thin entry points that build the matching view and share every /// downstream check — no serialization, no duplicated logic. pub trait IsStarkVerifier< - Field: IsSubFieldOf + IsFFTField + Send + Sync, - FieldExtension: Send + Sync + IsField, + Field: IsSubFieldOf + IsFFTField + Send + Sync + 'static, + FieldExtension: Send + Sync + IsField + 'static, PI, + H: StarkHash, > where Field::BaseType: math::field::element::NativeArchived, FieldExtension::BaseType: math::field::element::NativeArchived, @@ -574,17 +585,17 @@ pub trait IsStarkVerifier< where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, - E: IsField, + E: IsField + 'static, E::BaseType: math::field::element::NativeArchived, Field: IsSubFieldOf, { // Two-slice leaf hash: the committed leaf is `evaluations ‖ evaluations_sym`, // hashed without allocating the concatenation (see `hash_data_from_slices`). - let leaf_hash = BatchedMerkleTreeBackend::::hash_data_from_slices( + let leaf_hash = as IsStreamingLeafBackend>::hash_data_from_slices( opening.evaluations(), opening.evaluations_sym(), ); - verify_merkle_path_from_leaf_hash::>( + verify_merkle_path_from_leaf_hash::>( opening.merkle_path(), root, iota, @@ -663,12 +674,14 @@ pub trait IsStarkVerifier< { let composition_poly = deep_poly_openings.composition_poly(); // Two-slice leaf hash of `evaluations ‖ evaluations_sym`, no concat alloc. - let leaf_hash = BatchedMerkleTreeBackend::::hash_data_from_slices( + let leaf_hash = as IsStreamingLeafBackend< + FieldExtension, + >>::hash_data_from_slices( composition_poly.evaluations(), composition_poly.evaluations_sym(), ); - verify_merkle_path_from_leaf_hash::>( + verify_merkle_path_from_leaf_hash::>( composition_poly.merkle_path(), composition_poly_merkle_root, *iota, @@ -720,7 +733,7 @@ pub trait IsStarkVerifier< vec![evaluation.clone(), evaluation_sym.clone()] }; - verify_merkle_path::>( + verify_merkle_path::>( auth_path_sym, merkle_root, iota >> 1, @@ -1665,7 +1678,11 @@ pub trait IsStarkVerifier< let security_bits = air.context().proof_options.grinding_factor; if security_bits > 0 { let nonce_is_valid = proof.nonce().is_some_and(|nonce_value| { - grinding::is_valid_nonce(&challenges.grinding_seed, nonce_value, security_bits) + grinding::is_valid_nonce::>( + &challenges.grinding_seed, + nonce_value, + security_bits, + ) }); if !nonce_is_valid { diff --git a/executor/programs/asm/test_blake3.s b/executor/programs/asm/test_blake3.s new file mode 100644 index 000000000..d066f4645 --- /dev/null +++ b/executor/programs/asm/test_blake3.s @@ -0,0 +1,60 @@ + .attribute 5, "rv64i2p1_m2p0_zmmul1p0" + .globl main +main: + # 176 bytes on the stack for the BLAKE3 state region (22 x u64): + # h[4 dwords] | m[8] | t[1] | block_len,flags[1] | out[8]. + addi sp, sp, -176 + + # Deterministic non-zero seed over the 14 input dwords: dword[k] = k + 1. + # (t therefore = 13, block_len = 14, flags = 0 — arbitrary but fixed.) + mv t0, sp + li t1, 1 + li t2, 15 +.Linit_loop: + sd t1, 0(t0) + addi t0, t0, 8 + addi t1, t1, 1 + bne t1, t2, .Linit_loop + + # First compression. + # a0 = pointer to the 176-byte region (8-aligned) + # a7 = syscall number (u64::MAX - 2 = -3) + mv a0, sp + li a7, -3 + ecall + + # Chain: copy out (8 dwords at sp+112) over m (8 dwords at sp+32), so the + # second call consumes the first call's output AND its out-region write has + # non-zero previous content. + li t1, 0 +.Lcopy_loop: + slli t2, t1, 3 + addi t3, sp, 112 + add t3, t3, t2 + ld t4, 0(t3) + addi t3, sp, 32 + add t3, t3, t2 + sd t4, 0(t3) + addi t1, t1, 1 + li t2, 8 + bne t1, t2, .Lcopy_loop + + # Second compression. + mv a0, sp + li a7, -3 + ecall + + # Commit the final 64-byte output. + li a0, 1 + addi a1, sp, 112 + li a2, 64 + li a7, 64 + ecall + + # Restore stack and halt. + addi sp, sp, 176 + li a0, 0 + li a7, 93 + ecall +.Lfunc_end0: + .size main, .Lfunc_end0-main diff --git a/executor/programs/rust/keccak_transcript_pattern/Cargo.lock b/executor/programs/rust/keccak_transcript_pattern/Cargo.lock index 4e5afb1bd..0b59195aa 100644 --- a/executor/programs/rust/keccak_transcript_pattern/Cargo.lock +++ b/executor/programs/rust/keccak_transcript_pattern/Cargo.lock @@ -88,8 +88,6 @@ dependencies = [ "digest", "lambda-vm-syscalls", "math", - "rand 0.8.7", - "rand_chacha 0.3.1", "serde", "sha3", ] @@ -240,7 +238,7 @@ dependencies = [ "getrandom 0.2.17", "getrandom 0.3.4", "lazy_static", - "rand 0.9.5", + "rand", "riscv", "thiserror", ] @@ -270,7 +268,6 @@ dependencies = [ "getrandom 0.2.17", "num-bigint", "num-traits", - "rand 0.8.7", "rayon", "serde", "serde_json", @@ -361,33 +358,14 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rand" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core", ] [[package]] @@ -397,15 +375,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.5", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" - [[package]] name = "rand_core" version = "0.9.5" diff --git a/executor/src/tests/blake3_tests.rs b/executor/src/tests/blake3_tests.rs new file mode 100644 index 000000000..98bd9af5d --- /dev/null +++ b/executor/src/tests/blake3_tests.rs @@ -0,0 +1,319 @@ +//! Tests for the BLAKE3 6-round compression and its accelerator syscall. +//! +//! Ground truth is the validated oracle (`thoughts/blake3/blake3-oracle/`): +//! the pinned canonical 6-round vectors below were emitted by its harness and +//! checked against the official `blake3` crate at the official test-vector +//! parameters. The `t` values exercise the full 64-bit counter range, pinning +//! the `t_lo → v[12]` / `t_hi → v[13]` split order. + +use crate::vm::instruction::decoding::Instruction; +use crate::vm::instruction::execution::{ + BLAKE3_SYSCALL_NUMBER, ExecutionError, blake3_compress_6round, +}; +use crate::vm::memory::Memory; +use crate::vm::registers::Registers; + +/// One pinned 6-round vector from the validated oracle. +struct Blake3Vector { + h: [u32; 8], + m: [u32; 16], + t: u64, + block_len: u32, + flags: u32, + out: [u32; 16], +} + +/// The 10 canonical 6-round vectors, generated from +/// `thoughts/blake3/blake3-oracle/canonical_6round_vectors.json` (which the +/// oracle harness regenerates and which was validated against the official +/// `blake3` crate). Do not edit by hand. +const CANONICAL_6ROUND_VECTORS: &[Blake3Vector] = &[ + Blake3Vector { + h: [ + 0xd82c07cd, 0x6baa9455, 0x82e2e662, 0x7a024204, 0xe87a1613, 0x81332876, 0x48268673, + 0xc17c6279, + ], + m: [ + 0xe6f4590b, 0x4f65d4d9, 0xbad640fb, 0xaf19922a, 0x19c78df4, 0x6f25e2a2, 0xe9bb17bc, + 0x7a1d5006, 0x42af9fc3, 0x03983ca8, 0xde1b372a, 0xded733e8, 0x9148624f, 0xf7b0b7d2, + 0x72ae2244, 0xeece328b, + ], + t: 0xb4e1357d4a84eb03, + block_len: 42, + flags: 52, + out: [ + 0xced9d1ff, 0xc248eeab, 0xbd109b7f, 0x911b48f6, 0x923d62c0, 0xd804903f, 0x5974223e, + 0xaa4f0c80, 0xad61007f, 0xb50b8ddb, 0xe7372be1, 0x33d3d6c3, 0x42aa284b, 0xc5a25f28, + 0x79ac8370, 0xb75f3915, + ], + }, + Blake3Vector { + h: [ + 0xc386bbc4, 0x414c343c, 0x7311d8a3, 0xa6cecc1b, 0xc9e9c616, 0x18072e8c, 0xd5f4b3b2, + 0x7204e52d, + ], + m: [ + 0xf1fd42a2, 0xe6c3f339, 0x07d4bedc, 0x8a9a021e, 0x3bab6c39, 0x05805975, 0xa46d6753, + 0xdc2574bd, 0xab99254a, 0x4da98f1d, 0xe1ea24c4, 0x815a47c5, 0x08d6af57, 0xcc22af58, + 0x2c4a3698, 0x5fec898f, + ], + t: 0xc74803e31ba16215, + block_len: 50, + flags: 94, + out: [ + 0xf2a972e9, 0x81fdb8ec, 0x40c50ebc, 0x4ba1caf9, 0x9ee9e930, 0x6b1a16b2, 0xe9156f47, + 0xa89fb436, 0xa2f616b3, 0x12874c12, 0x30768035, 0xe01a17d9, 0xbee5c17c, 0xd61c0be0, + 0x3041ff46, 0xdfb91125, + ], + }, + Blake3Vector { + h: [ + 0x0e7a269f, 0x15ba2bdd, 0xd5e34124, 0x4ee207f8, 0x9b1f282e, 0x9b575bd1, 0xf30b94fa, + 0x0706a045, + ], + m: [ + 0x6148a86f, 0x8697bbd0, 0x8f7d9b78, 0x3c729578, 0x061b9030, 0x533c9135, 0x829e07b0, + 0xe4c11ab2, 0xcbf87544, 0xc34c769f, 0x5a91c89b, 0xf63f23d0, 0xc1066932, 0x87c56473, + 0x7d718d73, 0xecc1cb63, + ], + t: 0x7604e4b4e73695c3, + block_len: 58, + flags: 124, + out: [ + 0x5aa6b114, 0xc9d6740c, 0x8738caf4, 0xac5f4b72, 0x9fc6b9de, 0x3f2efb8f, 0x8cb7a912, + 0xf497a285, 0x3d062266, 0x7f22380c, 0xafd468fa, 0x122cba80, 0x446b156d, 0xb239d8c2, + 0xc3eab2cf, 0x775f2f92, + ], + }, + Blake3Vector { + h: [ + 0x8b529b4a, 0x9a9a80fd, 0xd6645fa9, 0x3bfd1d33, 0x79f248b0, 0x268ecc45, 0xa2863a7f, + 0x85ef3430, + ], + m: [ + 0xbdc2ae99, 0x10645d51, 0x97524d6a, 0xdd933160, 0xe0f9e038, 0xebcd1f5e, 0xef829c88, + 0xe0fd67dd, 0x18f2c41c, 0x22cedafb, 0x378c74dc, 0x4d100d8f, 0x95c76ab4, 0x95918694, + 0xe779c470, 0xedcf6109, + ], + t: 0x92d3043afcf249f3, + block_len: 36, + flags: 31, + out: [ + 0xeed92fab, 0x138d9358, 0x915bfe3c, 0x13718b01, 0xb506e277, 0xbe4007cd, 0x35847e06, + 0xce1c6896, 0x52fa01b5, 0x4aa26af8, 0xb1078a61, 0x2c517aed, 0xa08867a0, 0xea6ecfea, + 0x6d33d3b0, 0xdc293166, + ], + }, + Blake3Vector { + h: [ + 0x3c6da5d7, 0x656412a9, 0x27ac435a, 0x11072231, 0xeaff1a09, 0xc3e1b258, 0x8963dc6e, + 0x1b2ed40e, + ], + m: [ + 0xed6f0b09, 0xce80c4b0, 0xccea2645, 0x3184ff27, 0x4f5253a0, 0xe14b0190, 0x9b191bf4, + 0xabf4a07c, 0x81862fc9, 0x2d83a823, 0x793d0e45, 0x4cdce7a6, 0xe8abb93f, 0xe1df8af9, + 0x8224b122, 0x69f85e31, + ], + t: 0x49c7b59b995253fd, + block_len: 57, + flags: 41, + out: [ + 0xca00bda3, 0x84239a3a, 0xe7c88e6d, 0x33a8a3d6, 0x09dcd1ce, 0xa1b10212, 0xf48e1156, + 0x8f039915, 0x8a055eaa, 0xff5b11d5, 0xb725085b, 0x2e1ab267, 0x6ae7323d, 0xb2ff6fa8, + 0x7102c8a1, 0x7561eb37, + ], + }, + Blake3Vector { + h: [ + 0x9f767c45, 0xbde5c099, 0xf17fd374, 0xa6233255, 0xe6a16a3b, 0x1cfb10f6, 0x3f1f65a8, + 0x8b33e968, + ], + m: [ + 0x92edcf45, 0x377b9aa2, 0x478c281d, 0xc4069545, 0xcc11d357, 0x9e115e4b, 0x206f5c66, + 0xdf1461aa, 0xfb7ff337, 0xdf561d80, 0x4a0fe75d, 0xf6236bf2, 0x346c6e2b, 0xb0cde917, + 0xe4cc4132, 0x4c7d6df0, + ], + t: 0x6a3753915c76f18a, + block_len: 18, + flags: 67, + out: [ + 0x14a9f66f, 0x101bdfe8, 0x9b0a50dd, 0xee4bb45b, 0x7a914502, 0x77b3486b, 0x59bfc114, + 0xa1ad2afd, 0xc194dde6, 0x894ec54d, 0xad36c805, 0x9018f3f5, 0x165af5d8, 0x3e85b598, + 0x78e76653, 0xbb7a485d, + ], + }, + Blake3Vector { + h: [ + 0xd26b9496, 0x42f9a039, 0x001d9a88, 0x5f877031, 0xc527e279, 0x45cf8aa4, 0xcd4a5557, + 0xae9af169, + ], + m: [ + 0xaf895f5b, 0xd822e2f9, 0x17d7ab26, 0xccdf540b, 0xce06294d, 0x4a8b0188, 0xf38d2e64, + 0x5c41d5c5, 0xe8d5b9e3, 0x5c832a51, 0x9a0c1b76, 0x4de8344e, 0x96d2f9e0, 0x8677a5f2, + 0xa9a967c1, 0x323bbeaf, + ], + t: 0x390567c27bd6aa42, + block_len: 26, + flags: 3, + out: [ + 0x32a6ff70, 0xc30560bc, 0xd1c777c8, 0xf1871821, 0x7207ab54, 0x9f5b83c7, 0xb6561c5d, + 0x991e738f, 0xb38b62b9, 0x0ef6d156, 0x994becb1, 0x09a85d0e, 0x32221741, 0xada3cc5f, + 0x5b654ed6, 0x2a7a62b2, + ], + }, + Blake3Vector { + h: [ + 0x269e0d37, 0xa6a3a450, 0x892f902b, 0x81e74ef5, 0x099950d8, 0x6f03675a, 0x11e20b8f, + 0x6cad4a26, + ], + m: [ + 0xf29d0da9, 0x658cda14, 0xf9ebdacc, 0xdbc496cb, 0x4a23d596, 0x2e44158b, 0xa38fd547, + 0x5f557203, 0x34b9b5df, 0x506bf2ef, 0x7403e430, 0x4cbd87ad, 0xcb5c7427, 0x3e7d1bfb, + 0x930d6eaf, 0x86734721, + ], + t: 0x12bd4acefaecbd38, + block_len: 53, + flags: 42, + out: [ + 0xa632ad45, 0x12ce41f4, 0xd21b2cbd, 0x76795c62, 0x6bec36c1, 0xdafafcde, 0x53ca87b7, + 0x92e8465b, 0x7b424f5d, 0xe1e6ad7f, 0x753ba387, 0xccc50824, 0x69aedf6d, 0xbbbbf253, + 0x78d04883, 0xf3f33689, + ], + }, + Blake3Vector { + h: [ + 0x3a096533, 0xf658f7a7, 0x205738d1, 0xb46ee1da, 0x15ceb3a1, 0x359b1548, 0xa4517d6c, + 0x7589ca4a, + ], + m: [ + 0x74007cb4, 0xd49d0ac1, 0x16edc5d4, 0x685ca8af, 0x4223aa56, 0x10269470, 0x60908405, + 0xa92d04a3, 0x56a3e957, 0xb0f91306, 0xe6c08269, 0xf2306d4a, 0x31a06a7c, 0x9436d6f6, + 0xe18692e2, 0xe0c99f3e, + ], + t: 0x329911da9fbd8735, + block_len: 19, + flags: 91, + out: [ + 0x913b2ae1, 0xc7f73082, 0x45e1c023, 0x6f1f3f82, 0x20aee6f5, 0xdaf21d94, 0xf2c1e4af, + 0xd4f7d4ac, 0x44a45f87, 0xf4c40ce5, 0x613e9b94, 0x08ce53de, 0x4ff07aa4, 0x456bf2e2, + 0x2066ea7f, 0x3c5a654b, + ], + }, + Blake3Vector { + h: [ + 0x5f915ef0, 0x237751aa, 0x01a5ba50, 0x80b65386, 0x14b044d7, 0x61076dc3, 0xb99de255, + 0x283b73a6, + ], + m: [ + 0x3cee5e2c, 0x1c670ea9, 0x972651da, 0x4a8aa593, 0xac9abb0c, 0x35bb5c11, 0x47fbb3b4, + 0xcf3c17e5, 0xe2eb17c8, 0xe11e99fb, 0x7de0d208, 0x0602fe0c, 0x98cae043, 0x9425b3e2, + 0x33fb4b4f, 0x15607df9, + ], + t: 0xeaeb999b8a2e547e, + block_len: 64, + flags: 21, + out: [ + 0xf5ee9114, 0x856cabb8, 0x29be2cf1, 0x603be91c, 0x94a7dd0e, 0x28fc3e27, 0xb64e2cc8, + 0x2d2c67ff, 0x69fac1ba, 0x0c949090, 0xd68de435, 0xce91a527, 0xe80c1815, 0x6d44efe6, + 0x87c7b175, 0xd18a8b94, + ], + }, +]; + +#[test] +fn test_blake3_6round_canonical_vectors() { + for (i, v) in CANONICAL_6ROUND_VECTORS.iter().enumerate() { + let out = blake3_compress_6round(&v.h, &v.m, v.t, v.block_len, v.flags); + assert_eq!(out, v.out, "canonical 6-round vector {i} mismatch"); + } +} + +#[test] +fn test_blake3_syscall_matches_vectors() { + for (i, v) in CANONICAL_6ROUND_VECTORS.iter().enumerate() { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + let addr = 0x1000u64; + + // Lay out the 176-byte state region: h | m | t | (block_len, flags) | out. + let mut words = [0u32; 28]; + words[0..8].copy_from_slice(&v.h); + words[8..24].copy_from_slice(&v.m); + words[24] = v.t as u32; + words[25] = (v.t >> 32) as u32; + words[26] = v.block_len; + words[27] = v.flags; + for k in 0..14 { + let dw = (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32); + memory.store_doubleword(addr + (k as u64) * 8, dw).unwrap(); + } + // Pre-fill the out region so the test catches a partial write. + for k in 14..22 { + memory + .store_doubleword(addr + (k as u64) * 8, 0xDEAD_BEEF_DEAD_BEEFu64) + .unwrap(); + } + + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + registers.write(10, addr).unwrap(); + Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap(); + + let mut got = [0u32; 16]; + for k in 0..8 { + let dw = memory + .load_doubleword(addr + ((14 + k) as u64) * 8) + .unwrap(); + got[2 * k] = dw as u32; + got[2 * k + 1] = (dw >> 32) as u32; + } + assert_eq!(got, v.out, "syscall output mismatch on vector {i}"); + + // The 112 input bytes must be untouched. + for k in 0..14 { + let dw = memory.load_doubleword(addr + (k as u64) * 8).unwrap(); + let expected = (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32); + assert_eq!(dw, expected, "input dword {k} clobbered on vector {i}"); + } + } +} + +#[test] +fn test_blake3_syscall_rejects_unaligned_state_addr() { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + registers.write(10, 0x1004).unwrap(); + + let err = Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap_err(); + assert!(matches!( + err, + ExecutionError::UnalignedBlake3StateAddress(0x1004) + )); +} + +#[test] +fn test_blake3_syscall_rejects_overflowing_state_range() { + let mut pc = 0; + let mut registers = Registers::default(); + let mut memory = Memory::default(); + + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + // 22 dwords = 176 bytes; addr + 175 must not overflow. u64::MAX - 167 is + // 8-aligned and the last byte lands at u64::MAX + 8 → overflow. + registers.write(10, u64::MAX - 167).unwrap(); + + let err = Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap_err(); + assert!(matches!( + err, + ExecutionError::Blake3StateAddressOverflow(addr) if addr == u64::MAX - 167 + )); +} diff --git a/executor/src/tests/mod.rs b/executor/src/tests/mod.rs index 244447b22..662a116d5 100644 --- a/executor/src/tests/mod.rs +++ b/executor/src/tests/mod.rs @@ -1,3 +1,4 @@ +pub mod blake3_tests; pub mod ecsm_tests; pub mod flamegraph_tests; pub mod hint_tests; diff --git a/executor/src/vm/instruction/execution.rs b/executor/src/vm/instruction/execution.rs index 592af95e8..681d6c7c7 100644 --- a/executor/src/vm/instruction/execution.rs +++ b/executor/src/vm/instruction/execution.rs @@ -12,6 +12,8 @@ pub enum SyscallNumbers { KeccakPermute = 0, Print = 1, Panic = 2, + // Placeholder discriminant. The actual syscall value is BLAKE3_SYSCALL_NUMBER. + Blake3Compress = 3, Commit = 64, Halt = 93, // Placeholder discriminant. The actual syscall value is ECSM_SYSCALL_NUMBER. @@ -27,6 +29,30 @@ pub enum SyscallNumbers { pub const KECCAK_SYSCALL_NUMBER: u64 = u64::MAX - 1; const KECCAK_STATE_BYTES: u64 = 25 * 8; +/// Syscall number for the BLAKE3 6-round compression accelerator +/// (u64::MAX - 2 = 0xFFFF_FFFF_FFFF_FFFD). +/// +/// This is the **6-round internal variant** of the BLAKE3 compression function +/// (`thoughts/blake3/blake3-chip/DESIGN.md`), intended for in-house Merkle / +/// Fiat–Shamir use — it is NOT standard 7-round BLAKE3 and its security rests +/// on the named 6-round assumption recorded in the design. +/// +/// ABI: `x10` = 8-byte-aligned pointer to a 176-byte state region laid out as +/// consecutive little-endian dwords at `addr + 8k`: +/// +/// | dword k | contents | +/// |---------|--------------------------------------------| +/// | 0..=3 | `h[0..8]` chaining value (2 u32 words/dword) | +/// | 4..=11 | `m[0..16]` message block | +/// | 12 | `t` counter (`t_lo = low u32 → v[12]`, `t_hi = high u32 → v[13]`) | +/// | 13 | `block_len` (low u32) \| `flags` (high u32) | +/// | 14..=21 | `out[0..16]` — written by the syscall | +pub const BLAKE3_SYSCALL_NUMBER: u64 = u64::MAX - 2; +/// Bytes of the BLAKE3 state region: 112 input + 64 output. +const BLAKE3_STATE_BYTES: u64 = 22 * 8; +/// Dword offset of `out[0..16]` inside the BLAKE3 state region. +const BLAKE3_OUT_DWORDS: u64 = 14; + /// Syscall number for the ECSM (elliptic-curve scalar multiply) accelerator. /// /// The spec uses ECALL number `-11`; interpreted as an unsigned 64-bit value that is @@ -87,6 +113,7 @@ impl TryFrom for SyscallNumbers { 64 => Ok(SyscallNumbers::Commit), 93 => Ok(SyscallNumbers::Halt), v if v == KECCAK_SYSCALL_NUMBER => Ok(SyscallNumbers::KeccakPermute), + v if v == BLAKE3_SYSCALL_NUMBER => Ok(SyscallNumbers::Blake3Compress), v if v == ECSM_SYSCALL_NUMBER => Ok(SyscallNumbers::Ecsm), v if v == HINT_SYSCALL_NUMBER => Ok(SyscallNumbers::Hint), _ => Err(()), @@ -98,6 +125,7 @@ impl TryFrom for SyscallNumbers { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Accelerator { Keccak, + Blake3, Ecsm, } @@ -108,6 +136,7 @@ impl SyscallNumbers { pub fn accelerator(self) -> Option { match self { SyscallNumbers::KeccakPermute => Some(Accelerator::Keccak), + SyscallNumbers::Blake3Compress => Some(Accelerator::Blake3), SyscallNumbers::Ecsm => Some(Accelerator::Ecsm), SyscallNumbers::Print | SyscallNumbers::Panic @@ -517,6 +546,41 @@ impl Instruction { } src2_val = state_addr; } + SyscallNumbers::Blake3Compress => { + // BLAKE3 6-round compression on a 176-byte region at the + // address in x10 (layout: see BLAKE3_SYSCALL_NUMBER docs). + let state_addr = registers.read(10)?; + if !state_addr.is_multiple_of(8) { + return Err(ExecutionError::UnalignedBlake3StateAddress(state_addr)); + } + state_addr + .checked_add(BLAKE3_STATE_BYTES - 1) + .ok_or(ExecutionError::Blake3StateAddressOverflow(state_addr))?; + + // Input: 14 dwords = h[8] | m[16] | t | (block_len, flags), + // each dword two little-endian u32 words. + let mut words = [0u32; 28]; + for k in 0..14 { + let dw = memory.load_doubleword(state_addr + (k as u64) * 8)?; + words[2 * k] = dw as u32; + words[2 * k + 1] = (dw >> 32) as u32; + } + let h: [u32; 8] = words[0..8].try_into().unwrap(); + let m: [u32; 16] = words[8..24].try_into().unwrap(); + let t = (words[24] as u64) | ((words[25] as u64) << 32); + let block_len = words[26]; + let flags = words[27]; + + let out = blake3_compress_6round(&h, &m, t, block_len, flags); + for k in 0..8 { + let dw = (out[2 * k] as u64) | ((out[2 * k + 1] as u64) << 32); + memory.store_doubleword( + state_addr + (BLAKE3_OUT_DWORDS + k as u64) * 8, + dw, + )?; + } + src2_val = state_addr; + } SyscallNumbers::Ecsm => { // ECSM(-11): k×G on secp256k1. // x10 = addr to write xR, x11 = addr of xG, x12 = addr of k. @@ -762,6 +826,10 @@ pub enum ExecutionError { UnalignedKeccakStateAddress(u64), #[error("Keccak state address range overflows: {0:#018x}")] KeccakStateAddressOverflow(u64), + #[error("Unaligned BLAKE3 state address: {0:#018x}")] + UnalignedBlake3StateAddress(u64), + #[error("BLAKE3 state address range overflows: {0:#018x}")] + Blake3StateAddressOverflow(u64), #[error("ECSM address range overflows the lower 32-bit limb")] EcsmAddressOverflow, #[error("ECSM xG and k operand ranges overlap")] @@ -855,3 +923,106 @@ pub fn keccak_f1600(state: &mut [u64; 25]) { state[0] ^= rc; } } + +// ============================================================================= +// BLAKE3 6-round compression (internal variant) +// ============================================================================= +// +// A Rust port of the validated oracle `thoughts/blake3/blake3-oracle/blake3_ref.py` +// with `rounds = 6` fixed. This is the **6-round internal variant** — NOT +// standard BLAKE3 (7 rounds); its security rests on the named 6-round +// assumption recorded in `thoughts/blake3/blake3-chip/DESIGN.md`. Differentially +// tested against the oracle's canonical 6-round vectors (pinned in +// `thoughts/blake3/blake3-oracle/canonical_6round_vectors.json`, themselves +// validated against the official `blake3` crate). + +/// The BLAKE3 IV (identical to SHA-256's initial state). `IV[0..4]` seeds +/// `v[8..12]` of the compression working state. +pub const BLAKE3_IV: [u32; 8] = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +]; + +/// The BLAKE3 message-schedule permutation, applied between rounds +/// (`m'[i] = m[MSG_PERMUTATION[i]]`). +pub const BLAKE3_MSG_PERMUTATION: [usize; 16] = + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8]; + +/// Rounds of the internal variant. 6, per the design; standard BLAKE3 is 7. +pub const BLAKE3_ROUNDS: usize = 6; + +/// The BLAKE3 quarter-round G (spec §2.1). +#[inline] +fn blake3_g(v: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, mx: u32, my: u32) { + v[a] = v[a].wrapping_add(v[b]).wrapping_add(mx); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(12); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(my); + v[d] = (v[d] ^ v[a]).rotate_right(8); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(7); +} + +/// The BLAKE3 compression function `f` at 6 rounds (spec §2.2, oracle §2.4). +/// +/// State init: `v[0..8] = h`, `v[8..12] = IV[0..4]`, `v[12] = t as u32`, +/// `v[13] = (t >> 32) as u32`, `v[14] = block_len`, `v[15] = flags`. Six +/// rounds of 8 G-calls (4 columns then 4 diagonals), permuting the message +/// schedule between rounds (`r < rounds - 1`, i.e. 5 permutes — the trailing +/// permute is never consumed). Feed-forward: `out[i] = v[i] ^ v[i+8]`, +/// `out[i+8] = v[i+8] ^ h[i]`. The truncated chaining value is `out[0..8]`. +pub fn blake3_compress_6round( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, +) -> [u32; 16] { + let mut v: [u32; 16] = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + t as u32, + (t >> 32) as u32, + block_len, + flags, + ]; + + let mut m = *m; + for r in 0..BLAKE3_ROUNDS { + // Mix the columns. + blake3_g(&mut v, 0, 4, 8, 12, m[0], m[1]); + blake3_g(&mut v, 1, 5, 9, 13, m[2], m[3]); + blake3_g(&mut v, 2, 6, 10, 14, m[4], m[5]); + blake3_g(&mut v, 3, 7, 11, 15, m[6], m[7]); + // Mix the diagonals. + blake3_g(&mut v, 0, 5, 10, 15, m[8], m[9]); + blake3_g(&mut v, 1, 6, 11, 12, m[10], m[11]); + blake3_g(&mut v, 2, 7, 8, 13, m[12], m[13]); + blake3_g(&mut v, 3, 4, 9, 14, m[14], m[15]); + // Permute between rounds; the permute after the last round is never + // consumed (oracle: `r < rounds - 1`). + if r < BLAKE3_ROUNDS - 1 { + let prev = m; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + m[i] = prev[p]; + } + } + } + + let mut out = [0u32; 16]; + for i in 0..8 { + out[i] = v[i] ^ v[i + 8]; + out[i + 8] = v[i + 8] ^ h[i]; + } + out +} diff --git a/others/falsify_assembly.sh b/others/falsify_assembly.sh new file mode 100755 index 000000000..c4f9d52ef --- /dev/null +++ b/others/falsify_assembly.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Falsification harness for the assembly leg. +# +# Applies one deliberate defect at a time to prover/src/lfm/epoch.rs, runs the +# named test, and reports PASS (test still green = the defect is INVISIBLE, a +# hole in the suite) or FAIL (the defect was caught). The verdict is read from +# the `test result:` summary line, because per-test FAILED lines do not appear +# in `cargo test -q` output — the trap the fri-emitter leg hit. +set -u +cd "$(dirname "$0")/.." +FILE=prover/src/lfm/epoch.rs +TEST=${2:-lfm::epoch_tests} +cp "$FILE" /tmp/epoch.rs.bak + +restore() { cp /tmp/epoch.rs.bak "$FILE"; } +trap restore EXIT + +run() { + local label="$1" + local out + out=$(cargo test -p lambda-vm-prover --lib "$TEST" 2>&1 | grep "test result:") + if echo "$out" | grep -q "FAILED"; then + echo "CAUGHT $label ($out)" + elif echo "$out" | grep -q "ok\."; then + echo "INVISIBLE $label ($out)" + else + echo "ERROR $label (build failure or no result: $out)" + fi + restore +} + +case "${1:-all}" in + fri_order) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" zetas.push(t.sample_ext(b)); + t.append_halves(&root.halves());""",""" t.append_halves(&root.halves()); + zetas.push(t.sample_ext(b));""") +open(p,'w').write(s) +PY + run "FRI: absorb the layer root BEFORE sampling its zeta" + ;; + fri_drop_root) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" zetas.push(t.sample_ext(b)); + t.append_halves(&root.halves());""",""" zetas.push(t.sample_ext(b)); + let _ = root;""") +open(p,'w').write(s) +PY + run "FRI: never absorb the committed layer roots" + ;; + fri_drop_final) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" if shape.fri.total_folds() > 0 { + zetas.push(t.sample_ext(b)); + }""",""" if false { + zetas.push(t.sample_ext(b)); + }""") +open(p,'w').write(s) +PY + run "FRI: skip the final-fold zeta draw" + ;; + fri_drop_coeffs) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" for c in absorbs.fri_coeffs { + append_ext_cell(b, t, *c); + }""",""" for c in absorbs.fri_coeffs { + let _ = c; + }""") +open(p,'w').write(s) +PY + run "FRI: never absorb the terminal polynomial coefficients" + ;; + ood_row_major) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" for col in 0..width { + for row in 0..height { + append_ext_cell(b, t, block[row * width + col]); + } + }""",""" for row in 0..height { + for col in 0..width { + append_ext_cell(b, t, block[row * width + col]); + } + }""") +open(p,'w').write(s) +PY + run "Round 3: absorb the OOD blocks ROW-major" + ;; + ood_order) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" (shape.ood_current_dims, absorbs.ood_current), + (shape.ood_next_dims, absorbs.ood_next),""",""" (shape.ood_next_dims, absorbs.ood_next), + (shape.ood_current_dims, absorbs.ood_current),""") +open(p,'w').write(s) +PY + run "Round 3: absorb the next-row OOD block before the current-row one" + ;; + nonce_absorb) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" emit_grinding_check(b, seed, halves, shape.grinding_factor); + t.append_halves(&halves);""",""" emit_grinding_check(b, seed, halves, shape.grinding_factor);""") +open(p,'w').write(s) +PY + run "Grinding: never absorb the nonce" + ;; + grinding_check) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" emit_grinding_check(b, seed, halves, shape.grinding_factor); + t.append_halves(&halves);""",""" let _ = seed; + t.append_halves(&halves);""") +open(p,'w').write(s) +PY + run "Grinding: emit no proof-of-work check at all" + ;; + z_guard) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" let one = b.ext_const(&FEE::one()); + assert_ne_ext(b, z_pow_trace, one);""",""" let one = b.ext_const(&FEE::one()); + let _ = one;""") +open(p,'w').write(s) +PY + run "z_ood: drop the trace-domain non-membership guard" + ;; + fork_separator) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" if num_tables > 1 { + fork.append_const_bytes(&(index as u64).to_le_bytes()); + }""",""" if false { + fork.append_const_bytes(&(index as u64).to_le_bytes()); + }""") +open(p,'w').write(s) +PY + run "Fork: omit the per-table domain separator" + ;; + contribution) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" if let Some(l) = absorbs.contribution { + append_ext_cell(b, t, l); + }""",""" if let Some(l) = absorbs.contribution { + let _ = l; + }""") +open(p,'w').write(s) +PY + run "Phase C: never absorb the bus contribution L" + ;; + aux_root) + python3 - <<'PY' +p='prover/src/lfm/epoch.rs'; s=open(p).read() +s=s.replace(""" if let Some(root) = absorbs.aux_root { + t.append_halves(&root.halves()); + }""",""" if let Some(root) = absorbs.aux_root { + let _ = root; + }""") +open(p,'w').write(s) +PY + run "Phase C: never absorb the aux trace root" + ;; + *) + echo "usage: $0 [test-filter]" + echo "defects: fri_order fri_drop_root fri_drop_final fri_drop_coeffs ood_row_major" + echo " ood_order nonce_absorb grinding_check z_guard fork_separator" + echo " contribution aux_root" + ;; +esac diff --git a/others/lfm-RESUME.md b/others/lfm-RESUME.md new file mode 100644 index 000000000..361b3bb6e --- /dev/null +++ b/others/lfm-RESUME.md @@ -0,0 +1,395 @@ +# RESUME HERE — Phase R, keccak recursion + +Written 2026-07-30 as a compaction-survival doc. If you are picking this up +with no memory of the session, read this file first, then +`lfm-standing-decisions.md`, then `lfm-target-shape.md`. Everything else is +reference. + +## The goal, in one paragraph + +Make the LFM (Lambda Field Machine — a straight-line, field-native recursion +machine living in `prover/src/lfm/`) verify a real Lambda VM **continuation +epoch proof**, using **keccak** as the hash. Keccak is explicitly the hash we +do NOT expect to ship; it is first because it needs zero changes to the inner +prover (post-#841 the verify path is keccak-only). Once the e2e works, the +same e2e becomes a **hash test matrix**: blake (most probable final choice) +and Poseidon behind the same socket, giving measured cells-per-verify per +candidate as input to the ecosystem hash decision. + +## Where the code is + +- Branch **`feat/lfm`**, worktree `/Users/maurofab/workspace/lambda_vm_3-lfm`. + Never pushed. Based on `origin/main` (includes #841). +- Side branches, both merged into `feat/lfm` as of this writing: + `feat/lfm-chunking`, `feat/phase0-constraint-ir`, `feat/lfm-constraint-emitter`. +- ⚠ **One worktree per agent, always.** Merging into a worktree an agent is + live in produced a merge commit wearing that agent's message (`af5ea7c4`). + Both agents were down when the latest consolidation happened; that is the + only safe time. + +## What works today (all committed, all green) + +The machine proves and verifies, end to end, through the registry: + +1. **Keccak family hosted unchanged** — production `KECCAK_RND`/`KECCAK_RC`/ + `BITWISE` AIRs driven by an LFM adapter chip speaking their bus contract. + `KECCAK_RND` is chunkable (chunk count is program shape, in the digest). +2. **`keccak256` bit-exact** vs `PlatformKeccak256` at eight boundary lengths. +3. **Transcript replay** bit-exact vs the real post-#841 `DefaultTranscript` + (squeeze buffer, absorb invalidation, canonicity guard, zero-rejection). +4. **Continuation-epoch statement + Phase A** → publishes the real `(z, α)`, + verified against production's own `absorb_statement_with_digest`. +5. **A real Merkle opening authenticated under production keccak** — real + proof, real committed root, real path. +6. **Constraint evaluation** — a serialized `ConstraintArtifact` lowered to + machine instructions, all 28 AIRs vs the production evaluator, plus a + real-proof composition check. 57,252 instructions/epoch, 9.7% under budget. +7. **DEEP slice 1** — composition-polynomial reconstruction at a query point, + vs production's own reconstruction. +8. **Chaining (ii) and (iii)** — cross-epoch L2G root binding, and the + attestation's `program_id` fold bit-exact vs production. + +## What is left, in order (updated 2026-07-31) + +1. ~~**Chaining (i)**: REGISTER preprocessed commitment from `reg_fini`~~ — + **DONE** (reg-tree, merged at 69b4a915). Prediction confirmed exactly: + 255/511/1023 permutations, 0.0182%/0.2224% noise. No second hashing + gadget — `keccak_hash_pair` unwelded from the walk sufficed. ⚠ Left an + OPEN assembly obligation: the `reg_fini` felt-width gap — see + `lfm-assembly-obligations.md`, which is now the ledger every leg's + deferral goes into. +2. ~~**DEEP across a full sub-proof**, wired to R1f's Merkle + authentication~~ — **DONE** (deep-join, merged at 703f742b). Join is + structural: same arena cells, index bits bound (a hinted point is the + same gap one level over), control programs run the denied attacks. + Cost inversion found: authentication is 99.0% of the leg, DEEP 1.0%; + 213,744 permutations/epoch at blowup 8 for openings (~46% of the + predicted epoch keccak bill). Shared-commitment lever measured at 48% + collapse (111,471) — parked, see + `lfm-team-lead-shared-commitment-ruling.md`. +3. ~~**FRI folding leg**~~ — **DONE, leg CLOSED** (fri-emitter, merged at + 5a246ba5; spec now carries Addendum 2). Emitter (per-layer walk + fold + chain + terminal check) differentialled entirely against REAL + production proofs that fold: the leg's blindness premise was FALSE — + the L2G fixture's trace sizes with boundary count, so 512/1024/2048 + boundaries give real proofs with 1/2/3 committed layers in under a + second. Measured = predicted on all six pinned numbers (174/186/198 + perms/query; 38,106/20,460/14,454 per sub-proof at blowup 2/4/8). + Proves+verifies end to end. Two approved deviations: the terminal + check EVALUATES at υ^(2^total_folds) instead of emitting the FFT (a + codeword lookup is a 1,023-wide Select tree on this machine — the + guest's economics do not transfer; equivalence checked at 876 + index/shape points, and the zero-fold branch unifies), and + `fri_fold`'s mul association differs from production's while the + field element does not. The OWED leaf-gadget byte check discharged + executably vs BOTH production backends. Left ledger entries 4 (FRI + challenges from the transcript in production's interleaved order; + coefficients+roots are proof DATA the transcript must absorb — + nothing leg-side can catch this) and 5 (informational: + isolation-driver index width). +4. ~~**LogUp closure**~~ — **DONE, leg CLOSED** (deep-join, 7 slices, + merged at 1145041a; handoff `lfm-logup-handoff.md`). Closure built + against production's own oracles; found and closed THREE soundness + gaps (L two-consumer split, hinted alpha powers — instance 3, worse in + degree — and the earlier DEEP/auth parallel-copy class), witnessed + per-chunk accumulation with a ≥2-chunk fixture, resolved + has_trace_interaction by reading. The one unknown it left is now + **SETTLED** (zerorow, 2026-08-03): a zero-row fixed table reports + `Some(zero)`, measured on a real accepted epoch — five of them + (KECCAK, KECCAK_RND, KECCAK_RC, ECSM, ECDAS) — and stripping the field + makes the proof fail, so `Some` is forced. Same test closes the + table-set-LENGTH gap (closure run over a real epoch's 24 + contributions) and found that three of the five have NON-blank traces + with every multiplicity column zero: "unused" ≠ "blank". +5. **Assembly** into one epoch-verifier program — **SPINE + LEGS DONE; the + whole verifier runs on a real epoch** (assembly waves 4 and 5, branch + `feat/lfm-assembly`). + - **DONE**: the Fiat-Shamir spine RUNS on a real 24-sub-proof continuation + epoch that production accepts. `prover/src/lfm/epoch.rs` replays the + fork, Phase C, and rounds 2-4 in production's order; + `epoch_tests::the_epoch_challenge_spine_matches_production` matches + production's own `replay_rounds_after_round_1` on all 111 challenges + (shared z/α, then per table β, z, γ, every ζ, every query index), and the + LogUp closure on top reaches production's COMMIT-bus target. Ledger + entries 4, 5 and 6 DISCHARGED; 2 half (the cell is right, the derivation + is not built); 3 partially (one cell + two views is now a construction, + but the second consumers are not wired). + - **LEGS DONE** (assembly-w5, `feat/lfm-assembly` @ a1f32859): the whole + verifier RUNS. `prover/src/lfm/epoch_verify.rs` is the seam — per sub-proof + it rebuilds the OOD grid from the two pruned blocks the transcript + absorbed, runs the constraint evaluation and quotient check at the spine's + own `z`/`beta`, and takes each query's `iota_bits` straight into the Merkle + walk, the DEEP fold and the FRI chain. `epoch_challenge_program` became + `epoch_program(e, with_legs)` so ONE spine emitter serves both programs and + the leg program cannot drift from the one the 111-challenge differential + covers. Ledger entry 3 DISCHARGED (the absolute hinted-once count now runs + over a program that HAS both consumers of every value, with a positive + control that it declares strictly more arena words than the spine); 21 + tamper vectors rejected. + * MEASURED, min preset (blowup 2, 1 query/table, grinding 1, 24 + sub-proofs): **spine 1,095,553 instructions / 1,211 permutations / 5,716 + arena words -> ASSEMBLED 2,184,360 / 2,616 / 16,478.** Legs alone + 1,088,807 / 1,405 / 10,762, so the verifier is ~50/50 Fiat-Shamir and + verification at this preset. + * Leg permutations match a closed form over the shapes EXACTLY (927 leaves + + 304 Merkle levels + 174 FRI = 1,405). Constraint lowering reproduces + the design's 54,358 ALU rows to the digit (63,393 unfused likewise); + recombination 2,431 against 2,894, the gap being zerofier squarings at + this epoch's real trace lengths. FRI at blowup 8 lands on the pinned + 14,454 exactly. + * WARNING: the composed OPENING predictions assumed a UNIFORM 2^20 per + sub-proof. A real intermediate epoch is `[2 x14, 3, 4 x4, 5 x3, 7, 20]` — + one big table and 23 tiny ones — so its openings are 1.88x cheaper than + the uniform model (100,959 against 189,727 at blowup 8). 213,744 stands + as a model of a PRODUCTION-sized epoch, not of this one; ledger entry 10. + - **NEW ledger entries** 7 (the preprocessed commitments are hinted — and + wave 5 CORRECTED its taxonomy: DECODE is ELF-dependent like PAGE, so the + split is 2 constants + 2 ELF-dependent + 1 derived, with a proposed + resolution awaiting a ruling), 8 (the OOD absorb ORDER has no production + witness: every OOD block of all 24 sub-proofs is one row tall), 9 (the + constraint leg's frame-STEP view of the grid is invisible at + `step_size = 1` — same witness as 8) and 10 (per-epoch numbers must name + their epoch shape). + - **STILL NOT DONE**: entry 7's wiring (intern BITWISE + KECCAK_RC, call + `programs::emit_register_commitment` from Phase A, rule on DECODE/PAGE) and + therefore entry 2's derivation; entry 8's synthetic AIR. +6. ~~**The wrap run**~~ — **DONE, and the machine PROVES its own epoch verifier** + (assembly-w7, `feat/lfm-assembly`). Run LOCALLY, not on the box: the box was + occupied by an ethrex continuation campaign at all three check points (18:43, + 18:52 and 19:12 UTC, load 22-29 on 32 cores, three different `cli prove` + invocations), so per the brief nothing was started on it. Every number names the epoch profile + `[2 x14, 3, 4 x4, 5 x3, 7, 20]`; the full table is in + `lfm-assembly-obligations.md` entry 10, now SATISFIED. + - **Slice 0** (min preset): the assembled verifier proves in 19.5 s and + verifies in 0.09 s, 30,707,816-byte proof, 14 sub-proofs, 15.1 GiB peak. + 220,107,920 main + 87,073,068 aux ext cells. + - **Slice 1a** (inner blowup 8, 1 query — the GEOMETRY, 2^23 LDE, 22 Merkle + levels, 12 committed FRI layers): proves in 23.3 s, verifies in 0.09 s, + 31,147,664-byte proof, 15.5 GiB peak. + - **Slice 1b** (inner blowup 8, 73 queries — the PRODUCTION SHAPE, emitted and + censused, not proved): 76,501,118 instructions / 118,080 permutations / + 817,101 arena words / 6 KECCAK_RND chunks and **5,077,422,224 main + + 2,029,461,548 aux ext = 11,165,806,868 base-field-equivalent cells per epoch + verify**. Openings **100,959** and FRI **14,454** — both pinned predictions + hit exactly. + - ★ **84.0% of the cells are the keccak family**, 36,256 main + 13,912 aux per + permutation. The hash matrix's other columns therefore decide the machine's + SIZE; its structure is already settled. + - ⚠ **The production-shaped wrap is not provable at 124 GiB**: 350.6 GiB + projected peak from a coefficient measured twice (33.7 bytes per + base-field-equivalent cell; a 15.9 GiB projection came in at 15.5 GiB). The + three ways out are a cheaper hash, disk spill, or splitting the wrap — a + decision, not a debt. + - Falsified both directions, end to end: a tampered inner proof makes the wrap + **unbuildable** (execution dies at `DivByZero` in the root compare — a false + assert has no witness), while a moved claimed public word or a moved program + digest makes an honest proof **unverifiable**. + +## Decisions already made — do not relitigate + +- **Prove the inner proof at BLOWUP 8.** Three independent legs point + there: DEEP scales with query count (73 vs 219 ⇒ ~3×), the keccak bill + does too (~460k vs ~1.4M permutations), and FRI is 2.6× cheaper (14,454 + vs 38,106 permutations — query count falls 3× while per-query cost rises + only 14%; reg-tree, FRI slice 0, derived from the verified spec). +- **The REGISTER derivation IS the binding.** `VmAirs::new`'s + `register_preprocessed` parameter looks like unfinished plumbing; it must + stay unwired. Computing the commitment from `reg_fini` is what ties the + values to it. +- **Shape-static values are program constants, never arena reads** — and + next-row pruning likewise, because the verifier reconstructs an undeclared + column as ZERO. +- **The uniform promotion (`epoch_label`/`page_base`) is PARKED** — off the + critical path, design complete in `lfm-page-base-uniform-proposal.md`. +- **Zero-rejection transcript** is forced by straight-line shape, not a + choice; completeness cost < 1e-6/proof. + +## Open items needing the USER, not an agent + +- **The prover determinism fix.** Root-caused: six dedup tables assign row + indices by std `HashMap` iteration order (`lt.rs:163/168/177` and five + siblings), plus grinding's `find_any`. Fix is a contained ~7-file change + (insertion-ordered index map; `find_first`), no soundness risk, and it + would restore byte-reproducible proofs and enable a decisive experiment on + the long-standing ±100k recursion-bench noise. **Offered, not started.** +- **The `check_attestation` gap.** The consumer-side recompute that binds + supplied roots to a trusted ELF has ZERO production call sites; there is a + committed PoC (`prover/src/tests/recursion_soundness_gap_poc.rs`). Working + as designed ("not self-enforcing"), but the design assumes a consumer who + performs the ritual and nothing in the CLI does. + +## How to restart the work + +(Updated 2026-08-03 after wave 2.) Every wave so far ended the same way — +worker agents hit session limits, so a restart is always a cold start: +nothing to resume, only to re-spawn against the committed briefs. What +worked, twice now: + +- **One agent per leg, one worktree per agent.** Create the worktree off + `feat/lfm` first (`git worktree add -b feat/lfm`, then + symlink `executor/program_artifacts` from the main checkout, or prover + tests fail on missing fixtures). +- **Brief with pointers, not content**: this file, then + `lfm-standing-decisions.md` (binding), then the leg's own section above. + Tell the agent to verify ground truth (`cargo test -p lambda-vm-prover + --lib lfm`) before writing anything. +- **Have them merge `feat/lfm` INTO their branch** as it moves, never the + other direction, and consolidate only when no agent is live. +- **Ask for the report format** the phase used: headline, what landed, tests + verbatim, measurements vs prediction, deviations with reasoning, + surprises, falsification runs. The measurements-vs-prediction line is what + caught most of the errors. +- Agents append to `lfm-agent-status.log` at slice boundaries; that log is + the history if a mailbox message is lost, which happened repeatedly. + +Wave 3 CLOSED 2026-08-03 (both legs same day, both agents stood down +cleanly — first wave that did not end at a session limit). 188 green, +lint 0. + +Wave 4 (assembly) SPAWNED and ABORTED same day: the agent hit the +session token limit ~25 minutes in (reset 16:40 America/Buenos_Aires), +branch untouched. Its worktree is ALIVE and clean — reuse it, do not +create another: +`/private/tmp/claude-501/-Users-maurofab-workspace-lambda-vm-3/0f390d07-adf0-4a3e-a1b5-d6a58e444fae/scratchpad/wt-assembly`, +branch `feat/lfm-assembly` @ 35845e4c (artifacts symlink in place). +One deliverable survived and is COMMITTED: +`lfm-team-lead-start-index-research.md` answers ledger entry 2 — +production binds `start_index` (x254, reg slot 64) by REBUILDING epoch +N's REGISTER preprocessed commitment from epoch N−1's FINI vector; no +arithmetic start+len check exists anywhere; the LFM analogue is binding +the arena word to `reg_fini[64]`, which the reg leg already handles. +Bonus: FINI's u32 commitment forces `start_index < 2^32`, which bears +on ledger entry 1 (may upgrade the REG-C2 argument route over the +range check). + +Wave 4 (assembly) RAN 2026-08-03 on `feat/lfm-assembly` (3 commits off +35845e4c). Suite 195 green (188 + 7), `make lint` exit 0. See item 5 above for what +landed. `lfm-team-lead-start-index-research.md` was originally committed as +a raw 518 KB JSONL session transcript under a `.md` name; the team lead +replaced it (post-wave-4) with the research agent's final report extracted +verbatim from that transcript. The raw session survives in git history at +e105dea2 if ever needed; findings are also summarised in ledger entry 2. + +Wave 6 CLOSED 2026-08-04 (`feat/lfm-assembly`, 3 commits off 3766214a; suite +208 green / 1 ignored, `make lint` exit 0). **The assembly ledger is now empty +of debts**: entries 1, 2, 7, 8 and 9 all discharged, leaving only entry 10, +which is the wrap run's own reporting rule and not a debt. + +- **Entry 7 + 2**: every preprocessed root now comes from the source its + provenance admits, chosen by a classifier that recomputes production's + candidate functions (so an unknown provenance PANICS instead of being hinted + unbound). Options-only roots intern as program text; REGISTER is derived in + Phase A from the register boundary, which is what binds `start_index`; DECODE + stays an arena cell bound by the attestation join, with the `program_id` fold + emitted on the same cell and differentialled against production. The join is + denied structurally by a hinted-once guard PLUS an exact arena schema, and + falsified with a coherent forgery — a split-cell control program runs the + substitution and attests to another program's id. +- **Entries 8 + 9**: witnessed by TWO fixtures, not one. The brief's single AIR + is unbuildable — `AirWithBuses` hardcodes two transition offsets, and + `step_size > 1` is unprovable (a framework ceiling, measured). Entry 8 needed + no synthetic AIR at all: `FibonacciMultiColumnAIR` already has three offsets, + giving a 3×2 next-row block. Entry 9 needed no proof: production's own + `into_frame` is the oracle for the grid→frame-step mapping. +- **Entry 1** discharged by its own stated default (emit the range check), which + slice 1 triggered by making the boundary vectors live arena data. +- **Cost**: assembled verifier 2,184,360 → 2,244,094 instructions, 2,616 → + 2,872 permutations at the min preset. The +256 is exactly 255 (REGISTER tree at + blowup 2) + 1 (the `program_id` fold). + +⚠ TWO THINGS FOR THE USER, both always-stop items: +1. **The `step_size > 1` framework ceiling.** `RowFrame::from_lde` asserts + single-row steps (`frame.rs:38`, reached from `evaluator.rs:72`). From reading + only, the assert looks over-strict for the access pattern that exists — the + general `Frame::read_from_lde` already handles multi-row steps and constraint + bodies only ever read row 0 of a step — so it is plausibly a one-line + relaxation in `crypto/**`. Lifting it would let entry 9 have an end-to-end + witness. +2. **PAGE's preprocessed roots are the GLOBAL proof's, not an epoch's.** No + continuation epoch of any guest carries a PAGE sub-proof (`prove_epoch` + rejects one). This overturns the entry-7 ruling's condition (b), which asked + for a witness epoch that cannot exist; the obligation migrates to a + global-proof verifier. + +Wave 7 CLOSED 2026-08-04 (`feat/lfm-assembly`; suite 209 green / 5 ignored, +`make lint` exit 0). **The wrap run happened: the LFM prover proves the assembled +epoch verifier and the LFM verifier accepts it.** Item 6 above has the numbers and +entry 10 has the table. The three things wave 8 inherits: + +1. **The hash matrix, which is now the whole remaining question.** Keccak is 84.0% + of cells per verify, so blake and Poseidon behind the same socket are not a + refinement of the number — they ARE the number. The e2e that measures them + exists and is one function call parameterised by options + (`lfm::wrap_tests::wrap_run`). +2. **A resource ceiling, measured**: the production-shaped wrap (inner blowup 8, + 73 queries) is 11.17 billion cells and needs a projected 350.6 GiB. Nothing + about the machine blocks it; a box or a cheaper hash does. +3. **The box was never used.** It was busy both times it was checked. A run there + buys a bigger provable RUNG (4 queries ≈ 70 GiB projected), not the headline. + +Superseded — the wave-6 hand-off line, kept for the record: "Ready to start next +(wave 7): the wrap run, whose numbers must state their epoch's trace-length +profile (entry 10)." + +Superseded — the wave-6 order of work, kept for the record: +- **Ledger entry 7's wiring, and entry 2 with it.** `programs::emit_register_ + commitment` now exists (extracted in wave 5); Phase A must call it on the + register-boundary arena the spine already declares, so REGISTER's root is + COMPUTED and `start_index` is bound. BITWISE and KECCAK_RC intern as program + constants. DECODE and PAGE need a RULING — wave 5 found DECODE is + ELF-dependent, so the entry's own taxonomy was wrong; the proposal is in + ledger entry 7 and it touches program identity, which is an always-stop item. +- **Ledger entries 8 and 9 together** — one synthetic AIR, proved by the + PRODUCTION prover, with three transition offsets AND `step_size > 1`. Entry 8 + is the OOD absorb ORDER (column- vs row-major), entry 9 is the constraint + leg's frame-STEP view of the grid; a witness built for one does not close the + other unless it exercises both. +- **Then the wrap run**, whose numbers must state their epoch's trace-length + profile (ledger entry 10). + +DONE in wave 5 (kept for the record): +- ~~**Assembly, part 2 — hang the legs off the spine.**~~ The seam already + exists: `epoch::TableAbsorbs` carries every proof-carried cell and + `epoch::TableChallenges` every derived challenge, per table. What is + needed is, per sub-proof: reconstruct the full OOD grid from the two + pruned blocks with program-constant zeros, run the constraint + evaluation and quotient check at the spine's `z` and `β` powers, then + per query take `TableChallenges::iota_bits` straight into + `sub_proof::emit_query_with_bits` and `fri::emit_query_fri`. Only then + do the composed per-epoch numbers become measurements. Start from + `epoch_tests::epoch_challenge_program`, which is the assembled program + minus exactly these legs. +- **Ledger entries 7 + 2, which close together**: intern the three + constant preprocessed commitments, wire reg-tree's derivation into + Phase A so REGISTER's root is computed from the register-boundary + arena the spine already declares (which is what binds `start_index`), + and decide what to do about PAGE's ELF-dependent commitment. +- **Ledger entry 8** needs a synthetic AIR with three transition offsets + (or `step_size > 1`), proved by the production prover, or the OOD + absorb order stays unwitnessed. + +After that: the wrap run on the box. + +## How to work here + +`lfm-standing-decisions.md` is binding: six method rules, the +pre-authorization list, and the always-stop list. The rules exist because +each one caught something. The highest-yield pattern of the phase, stated +generally: + +> When all production instances share a degenerate parameter value, a +> differential over production data cannot distinguish implementations that +> differ only off that value. The synthetic case is the only witness. + +Three members so far: next-row pruning, the DEEP coefficient stride, and the +`step_size = 1` collapse. Expect more — but check the premise first: the +FRI leg's "no real proof can witness the fold" turned out to be a claim +about the FIXTURES ON HAND, not about the prover, and fell to a +one-parameter change (boundary count) that made real folding proofs in +under a second. "All production instances share the value" and "all +fixtures we happen to have share the value" are different claims; only +the first forces a synthetic witness. + +Second-highest: **falsify your own test guards, not just the mechanism.** +Three separate agents found real holes that way — including a tamper suite +whose every vector hit byte 0, so a digest's second word was never checked. diff --git a/others/lfm-agent-handoff.md b/others/lfm-agent-handoff.md new file mode 100644 index 000000000..64ae05827 --- /dev/null +++ b/others/lfm-agent-handoff.md @@ -0,0 +1,292 @@ +# R1d handoff — keccak-probe → successor + +Written 2026-07-29 ~20:30Z. R1d is PARTIAL. Foundations are built and verified; +the `TranscriptReplay` emitter is not started. Handing off on context, per the +team lead's "quality over completion" instruction. + +**State: 69/69 `cargo test -p lambda-vm-prover --lib lfm`, `make lint` 0, +nothing committed.** Worktree at `e0add1d5` (post-merge, #841 present). Tracked +diff is 19 lines (`prover/src/lib.rs` +1, `prover/src/tables/types.rs` +18) — +both pre-existing, not mine. Everything else of ours is untracked: +`prover/src/lfm/`, `prover/src/bin/compute_lfm_registry.rs`, `others/`. + +--- + +## 1. What is DONE (R1d) + +### 1a. Reversed-digest primitive — `sample()` replayed and PROVED +- `layout::keccak`: `REV_ADDR0/1`, `REV_MULT0/1` (prep width 52 → 56). +- `chips::keccak`: two extra `LfmMem` sends whose lanes are reversed-coefficient + `Linear`s over the **existing** OUT byte columns — + `reversed half h = Σ_k OUT[31 − 4h − k]·256^k`. Zero new value columns. +- `instr::KeccakOperands.rev: Option`, + `LfmBuilder::keccak_absorb_rev`, `edsl::keccak256_rev`, + `programs::keccak_sample_program(len)`. +- Tests: `machine_reversed_digest_matches_default_transcript_sample` + (execute-only, vs the REAL `DefaultTranscript`, lengths 0/1/135/202) and + `machine_proves_the_sample_replay` (prove+verify, lengths 0/135/202). + +### 1b. Host model — `keccak_host::TranscriptModel` +Mirrors post-#841 `DefaultTranscript` (`segment` / `buf` / `pos`). Verified by +`transcript_model_matches_default_transcript` across: draining a squeeze then +forcing a refill on the 5th candidate, an absorb mid-buffer, a raw `sample()` +(which ALSO invalidates), and absorbs of length 1/135/136/200. + +### 1c. ★ The reversal-cancellation identity — `candidate_from_state` +**Candidate `i` of a squeeze is the plain digest's `u64` lane `3 − i`.** +`Σ_{k<8} reversed[8i+k]·2^(8(7−k))` with `reversed[j] = digest[31−j]`, sub +`m = 7−k` ⇒ `Σ_{m<8} digest[24−8i+m]·2^(8m)` = the LE u64 at digest byte offset +`24−8i` = state lane `3−i`. The BE read and the byte reversal cancel. +Verified by `be_candidates_are_plain_state_lanes`. **This is now the spec** (team +lead accepted it): sampling needs no reversal, no extra `Linear`, no BitDec. + +--- + +## 2. What REMAINS (against the R1d spec) + +1. **`edsl::TranscriptReplay`** (new file `transcript_replay.rs` suggested). + Host-side emit-time state mirroring `TranscriptModel`: the segment as a + `Vec` of `u32` halves **plus a byte length** (padding is length-driven), + the current squeeze's 8 half-cells, and `out_pos`. + - `append_halves(...)` / `append_word(...)`: extend the segment, set + `out_pos = SQUEEZE_LEN`. + - `sample(&mut self, b) -> [Cell; 2]`: emit `keccak_absorb_rev` over the + segment; the row yields BOTH the plain digest (state words 0,1 — the + candidate source) and the reversed digest (the re-absorb prefix). Set the + next segment to the reversed digest's 8 halves; set `out_pos = SQUEEZE_LEN`. +2. **Candidate extraction.** `unpack` the two PLAIN digest words → 8 half-felts + `h[0..8]`. Candidate `i` = `(lo = h[6−2i], hi = h[7−2i])`. Two `unpack`s per + squeeze. Refill when `out_pos + 8 > 32`. +3. **`sample_field_element`** with the canonicity guard. `candidate ≥ p` iff + `hi = 2^32−1 ∧ lo ≠ 0` (derivation: `p−1 = (2^32−1)·2^32`; if `hi < 2^32−1` + the max is `(2^32−1)·2^32 − 1 < p`). Emit `g = (2^32−1) − hi`, `z = is_zero(g)` + via hinted-and-verified inverse (`z·g = 0` and `z + g·ginv = 1` pin `z` + uniquely), then `assert z·lo = 0`. Felt = `hi·2^32 + lo`. + - **Arena rule note to put in a comment:** hinting `z`/`ginv` is sound because + they are VERIFIED in-circuit; the rule bans unverified TRANSCRIPT inputs, + not verified auxiliary witnesses. +4. **Zero-rejection variant + completeness doc.** The emitted program cannot + prove an inner proof whose transcript ever rejected a candidate + (p ≈ 2⁻³² per draw). **Fold in the ext3 correction: an extension draw is 3 + independently rejection-sampled candidates, so ~3× the per-draw figure.** + State the resulting **per-proof bound at real draw counts**, not a ratio. + Structure so a k-rejection variant is an emitter PARAMETER later, not a + redesign. +5. **`sample_u64_pow2(nbits)`**: low `nbits` of the candidate. Confirmed from + source: `threshold = upper_bound.wrapping_neg() % upper_bound` is 0 at powers + of two, so it never rejects and returns `candidate % 2^n`. For `nbits ≤ 32` + only `lo` matters → `b.bit_dec(lo, nbits)` then recombine `Σ 2^i·b_i`. + **Assert `nbits ≤ 32`** rather than silently mishandling more (team lead: FRI + query bounds are ≤ 2^25, so the bound is real). +6. **Acceptance**: a scripted interleaving (absorbs of several lengths / + 3× `sample_field_element` incl. one ext3 / absorb / `sample_u64(1<<20)` / + `sample_field_element`) producing IDENTICAL values to a host + `DefaultTranscript`, proved+verified e2e with sampled values `public`ed; + plus tamper one absorbed half → reject. Register `TranscriptReplayV0`, + regenerate the registry, add a drift test. + +--- + +## 3. Non-obvious decisions and WHY (not visible in the diff) + +- **`sample()` returns the SAME 32 bytes it re-absorbs.** One value serves as + both the challenge and the next segment's prefix — do not emit two. +- **Reversed digest is scoped to RE-ABSORB ONLY** after the cancellation finding. + Do not use it for candidates. +- **`sample()` invalidates the buffer too**, not just `append_bytes` / + `append_field_element`. All three set `out_pos = SQUEEZE_LEN`. +- **Prep-column growth moves ALL registry digests.** Any layout change ⇒ + stub `LFM_REGISTRY` to `&[]`, `cargo run --release --bin compute_lfm_registry`, + paste, rebuild. The bin cannot build while the table is stale — that is why + the stub step exists. +- **`KeccakF` is boxed** (`Instr::KeccakF(Box)`): inline, its + 312-byte payload quadrupled the whole `Instr` enum and failed clippy's + `large_enum_variant`. Keep it boxed when adding fields. +- **Slot 11 (`KECCAK_RND`) has no preprocessed columns** — all-zero sentinel root, + height 0. `LfmAirs::new`'s roots array is a PARTIAL FUNCTION. `build_air_no_prep` + exists solely for it. +- **The digest binds the static `KECCAK_RC` / `BITWISE` roots**, so a change to + those production tables moves every LFM program digest. Deliberate. +- **`verify_against(roots, program_id, …)`** exists so per-shape programs can be + proved AND verified without a registry entry. It is NOT a registry off-switch; + `lfm_verify` still hard-errors on a miss. Keep that distinction in comments. +- **Length is program shape.** Each message length is a distinct program and + identity; register one representative, verify the rest via `verify_against`. +- **Rate-region pass-through constraint** `MODE_PERM·(PERM_IN − STATE) = 0` is + load-bearing and NO bus catches it (see §4). + +--- + +## 4. Test-oracle gotchas — read before writing tests + +**(a) Execute-only tests are VACUOUS with respect to the chip.** +For any value the executor computes host-side AND the chip recomputes on the +bus, `execute()` never evaluates the bus interaction — so chip-side corruption +cannot move the result. I hit this exactly: my reversed-digest bit-exactness +test was execute-only, and neutralising the chip's reversed-coefficient `Linear` +left it GREEN. Fix was `machine_proves_the_sample_replay` (prove+verify), which +fails under the same neutralisation. **Both tests are kept on purpose** — the +execute-only one validates the executor mirror against the real +`DefaultTranscript`, the proving one validates chip-vs-executor agreement. +Anything R1d adds to the adapter needs a PROVING test. + +**(b) Scrutinise the oracle as hard as the thing under test.** +`transcript_model_matches_default_transcript` failed on first run and the MODEL +was right — my comparison was wrong. `sample_u64(2^n)` returns `candidate % 2^n` +(threshold 0 at powers of two), not the raw candidate; the delta was exactly +2^63. Mask the model's raw candidate, and compare raw 32-byte squeezes via +`sample()` separately. + +**(c) Coherent forgeries, not trace tampering, find constraint holes.** +The permute-mode hole (R1c) is invisible to trace tampering — tampering desyncs +the round chip and the bus catches it first. It took building a forgery where +KECCAK_RND, BITWISE multiplicities, the reply token, the output words AND the +claimed public words were all internally consistent, so every bus balanced and +only the constraint stood in the way. Then neutralise the constraint and confirm +the forgery is ACCEPTED. That pattern is the standard here; see +`permute_row_cannot_substitute_the_permuted_state`. + +**(d) Falsify every new mechanism.** Every load-bearing piece in R1a–R1d was +confirmed by breaking it and watching the right test fail: R1a token lane order, +R1b half-recomposition byte order, R1c block byte mapping + the rate-equality +constraint, R1d the chip's reversal. If a falsification does NOT fail, the test +is vacuous — see (a). + +--- + +## 5. Status log + +`others/lfm-agent-status.log`, one line per slice boundary. Last line is marked +`R1d-PARTIAL-HANDOFF`. Append at each slice; the team lead polls it if the +mailbox goes quiet. + +## 6. Process notes + +- Mailbox messages crossed repeatedly. The team lead now uses + `others/lfm-team-lead-*.md` for anything authorization-shaped; **check this + directory when a blocker answer seems overdue.** +- `git stash list` has a pre-existing unrelated `bench-keccak-vs-leanvm WIP` + entry. Leave it alone. +- Nothing is committed and nothing should be without the user's say-so. +- `make lint` from the repo root is the gate (`cargo fmt --check` is not a + substitute); it runs four clippy configurations. + +--- + +## 7. File map — everything added or changed + +All under `prover/src/lfm/` unless noted. Nothing is committed; every file below +except the two tracked ones is UNTRACKED. + +**New files** +- `keccak_adapter.rs` (517L) — the raw keccak-family contract: the two + `BusId::Keccak` tokens, `KECCAK_RND`/`RC`/`BITWISE` trace drivers, the + per-round BITWISE feed (forked from `trace_builder::collect_bitwise_from_keccak`), + the `u32`-half state↔words conversion, the absorb XOR feed, and the host mirror + of the reversed digest. +- `keccak_probe.rs` (290L) — standalone probe of the UNCHANGED production family. + Deliberately untouched since R1a; it documents the raw contract including the + live tag-swap hazard. +- `keccak_host.rs` (198L) — byte-stream packing convention, `pad10*1`, + `PlatformKeccak256` reference wrapper, `TranscriptModel`, `candidate_from_state`. +- `others/lfm-agent-status.log`, `others/lfm-agent-handoff.md` (this file). + +**Changed files** +- `layout.rs` — `mod keccak`: prep layout (tags, 13+13 addrs, mults, 9 block + addrs, 2 mode selectors, 2 rev addrs + 2 rev mults = 56 wide) and `tag_for_row`. +- `instr.rs` — `Instr::KeccakF(Box)`, `KeccakMode`, + `KeccakReversedDigest`; `writes()`/`reads()` arms. +- `builder.rs` — `keccak_f`, `keccak_absorb`, `keccak_absorb_rev`, shared + `emit_keccak`. +- `compiler.rs` — keccak group emission (tag = row ordinal, mode one-hot, block + and rev addrs) + multiplicity backfill. +- `executor.rs` — the `KeccakF` arm (permute and absorb), `KeccakRow` record, + `NotU32Half` / `KeccakSpareLaneNonZero`. +- `validator.rs` — keccak partition count, mode one-hot, padding; **check 8** + (`check_keccak_tags`) with `DuplicateKeccakTag` / `MalformedKeccakTag`. +- `chips.rs` — the `LFM_KECCAK` chip: 788 columns, 173 interactions, + `KeccakAdapterConstraints` (201 constraints, degree 2). +- `trace.rs` — the keccak chip trace plus the three production family traces. +- `airs.rs` — 10 → 14 chips, `build_air_no_prep`, `KECCAK_RND_SLOT`, + `keccak_rnd_rows`, extended `lfm_cell_counts`. +- `registry.rs` — `build_artifacts` over 14 slots with the slot-class doc, + `KeccakChainV0` / `KeccakSpongeV0` kinds, regenerated table (4 entries). +- `programs.rs` — `keccak_chain_program`, `keccak_sponge_program(len)`, + `keccak_sample_program(len)`, `KECCAK_SPONGE_LEN = 202`. +- `edsl.rs` — `keccak256`, `keccak256_rev`, shared `keccak256_absorb_all`. +- `proof.rs` — split out `prove_traces` and `verify_against`. +- `machine_tests.rs` (1008L) — all R1a–R1d tests. +- `mod.rs` — module registrations. +- `prover/src/bin/compute_lfm_registry.rs` — the three new programs. +- TRACKED (pre-existing, not mine): `prover/src/lib.rs` +1 (`pub mod lfm;`), + `prover/src/tables/types.rs` +18 (BusId 32/33/34). + +## 8. Guard-test map — what each test pins + +| Test | Hole it pins | +|---|---| +| `keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` | Documents the LIVE forgery on the raw family — **asserts it SUCCEEDS**. If it ever starts failing, something began binding request→reply; re-derive before relaxing. | +| `preprocessed_tags_close_the_output_swap_hazard` | Closes the above, 3 legs: distinct tags / swap now rejects / prover can't collide tags (`PrecomputedCommitmentMismatch`). | +| `duplicate_keccak_tags_fail_admission` | The registrar's independent gate on tag uniqueness. | +| `permute_row_cannot_substitute_the_permuted_state` | ★ The permute-mode hole. NO bus catches it; only `MODE_PERM·(PERM_IN−STATE)=0` does. Built as a coherent forgery. | +| `tampered_absorb_xor_rejects` | The absorb XOR is pinned by the 136 BITWISE lookups. | +| `keccak_rejects_non_u32_half`, `keccak_rejects_nonzero_spare_lane` | Executor guards on the `u32`-half word convention and the 2 spare slots. | +| `tampered_keccak_input_half_rejects` / `_output_half_rejects` | State byte columns bound to memory and to the family. | +| `keccak256_matches_platform_hasher` | Bit-exactness vs the production hasher, 8 boundary lengths. | +| `machine_proves_the_sample_replay` | Chip-vs-executor agreement on the reversed digest (the execute-only sibling canNOT see this — see §4a). | +| `transcript_model_matches_default_transcript` | The emit-time oracle is correct. | +| `be_candidates_are_plain_state_lanes` | The reversal-cancellation identity the emitter rests on. | +| `registry_drift_*` (4) | Program identity; investigate, never re-bless. | + +## 9. Packing conventions (get these wrong and nothing balances) + +- **State**: 25 `u64` lanes → 50 `u32` halves → 13 words. Half `h` = low (`h` + even) / high (`h` odd) 32 bits of lane `h/2`. Word `j` carries halves `4j..4j+3`. + Last word's top **two** slots are unused, pinned zero as tuple constants. +- **Byte columns** are lane-major: `STATE + lane*8 + b`, little-endian in-lane. + Note `(h/2)*8 + 4*(h%2) == 4h`, so half `h` starts at byte column `4h`. +- **The column-major trap is TOKEN-order only.** Keccak bus element + `3 + 8(5x+y) + b` is byte `b` of lane `x+5y` (lanes visited 0,5,10,15,20,1,…). + The COLUMN layout is plain lane-major — block byte `k` pairs with state byte `k`. +- **Rate block**: 136 bytes = 17 lanes = 34 halves = 9 words, 2 spare slots. +- **Byte streams** (`keccak_host::pack_stream`): `u32` halves, 4 bytes each, LE, + final partial half ZERO-PADDED. The emitter's `stream_half + pad_const` equals + a bitwise merge only because of that zero-padding — + `assert_high_bytes_zero` states the obligation executably. +- **Digest**: first 32 state bytes = halves 0..7 = words 0,1. Digest byte `j` = + byte `j%4` of half `j/4`. Matches `PlatformKeccak256` output order exactly. + +## 10. Half-formed emitter intentions (what I would have done next) + +- **New file `transcript_replay.rs`**, not more `edsl.rs`. `edsl.rs` is already + the FRI/sponge library; the transcript is its own concern. +- **Shape**: `TranscriptReplay { segment: Vec, segment_bytes: usize, + buf: Option<[Felt; 8]>, out_pos: usize, hints: ArenaId, hint_cursor: u32 }`. +- **Carry a `TranscriptModel` alongside and `debug_assert` they agree at every + step.** The consumption schedule is static, so a divergence is a BUILD-time + bug; catching it at emit time beats discovering it as a failed proof. +- **Host hint generation next to `TranscriptModel`** in `keccak_host.rs`, so the + `z`/`ginv` vector is produced by the same code that models the schedule — + one source of truth for ordering. +- `sample_field_element` → `felt = hi·2^32 + lo` as a single `mul_add` against an + interned `2^32` constant. ext3 = three consecutive draws, assembled with + `pack_ext`. +- **★ THE ONE REAL DESIGN DECISION I DID NOT RESOLVE — partial-half appends.** + The production transcript absorbs ARBITRARY byte lengths, but our segment is a + vector of 4-byte halves. An append whose length is not a multiple of 4 puts a + partial half in the MIDDLE of a segment, where the next append's bytes must + continue inside that same half. That is the mixed-half problem from R1c padding, + except it can recur mid-stream instead of only at the end, and the + `stream_half + pad_const` trick does not generalise (the later bytes are not + known-zero-padded, they are real data). Options I weighed: + (a) restrict `append` to whole halves and ASSERT it — fine for the FRI verifier, + whose absorbs are digests and field elements (all multiples of 4/8 bytes), + and I would start here; + (b) carry a partial-half accumulator in the emitter and merge with an in-machine + `mul_add` when the next append arrives — correct in general, more instructions; + (c) re-pack the whole segment per sample — simplest, most wasteful. + **Recommendation: (a) with a loud assert, then (b) only if a real caller needs + it.** Do not silently truncate or pad — that would diverge from the host + transcript in a way no test in this suite would catch unless it specifically + exercises a non-multiple-of-4 append. If you take (a), ADD a test that the + assert fires, so the limitation is pinned rather than latent. diff --git a/others/lfm-agent-status.log b/others/lfm-agent-status.log new file mode 100644 index 000000000..baa7434d5 --- /dev/null +++ b/others/lfm-agent-status.log @@ -0,0 +1,228 @@ +# LFM agent status log — one line per slice boundary. +# Format: | | | +2026-07-29T15:20Z | R1a keccak adapter probe | lfm 44/44 | DONE (uncommitted) +2026-07-29T16:45Z | R1b KeccakF in the machine | lfm 55/55 | DONE (uncommitted) +2026-07-29T18:20Z | R1c keccak256 sponge | lfm 64/64 | DONE (uncommitted) +2026-07-29T18:55Z | R1d transcript replay | lfm 64/64 | BLOCKED — worktree predates #841; awaiting merge/spec-only call. +2026-07-29T19:40Z | R1d groundwork (sample()) | lfm 67/67 | PARTIAL — reversed-digest primitive DONE + bit-exact vs real DefaultTranscript::sample(); buffered-candidate work still blocked on the #841 call. +2026-07-29T19:55Z | R1d merge protocol | lfm 67/67 | DONE — clean ff 0893b6b5->e0add1d5 (#841 in history), stash pop clean, our 19 lines intact. LFM digests SURVIVED (all 4 drift tests green, no regen). lint 0. +2026-07-29T20:20Z | R1d foundations | lfm 69/69 | DONE — post-merge. Host TranscriptModel matches real DefaultTranscript (refill+absorb-invalidation+raw-sample+multi-length). KEY: BE candidates == plain state lanes 3,2,1,0, so sampling needs NO reversal. +2026-07-29T20:30Z | R1d-HANDOFF-READY | lfm 69/69 | HANDOFF — emitter (TranscriptReplay, canonicity guard, sample_u64_pow2, scripted e2e, registry entry) NOT started. See others/lfm-agent-handoff.md. lint 0, nothing committed, tree clean apart from the uncommitted LFM work. +2026-07-29T21:55Z | R1d TranscriptReplay emitter | lfm 85/85 | DONE — emitter + canonicity guard + sample_u64_pow2 + registered TranscriptReplayV0 proved e2e vs the REAL DefaultTranscript. Guard is ONE div (LFM_BALU's B*OUT=A), no hint needed — supersedes the hinted-inverse plan. 10 falsifications (F1-F9) all failed correctly (incl. execute-green/prove-red on the chip reversal, and a coherent forgery at candidate p accepted once emit_base(3) is neutralised). SOUNDNESS.md §6.3: zero-rejection is FORCED by straight-line shape, NOT a future emitter parameter. lint 0, nothing committed. +2026-07-29T22:40Z | R1d ruling + true-up (team-lead) | lfm 88/88 | DONE — trails the 21:55Z line, which disk had already outrun. Landed by the emitter agent before the outage: partial-half ruling IMPLEMENTED (SegPiece::{Const,Halves}, packing deferred to squeeze time so constant runs concatenate across append boundaries; TranscriptReplay::new no longer takes a builder), 3 new tests (constant_appends_concatenate_across_append_boundaries, machine_data_may_follow_constants_that_together_align, squeeze_economics_match_the_verified_draw_schedule), §6.3 rewritten with the verified draw schedule (E = 4 + T(3+L_t), L_t = max(trace_bits-7,0), blowup-independent; per-table increment 1.05e-8 = 15x the per-DRAW figure — a conflation corrected in review), F10 falsification (per-append packing regression is caught). Registry did NOT drift. Landed by team-lead now, during a platform classifier outage that blocked the agent's write path: stale #[should_panic] string fixed (test renamed machine_data_after_a_misaligned_constant_is_rejected, expects "must start on a 4-byte boundary" per the live assert) + §6.3 sentence that constant-consumption sampling should ride the ecosystem hash migration's transcript rebuild. HONEST RECORD: the tree was 87/88 (one cosmetic red, mechanism correct) from ~22:05Z until this line. lint 0, nothing committed. NEXT: R1e slice a (append_field_element BE + ext variant; splice s in {1,2,3}; two statement misalignment points). +2026-07-29T23:20Z | R1e slice a (BE field elems) | lfm 92/92 | DONE — append_felt (8B BE) + append_ext (24B, coords 0,1,2) + felt_be_halves gadget: 1 BitDec + 64 BALU rows/felt, byte permutation folded into interned weights (2^0..2^31, shared by both halves). Bit-exact vs REAL append_field_element (9 base values incl. p-1/p-2, 3 ext triples); ext PROVED e2e. Coord order 0,1,2 VERIFIED from source — the reversed 2,1,0 impl in the same file is for raw [FpE;3], a different type; do not "fix". F11 no-swap / F12 halves-swapped / F13 ext-coords-reversed all failed correctly (F13 fails ext only, felt green => ext test covers coord order specifically). lint 0. +2026-07-29T23:25Z | ⚠ BASELINE CHANGE (not by me) | lfm 92/92 | CORRECTED 2026-07-30: the TEAM LEAD committed the whole LFM tree as 77bcc5e6 "feat(lfm): field-native recursion machine with the keccak family hosted", on the user's authorization, signed with the user's identity (Mauro Toscano) because the repo requires verified signatures and this machine's default git identity cannot produce one. My original line here read "the USER committed" — inferred from authorship alone, which does not distinguish author from actor, branch feat/lfm now +1 ahead of origin/main. Working tree is IDENTICAL to HEAD; slice-a work is inside the commit; nothing lost. others/ deliberately NOT committed (still untracked). The standing "nothing committed" rule is SATISFIED, not broken — a user commit IS the user's say-so — but the invariant no longer holds: R1e slices b/c/d will show as a diff on top of 77bcc5e6, not as untracked files. +2026-07-30T00:10Z | R1e slice b (byte splice) | lfm 97/97 | DONE — splice_half + a byte-granular Packer replacing the run/flush packer. DEVIATION from the spec signature: no splice_misaligned(constant_prefix_len, dynamic_halves) — the real statement ALTERNATES const/dynamic runs and the shift CHANGES mid-stream (the 1-byte fri field takes it 2->3), so a one-prefix-one-run helper cannot express it. Instead the packer tracks a byte cursor and splices whenever a machine half lands misaligned; append_halves keeps its loud assert, append_halves_misaligned opts in. ALIGNED PATH PROVEN FREE (0 bitdec / 0 balu) and all 5 registry drift tests still green => no digest moved. Cost 1 BitDec + ~34 BALU per spliced half (8 halves = 8/273). split_half's recomposition assert pins d < 2^32 (bit_dec only bounds by p). F14 no-assert / F15 lo-hi-swapped / F16 shift-latched all failed correctly; F14 and F16 each fail ONLY their target test. lint 0. +2026-07-30T01:05Z | R1e slices c+d (statement+PhaseA) | lfm 104/104 | DONE — new lfm/statement_replay.rs: absorb_epoch_statement (ContinuationEpoch, all 10 fields, tag from crate::statement made pub(crate) so the literal is NOT duplicated) + replay_phase_a (per-air optional prep root then main root, then z,alpha as ext). Registered StatementReplayV0; registry regenerated, pre-existing 5 entries UNCHANGED. ACCEPTANCE: machine (z,alpha) == the REAL absorb_statement_with_digest + Phase A, execute AND proved+registry-verified. Both tamper vectors reject (statement byte, Phase-A root half). MEASURED 2016 instrs, keccak 5, bitdec 53 (= exactly one per spliced half: 8 elf + 3 output + 2 label + 40 root), balu 1825. All 10 statement fields are u64 LITTLE-endian => no byteswap needed; slice-a's BE work is for the FRI leg's OOD values, not this one. Statement byte_len = 207 + output + 16*ranges ≡ 3 (mod 4), PINNED by a test => every Phase-A root absorb is spliced at shift 3. F17 prep-root-unconditional / F18 z-alpha-swapped / F19 epoch_label-dropped / F20 pages-field-omitted all failed correctly. lint 0. +2026-07-30T02:00Z | R1e ruling 2 (masked tail) | lfm 105/105 | DONE — ruling 2 VERIFIED not assumed: public_output_bytes is built ONE BYTE PER COMMIT OP (trace_builder.rs:3012), so an epoch's length has NO alignment guarantee => per the ruling, BUILT rather than deferred. append_bytes_misaligned + Packer::push_masked (split at len%4, PIN the high part to zero) + SegPiece::Partial; push_half/push_partial unified (aligned fast path still 0 instructions, entries 1-5 of the registry unchanged). ⚠ TWO CORRECTIONS to my own earlier claims, both now machine-checked: statement is 207+L+16R (not 223), and the Phase-A shift is (3+L) mod 4 — NOT unconditionally 3; it is ZERO when L≡1 mod 4, so the rider's cost is workload-dependent and free for ~1 workload in 4. Riders file entry 2 corrected. Acceptance shape moved to L=14 so it exercises BOTH the mask and a spliced Phase A (shift 1). F21 (drop the zero-pin) correctly accepts garbage past the length prefix. StatementReplayV0 re-registered: 2051 instrs, keccak 5, bitdec 54, balu 1859. lint 0. +2026-07-30T02:35Z | R1f recon + HOLD checkpoint | lfm 105/105 | RECON DONE, LEG NOT STARTED — see others/lfm-r1f-handoff.md. ★ SPEC CORRECTION: edsl::merkle_walk uses b.compress() = LFM_HASH/TestPermutation, the deliberately NON-CRYPTO Milestone-C placeholder; it authenticates the fixture tree only and CANNOT authenticate production-committed data. Production Merkle is keccak throughout: leaf = keccak(row PAIR 2i,2i+1 column-major, every element write_bytes_be) per commitment.rs ROWS_PER_LEAF=2; parent = keccak(left||right), 64B, no domain sep / no ordering flag (merkle_tree/backends/field_element.rs:41). R1f therefore needs a NEW edsl::keccak_merkle_walk (2 selects/level since a digest is 2 words, then keccak256 over 16 halves = 1 permutation/level). Slice a's BE gadget IS needed here after all — 2*cols elements per leaf, so leaf cost is dominated by byteswapping (~100 BitDec + ~6.4k BALU for a 50-col table) not by hashing. Honouring the chunking HOLD: no edits to airs/trace/registry/proof; R1f programs should use verify_against and NOT be registered until the merge lands. Ruling 3 (public_output length) was already discharged in R1e — no assert remains to annotate. lint 0, nothing uncommitted in prover/. +2026-07-30T02:55Z | R1f(b) API blocker found | lfm 105/105 | BLOCKED ON A DECISION, not on work — ContinuationProof.epochs (continuation.rs:431) is a PRIVATE field with only num_epochs() public, and lfm is a SIBLING module, so a fixture generator cannot reach an epoch's MultiProof/public_output/roots at all. prove_and_verify_continuation returns committed bytes, not the structure. Two routes, recorded in lfm-r1f-handoff.md §3b: (1) pub(crate) accessor on continuation.rs — additive, precedent = CONTINUATION_EPOCH_TAG in R1e, but it is a production file adjacent to the chunking merge so needs a team-lead call; (2) generate inside continuation's reachable surface and serialize to a fixture file so lfm reads only bytes — which also satisfies the spec's own "cache it as a file" instruction. MUST be decided BEFORE slice (a): it fixes whether the arena filler consumes a MultiProofView or a byte blob, i.e. the filler's entire input type. No code written; prover/ untouched since 50227c67. +2026-07-30T03:40Z | R1f slice b (real proof bytes) | lfm 119/119 | DONE — new lfm/proof_fixture.rs. Route 2 per the ruling: the fixture IS the guest's wire format, produced by the SAME encoder (prove_continuation -> encode_continuation_guest_input, both pub) rather than an invented one. The existing dump path (test_dump_recursion_input) is #[ignore]d + driven by 5 env vars + writes a fixed /tmp path, so unusable from a deterministic test — reused its two encoder calls and nothing else, as the ruling directs. MEASURED epoch sweep (not guessed): fibonacci gives 1 epoch at log2 6/8/10 and 2 at log2 4, so it runs 17..64 cycles; blob 310,212 B at 1 epoch, 587,188 B at 2. FIXTURE_EPOCH_LOG2=4, preset min. Cache lives in temp_dir (NOT checked in: a committed binary drifts from the encoder silently), so a cold run exercises the GENERATION path. lint 0. +2026-07-30T04:30Z | R1f slice a (arena filler, roots) | lfm 120/120 | PARTIAL — new lfm/proof_arena.rs reads REAL committed roots out of the guest wire-format blob, in place. ARCHIVED-ACCESSOR WALL WAS REAL and confirmed by compile probe: ArchivedContinuationProof.epochs is private. Used the PRE-APPROVED archived-path fix, but as METHODS not field visibility — rkyv mirrors the source field's visibility onto the archived struct, so relaxing `epochs` would have opened the OWNED type too (= route 1, rejected). Added pub(crate) num_epochs/epoch_proof/epoch_public_output on ArchivedContinuationProof only; owned type untouched. MEASURED on the real fixture: epoch 0 = 24 sub-proofs / 8-byte output, epoch 1 = 25 / 0-byte — independently confirms T_epoch = counts + (10 final | 9 intermediate) + pages + 1, and confirms T=24 as the structural minimum used in the SOUNDNESS §6.3 bound. ⚠ FINDING for the Phase-A leg: the PREPROCESSED root Phase A absorbs is NOT in the proof — it comes from air.precomputed_commitment(), so replaying Phase A over a real proof needs the epoch's AIRs rebuilt, not just its bytes. Openings/sibling extraction NOT done (view API located: query_list_len/query(i)->FriDecommitmentView/deep_poly_openings_len at crypto/stark/src/proof/view.rs:409-423). lint 0. +2026-07-30T05:15Z | R1f-PARTIAL-HANDOFF | lfm 121/121 | HANDOFF — (b) + half of (a) DONE and committed; (c) keccak_merkle_walk and (d) tamper NOT started. See others/lfm-r1f-handoff.md. VERIFIED the preprocessed-root ruling rather than taking it: ContinuationGuestInput carries decode_commitment (present, nonzero) and page_commitments as pub fields, so no build_epoch_airs access is needed — BUT this fixture has ZERO page commitments (fibonacci touches no data pages), so that path is present-but-unexercised. TWO refinements the ruling did not cover, both read from EpochProof (continuation.rs:394): REGISTER is DERIVED from reg_fini (the register FILE is in the blob; the root is not), and runtime_page_ranges is ALWAYS EMPTY for continuation epochs => the REAL statement has R=0, length 207+L, Phase-A shift (3+L) mod 4 with no 16R term (R1e's synthetic shape uses R=2). SOUNDNESS §6.3 updated: T=24 now says MEASURED (24 intermediate / 25 final on a real proof), not assumed. lint 0. +2026-07-30T14:05Z | R1f slices c+d (real Merkle opening) | lfm 125/125 | DONE — committed a4711c63. New edsl::keccak_merkle_walk + keccak_leaf_hash + keccak_digest_halves; proof_arena::MainTraceOpening (extract/leaf_hash/verifies_at/indices_that_verify/arena packers) + walk_to_root; programs::keccak_merkle_opening_program(MerkleOpeningShape). ★ THE MACHINE AUTHENTICATED PRODUCTION-COMMITTED DATA: epoch 0 / table 0 / query 0 of the real 2-epoch fixture — 10 columns, 20-value row pair, depth 20, index 379880 — walked to the proof's own lde_trace_main_merkle_root, PROVED and verified (published root == committed root, verify_against true). Conventions re-verified from source: leaf = keccak(row PAIR column-major, every elem write_bytes_be 8B) per commitment.rs ROWS_PER_LEAF=2 + stream_bytes = canonical_u64().to_be_bytes(); parent = keccak(l||r) 64B no domain sep / no ordering flag (both hash_new_parent paths agree); walk order from verify_merkle_path_from_leaf_hash (index even => H(cur,sib), merkle_path[0] = LEAF level — the Proof doc comment says the reverse, the CODE is what we mirror). IOTA IS NOT IN THE PROOF (transcript challenge, needs statement+AIRs) => recovered by exhaustion against PRODUCTION's own checker, ~3.9s at depth 20, OnceLock-shared. TARGET CHOICE IS LOAD-BEARING: 47 of 49 sub-proofs are mostly-padding tables whose identical rows give identical leaves so EVERY index verifies — on those the index-tamper vector is vacuous; table 0 is the only deep+unique one, pinned by real_opening_is_a_usable_tamper_target. Tamper x3 (sibling/index bits/leaf value) x2 modes: incoherent trips the in-machine root assert (DivByZero), COHERENT (claim the root the tamper really folds to, method rule 4) proves fine then rejects on the published root. ⚠ MEASURED REFUTATION of the R1f handoff's headline prediction: byteswapping does NOT dominate. Row counts (20 bitdec + 1280 balu vs 22 perms) are right; the conclusion is wrong because chip rows are not comparable units — LFM_BALU row = 4 non-prep cols, one permutation = 1 LFM_KECCAK row (736) + 24 KECCAK_RND rounds x 1480 = 36,256 main cells vs 322 per byteswap = 113x. Hashing dominates at EVERY width: 124x @10 cols, 8.9x @511, 7.4x @1480, asymptote ~6.6x (both terms linear in c). NO crossover exists; byteswap chiplet is not the lever. Stale claim corrected in edsl docs too. F22-F28 (select order swapped / only word 0 swaps / leaf halves reversed / parent right||left / bits high-to-low / leaf = one row not the pair / digest words hi||lo) all failed the RIGHT test; F29-F30 falsify the TEST GUARDS (degenerate table -> uniqueness assert fires; different table -> shape pin fires). SOUNDNESS §6.3 T=24 item was already landed by team-lead in 82311178 — not redone. lint 0. +2026-07-30T15:10Z | R1f fixture non-reproducibility | lfm 125/125 + 1 ignored | DONE — commit af5ea7c4. ⚠⚠ TWO THINGS THE TEAM LEAD SHOULD READ. (1) FINDING: prove_continuation is NOT reproducible. Two generate() calls on IDENTICAL inputs (same ELF, same empty input, same epoch_log2, same options) differ in ~65k of 587k bytes and the divergence is SEMANTIC not rkyv padding: epoch-0 main roots differ across runs => Fiat-Shamir challenges move => different leaves opened. Caught because the recovered leaf index changed 379880 -> 655761 between two runs of my own test while depth/columns held. Tree SHAPE is stable; VALUES are not. Consequence, now recorded on load_or_generate: nothing derived from a specific blob may be pinned as a constant — R1f pins shape and RECOVERS the index, which was a judgement call and is now a rule with evidence. Standing evidence = machine_tests::fixture_generation_is_not_reproducible (#[ignore]d, ~28s, asserts the divergence is semantic so it fails loudly if the prover is ever made reproducible). Also made the cache write ATOMIC (temp+rename): continuation_fixture_generates_two_epochs regenerates the shared /tmp path in parallel with readers, and since blobs legitimately differ, "it worked last time" was never evidence the race was safe. ROOT CAUSE OF THE PROVER NONDETERMINISM NOT INVESTIGATED — out of this leg's scope, but it is a real property of the continuation prover and someone should own it. (2) ⚠ GIT ACCIDENT, no work lost, needs a decision: af5ea7c4 is a MERGE commit. A merge of the constraint-lowering line (d2fb95c9..7b966d01) was in progress in this worktree (MERGE_HEAD set by another actor) when I ran git commit, so my commit COMPLETED THAT MERGE under MY message. Both sides are intact and verified present in HEAD; lfm 125/125 and make lint 0 on the merged tree. I did NOT rewrite it — "never rewrite history" is a standing rule and another agent was mid-operation in this worktree. If you want it clean: reset --soft a4711c63, re-commit the merge with a merge message, then re-commit prover/src/lfm/{machine_tests,proof_fixture}.rs as its own change. My R1f slice proper is the CLEAN commit a4711c63. +2026-07-30T15:40Z | R1f width-comment fix + pin | lfm 126/126 + 1 ignored | DONE — commit 3dd4556c. chips::keccak::cols inline comments (52/252/388/588/788) were STALE BY 4 since R1d widened PREP_WIDTH for the reversed-digest columns; real values 56/256/392/592/792. The constants were always right (they are derived) but I read the comments while building the R1f cost model and got a wrong per-permutation figure on the first pass — so the widths the model depends on now have an assertion instead of a comment: cost_model_widths_are_what_the_chips_declare pins LFM_KECCAK 792/56, LFM_BALU 4, LFM_BITDEC 66, KECCAK_RND 1480, and the two derived figures 322 cells/byteswap and 36,256 cells/permutation. A wrong width silently rescales every number in keccak_merkle_opening_cost. +2026-07-30T16:05Z | R1f CORRECTION to my own claim | lfm 126/126 + 1 ignored | ⚠ SELF-CORRECTION, measured. My 14:05Z line and commit a4711c63's message both say table 0 is "the only one of the fixture's 49 sub-proofs that combines a deep tree with a unique index", and the 14:05Z line says "47 of 49 sub-proofs are mostly-padding tables". BOTH WRONG — I eyeballed a probe instead of counting. MEASURED on the current blob: 49 sub-proofs, 24 with a UNIQUE verifying index, 25 degenerate (not 47). The two deepest trees are epoch 0 AND epoch 1 table 0, both depth 20; next deepest is 7; half the sub-proofs are depth 2. So table 0 is the right target for being the DEEPEST (and depth is shape, stable across blobs), not for being uniquely unique. Doc comment on R1F_EPOCH rewritten to say exactly that and to flag the unique/degenerate split as BLOB-DEPENDENT (hence asserted at run time by real_opening_is_a_usable_tamper_target, never pinned). a4711c63's message cannot be amended — it is behind later commits and history is not to be rewritten — so this line is the correction of record. +2026-07-30T17:20Z | R1g recon + slice (ii) L2G binding | lfm 129/129 + 1 ignored | DONE — commit 31c59654. ★ RECON FINDING on obligation (i), reported to team-lead: it is NOT a comparison, it is a DERIVATION. continuation.rs's chaining loop carries register_init = epoch.reg_fini() forward and build_epoch_airs (line 636) CONSTRUCTS the next epoch's REGISTER preprocessed commitment via register::compute_precomputed_commitment_with_fini — 3 columns x 128 rows, interpolate_fft, LDE, then commit_bit_reversed which builds the ENTIRE TREE. Its output IS the preprocessed root Phase A absorbs, closing the predecessor's "not in the proof" gap; so it is mandatory, not optional. DERIVED cost (constants read, not yet machine-measured): NUM_REGISTER_ADDRESSES=67 -> 128 rows, ROWS_PER_LEAF=2, leaf=48B=1 block, tree=2L-1 perms => blowup2 255 perms ~9.5M cells (~12x the WHOLE R1f opening leg), blowup4 511, blowup8 1023 ~37M (~46x). FFTs are ~0.5% of it — same lesson as R1f, cost is hashing, because a TREE is 2L-1 perms while a PATH is log L. All straight-line/shape-static, so nothing un-emittable. REGISTER is the ONLY per-epoch derived preprocessed commitment (enumerated every compute_precomputed_commitment call site): DECODE supplied, PAGE skipped for epochs, BITWISE/KECCAK_RC static. Flagged the supply-the-root alternative as a QUESTION not a proposal — it may be unsound since the derivation is what ties reg_fini VALUES to the commitment. SLICE (ii) LANDED: 2 archived-only METHODS (epoch_l2g_root, global_proof) per the standing rkyv rule; proof_arena::{epoch_l2g_roots, global_l2g_roots, commitments_to_arena}; programs::l2g_binding_program(n). Oracle verified on real data BEFORE building: 2 epochs / 4 global sub-proofs, epoch i root == global sub-proof i main root, roots pairwise DISTINCT (so the position-swap tamper is real). Tamper = wrong epoch root / wrong global root / one-side swap (all trip the assert) + COHERENT both-side swap (proves, then rejects on published order). ⚠ F32 FOUND A REAL HOLE IN MY OWN VECTORS: a digest is TWO words needing two asserts, but every tamper byte was byte 0, so deleting the second assert left all 5 tests GREEN. Vectors now straddle both words (byte 0 + byte 31); F32 fails correctly after the fix. F31 (asserts removed) / F33 (epoch vs itself) / F34 (off-by-one index) fail the right test; F35b (both sides made to share a root) fires the distinctness guard verbatim. lint 0. +2026-07-30T17:45Z | ⚠ R1g(i) COST FRAMING CORRECTED (team-lead) | lfm 129/129 + 1 ignored | MY BASELINE WAS WRONG; THE CONCLUSION REVERSES — BUILD THE DERIVATION. I sized the REGISTER tree against R1f's opening program (22 perms) and reported "12-46x the opening leg", implying it was expensive. But R1f was ONE query on ONE table — roughly 1/(219 queries x 24 tables) of the epoch's opening work, i.e. a SAMPLE of a leg, not the leg. Correct baseline (lfm-target-shape.md Scale): ~1.4M keccak permutations per epoch verify at blowup 2 / 219 queries, ~460k at blowup 8 / 73 queries. So the register tree is 255/1,400,000 = 0.018% at blowup 2 and 1023/460,000 = 0.22% at blowup 8 — noise at both ends (the two ratios differ by ~10x, so I record both rather than calling them the same order). It does not compete with the hash decision; "worth doing before the hash lands?" resolves to too-small-to-matter-either-way. ★ GENERALISABLE, and it is the MIRROR of the byteswap error: there the trap was comparing ROWS across chips (fix: compare cells); here it is comparing a gadget against a SAMPLE of another leg rather than the whole (fix: size against the epoch total). Both give ratios that are arithmetically right and decision-relevant in the WRONG DIRECTION. Both now recorded as a sizing rule in lfm-target-shape.md, with the epoch-total baseline, so the next person sizing a gadget has it. ★ ALSO SETTLED: the standing Phase-0 item "wire the REGISTER verify-side supply route" is now a DECIDED DESIGN POINT — it must stay unwired. VmAirs::new's register_preprocessed: Option<(Commitment, usize)> parameter (every verify caller passes None) looks like unfinished plumbing but is not: THE DERIVATION IS THE BINDING. Computing the commitment from reg_fini ties the VALUES to the commitment; supply the root instead and reg_fini has no remaining role, so a prover can offer a root consistent with a reg_fini it never honoured and the cross-epoch chain goes unenforced. The in-guest RV64 per-epoch recomputation is load-bearing, not wasteful — the opposite of how it has been described. Written up in lfm-target-shape.md Consequence 3. +2026-07-30T18:30Z | R1g slice (iii) program id | lfm 132/132 + 1 ignored | DONE — commit 74924545. The attestation fold emitted, BIT-EXACT vs production's own recursion::program_id_from_digest (the oracle is the production fn, not a local model), PROVED + verified. Layout: PROGRAM_ID_TAG || elf_digest || pc_start_le || decode_commitment || n_pages_le || (base_le || commitment)*. TAG IS 22 BYTES = 2 mod 4 => every machine value after it is spliced, same shape as R1e's 30-byte epoch tag; measured 102 bytes hashed at 0 pages, shift 2. NEW: transcript_replay::ByteString exposes the EXISTING byte-granular Packer for callers that hash a structured string directly rather than absorbing into a transcript — extraction is pure code movement (pack_pieces shared), and ALL 6 REGISTRY DRIFT TESTS STILL PASS so no emitted program moved. PROGRAM_ID_TAG made pub(crate) (CONTINUATION_EPOCH_TAG precedent) so the literal is not duplicated. ⚠ THE ZERO-PAGE CAVEAT HANDLED, NOT INHERITED: the fibonacci fixture has NO page commitments, so the sorted-page loop would have been present-but-untested (same caveat the team lead flagged for supplied roots). Driven with a SYNTHETIC shape at 1 and 3 pages against the same production oracle, and PROVED not executed — the page count changes the hashed byte length, hence every padding position, and only a proof sees the keccak chip agree with the executor about that. Tamper x6 all move the id and reject when the honest id is claimed: elf digest / entry point / decode root / page commitment / page base / page ORDER. TWO THINGS WRITTEN INTO THE PROGRAM DOCS rather than left inferable: (1) the attestation is NOT self-enforcing and emitting the fold does not change that — the guest uses supplied roots verbatim, binding happens at consumer-side check_attestation (native FFT+Merkle, once at top level, never in-VM); do not read "the machine folded the roots" as "the machine bound the roots"; (2) production SORTS pages by base and this folds them in supplied order, so sortedness is the arena filler's obligation — NOT a soundness hole (an unsorted fold just fails the consumer's compare, the prover breaks only their own attestation) but a COMPLETENESS one. F36 (tag dropped) / F38 (page-count field omitted) fail the right tests; F40 (page loop never runs) fails ONLY the page test and leaves the 0-page one green — so the page test genuinely bites rather than merely compiling. lint 0. NEXT: (i), the REGISTER derivation — report measured permutations against the 255/511/1023 prediction. +[emitter] DEEP leg slice 1 DONE. prover/src/lfm/deep.rs reconstructs the deep-composition polynomial at one query point. Differential oracle is the PRODUCTION pair compute_query_invariant_deep_terms + reconstruct_deep_composition_poly_evaluation_pair, fed a real L2G_MEMORY proof's own query openings; 3 queries x both points agree exactly. KEY STRUCTURAL FACT: the trace-term coefficients are one geometric run of a single gamma, so every sum is a Horner fold — no coefficient table to store, hint or authenticate. HAZARD FOUND AND HANDLED: build_pruned_trace_term_coeffs walks COLUMN-MAJOR, so along a fixed row the exponent advances by the block's row count, not by one. Every production AIR has step_size 1 and one next row, which collapses the stride to 1 — a plain Horner would pass every test we have and be wrong for the first widened step. Stride is carried explicitly and falsified at step_size 2 against the verifier's own coefficient table. MEASURED: DEEP is ~2.3M rows at 219 queries summed over all 28 AIRs (order-of-magnitude, not an epoch) vs 57,252 for the whole constraint leg — DEEP is ~40x the constraint leg and is the dominant term. Per query point: L2G_MEMORY 26 rows, CPU 62, ECDAS 729, KECCAK_RND 2,070. lfm suite 142 passed / 0 failed, make lint 0. +[emitter] slice (a)+(b) DONE. prover/src/lfm/constraints.rs lowers a ConstraintArtifact to LFM instructions; constraint_tests.rs is the differential. All 28 production AIRs, 4 random all-extension OOD frames each, machine execution == eval_program_verifier on the deserialized artifact. lfm suite 133 passed / 0 failed (was 125), clippy -D warnings clean. Cost census reproduces the design's §8.1 table EXACTLY (unfused total 64,187); emitted after fusion 55,147. Three design corrections measured: fusion saves 9,040 not 9,069 (9,113 candidate operand pairs, capped by one multiply per row); "3 dead nodes" are non-arithmetic (0 arithmetic orphans, so DCE saves 0 rows on production artifacts and its test injects one); MulBase is cost-neutral vs Mul, not a 4x obligation. +[emitter] slices (c)+(d) DONE. emit_quotient adds the shared zerofier Z = ζ^N−1 by repeated squaring, the β-power Horner over the constraint values, ONE division per AIR (boundary terms pre-scaled by Z so they keep their own β powers inside the same fold), and the claimed-parts Horner. Checked against a REAL STARK proof of L2G_MEMORY: challenges replayed through the production verifier's own replay_rounds_after_round_1, OOD grid reconstructed by the verifier's own OodLayout, and the machine's in-program assert_eq_ext(claimed, composition) executes. Six tamper vectors all reject (frame value, LogUp challenge, ζ, β, claimed part, and a ζ on the trace domain that zeroes the zerofier). The program PROVES and VERIFIES via verify_against; a mismatched claimed public word is rejected. lfm suite 138 passed / 0 failed, constraint_ 64 / 0. DEVIATION: not added to LFM_REGISTRY — one AIR's leg is not the epoch verifier and its digest must move once DEEP/openings land; determinism (the property registration pins) is tested instead. MEASURED per intermediate continuation epoch: 54,358 leg + 2,894 recombination = 57,252 over 24 sub-proofs, vs the 63,393 budget (design's number reproduces exactly from the unfused counts). +[phase0] 94% of the epoch leg is the fixed block => sharpest statement of the workload-independence correction. +[phase0] ACCEPTANCE (3): (1) parameterized_airs_vary_per_parameter_value becomes deletable for the L2G pair, deleted only after failing FOR THE RIGHT REASON; (2) test_split_verify_rejects_reordered_epochs and test_split_verify_rejects_dropped_last_epoch (both VERIFIED to exist, continuation.rs:1693/:1711, they pop and swap epochs) must pass UNCHANGED — a promotion that required editing them broke something; (3) a new negative test: a uniform disagreeing with the positional derivation must be rejected — and if it cannot be, that is the finding, meaning the invariant needs a mechanism not a review rule. +[phase0] ALSO: the 2,916 verify-time-base nodes are constant-only subtrees => fold at BUILD time, zero instructions. +[phase0] Built: crypto/stark/src/constraint_ir/artifact.rs (ConstraintArtifact = flat program + ConstraintMeta + AirShape + composition degree multiplier, rkyv codec, lift/validate); artifact_tests.rs (17 tests); prover/src/tests/constraint_artifact_tests.rs (4 tests, 25 AIRs x 100 frames x 3 eval paths); prover/src/bin/compute_constraint_artifacts.rs. +[phase0] Constants interned PROGRAM-WIDE, not per AIR: 655 summed pools -> 315 distinct 4-lane words. More than half the apparent constant cost was cross-table duplication. +[phase0] ConstraintArtifact::program() becomes program_with_uniforms(&[..]); program() retained for the count==0 case and ERRORS otherwise, so "forgot to supply the uniform" is loud rather than a silent zero. +[phase0] Cost model: 1 instruction = 1 row on 1 chip; group heights pad to next_power_of_two().max(4), so padded-cell cost needs the whole program's per-chip distribution (airs::lfm_cell_counts) and cannot be costed for this leg alone. +[phase0] DOCS: artifact.rs now states the uniform-zerofier consequence per team-lead framing (all 28 tables emit RowDomain::ALL => one zerofier group per AIR; the GPU path's uniform-zerofier precondition holds in fact, not by luck; a consumer needs one zerofier per AIR not one per exemption value). ExemptConstraints coverage kept so the field cannot rot into being untested. Also documented trace-length and preprocessed-commitment as deliberate exclusions. +[phase0] Doc now separates what I verified myself (op inventory, Neg/Embed/ConstExt, no narrowing, all counts) from what I took on report (row parity facts, interning, padding) — sec 2.3. +[phase0] EXPRESSIBILITY: nothing blocked. IR is a pure DAG, nodes[i] references only < i — IDENTICAL to the machine's acyclicity premise (A), so dense address assignment in node order satisfies it by construction. Static fanout gives mult directly (max 1,632; 3 dead nodes need DCE). No Div in the constraint algebra. +[phase0] FALSIFIED: lift SUB->ADD => structural wire check fails; device_program const swap => evaluation differential fails (the path with no structural check). Both reverted. +[phase0] FINDING (identity, escalated): ALL FOUR of PAGE, GLOBAL_MEMORY, L2G_GLOBAL, L2G_MEMORY are parameterized. Two axes, not one: page_base (PAGE, GLOBAL_MEMORY) and epoch_label (both L2G tables, via BusValue::constant(epoch_label) at local_to_global.rs:361 and LinearTerm::Constant(epoch_label-1) at :447). +[phase0] FINDING (my own hypothesis falsified by my own test): the variation is NOT confined to constant VALUES. Constant interning means a parameter value already in the table costs no node while a fresh one appends, shifting later node ids and the constraint ROOTS. PAGE/GLOBAL_MEMORY: 63/43 nodes both values, roots stable. L2G_GLOBAL: 47 vs 48 nodes, roots MOVED. L2G_MEMORY: 93 vs 95 nodes (2 nodes for 1 constant - a CSE collision on the enclosing Add), roots MOVED. So "swap one constant per page" is NOT an available fix; runtime-uniform promotion is, because the algebra (shape/meta/num_base/constraint count) is invariant - asserted. +[phase0] FINDINGS: (a) PAGE's constraints are parameterized by page_base (folded into IR constants) - not one static blob per continuation proof. (b) No production constraint uses end_exemptions; all 25 are RowDomain::ALL. (c) No production constraint uses an extension constant (ext_consts = 0 everywhere). (d) composition max_degree is absent from both AirContext and ConstraintMeta. +[phase0] GATE CLEARED for page_base, and my premise was wrong in my favour: the constant was NEVER a binding (not in the preprocessed commitment, not absorbed in the transcript, program_id only for ELF-backed data pages). Conclusion survives, reason was backwards. Proposal now carries §4.1 as a LOAD-BEARING INVARIANT: the uniform must come from page_configs / canonical_page_bases(bundle.touched_page_bases), NEVER from proof or trace — because with no binding, nothing downstream would catch a prover-chosen base. +[phase0] Global proof: 27 instr/epoch (L2G_GLOBAL) + 25 instr/page (GLOBAL_MEMORY) — settles page-base as identity-only, never size. +[phase0] HANDOFF written: others/lfm-phase0-handoff.md — state, the 3 sections to read before coding, the mandatory falsifications, the 5 instruments left behind, and the things a successor would otherwise rediscover. +[phase0] HAZARD the refinement creates, documented not hidden: ConstraintProgram becomes a hybrid of program identity (base_consts) and per-instance values (base_uniforms). If anything ever hashed a ConstraintProgram including uniforms, the digest varies per epoch again — the exact bug, one layer down. Latent today (only the artifact is hashed and it stores the count only). Flagged for review; not decided unilaterally. +[phase0] Hash-consing-vs-fusion trap moved into the CODE (crypto/stark/src/constraint_ir/artifact.rs, ConstraintArtifact doc) per ruling, not just the design doc. +[phase0] IMPLEMENTATION NOT STARTED. Stopping here deliberately rather than half-building a multi-file semantics-adjacent change: the refinement above is worth more than a partial implementation, and §5.1 needs the lead's agreement before it lands. +[phase0] INVARIANT: epoch_label uniform MUST be derived positionally from the verifier's enumerate() (continuation.rs:1293-1295), NEVER read from the bundle. Easier to honour than the page_base one (it is a loop counter) — written down so nobody adds a supply route. +[phase0] KEY CORRECTION: the IR's dim tags are PROVER-side. The machine runs verifier semantics where the OOD frame is all-extension, so a node is base only if its whole subtree is constants. IR declares 42,137 base arithmetic nodes; only 2,916 are base at verify time — a 14x discrepancy. Budgeting from declared dims would understate extension traffic badly. MulBase-eligible drops 9,413 -> 5,041 for the same reason. +[phase0] LESSON recorded in the census's own doc comment: the node census CANNOT see how sub-proofs are assembled, so workload/epoch/sub-proof-count inferences are outside what it supports. Names the exact false claim it produced. +[phase0] MAPPING TOTALITY (11 ops): total, with two non-obvious arms. Op::Neg has NO instruction (ExtOp = Add|Sub|Mul|Div|MulAdd|MulBase) -> lowers to Sub from pooled zero. Op::Embed is FREE under [F;4] lane-3-zero (a base value (v,0,0,0) IS its own embedding) — and measured 0 in production, as is ConstExt. +[phase0] MEASURED (28 AIRs): 73,722 nodes = 5,964 leaves (addresses, free) + 655 pooled constants + 67,103 arithmetic. Constraint-leg instr 64,842 + 2,150 beta-folds = 66,992; 57,923 after MulAdd fusion (9,069 fusable). Design doc claimed ~69K at 25 AIRs — BUDGET HOLDS. +[phase0] MEASURED (28 tables): 73,722 nodes / 1,223,896 bytes (1195.2 KiB). Continuation tables are tiny: L2G_GLOBAL 47 nodes/968 B, L2G_MEMORY 93 nodes/1,768 B, GLOBAL_MEMORY 43 nodes/904 B — +3,640 B total. PAGE 63 nodes/1,240 B. +[phase0] MEASURED, 2^20-cycle epoch: 16 chunked sub-proofs (CPU x2, MEMW_R x2), 26 sub-proofs total, 64,035 instr. vs the 24-sub-proof/63,393 minimum (<=2^19). Doubling the epoch past CPU's 2^19 bound costs 642 instr — that is the WHOLE growth term. So the leg is ~63-65K across any plausible epoch size; §8.2's monolithic 1.49x at 20M was an over-estimate for an epoch, which is capped. +[phase0] MEASURED: 73,539 nodes / 1,220,256 bytes (1191.7 KiB) over 25 tables. ECDAS 404KB + ECSM 368KB + KECCAK_RND 271KB = 85.5% of total. +[phase0] MulBase reframed: it is 1 XALU row, same as Mul — NOT a reduction. It is a ROUTING OBLIGATION (4+ rows if lowered by hand as 3 base muls + repack). 5,041 sites; would be 9,413 and wrong from prover dims. +[phase0] NEW INSTRUMENT: constraint_op_census in prover/src/tests/constraint_artifact_tests.rs — per-AIR node/leaf/const/fold/ext/mulbase/instr breakdown, standing (printed, loose ceiling only). +[phase0] NEW INSTRUMENT: epoch_chunk_multiplier — builds real traces so chunk counts are the prover's own splitting. MEASURED: fib_1M 64,712 instr (1.01x), fib_2M 65,996 (1.03x), array_multipass_20M 95,532 (1.49x). So the leg is ~65K/epoch at 1-2M cycles, ~96K at 20M. +[phase0] NEW: continuation_epoch_chunk_counts_measured — drives the real path (Executor::resume_with_limit for one epoch's cycles, then Traces::from_image_and_logs). No proving needed: epoch 0's register_init comes from the entry point, and every intermediate epoch runs exactly epoch_size cycles by construction. +[phase0] NEW: continuation_epoch_constraint_leg. INTERMEDIATE epoch 63,393 instr over 24 sub-proofs; FINAL 64,094 over 25. Composition = 14 split families (3,640, min 1 chunk each) + 9 fixed no-HALT (59,688) + 1 L2G_MEMORY (65). Test ASSERTS the 24/25 count so the shape is pinned, not inferred — if the epoch composition changes the arithmetic stops matching and it fails. +[phase0] NOT MEASURED: distinct page count for a realistic workload. Page size 1<<18 = 256 KiB (page.rs:50) and MAX_EPOCHS = 1<<20 (local_to_global.rs:83) are verified; the page count is not, and I did not guess one. +[phase0] NOT VERIFIED: chunk counts per family (=> no per-epoch multiplier); instruction->trace-row factor; MulBase row shape taken from SOUNDNESS.md not the chip. +[phase0] PROPOSAL (no semantics touched): others/lfm-page-base-uniform-proposal.md — promote page_base AND epoch_label to base-field runtime uniforms. New Op::BaseUniform{idx} / OP_BASE_UNIFORM=11 (additive: tags 0..10 unchanged, DeviceNode stays 16 bytes, existing artifacts stay valid). MUST be base-dim not ext: reusing the existing ext uniform machinery would flip downstream node dims and make eval_program panic in as_base(). CPU walker = 1 match arm + 1 &[u64] param; CUDA = 1 switch case + 1 buffer. Plumbing rides the AIR (per-AIR value), NOT TransitionEvaluationContext (per-proof). GATE: the soundness obligation that page_base/epoch_label are verifier-derived and never read from the proof is stated as an obligation, NOT claimed as verified. +[phase0] Page counts from the gate trace: 11 distinct ELF page bases (PT_LOAD headers, 1<<18) + 1 private-input page per fixture. Continuation touched-set size NOT recorded, NOT statically derivable — labelled inference. +[phase0] REGISTRY IMPLICATION now stated in §8.2.1 (was buried in an erratum): a ~94%-fixed leg means the emitted program barely varies with workload, so the profile ladder is ONE-DIMENSIONAL (epoch size) instead of the feared cross-product (workload classes x shapes). Composes with the page_base uniform promotion, which removes the other source of workload-dependence. +[phase0] REMAINING INFERENCE (narrowed): chunk GROWTH for a large continuation epoch is still monolithic-derived. Shape of growth is the same (cheap AIRs chunk, expensive ones do not) so ~96K is the ceiling, but the exact continuation curve is not run. +[phase0] RETARGETED: on the continuation path create_page_air is NEVER called (page_configs = &[]). GLOBAL_MEMORY is the AIR on the critical path. My census did measure both, but the proposal had foregrounded PAGE. +[phase0] REVISED ESTIMATE: 66,652 upper bound - 9,069 fused = 57,583. The doc's ~69K assumed ~1:1 with nodes; MulAdd costs the SAME row as Mul, so fusion is mandatory and the real number lands 16.5% UNDER. +[phase0] Refactor: the 25-AIR list was hand-copied into 3 test suites; now test_utils::production_airs() once. +[phase0] SCALING FLAG: 66,992 is per DISTINCT AIR, not per epoch — each sub-proof needs its own evaluation and chunking gives a family several. lfm-design.md 5.2's 69K line reads per-epoch but is per-distinct-AIR. Also workload-shaped: ECDAS+ECSM+KECCAK_RND = 86.9% of the total; no-EC workloads drop 65%. +[phase0] SIDE FINDING: the 3 continuation AIRs were also outside ood_window_ir_tests, which guards a real soundness bug (next-row columns pruned to zero). They pass with exact=true, so no bug — but they were unguarded until now. +[phase0] Scope: added the 3 continuation-only AIRs (l2g_global_air, l2g_memory_air, global_memory_air) now pub(crate) + L2gMemoryConstraints pub(crate). NUM_PRODUCTION_AIRS = 28, asserted by every suite that iterates the list. +[phase0] TESTED: artifacts_are_invariant_across_trace_length - the axis is structurally absent (no AIR ctor takes a trace length), so the test sweeps composition_poly_degree_bound(n) == k*n for n = 2^4..2^24 on all 28, which is the only route trace length could reach the artifact. Passes. Plus capture determinism. +[phase0] TESTED: global_memory_private_input_is_a_second_shape_not_a_second_program - is_private_input is a SECOND axis but an enumerable one (boolean): same program, differs only in is_preprocessed/num_precomputed_columns. +[phase0] THREAT MODEL now on record (§4.3), and it is SHARPER than the page case: epoch_label pins an epoch's POSITION in the chain — IsB20[epoch_label - 1 - init_epoch] is the cross-epoch ORDERING check (local_to_global.rs:447), and BusValue::constant(epoch_label) (:360) is the fini_epoch chain link itself. Today the verifier builds the AIR from its own enumerate() index so a prover cannot assert a different position. If the uniform were ever bundle-sourced: inflating the label RELAXES the ordering range check, and free choice of labels permits epoch REPLAY (two epochs claiming one position) or REORDER. page_base risks a wrong address; this risks the integrity of the chain. +[phase0] TWO THINGS FELL OUT: (a) fib_iterative_2M and array_multipass_20M give IDENTICAL chunk counts for their first 2^20 cycles — workload-independence visible directly, not argued from FIXED_TABLE_COUNT. (b) the test asserts traces.page_configs.is_empty(), so "an epoch never builds PAGE" is pinned by a run, not read off a comment. +[phase0] Trait: AIR::precaptured_constraint_program() added (never captures, guest-safe); AIR::constraint_program() default still panics. AirWithBuses::with_precaptured() supplies a build-time program. +[phase0] UNIFORM ZEROFIER CASHED: all 28 emit RowDomain::ALL => Z = zeta^N - 1 per AIR, division factors out of the beta sum. Once-per-AIR vs main's once-per-constraint saves ~50,900 instructions, ~44% of the unfused leg. +[phase0] WHY THE MULTIPLIER IS SMALL (structural): chunking multiplies the CHEAP AIRs (CPU 489, MEMW_R 153). The expensive ones (ECSM 19,264 / ECDAS 22,718 / KECCAK_RND 14,016) are never chunked — one sub-proof each. +[phase0] WHY: an epoch proof = 14 split families + 9/10 fixed + 1 L2G_MEMORY. No PAGE (page_configs = &[]), no GLOBAL_MEMORY (that is the GLOBAL proof). So the ONLY parameterized AIR in an epoch proof is L2G_MEMORY, parameter epoch_label = index + 1 => unpromoted, the registry needs one program PER EPOCH INDEX and the ladder grows LINEARLY with epoch count. That is exactly the workload-dependence the ~94%-fixed constraint leg was just shown not to have — winning it structurally and losing it to a bus constant. +[phase0] base->ext confirmed FREE => Op::Embed emits nothing (my independent analysis agreed). ext->base costs 1 LANES row but this leg NEVER needs it: nothing in the IR narrows, Dim only widens through binop's join. +[phase0] epoch_label is NOT symmetric with page_base: it is verifier-derived from enumerate() position, no supply route to get wrong. My "move them together, same risk" recommendation was wrong; revised to epoch_label-first as lower risk. +[phase0] §5.1 NEW: my first sketch threaded a &[F] uniform slice through every eval entry point (eval_program, eval_program_verifier, eval_device_program, interp::run) — large churn across both walkers, the CUDA host side and every caller, for a value that behaves exactly like a constant at eval time. BETTER: resolve uniforms INTO the program struct next to the constants — ConstraintProgram.base_uniforms / DeviceProgram.base_uniforms, with the ARTIFACT storing only num_base_uniforms. OP_BASE_UNIFORM's `a` indexes it exactly as OP_CONST_BASE indexes base_consts. Result: ZERO evaluation signature changes; the CUDA kernel gains a buffer uploaded like base_consts, not a new host parameter; the AIR fills the table at construction from its verifier-derived value, which is the natural place. +[phase0] ⚠ SELF-CORRECTION: my "the leg is workload-shaped, a no-EC epoch drops 65%" claim was FALSE. FIXED_TABLE_COUNT=10 is documented as tables that always contribute exactly one sub-proof REGARDLESS of TableCounts — bitwise, decode, halt, commit, keccak, keccak_rnd, keccak_rc, register, ecsm, ecdas. The fib fixtures use no EC and no keccak and still carry the full 60,389-instruction fixed block. The leg is essentially workload-INDEPENDENT (~94% fixed). I asserted the reverse from the census alone; the census cannot see how sub-proofs are assembled. +[phase0] 2026-07-29 slice 1 — constraint artifact ("constraints as data") landed on feat/phase0-constraint-ir off origin/main e0add1d5. +[phase0] 2026-07-29 slice 2 — extended to all 28 AIRs per team-lead ruling; two new invariance/characterization tests. +[phase0] 2026-07-30 slice 10 — DESIGN REFINEMENT found while planning implementation, plus handoff. +[phase0] 2026-07-30 slice 3 — docs + written proposal. +[phase0] 2026-07-30 slice 4 — constraint-lowering design (design alpha) + instruction census instrument. +[phase0] 2026-07-30 slice 5 — lowering design REVISED against the ISA inventory's cost facts. +[phase0] 2026-07-30 slice 6 — gate ruling absorbed; per-epoch multiplier MEASURED; self-correction. +[phase0] 2026-07-30 slice 7 — continuation-epoch leg computed and PINNED against the measured 24/25 sub-proof count. +[phase0] 2026-07-30 slice 8 — §8.2 is now FIRST-HAND for the continuation path; registry implication surfaced. +[phase0] 2026-07-30 slice 9 — PRIORITY REORDERED (team-lead, drawn from my own composition which I had not taken the step from): epoch_label FIRST, page_base after. +[reg-tree] 2026-07-31 slice 1 — R1g(i) REGISTER derivation LANDED on feat/lfm-register-derivation off feat/lfm 7bf0e157. Emits `compute_precomputed_commitment_with_fini` in the machine: 2 arenas (R_i, R_{i+1}), coset-decomposed LDE, row-pair leaves, full keccak tree, root published. PREDICTION CONFIRMED EXACTLY — 255/511/1023 permutations at blowup 2/4/8 (= 128·blowup − 1), 0.0182% / 0.2224% of an epoch's hashing. New: `lfm/lde.rs` (LDE emitter), `edsl::keccak_hash_pair` + `keccak_merkle_tree_root`, `proof_arena::register_boundary`, `ArchivedContinuationProof::epoch_reg_fini`. +[reg-tree] GADGET QUESTION ANSWERED: no second hashing gadget. The tree needs the PARENT step unwelded from the walk's Select, nothing more — `keccak_hash_pair` is now that step with `keccak_merkle_walk` and `keccak_merkle_tree_root` as its two callers. (I did NOT check what FRI wants — a verifier receives layer roots and authenticates against them, so it plausibly wants the WALK at a different leaf width, not a build; `keccak_leaf_hash` is already width-parameterized. That leg should settle it.) Sizing rule applied: a Select is 17 main cells against a permutation's 36,256, so the case for a separate driver is structural (a walk visits one node per level, a tree visits 2^k), not economic. Leaf gadget is `keccak_leaf_hash` reused unchanged. +[reg-tree] ⚠ THE FFT HALF OF THE DESIGN'S COST MODEL POINTS AT THE SMALL HALF. "3 columns, an inverse FFT and an LDE FFT each" is 8.5% of the derivation's arithmetic at blowup 8 (18,176 LFM_BALU rows of 214,784); 91.5% is BYTE SWAPPING the extended values into the leaves. Also only TWO columns need a transform — OFFSET is fixed, so its extension is interned constants computed at build time by production's own transform (which makes a matching root pin the emitter against the function it emits). Pre-swapping the constant column would drop 32,768 rows and save ZERO committed cells: `padded_rows` rounds to 2^18 either way. Measured, left undone. +[reg-tree] FOURTH DEGENERATE-PARAMETER INSTANCE, DEMONSTRATED. The fixture's epoch-0 boundary is 3/67 nonzero INIT, 10/67 nonzero FINI, 9 rows differing — blind over 57 of 67 rows. Deliberately dropping row 40 from the emitted columns PASSED the real-fixture differential and was caught ONLY by a synthetic all-distinct register file. Six falsification runs total, all caught: leaf column order, bit-reversal, coset interleaving, inverse twiddle direction, parent order, padding value. (Inverse-twiddle direction is itself invisible at n=2, where ω = ω⁻¹.) +[reg-tree] ⚠ FLAGGED, NOT FIXED: production's `reg_fini` is `Vec` and the type is the whole enforcement; an LFM arena is untyped felts, so the machine's accepted set is WIDER. `the_derivation_extends_a_non_u32_register_value_demonstrating_hazard` asserts the hazard succeeds. Not a hole in the derivation (such a root matches no production epoch commitment), but the assembly owes either a 67-per-column range check or the argument that no epoch proof can exist over such a column — the latter is plausible via REG-C2's Memory-bus value word and is UNVERIFIED. +[reg-tree] Chunking is not a constraint here: 1023 permutations at blowup 8 against a 2^19-ROW ceiling, one KECCAK_RND chunk at every blowup. `VmAirs::new`'s `register_preprocessed` left unwired, as decided in 236c0f4f. +[deep-join] 2026-07-31 slice 1 — DEEP across a FULL sub-proof, joined to the Merkle authentication: same cells, four committed matrices, query point derived from the walk's own index bits. 219/219 queries vs the production reconstruction; 32 tamper vectors; two control programs run the attacks the join denies. MEASURED: authentication is 99.0% of the joined leg's instructions (DEEP is 1.0%), 213,744 permutations per epoch's sub-proofs at blowup 8. +[deep-join] 2026-07-31 slice 2 — the precomputed group is a degenerate parameter: built a PREPROCESSED single-table fixture (four committed matrices) and witnessed both halves — the machine matches production on it, and production's own reconstruction with the base slices SWAPPED disagrees at every query. Falsified: with precomputed/main swapped in the emitter, only this test fails; the 219-query real-proof differential and all five other join tests pass against the wrong emitter. +[deep-join] 2026-07-31 slice 3 — deepened the preprocessed fixture to 64 rows (depth 6, the only executed multi-level walk over all four matrices) and swept the tamper suite over BOTH fixtures: 60 vectors total, and the precomputed group's own leaf and path are now tampered rather than merely emitted. +[deep-join] 2026-07-31 slice 4 — merged feat/lfm in (post-consolidation, 163 green) and answered the shared-commitment ruling's open question EXACTLY: the collapse is 48%, not the pinned 70% — 213,744 -> 111,471 permutations per epoch. Walks do collapse 69% (1,958 -> 616/query) but are only two thirds of the bill. Leaf widening costs NOTHING (absorbs 970 -> 911): total leaf bytes are unchanged by sharing, so only the vanished leaves' padding moves. +[deep-join] 2026-07-31 slice 5 — LogUp closure, and it CLOSED A LIVE GAP in the constraint leg: L had two consumers (the accumulator's L/N, the bus sum) hinted as independent arena words, so a prover could supply a truthful L/N and a fabricated L and both legs pass in isolation. constraints::emit_table_offset now derives L/N from the closure's own L (one MulBase, N is shape). Split-arena control RUNS the forgery it permits; the derived shape rejects all 4 deltas. Also: COMMIT-bus target vs production over 28 (length,start) combinations, a deliberate fingerprint collision proven unprovable, and a real sender/receiver pair whose bus multi_verify accepts at target zero. +[deep-join] 2026-07-31 slice 6 — two-consumer AUDIT (team-lead request) found instance 3, of the same class and worse in degree: alpha_powers were HINTED one word each, and Op::AlphaPow feeds every LogUp fingerprint, so a prover choosing them chooses the fingerprints. Fixed by constraints::emit_alpha_powers (chain from the one alpha; one ExtAlu per power). Also found: the guard test `challenges_are_not_an_arena_in_the_assembled_verifier`, cited by constraint_tests.rs:165, DOES NOT EXIST. Added an ABSOLUTE structural guard (method rule 7): L/N and every alpha power are asserted to be computed cells, not Hint outputs, with the hinted L and raw challenges as positive controls. Falsified both branches independently. +[reg-tree] 2026-07-31 FRI slice 0+1 — recon, spec corrections, index-bits exposure (8b8e55bf), and the fold-layout shape. ★ BLOCKING INSTRUMENT FINDING: the proof fixture carries ZERO committed FRI layers (measured: fri_layers_merkle_roots = 0, 4 terminal coeffs, 219 decommitments), because the min preset over a 2^4-step epoch gives log2(lde) = 3 and terminal_log = min(1+7, 3) = 3. A differential over the real proof therefore cannot see the fold loop, the per-layer walks, or the terminal check AT ALL. This is the degenerate-parameter family past its previous limit: earlier members were "every production instance shares one value, so a differential cannot separate two implementations"; here the production instance exercises NONE of the mechanism. Pinned by `the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold`. +[reg-tree] DEMONSTRATED, not argued: deleting the `saturating_sub(1)` from `num_committed` — the off-by-one that makes a verifier authenticate one layer FEWER than the proof commits — fails both synthetic tests and PASSES the real-proof differential, because the fixture's total_folds = 0 makes 0 and 0-saturating-1 the same number. The most soundness-relevant constant in the leg is invisible to the only real data available. +[reg-tree] Two of my own claims were WRONG and corrected: (a) I reported the FRI leaf backend as "PairKeccak256Backend, not BatchedMerkleTreeBackend" — true prover-side, FALSE of the verify path the machine emits (verifier.rs:643 uses BatchedMerkleTreeBackend over a 2-element vec; the scout's "both, byte-identical" is right). The verify-side reading also surfaced the parity-dependent leaf ordering (verifier.rs:637-641) a prover-side-only reading would have shipped wrong. (b) A hand-derived layout table expanded total_folds as trace_bits - blowup_log - k, subtracting the blowup twice; correct is trace_bits - k, the blowup cancelling between n and terminal_log — which is why num_committed = trace_bits - 8 holds across every preset. The module was right and anchored externally (reproduces spec §8's worked example and the fixture's real vector lengths); the hand table was wrong. +[reg-tree] METHOD RESULT worth generalising: my first join-guard test was VACUOUS and only falsification said so. It compared the program `emit_sub_proof` emits against `emit_sub_proof_with_bits`, but the former now DELEGATES to the latter, so both sides move together and any defect cancels; injecting the exact failure it denied left it green. Replaced with an absolute property (every returned bit must be read by some Select). General rule: a differential between two code paths dies the moment one is implemented in terms of the other — and the refactor that makes an API additive is exactly what kills the test policing it. +[reg-tree] PREDICTION PINNED BEFORE MEASURING (team-lead targets): 174/186/198 permutations per query and 38,106 / 20,460 / 14,454 FRI permutations at blowup 2/4/8, trace_bits 20. Recorded as `the_fri_sizing_prediction`. FRI is 2.6x cheaper at blowup 8 than blowup 2 — third independent leg of the blowup-8 decision. coset_offset plumbing condition discharged: `FriShape::from_options` reads the offset from ProofOptions, asserted against it on the real proof. +[deep-join] 2026-07-31 slice 7 (FINAL) — degenerate-parameter witnesses + handoff. Per-CHUNK accumulation WITNESSED both halves: a 3-table fixture (one sender, TWO receiver chunks of one family) closes, all three 2-term readings are nonzero, and a closure compiled for 2 tables rejects both chunk drops — on any 1-chunk-per-family fixture the two readings agree, which is why it needed building. has_trace_interaction() resolved by reading: production checks AIR-vs-proof presence BOTH ways (verifier.rs:1238/1244), so the contributing count is shape; a short arena is rejected. Zero-row fixed tables NOT witnessed — precise statement of what remains, and the cheap experiment, in others/lfm-logup-handoff.md. Context exhausted; handing off. +[fri-emitter] slice 0: ground truth green (176 passed / 0 failed / 1 ignored). ★ FINDING: the leg's blindness premise is FALSE — the L2G fixture's trace is boundaries.len().next_power_of_two(), so asking for 512/1024/2048 boundaries gives REAL production proofs with committed FRI layers C=1/2/3 (n=10/11/12, 219 queries, real paths, real terminal coeffs, zetas from the verifier's own replay), in 0.45s. No synthetic codeword needed anywhere. Emitter written (fri.rs +~330 lines): per-layer walk, fold chain, terminal check. OWED leaf-gadget check discharged by reading (stream_bytes = components 0,1,2 big-endian; felt_be_halves + LE half-packing composes to big-endian) and now under executable test vs BOTH production backends. +[zerorow] SETTLED: a zero-row fixed table reports Some(zero), NOT None — measured on a real accepted intermediate epoch (5 such tables: KECCAK/KECCAK_RND/KECCAK_RC/ECSM/ECDAS, each L=0); inference HELD, no LogUp-closure change needed. Stripping bus_public_inputs from any of them makes the proof FAIL, so Some is forced not just observed. Bonus: closure now runs over a real epoch's 24 contributions (closes the sum-LENGTH gap). Finding: unused != blank — 3 of the 5 have non-blank padded traces with every multiplicity column zero. feat/lfm-zerorow-experiment @ 2c03c100 (test) + 09f1966e (docs); lfm suite 177 green, make lint exit 0. +[fri-emitter] slice 1 DONE (bc9f2175): emitter + 10 tests, full lfm suite 186 passed / 0 failed / 1 ignored, make lint clean. MEASURED = PREDICTED exactly: 174/186/198 perms/query and 38,106/20,460/14,454 per sub-proof at blowup 2/4/8; on executed real proofs at n=10/11/12, 1,971/4,161/6,570 for 219 queries. Two spec deviations flagged: terminal check EVALUATES at υ^(2^total_folds) instead of emitting the FFT (§5 overridden — sim/24 does not transfer, a codeword lookup is a terminal_len-wide Select tree here; equivalence checked vs production's own FFT at 876 index/shape points, and it unifies the zero-fold branch), and fri_fold's mul association differs from production's while the field element does not. +[fri-emitter] slice 2 DONE (79cd8a3b): ★ my own structural guard was VACUOUS — selects(joined)−selects(trace_only) is a differential in disguise (both sides call the defective function), and the injected second-point-derivation left it green. Replaced with a closed form over the shapes; re-falsified, now fails with a surplus of exactly index_bits. New rule-7 instance for the standing decisions: a difference of two counts from our own emitter is still a relative test. Second trap, same session: the falsification harness reported all 7 breakages as "nothing failed" because cargo test -q names failures only in the trailing summary block — rule 3 applies to instruments too. 10/10 deliberate breakages now fail the right tests. Ledger gains entries 4 (FRI zetas must come from the transcript; coeffs+layer roots must be absorbed in production's order) and 5 (isolation driver's hinted index is wider than production's; assembled machine is fine by construction). +[team-lead] 2026-08-03 wave 3 CLOSED: fri-emitter (3 commits, leg CLOSED, measured=predicted on all six numbers, blindness premise FALSE — real folding proofs via boundary count) + zerorow (Some(zero) SETTLED, sum-length gap closed) merged into feat/lfm @ 6d5f197f; suite 188 passed / 0 failed / 1 ignored, make lint exit 0; ledger now 6 OPEN entries; next = wave 4 assembly. +[team-lead] 2026-08-03 wave 4 ABORTED at spawn: assembly agent hit session limit ~25min in (resets 16:40 America/Buenos_Aires), branch feat/lfm-assembly @ 35845e4c untouched, worktree wt-assembly alive. Its start_index research subagent DID finish; result preserved at others/lfm-team-lead-start-index-research.md — production binds start_index by AIR-reconstruction from prev epoch's FINI (bind arena start_index to reg_fini[64]; do NOT invent a start+len equation); FINI committed as u32 forces start_index < 2^32 (bears on ledger entry 1); one UNVERIFIED note: RV instructions cannot address word 508. Respawn assembly against the same brief after reset. +[assembly] 2026-08-03 slice 1+2 — THE FIAT-SHAMIR SPINE OF A REAL EPOCH RUNS. New `prover/src/lfm/epoch.rs` (fork + rounds 2-4 challenge replay) and `epoch_tests.rs` (4 tests). `the_epoch_challenge_spine_matches_production` builds a REAL continuation epoch (24 sub-proofs, production accepts it), replays statement + Phase A + 24 forks in the machine, and matches production's own `replay_rounds_after_round_1` on all 111 published challenges: shared z/alpha, then per table beta, z, gamma, every zeta and every query index. Ledger entries 4, 5, 6 DISCHARGED; 3 partially (one cell + both views now a construction — `RootCells`/`TableAbsorbs` — but the second consumers are not wired yet). +[assembly] FALSIFICATION: 12 deliberate defects, 10 CAUGHT (FRI root-before-zeta, no root absorb, no final-fold draw, no coeff absorb, OOD blocks in the wrong ORDER, no nonce absorb, no L absorb, no aux-root absorb, no fork separator [needs the 24-table epoch — invisible on the single-table fixture], no z-domain guard [needs the hinted-z driver]). 2 INVISIBLE and both understood: the grinding CHECK moves no challenge (closed by a separate test — the proof's nonce runs, 8 neighbours are rejected at factor 20) and the OOD absorb ORDER has NO production witness at all. +[assembly] ★ NEW LEDGER ENTRY 8, degenerate-parameter family, MEASURED: every OOD block of all 24 real sub-proofs is ONE ROW tall, so column-major and row-major absorbs coincide. The current block's height IS `step_size` (ood.rs:110-114) and the next block's is `num_eval_points - step_size` = 1 at two transition offsets. Premise checked as the RESUME asks: this is a claim about PRODUCTION, not about fixtures on hand. Needs a synthetic AIR with 3 transition offsets, proved by the production prover. +[assembly] ★ NEW LEDGER ENTRY 7: the preprocessed commitments are HINTED in the spine and four of five have no in-machine derivation. BITWISE/DECODE/KECCAK_RC are compile-time constants and can be interned; REGISTER's derivation exists (reg-tree); PAGE's CANNOT be a constant — it is a function of the inner ELF, which is per-proof arena data, so baking it would make program identity proof-dependent (an always-stop item). PAGE needs a derivation of REGISTER's family and does not have one. +[assembly] MEASURED on the real epoch (min preset, blowup 2, 1 query, grinding 20... factor 1): 24 sub-proofs; CPU is log2_trace 20 with 12 COMMITTED FRI LAYERS (a far better fold witness than the 0/1/2/3-layer single-table fixtures); the other 23 are log2_trace 2-7 with ZERO layers; OOD widths run 6 to 2,056 columns; 2 composition parts on 21 tables, 1 on two. +[assembly] 2026-08-03 slice 3 — the LogUp CLOSURE now hangs off the spine and reaches production's own COMMIT-bus target on the real epoch. Three joins made structural: the 24 `L` cells the closure sums are the cells their own forks absorbed; the public output is ONE arena (halves) with the bytes DERIVED (`epoch::emit_output_bytes`, whose recomposition assert doubles as the `< 2^32` range check and whose trailing mask pins the bytes past the length prefix); `start_index` is read from a register-boundary arena declared at production's width, slot 64 — the same cell the REGISTER derivation will bind, which is ledger entry 2's answer per the start_index research (production has NO arithmetic start+len check; the binding is AIR reconstruction from the previous epoch's FINI). +[assembly] GUARDS: `the_spine_hints_each_proof_value_once` is an ABSOLUTE structural count (rule 7) — no arena word is read by two Hints, and the positive control is that every declared word bar the unread register file is read exactly once. Falsified by adding one duplicate hint: it names `((3, 64), 2)` and fails. `the_closure_rejects_a_moved_index_or_output` moves start_index by 1/2/7 and every public-output half; all rejected. +[assembly] ★ MEASURED, and it is a cost line nobody had: the epoch spine is 1,095,553 instructions / 1,211 keccak permutations / 5,716 arena words for 24 sub-proofs at the min preset. 98.1% of its 16,621 BitDec rows are the BIG-ENDIAN felt streams of absorbed extension values (5,437 ext values x 3 coordinates = 16,311). So the spine's cost is essentially "byte-swap every OOD value into the transcript", it scales with total trace WIDTH (~4,863 OOD columns across the 24 tables, up to 2,056 on one), and it is independent of blowup and query count. For scale: the whole constraint-evaluation leg was measured at 57,252 instructions/epoch. +[assembly] ⚠ SCOPE, stated plainly: the verification LEGS are not wired onto the spine. No opening authentication, no DEEP, no FRI walk, no constraint evaluation runs in the assembled program. So the composed per-epoch predictions (213,744 opening permutations at blowup 8, ~460k total) are NEITHER confirmed NOR falsified by this run — they are untouched. What this run confirms is the Fiat-Shamir spine and the closure. +[team-lead] 2026-08-03 wave 4 CONSOLIDATION PARTIAL: assembly report received (spine RUNS on real 24-sub-proof epoch, 111/111 challenges match production replay, LogUp closure reaches COMMIT-bus target; legs NOT wired — composed predictions untouched; 195 green, lint 0; ledger 4/5/6 discharged, 2 half, 3 partial, NEW 7 PAGE-commitment + 8 OOD-absorb-order-unwitnessed). start-index research file de-JSONL'd (report extracted verbatim, raw transcript in history at e105dea2). Merge feat/lfm-assembly -> feat/lfm BLOCKED by session permissions — deferred to user; harmless, branch is strictly ahead. Wave 5 (hang the legs off the spine) spawning into same worktree/branch. +[team-lead] 2026-08-03 wave 5 spawn 1 ABORTED at session limit ~19min in (reset 20:30 America/Buenos_Aires), no slice committed, no agent log entry — died mid-slice with uncommitted WIP (epoch.rs/fri.rs/sub_proof.rs modified + NEW epoch_verify.rs). WIP preserved as stash@{0} on feat/lfm-assembly ('wave-5 spawn 1 aborted...UNTESTED WIP'); tree restored clean at 05c086f9. Respawn agent: review the stash FIRST (git stash show -p stash@{0}), decide keep-or-drop explicitly in this log, and only run the ground-truth suite on a clean tree. Respawn armed for after the reset. +[assembly-w5] 2026-08-04 slice 0 — respawn (spawn 2 died on a transient API error, not a limit; no work had reached disk). STASH DECISION: **KEEP**. stash@{0} is not a half-built feature but the SEAM API the wiring needs, and it is the part that is cheap to review and expensive to re-derive: `epoch::emit_reconstruct_ood` (two pruned blocks -> the full grid, pruned cells as the pooled ZERO constant), `sub_proof::emit_query_from_bits` + `opening_words` (an arena with no index word, because the index is the transcript's bits), `GroupCommitment::from_lanes` / `LayerCommitment::from_lanes` (a root reaches the compare as the cells the spine absorbed — the two-consumer join), `fri::hint_layer_openings_from` (one caller-declared query arena), and an untracked 416-line `epoch_verify.rs` sketch that composes them. It is consistent with every standing decision and with the wave-4 seam; treating it as a sketch, not a baseline — nothing in it has ever compiled. +[assembly-w5] 2026-08-04 slice 1 — ★ THE LEGS RUN ON THE SPINE. New `prover/src/lfm/epoch_verify.rs` (the seam emitter) + `epoch_verify_tests.rs`; `epoch_tests::epoch_challenge_program` refactored into `epoch_program(e, with_legs)` so there is ONE spine emitter and the leg program cannot drift from the one the 111-challenge differential covers. Per sub-proof the OOD grid is rebuilt from the two pruned blocks (`epoch::emit_reconstruct_ood`, verified line-by-line against production's `ood::reconstruct_ood_full`), constraint evaluation + quotient check run at the spine's own z/beta, and each query's `iota_bits` go straight into the Merkle walk, the DEEP fold and the FRI chain. Every check is an in-program assert, so execution IS the verdict. +[assembly-w5] ★ MEASURED (min preset: blowup 2, 1 query/table, grinding 1, 24 sub-proofs): spine 1,095,553 instr / 1,211 perms / 5,716 arena words -> ASSEMBLED 2,184,360 instr / 2,616 perms / 16,478 arena words. Legs alone = 1,088,807 instr / 1,405 perms / 10,762 words, i.e. the verifier is ~50/50 Fiat-Shamir and verification at this preset. Leg permutations decompose EXACTLY against a closed form over the shapes: 927 leaves + 304 Merkle levels + 174 FRI = 1,405 emitted (asserted, not printed). +[assembly-w5] ★ RECONCILIATION, number by number. (a) CONSTRAINT LEG EXACT: 54,358 ALU rows measured = 54,358 predicted (`lfm-constraint-lowering-design.md:604`), and 63,393 unfused reproduces the design constant to the digit; the recombination half measured 2,431 against the design's 2,894, and the 463 gap is the ZEROFIER SQUARINGS — 89 across this epoch's real trace lengths against 480 for a uniform 2^20 (391 of the 463), the rest per-sub-proof constant interning. So 56,789 vs 57,252. (b) FRI EXACT: at blowup 8 / 73 queries this epoch's FRI bill is 14,454 permutations, precisely the pinned per-sub-proof figure — and it comes ENTIRELY from the one 2^20 sub-proof, because the other 23 have zero committed layers at blowup 8 (their LDE is already terminal). (c) OPENINGS: pinned 213,744 was 28 production AIRs at a UNIFORM 2^20 with no FRI; this epoch's 24 sub-proofs under the SAME uniform assumption give 189,727 (the residue is the table set: L2G instead of five other AIRs), and at their REAL trace lengths only 100,959. Openings+FRI at blowup 8 = 115,413. +[assembly-w5] ★ THE DEVIATION THAT MATTERS: the composed predictions assumed every sub-proof is a 2^20 table. A real INTERMEDIATE epoch is not shaped like that — measured trace lengths (log2) are [2 x14, 3, 4 x4, 5 x3, 7, 20]: ONE big table and 23 tiny ones. Openings fall 1.88x against the uniform model on this epoch. This does NOT retract 213,744 as a model of a production-sized (2^24-step) epoch, where most tables are large; it says the pinned number is a claim about a workload, and the fixture epoch is a different workload. Every per-epoch number must now name its epoch shape. +[assembly-w5] LEDGER: entry 3 DISCHARGED — `the_assembled_verifier_hints_each_proof_value_once` is the same ABSOLUTE count as the spine's but over the program that now HAS both consumers of every value, plus a positive control that the assembled program declares strictly more arena words than the spine (without it the guard would pass over the spine alone). All four staged two-consumer values are now joined by construction: the OOD blocks reach constraints AND DEEP as one grid, the parts reach the quotient AND DEEP's h_sum, z is the zerofier's and DEEP's, and every root reaches the absorb AND the Merkle compare through `RootCells::lanes`. +[assembly-w5] NEW premise DISCHARGED by measurement, and it was a real risk: the boundary-constraint list is a program constant only if production's own `AIR::boundary_constraints` — which takes the PROOF's bus public inputs — always returns the framework's `acc[0] = 0` and nothing else. `the_boundary_terms_are_program_shape` compares the rule against the call as SETS (a MISSED term would be a constraint the machine silently never checks) on all 24 sub-proofs: 24 of 24 agree. +[assembly-w5] ★ NEW DEGENERATE-PARAMETER INSTANCE, found while writing the seam and invisible to every test we have: `Op::Var{offset}` indexes the constraint frame's evaluation STEP, and production's own interpreter asserts `row == 0` (`constraint_ir/interp.rs:240-242`), so the constraint leg's view of the OOD grid is every `step_size`-th row while DEEP's is ALL rows. At step_size = 1 the two views are the same vector. The sketch passed the whole grid to both; corrected to a strided view (`TableVerifyShape::num_frame_steps`). NOTHING in the suite can tell the two apart, because every production AIR has step_size 1 — same family as ledger entry 8 and closed by the same synthetic witness. +[assembly-w5] FALSIFICATION: 21 tamper vectors over the assembled program, all rejected — opened values (first and last of a group, on three tables including the folding one), BOTH words of a sibling digest (a past suite in this phase touched only byte 0), claimed composition parts, OOD cells, and the FRI layer-0 sym plus both words of its sibling on the one sub-proof with 12 committed layers. +[assembly-w5] ★ LEDGER ENTRY 7 CORRECTED BY READING, and the correction changes the plan. Entry 7 says "BITWISE, DECODE and KECCAK_RC are compile-time constants of the AIR set and could simply be interned". DECODE is NOT: `VmAirs::new` builds it as `create_decode_air(opts).with_preprocessed(decode::commitment_from_elf(elf, opts), ...)` (`lib.rs:743-750`) — a function of the inner ELF, exactly like PAGE. BITWISE and KECCAK_RC really are options-only (`bitwise::preprocessed_commitment(proof_options)`, `tables::keccak_rc::preprocessed_commitment(proof_options)`, `lib.rs:707-713/771-774`). So the family split is 2 constants + 2 ELF-dependent + 1 derived, not 3 + 1 + 1. The corroborating evidence was in plain sight: `recursion::program_id_from_digest` folds `elf_digest`, `pc_start`, `decode_commitment` and every `(page_base, page_commitment)` — it folds precisely the ELF-dependent roots and none of the constant ones. +[assembly-w5] MEASURED INVENTORY of the real epoch (`the_preprocessed_commitments_of_a_real_epoch`): only 4 of 24 sub-proofs are preprocessed — index 0 (11 precomputed cols) = BITWISE, 1 (5) = DECODE, 5 (9) = KECCAK_RC, 8 (3 = NUM_PREPROCESSED_COLS_WITH_FINI) = REGISTER, per `VmAirs::air_refs`' fixed order (`lib.rs:610-625`). ★ There is NO PAGE sub-proof in this epoch at all (`num_private_input_pages = 0`), so this fixture cannot witness PAGE's half of entry 7 — a fifth degenerate-parameter instance, and it is about the FIXTURE not about production (a guest with private input pages has them), so per the RESUME's premise rule the witness is a differently-configured real epoch, not a synthetic AIR. +[assembly-w5] 2026-08-04 slice 2 — FALSIFICATION of the wave-5 mechanisms, harness self-validated first (the phase's own trap: a harness that parses per-test lines reports every breakage as "nothing failed", so the summary `failures:` block is what is read, and a deliberate self-check mutation is run before believing any result). 6 mutations: +[assembly-w5] ★ M1 FOUND A REAL HOLE IN MY OWN COVERAGE, now closed. Deleting `assert_eq_ext(q.claimed, q.composition)` — the quotient check, the single most load-bearing assert in the leg — failed NOTHING. It cannot be caught by an arena tamper either, and the reason is structural: every input to the quotient identity (the OOD grid, the claimed parts, z, beta) is transcript-absorbed, so moving any of them moves the challenges and the run dies at the Merkle walk for the wrong reason. Closed by `the_assembled_verifier_contains_every_composition_and_terminal_check`, an ABSOLUTE count: `assert_eq_ext` lowers to `ediv(diff, ZERO)` (`builder.rs:243-247`) and division by the interned zero is satisfiable only at a vanishing numerator, so an ext Div whose DIVISOR is the pooled zero IS an equality assertion and nothing else emits one. Expected count is arithmetic over the shapes — one per sub-proof plus, per query, ONE FRI terminal check when the codeword folds and TWO when it does not — and this epoch exercises both branches (1 folding + 23 zero-fold): 24 + 47 = 71, measured 71. Re-running M1 now fails exactly this test. +[assembly-w5] ★ M2 CONFIRMS LEDGER ENTRY 9 as predicted, by demonstration rather than argument: passing the WHOLE OOD grid to the constraint fold instead of the frame-STEP view fails nothing at all. That is the defect the wave-5 sketch shipped, and at step_size = 1 no test in the suite can see it. +[assembly-w5] M3 (boundary list forced empty) caught by THREE tests including the run — which is what proves the quotient assert is live and load-bearing, since only that assert notices a composition that no longer matches its claim. M4 (an openings word hinted twice) caught by the hinted-once structural guard, naming the duplicate. SELF_CHECK (closed-form permutation count off by one) caught by the run test, so the harness is known to detect breakage. M5 (a second bit-decomposition of the same index) INCONCLUSIVE — it does not compile in the obvious form, and on reflection it tests nothing: re-decomposing the SAME bits is functionally identical, so it is a cost redundancy and not a soundness hole; the index join is denied by construction (one `bits` vector reaches the walk, the points and the FRI query) and guarded absolutely on the FRI side by `fri_tests::the_fri_join_adds_no_second_point_derivation`. +[assembly-w5] VERIFIED BY READING, not assumed: `LOGUP_NUM_CHALLENGES = 2` and `LOGUP_CHALLENGE_ALPHA = 1` (`lookup.rs:102-105`), so the `&[z, alpha]` the legs receive as `rap_challenges` is exactly production's vector in production's order — the alpha-power chain would silently start from the wrong challenge otherwise. +[assembly-w5] STATE: full lfm suite 200 passed / 0 failed / 1 ignored; `make lint` exit 0. Committed at a1f32859 (legs) plus this slice. +[assembly-w5] 2026-08-04 STOPPING POINT, stated precisely (context, not a session limit). Suite 201 passed / 0 failed / 1 ignored, `make lint` exit 0, branch `feat/lfm-assembly` @ 43594fe6, tree clean, nothing stashed. +[assembly-w5] TASK 1 (hang the legs off the spine) — DONE, committed, measured, falsified. TASK 2 (ledger 7 + 2) — PARTIAL: `programs::emit_register_commitment` is extracted and committed (the derivation now takes INIT/FINI cells instead of owning two arenas, so the spine can call it), the taxonomy is CORRECTED by reading, and the DECODE/PAGE resolution is written up as a PROPOSAL in ledger entry 7 rather than decided, because it touches program identity — an always-stop item. TASK 3 (ledger 8's synthetic AIR) — NOT STARTED. +[assembly-w5] NEXT, in the order I would do it: +[assembly-w5] (1) Wire entry 7's easy three quarters — this is mechanical now and closes entry 2 with it. In `epoch_tests::epoch_program`, replace the single `a_prep_roots` arena with a per-table SOURCE decision: BITWISE (sub-proof 0) and KECCAK_RC (5) become interned constants absorbed with `t.append_const_bytes`, needing a `RootCells::constant` that interns the 8 halves as felt constants; REGISTER (8) calls `programs::emit_register_commitment` over the register-boundary arena the spine already declares plus a NEW `reg_fini` arena, and the resulting `[Cell; 2]` becomes `RootCells` by unpacking — needs a `RootCells::from_digest`. NOTE `RootCells::words` is written by `hint` and READ BY NOTHING (checked): only `lanes` is consumed, so the field can go, which makes both new constructors trivial. DECODE (1) stays hinted pending the ruling. +[assembly-w5] (2) Entries 8 AND 9 with ONE synthetic AIR — three transition offsets AND `step_size > 1`, proved by the PRODUCTION prover. Both defects are demonstrated-invisible today (entry 8 in wave 4, entry 9 by this wave's M2), and a witness built for one closes the other only if it exercises both. The falsification harness is at `/falsify.py` and re-running M2 against the new fixture is the acceptance test. +[assembly-w5] (3) The wrap run, whose numbers must carry their epoch's trace-length profile (entry 10). +[assembly-w5] ⚠ FOR THE TEAM LEAD, one decision blocks (1) from being complete: DECODE and PAGE are ELF-dependent, so interning them makes program identity a function of the inner ELF (one LFM program per guest). The proposal in ledger entry 7 is to leave both as arena cells JOINED to the attestation's `program_id` fold — which already folds exactly `decode_commitment` and every `(page_base, page_commitment)` — keeping one program per epoch SHAPE. Its honest weakness: `program_id`'s binding is only as strong as the consumer-side `check_attestation` compare, which the RESUME already records as having ZERO production call sites. So the proposal makes DECODE/PAGE exactly as bound as the existing chain and no more. +[team-lead] 2026-08-04 wave 5 CLOSED: legs report received (whole verifier RUNS on real 24-sub-proof epoch, 2,184,360 instr / 2,616 perms assembled at min preset; constraint 54,358 + FRI 14,454 EXACT vs prediction; openings model corrected — uniform-2^20 assumption vs real shape [2x14,3,4x4,5x3,7,20] = 1.88x, entry 10; entry 7 taxonomy corrected, DECODE is ELF-dependent; entries 3 discharged, 9+10 added; M1 found+closed own coverage hole on the quotient assert). RULING ISSUED on entry 7: others/lfm-team-lead-decode-page-ruling.md — proposal ACCEPTED (BITWISE+KECCAK_RC intern, REGISTER derive, DECODE+PAGE arena cells bound by structural attestation join), two conditions attached (structural join with falsified guard; PAGE witness epoch or explicit OPEN entry). Subject to user veto, flagged in report. Wave 6 spawning: entry 7 wiring + entry 2 close, entries 8+9 one synthetic AIR (three offsets AND step_size>1), stretch = PAGE witness epoch. +[assembly-w6] 2026-08-04 slice 1 — ★ LEDGER ENTRIES 7 AND 2 CLOSED, and the entry-7 ruling is AMENDED by reading. Every preprocessed root of the assembled epoch verifier now comes from the source its provenance admits, decided by a host-side classifier (`epoch_tests::prep_source`) that MATCHES the AIR's own commitment against production's candidate functions rather than by sub-proof index — a preprocessed table whose root matches none of them panics instead of being hinted unbound. `RootCells` grew `constant` (8 interned halves; `words` deleted, it was read by nothing) and `from_digest`; `PhaseATable::preprocessed_root` became `PhaseAPreprocessed::{Constant(&[u8;32]), Cells(&[Felt])}` so program text absorbs as LITERAL BYTES with no splice arithmetic. +[assembly-w6] ★ AMENDMENT TO THE RULING (condition (b) is unsatisfiable AND unnecessary): the ruling says PAGE's half is unwitnessed because of a FIXTURE property (`num_private_input_pages = 0`) and asks for a real epoch from a guest with private input pages. Neither would work. (i) Private-input pages are built NON-preprocessed (`lib.rs:800-828`), so they could never witness a PAGE preprocessed root. (ii) NO continuation epoch of ANY guest has a PAGE sub-proof: `prove_epoch` REJECTS one ("continuation epoch must have no PAGE configs (L2G bookend replaces PAGE)", `continuation.rs:695-702`) and both `build_epoch_airs` call sites pass `page_configs = &[]` (`continuation.rs:711-714, 815-818`). The ELF-data page genesis roots the attestation folds are the GLOBAL proof's GlobalMemory AIRs' preprocessed commitments (`continuation.rs:997-1010`), which are out of an epoch verifier's scope. So the epoch taxonomy is 2 constants + 1 derived + 1 ELF-dependent (DECODE), asserted by `the_preprocessed_commitments_of_a_real_epoch` (census 2/1/1) plus a guard that no sub-proof carries PAGE's preprocessed width. THIRD finding: PAGE's zero-init root IS options-only (`page::zero_init_preprocessed_commitment`), so it belongs in the CONSTANT family, not the ELF-dependent one — the classifier carries it for the global proof's sake. +[assembly-w6] ENTRY 2's derivation is built: Phase A calls `programs::emit_register_commitment` on the register-boundary arena the spine declares plus a new `reg_fini` arena, and `start_index` is no longer even a second READ of slot 64 — it IS `reg_init[X254_INDEX]`, the cell the derivation consumed. The differential is free and total: a wrong derivation moves the absorbed root, which moves all 111 challenges. New `the_derivation_binds_every_register_boundary_word` moves 10 words (first/last of both vectors, and slots away from 64) and every one rejects; 66 of those words were declared-and-never-read before this slice. +[assembly-w6] ★ THE ATTESTATION JOIN, and the falsification the ruling demanded. `programs::emit_program_id` (extracted from `program_id_program_source`, which now delegates) folds the STATEMENT's own `elf_digest` halves, a new `pc_start` arena, and the DECODE `RootCells` Phase A absorbed. Differentialled against production's `recursion::program_id_from_digest` inside the spine test. The join is denied structurally by a PAIR of absolute guards: hinted-once (a second READ) plus `the_assembled_verifier_declares_exactly_the_shape_words` (a second WORD), the latter a closed form over the epoch's shapes and not an emitter pass. Falsified with a COHERENT FORGERY, not a count: `epoch_program_with(split_decode = true)` gives the fold its own arena copy, and `a_split_decode_cell_forges_the_attestation` shows that program RUNS on a substituted DECODE root and publishes the OTHER program's id (with the honest root in the same surplus arena it publishes the honest id, so the forgery is a free choice and not a broken proof) — while on the joined program the substitution is inexpressible, because the cell it would move is the one the transcript absorbed. +[assembly-w6] ★ MEASURED, min preset, 24 sub-proofs (entry 10: trace lengths log2 [2 x14, 3, 4 x4, 5 x3, 7, 20]): spine 1,095,553 -> 1,155,296 instr, 1,211 -> 1,467 perms, 5,716 -> 5,779 arena words; ASSEMBLED 2,184,360 -> 2,244,094 instr, 2,616 -> 2,872 perms, 16,478 -> 16,541 words. The permutation delta is +256 and it is EXACTLY the prediction: 255 for the REGISTER tree at blowup 2 (= 128*blowup - 1, reg-tree's pinned closed form) plus 1 for the program_id fold (102 bytes, one rate block). The arena delta is +63 and it is exactly +67 (reg_fini) +2 (pc_start) -6 (three roots that stopped being arena data). Entry 7's wiring costs 59,743 instructions, 2.7% of the assembled verifier. +[assembly-w6] Suite 204 passed / 0 failed / 1 ignored (188 -> 195 -> 201 -> 204); `make lint` exit 0. Also fixed a pre-existing display bug in the assembled-verifier measurement print: the spine's published-word column read `len - (x - x)` and showed the assembled figure. +[assembly-w6] 2026-08-04 slice 2 — ★ LEDGER ENTRIES 8 AND 9 WITNESSED, with TWO fixtures rather than one, and the brief's single-AIR plan corrected by reading + measurement. New `prover/src/lfm/step_size_tests.rs`; `epoch_verify.rs` grew `frame_step_view` (the strided rule extracted out of the emitter so it can be differentialled). +[assembly-w6] ★ THE BRIEF'S "THREE OFFSETS AND step_size > 1" IS UNBUILDABLE, and each half fails for its own reason. (i) `AirWithBuses::new` HARDCODES `transition_offsets: vec![0, 1]` (`lookup.rs:922`), so three offsets needs an `AIR` impl, and every one outside `crypto/**`'s example tree is in that tree — writing one is an always-stop item. (ii) `step_size > 1` IS NOT PROVABLE AT ALL: the CPU transition evaluator borrows ONE row per offset (`RowFrame::from_lde`, called at `evaluator.rs:72`) and asserts the shape outright — `debug_assert_eq!(lde_step_size, blowup_factor, "RowFrame requires single-row steps (step_size 1)")`, and `lde_step_size = step_size * blowup_factor`, so the equality IS step_size == 1. MEASURED, not read: `the_prover_cannot_prove_a_step_size_two_air` is a `#[should_panic]` on that exact message, so the ceiling is recorded and self-updating (it fails the day someone lifts it). +[assembly-w6] ★ FRAMEWORK CEILING, reported per the standing rule. From reading only (NOT verified by running): the assert looks OVER-STRICT for the access pattern that exists. `ConstraintBuilder::main(offset, col)` resolves to row 0 of a step (`builder.rs:719-724`), `RowFrame::from_lde`'s index for step k is `row + offset*lde_step_size` = the same row the general multi-row-capable `Frame::read_from_lde` calls `initial_step_row`, and that general path already handles step_size > 1 correctly. So it is plausibly a one-line relaxation in `crypto/**` — an always-stop item, hence the USER's call. Until then NO end-to-end run of the assembled verifier at step_size > 1 is possible, from any AIR. +[assembly-w6] ENTRY 9 CLOSED WITHOUT A PROOF, and with a better oracle than a proof would have given. The defect is the machine's grid->frame-step mapping, and production has its own function for exactly that: `StarkTableView::into_frame(main_cols, step_size)` (`proof/view.rs:269-294`), which the real verifier calls at `verifier.rs:320-321`, is a PURE function of a grid and a step_size. `the_frame_step_view_matches_productions_own_frame_assembly` differentials `frame_step_view` against it at (offsets, step_size) = (2,1),(3,1),(2,2),(3,2),(2,4), main and aux columns both, on grids of distinct values. FALSIFIED (F1): making `frame_step_view` return the whole grid — wave-5's M2 defect verbatim — fails this test and NOTHING ELSE (206 passed / 1 failed). +[assembly-w6] ENTRY 8 CLOSED WITH A REAL PROOF of a THREE-OFFSET AIR. `stark::examples::fibonacci_multi_column::FibonacciMultiColumnAIR` already has `transition_offsets: vec![0,1,2]` and is generic over the extension, so at 3 columns / step_size 1 it gives `num_eval_points = 3` and a next-row block of **3 columns x 2 ROWS** — the phase's first block where column-major and row-major absorbs differ. Proved by `multi_prove_ram`, ACCEPTED by `multi_verify_views`, and the machine's `emit_table_challenges` replay differentialled against production's own `replay_rounds_after_round_1` on every challenge (beta, z, gamma, zetas, iotas). No `crypto/**` change, no synthetic AIR needed at all. +[assembly-w6] FALSIFIED (F2), and it re-proves entry 8's own claim as a by-product: swapping `emit_table_challenges`' absorb loop to ROW-major leaves the 24-sub-proof epoch spine differential, the assembled-verifier run and the single-table replay ALL GREEN (206 passed) and fails only the new three-offset test. Note the failure MODE honestly: the mutation is caught by the in-program GRINDING check (DivByZero) before the challenge comparison is reached, because a moved transcript state invalidates the nonce; the clean statement of the property is the test's own row-major CONTROL program, which stops at gamma and shows it moves against production's gamma. +[assembly-w6] NOT COVERED, stated: no test runs the ASSEMBLED verifier at step_size > 1, because no proof of that shape can exist. Entry 9's closure is about the emitter's grid indexing, against production's own mapping. Also unexercised and named rather than chased: an OOD grid with more than TWO blocks — three offsets at step_size 1 still yields two blocks, and nothing in the machine is shaped by the block count (`emit_reconstruct_ood` takes two because the proof carries two), so that is a framework property, not a machine one. +[assembly-w6] Suite 207 passed / 0 failed / 1 ignored; `make lint` exit 0. +[assembly-w6] 2026-08-04 slice 3 — ★ LEDGER ENTRY 1 DISCHARGED by its own stated default, and the ledger is now EMPTY OF DEBTS (only entry 10 remains, which is the wrap run's reporting rule, not a debt). Entry 1 said "emit the 67-per-column range check if the no->u32 argument is still unverified when assembly arrives". It is still unverified, and slice 1 is what made the boundary vectors live arena data in the assembled verifier — so `epoch::assert_u32` now runs on all 134 cells (one BitDec + one recomposition each). Placed at the ASSEMBLY call site, not inside `emit_register_commitment`: the isolated derivation's hazard guard (`the_derivation_extends_a_non_u32_register_value_demonstrating_hazard`) is RIGHT that an isolated derivation binds nothing, so it stays green. +[assembly-w6] ⚠ THE OBVIOUS TEST FOR ENTRY 1 IS VACUOUS AND I WROTE IT FIRST. "Set a boundary word to 2^32 and watch the epoch fail" fails with the check REMOVED too — a wide value moves the derived root and hence every challenge after Phase A, which is the same rejection `the_derivation_binds_every_register_boundary_word` already gets from moving a word by one. Replaced with a non-vacuous pair in `the_register_boundary_is_width_checked`: (a) `assert_u32` in ISOLATION admits [0, 2^32) and rejects [2^32, p); (b) a STRUCTURAL check that every register-arena Hint output is the input of a 32-bit BitDec. FALSIFIED (F3a): applying the check to a 3-cell PREFIX — exactly the defect a value tamper cannot see — fails (b) and nothing else (16 passed / 1 failed). +[assembly-w6] ★ SUBTLETY THE TEST FOUND, and it sizes the gap exactly: an arena word is a FIELD ELEMENT, so `FE::from(u64::MAX - 1)` is the felt `2^32 - 3` — a perfectly good u32. My first draft used it as an out-of-range value and reported the check broken when it was not. The widening entry 1 names is the interval [2^32, p) and nothing beyond; there is no felt at or above p. +[assembly-w6] ★ FINAL MEASUREMENT, min preset, 24 sub-proofs, trace lengths log2 [2 x14, 3, 4 x4, 5 x3, 7, 20] (entry 10): spine 1,159,852 instr / 1,467 perms / 5,779 arena words / 114 published; ASSEMBLED 2,248,650 / 2,872 / 16,541 / 162; legs alone 1,088,798 / 1,405 / 10,762 / 48. Against wave 5's 2,184,360 / 2,616 / 16,478 that is +64,290 instructions (+2.9%), +256 permutations, +63 words. Permutations decompose EXACTLY: 255 REGISTER tree at blowup 2 (= 128*blowup - 1) + 1 program_id fold. The width check is 4,556 of the instructions (134 cells x ~34), the rest is the register LDE + tree + fold. +[assembly-w6] DOCS: ledger updated (entries 1/2/7/8/9 closed with evidence, a STATUS AT WAVE 6 header naming the two USER items), RESUME's wave-6 section replaced by a wave-6-CLOSED section + "ready to start next (wave 7): the wrap run". +[assembly-w6] ⚠ FOR THE USER, two always-stop items surfaced rather than worked around: (1) the step_size > 1 framework ceiling in `crypto/**` (`frame.rs:38`), plausibly a one-line relaxation, which would give entry 9 an end-to-end witness; (2) the entry-7 ruling's condition (b) asks for a witness epoch that CANNOT EXIST — PAGE roots belong to the GLOBAL proof, so that obligation migrates rather than closing. +[assembly-w6] STATE: suite 208 passed / 0 failed / 1 ignored; `make lint` exit 0; branch feat/lfm-assembly, 3 commits off 3766214a, nothing stashed, not pushed. +[team-lead] 2026-08-04 wave 6 CLOSED, LEDGER EMPTY OF DEBTS: entries 1/2/7/8/9 discharged (208 green, lint 0, @ a222925f); provenance classifier sources every preprocessed root, REGISTER derived in Phase A (start_index IS the consumed cell), DECODE attestation-joined w/ coherent-forgery falsification; entries 8+9 witnessed by fibonacci_multi_column (3 offsets, prod proof) + frame_step_view differential vs StarkTableView::into_frame at steps 1/2/4. RULING AMENDED: condition (b) REVERSED — PAGE witness epoch cannot exist (prove_epoch rejects page configs, L2G bookend replaces PAGE; private-input pages non-preprocessed; ELF-data page roots live in the GLOBAL proof); PAGE migrates to global-proof-verifier scope, zero-init root reclassified options-only. TWO USER ITEMS surfaced: step_size>1 framework ceiling (RowFrame assert frame.rs:38, plausibly over-strict — would give entry 9 an e2e witness; crypto/** = always-stop) and check_attestation gap now MORE load-bearing (DECODE binding rests on it). Wave 7 = wrap run spawning. +[assembly-w7] 2026-08-04 slice 0a — THE WRAP'S INSTRUMENT, falsified before use. New `prover/src/lfm/wrap_tests.rs` (the wrap harness) plus a refactor of `airs::lfm_cell_counts` into `lfm_chip_census` + a summing wrapper, so the per-chip census and the cell TOTAL are one arithmetic and cannot disagree (verified behaviour-preserving: FriToyV0 prints 10,569,448 / 5,313,080 before and after). +[assembly-w7] ★ FIRST CELL MEASUREMENT, min preset, inner epoch trace lengths (log2) [2 x14, 3, 4 x4, 5 x3, 7, 20] (entry 10), 24 sub-proofs, blowup 2, 1 query/table: assembled verifier = 2,248,650 instr / 2,872 perms / 16,541 arena words (wave 6's line to the digit) and **220,107,920 main cells + 87,073,068 aux ext elements** = 481,327,124 base-field equivalents over 14 sub-proofs (1 KECCAK_RND chunk). KECCAK_RND alone is 193,986,560 main (88.1%) at 131,072 rows x 1,480 cols; BITWISE 10,485,760; LFM_BALU 8,388,608 at 2^21 rows. Spine alone 119,774,932 / 48,687,288, so the legs' marginal cells are 100,332,988 / 38,385,780. Fixed-machine floor (empty program) 10,560,752 main = 4.8% of the assembled verifier. +[assembly-w7] FALSIFIED the instrument itself, twice, before trusting a number: (F1) dropping the census' name-mapping shift across the KECCAK_RND slot fails `the_census_agrees_with_the_traces_the_prover_builds` on "the census and the AIR set disagree about the frozen chip order"; (F2) sourcing one chip's height from `real_rows` instead of `padded_rows` fails it on "the census height must be the trace's own". The test has THREE sides — census vs the traces `multi_prove` receives (heights) vs the AIRs `air_refs` builds (names and widths) — because heights alone cannot see a name mapping off by one. +[assembly-w7] Suite 209 passed / 0 failed / 3 ignored (208 + 1 new; the two wrap runs are #[ignore]d harnesses); `make lint` exit 0. +[assembly-w7] 2026-08-04 slice 0b — ★★ THE WRAP PROVES AND VERIFIES. The assembled epoch verifier went through `lfm_prove` + `verify_against` for the first time in the phase (everything before it was `execute` only, which method rule 2 says sees no chip). Inner epoch min preset, trace lengths (log2) [2 x14, 3, 4 x4, 5 x3, 7, 20], 24 sub-proofs: prove 19.5s / verify 0.09s / proof 30,707,816 bytes / 162 published words / 14 LFM sub-proofs / peak RSS 15.1 GiB / 16,228,499,456 bytes (11-core laptop, release). The WRAP's own options are blowup 2 / 219 queries / grinding 20, so the outer proof is at a 128-bit setting even where an inner rung's query count is reduced. Chip log-heights [11, 21, 17, 11, 15, 2, 12, 16, 15, 8, 16, 0, 5, 20] = the registry-entry shape record. +[assembly-w7] FALSIFIED IN BOTH DIRECTIONS, and the two failure MODES are different and both needed: (1) tampering an opened value makes the wrap UNBUILDABLE — `lfm_prove` fails inside `execute` with `DivByZero` at the root compare, because every check is an assert inside a straight-line program and a false assert has no witness (no branch to take, no error path to return); (2) an HONEST proof against a moved claimed public word, or against a moved program digest, is UNVERIFIABLE. A machine with only (1) would prove nothing about what its proof SAYS. +[assembly-w7] 2026-08-04 slice 1 — the box was NOT used and nothing was started on it: `app@195.154.218.198` was running an ethrex continuation campaign at both check points (18:43 and 18:52 UTC — two different `cli prove ... --continuations --epoch-budget-mib 51200` invocations, 36-50 GB RSS, load average 22 then 29 on 32 cores). Per the brief that is a stop-and-report, so slice 1 ran locally in two parts. +[assembly-w7] ★ SLICE 1a, the blowup-8 GEOMETRY proved end to end (inner blowup 8, 1 query): widest sub-proof 2^20 trace -> 2^23 LDE, 22 Merkle levels per group, 12 committed FRI layers, widest leaf 27,648 bytes. 2,425,718 instr / 3,816 perms / 16,893 arena words; 230,661,264 main + 92,350,764 aux ext cells; prove 23.3s / verify 0.09s / proof 31,147,664 bytes / peak 15.5 GiB. The query COUNT is what was reduced and it is separable: per-query cost is a closed form over the shapes, asserted against the emitted program in slice 1b. +[assembly-w7] ★★ SLICE 1b, THE PRODUCTION SHAPE, MEASURED (emitted + censused; NOT proved): inner blowup 8 with its real 73 queries, same epoch profile — **76,501,118 instructions / 118,080 keccak permutations / 817,101 arena words / 6 KECCAK_RND chunks / 5,077,422,224 main cells + 2,029,461,548 aux ext = 11,165,806,868 base-field-equivalent cells per epoch verify.** Emission alone takes 97.8s and 15.9 GiB. +[assembly-w7] MEASURED vs PREDICTED, number by number: openings **100,959 = 100,959** (wave 6's projection of THIS epoch, exact); FRI **14,454 = 14,454** (the pinned per-2^20-sub-proof figure at blowup 8, exact); legs total 115,413 = 100,959 + 14,454 (asserted as the closed form, not eyeballed); epoch bill 118,080 against the design's ~460,000, which is 3.9x apart because 460,000 models a production-SIZED epoch and this one has 23 tiny tables — entry 10's own point, now with the measurement on both sides. +[assembly-w7] ★ 84.0% OF THE CELLS ARE THE HASH: LFM_KECCAK + KECCAK_RND = 4,364,173,312 main + 1,672,478,720 aux ext; its fixed lookup tables (KECCAK_RC + BITWISE) another 0.2%; the verifier's own arithmetic is 15.7%. One permutation costs 36,256 main + 13,912 aux cells. So the hash matrix's other columns decide the machine's SIZE, not its structure — which is the finding the phase was built to produce. +[assembly-w7] ⚠ THE PRODUCTION-SHAPED WRAP IS NOT PROVABLE AT 124 GiB: 350.6 GiB projected peak at the measured 33.7 bytes per base-field-equivalent cell. The coefficient has TWO measured points and was checked before being extrapolated (slice 0: 15.1 GiB at 481.3M cells; slice 1a: a 15.9 GiB projection came in at 15.5 GiB actual), but it is a 22x extrapolation and is labelled a projection everywhere it appears. Ways out: a cheaper hash (84% of it), disk spill, or splitting the wrap. A decision, not a debt. +[assembly-w7] LEDGER: entry 10 SATISFIED with its table (every row naming [2 x14, 3, 4 x4, 5 x3, 7, 20]); the ledger is CLOSED. RESUME item 6 rewritten as DONE; wave-8 hand-off = the hash matrix, which `wrap_run(inner_options)` already parameterises. +[assembly-w7] STATE: suite 209 passed / 0 failed / 5 ignored (the 4 wrap harnesses + the pre-existing one); `make lint` exit 0; branch feat/lfm-assembly, not pushed. +[assembly-w7] 2026-08-04 slice 2 — RECONCILIATION, and two numbers that needed measuring rather than inferring. (a) The spine/legs split now prints per run and reproduces wave 6's line EXACTLY, all six numbers: spine 1,159,852 instr / 1,467 perms / 5,779 words, legs 1,088,798 / 1,405 / 10,762, assembled 2,248,650 / 2,872 / 16,541 at the min preset. (b) ★ THE SPINE GROWS WITH THE QUERY COUNT TOO (1,467 perms at blowup 2/1q, 2,235 at blowup 8/1q, 2,667 at blowup 8/73q), so per-query cost may only be taken from the DIFFERENCE — and taken that way the legs cost **1,581.0 permutations per query at ONE query and 1,581.0 at 73**, measured at both ends. That is what lets the 1-query geometry run stand in for the 73-query shape's geometry; my first pass computed 1,149/query from the total and would have published a wrong per-query figure. +[assembly-w7] NEW instrument, the recursion RATIO (kill-risk-3 asked of a real epoch at last): the inner epoch's own committed trace cells are 22,036,988 main + 5,248,588 aux ext = 37,782,752 base-field-equivalents, so verifying it costs 12.7x (min preset), 13.4x (blowup 8, 1 query) and 295.5x (blowup 8, 73 queries) its own trace cells. ⚠ NOT a machine constant and flagged as such in the ledger: the denominator is a 16-guest-cycle epoch — the smallest an epoch gets — while the numerator is set by query count and tree depth. Quote cells-per-verify, never this. +[assembly-w7] The GEOMETRY line also prints per run, so "blowup 8 was exercised" is evidence and not a label: widest sub-proof 2^20 trace -> 2^23 LDE (against 2^21 at blowup 2), 22 Merkle levels per group (against 20), 12 committed FRI layers, widest leaf 27,648 bytes. +[assembly-w7] Unit correction to my own earlier lines in this log: the peak-RSS figures were quoted in GiB where the byte counts are GB. Slice 0's peak is 16,228,499,456 bytes = 15.1 GiB (not 16.2 GiB); the b8 emission peak is 15.9 GiB. Fixed in place here, in the ledger and in RESUME. Run-to-run spread on peak RSS is about 5% (three slice-0 runs: 16.23 / 15.54 / 15.6 GB), and prove time 19.5-20.5s, so no figure here is quoted tighter than that. +[assembly-w7] FINAL STATE: suite 209 passed / 0 failed / 5 ignored; `make lint` exit 0; branch feat/lfm-assembly, 3 commits off 6dcdceab, nothing stashed, NOT pushed. The box was never touched. +[assembly-w7] BOX, third check 19:12 UTC — and a correction: the earlier two checks were at 18:43 and 18:52 UTC, which I mislabelled "UTC+2" above; the box clock is UTC. The third check found a THIRD distinct invocation, `--epoch-budget-mib 16384 -o ~/results/opportunistic-16gib-v2/out.proof`, load average 28. Three different `cli prove` runs across 30 minutes is an active sequential campaign, not a stray process, so the box was left alone. A run there would buy a bigger provable RUNG (4 queries at blowup 8 is ~70 GiB projected), never the 73-query headline (350.6 GiB). +2026-08-04T19:45Z | [hash-w8] slice 0 — hash-matrix SCOPE | lfm 209/209 + 5 ignored (clean 891f534f) | DONE — others/lfm-hash-matrix-scope.md. ★ THE MACHINE ALREADY HAS A HASH SWAP SURFACE, AND IT IS NOT THE SOCKET KECCAK USES. prover/src/lfm/hash.rs is titled "the machine's swap surface" and freezes LfmHasher (permute [FE;12]->[FE;12], compress_iv, default compress = 1 permutation of [a||b||IV] truncated), HASH_STATE_FELTS=12, HASH_DIGEST_FELTS=4, and the LFM_HASH bus tuples/opcodes; behind it sits TestPermutation, labelled NOT CRYPTOGRAPHIC (one degree-3 round, x->(x+rc)^3, M=I+J). Its own doc names the candidate set: "Poseidon2 is broken; candidates are Poseidon-original, RPO/XHash, Monolith and reduced-round Blake2s". edsl.rs:137-143 says the two sockets are "not interchangeable" — merkle_walk compresses with LFM_HASH/TestPermutation, keccak_merkle_walk authenticates production trees. So a candidate column is socket 2 carrying the epoch verifier's real workload for the FIRST time, not a variation on the keccak column. CENSUS FORMULA verified from source (airs.rs:122-128/234-236/246-249): main = padded_rows x (NUM_COLUMNS - PREP_WIDTH), aux = padded_rows x ceil(interactions/2), base-equiv = main + 3 x aux. Reproduces entry 10 EXACTLY: 5,077,422,224 + 3 x 2,029,461,548 = 11,165,806,868; hash 9,381,609,472 = 84.02%; residue 1,784,197,396; 118,080 x 77,992 x 1.01871 = the hash total (padding coefficient DERIVED, agrees with the ledger's independent 1.7% round-row waste). LFM_HASH today = PREP_WIDTH 11, 28 value cols (IN0..11, S8..11, OUT0..11), 6 LfmMem interactions -> aux_cols 3, max_degree 3, ONE row/permutation = 37 base-equiv cells/perm — a FLOOR WITH NO CRYPTOGRAPHIC CONTENT, flagged in the doc as the leg's worst available error. ★ TWO FINDINGS THAT MOVE THE DECISION: (1) 53.5% of keccak's per-permutation bill is AUX ALONE (3 x 13,912 = 41,736 of 77,992) and essentially all of it is KECCAK_RND's BITWISE lookups, which are bus interactions; an algebraic hash has NONE, so aux collapses to 3/row structurally rather than by estimate. (2) THE RATE PENALTY GOES THE WRONG WAY: keccak absorbs 17 felts/permutation (RATE_BYTES=136 at 8 bytes/felt, layout.rs:116 + keccak_host.rs:15) but the LFM sponge absorbs 8 (state 3 cells, rate 2, edsl.rs:16-17), so a candidate pays up to 2.125x MORE permutations — a consequence of the FROZEN HASH_STATE_FELTS=12, and the cleanest lever on P is widening it (team-lead call). Parent steps and FRI leaves are 1:1 either way (fri.rs:139-144, edsl.rs:149-155); only leaf hashes and the spine are absorption-bound. PREDICTION (est., falsifiable): Poseidon-original t=12 x^7 at degree 3 (two intermediate cols/S-box, 118 S-boxes) = m 600-1,100, a 3 -> 610-1,110 base-equiv/perm = 70-128x cheaper/perm than keccak; whole verify ~1.86-2.07 B cells vs 11.17 B = 5.4-6.0x smaller, ~58-65 GiB projected — INSIDE the 124 GiB box. Robust because once the hash is cheap the ALREADY-MEASURED residue dominates (1.78 B of ~1.9 B). Blake2s is ARX on 32-bit words = bit-oriented = keccak-CLASS in-AIR, NOT Poseidon-class, so the probable ship choice may not buy the 2.8x the wrap needs — the decision-critical column. ⚠ ROUND COUNTS ARE MY OWN DOMAIN KNOWLEDGE, not corpus-checked and not in-repo; flagged as the report's weakest link. ★ CHEAPEST NEXT SLICE NEEDS NO HASH AT ALL: epoch_verify::query_permutations (epoch_verify.rs:414-434) is already a closed form over shapes, so substituting ceil(felts/8) for num_blocks(bytes)=bytes/136+1 yields the candidate's P by arithmetic, and the keccak rate reproduces the emitter exactly = an absolute check (rule-7 safe). ⚠ WORKTREE COLLISION, reported to team lead: another agent is LIVE in wt-assembly (wrap_tests.rs mtime moved 16:17:32 -> 16:19:00 while I watched; others/lfm-assembly-obligations.md became dirty during my session; the in-flight edit adds wrap_run_with(inner, Option)). I did NOT checkout -b there — DEVIATION: made my own worktree wt-hash-matrix on feat/lfm-hash-matrix off 891f534f. Their 209/0/7 vs my clean 209/0/5 is the collision corroborated independently (their +2 = the_chunk_policy_prices_padding + the_wrap_proves_with_minimal_chunk_padding). NO code written yet — scope first, per brief. +2026-08-04T20:05Z | [hash-w8] slice A — the PERMUTATION axis, MEASURED | lfm 209/209 + 5 ignored, make lint exit 0 | DONE. epoch_verify.rs gains KECCAK_RATE_FELTS=17, LFM_HASH_RATE_FELTS=8, group_leaf_felts, blocks_at_rate, leaf_permutations_at_rate, query_permutations_at_rate; the assertions live in the existing the_assembled_epoch_verifier_runs beside the closed form it already checks. ★ THE CANDIDATE'S PERMUTATION COUNT IS MEASURABLE WITH NO HASH BUILT, because query_permutations was already a closed form over shapes: at blowup 8 / 73 queries on the fixture epoch's real trace lengths (profile [2 x14, 3, 4 x4, 5 x3, 7, 20]), keccak rate 17 = 115,413 permutations (67,671 leaves + 47,742 paths/FRI) and LFM_HASH rate 8 = 187,902 (140,160 leaves + same 47,742) = 1.6281x, leaf term alone 2.0712x, absorption-bound share 58.6%, widest leaf 3,456 felts. THE KECCAK SIDE REPRODUCES ENTRY 10 EXACTLY — 115,413 is the ledger's own legs figure (118,080 = 2,667 spine + 115,413 legs) — which is what makes the candidate side trustworthy: same function, different rate. So my slice-0 interval [1.0x, 2.125x] collapses to P_candidate in [190,569, 193,569] = 1.614-1.639x (spine bounded, +-0.8%, immaterial). RULE 7 AVOIDED DELIBERATELY: the new function is written through FELTS and the old through BYTES + keccak_host::num_blocks, and NEITHER delegates to the other, so their agreement at rate 17 is a real differential; making one delegate would have made the test vacuous at that moment. FALSIFIED BOTH ASSERTS (rule 1): F1 dropping the +1 padding block from blocks_at_rate trips "the felt-side closed form must reproduce the byte-side one at keccak's rate"; F2 making the Merkle PATH term rate-sensitive trips "only the leaf term may move with the rate" — and F2 SLIPS PAST the rate-17 differential entirely (blocks_at_rate(8,17)=1), so the decomposition assert is not redundant with it. Both reverted. ⚠ CORRECTED MY OWN SLICE-0 ARITHMETIC: I had carried a=3 into both candidate layouts, but aux_cells = rows x ceil(interactions/2) scales with ROWS, so a 30-rows-per-permutation layout pays 90 aux cells (270 base-equiv), not 3 — which flips the layout choice from "either" to clearly Layout B (unrolled, 617 base-equiv/perm vs 1,350). ⚠ ALSO FLAGGED: the 1.019 padding factor is KECCAK_RND's chunk padding and does NOT transfer — a 1-row-per-permutation candidate has trace height P ~ 192,000, which pads to 2^18 = 36.5% waste unless chunking.rs gets a sibling; a naive first measurement will read ~36% high on the hash term. PREDICTION NOW A BOX, NOT A POINT: layout x padding at both extremes gives total 1.905-2.162 B cells vs keccak's measured 11.166 B = 5.17-5.86x smaller, 59.8-67.8 GiB projected — EVERY cell inside the 124 GiB box, spread 1.13x against a win of 5.2-5.9x, so the conclusion survives being wrong about layout, about padding, and about m by 2x. Only estimated input left is the round count. ⚠ The four research legs I dispatched (inner-prover hash blast radius, in-tree AIR inventory, corpus Part I.7 candidate data, socket spec) had NOT returned when this slice closed — round counts remain my own domain knowledge, uncorroborated, and §2.2 says so. +2026-08-04T20:20Z | [hash-w8] slice 0b — in-tree inventory, done MYSELF after the dispatched leg went silent | no code change (doc only) | ★ A POSEIDON-ORIGINAL SKELETON ALREADY EXISTS IN-TREE: crypto/crypto/src/hash/poseidon/ (96+45 lines) — a Poseidon trait over PermutationParameters whose hades_permutation is N_FULL_ROUNDS/2 full -> N_PARTIAL_ROUNDS partial -> N_FULL_ROUNDS/2 full (mod.rs:28-41), i.e. HADES = Poseidon-ORIGINAL, which independently CONFIRMS the round SHAPE my §2.2 estimate assumed. Trait carries RATE/CAPACITY/ALPHA/N_FULL_ROUNDS/N_PARTIAL_ROUNDS/MDS_MATRIX/ROUND_CONSTANTS + default mix (parameters.rs:11-44). ⚠ BUT NO CONCRETE INSTANCE EXISTS — zero `impl PermutationParameters` anywhere, so no round constants, no MDS, no field binding: a generic skeleton, not a usable hash. ★ ALSO FOUND: TreePoseidon (merkle_tree/backends/field_element.rs:50-71) and BatchPoseidonTree

(field_element_vector.rs:206) ALREADY implement IsMerkleTreeBackend with Node=Data=FieldElement — a FIELD-ELEMENT tree beside the byte-oriented Digest-generic one. So the commitment layer is a trait with a field-native Poseidon impl behind it, which PARTLY OVERTURNS my slice-0 assumption that an inner-prover hash swap is necessarily invasive; whether the prover is generic over that trait or pins a concrete backend is UNVERIFIED and I did not establish it. ALSO: sha256 AIR SPECS exist (spec/src/sha256.toml + sha256round/msgsched/consts, 749 lines) with no generated Rust AIR found — the closest in-tree precedent for a bit-oriented hash AIR, so read it before costing blake. ABSENT in every spelling: blake, Rescue/RPO, Monolith, Griffin, Anemoi — no AIR, no software impl. ⚠ METHOD TRAP, recorded because it nearly cost me a false claim: `grep -r --include=*.rs` UNQUOTED makes the shell try to glob and fail with "no matches found", which is INDISTINGUISHABLE FROM GREP FINDING NOTHING — two of my "nothing exists" readings were shell errors, not evidence; re-ran quoted. Second trap: "monolith" matches 26 times in prover/src (statement/paged_mem/page/lib/recursion) and EVERY occurrence is the monolithic-PROOF concept, so a term-only search would have reported a Monolith-hash implementation that does not exist. CONSEQUENCE: the oracle risk is DOWNGRADED (differential against a reviewed in-tree HADES rather than against itself) but the remaining input is a PARAMETER SET, which is a cryptographic act and must come from a published reviewed source — and the skeleton is field-generic, so WHICH FIELD is itself an open input. Additive route that avoids always-stop: impl PermutationParameters for a LOCAL type inside prover/src/lfm/ (foreign trait on local type needs no crypto/** edit; adding it under crypto/** would be always-stop). REMAINING GENUINE GAPS for wave 9: the corpus's Part I.7 candidate cost data, and the inner-prover blast radius (transcript + grinding). +2026-08-04T20:55Z | [hash-w8] slice 0c — corpus data landed, MATRIX REVISED, I was WRONG about blake | lfm 209/209 + 5 ignored, make lint exit 0 | ★★ EVERY CANDIDATE FITS THE 124 GiB BOX, BLAKE INCLUDED — the hash decision is NOT cost-gated. Corpus extraction (my own subagent, reached me via team lead) gives a MEASURED anchor that normalizes onto our socket for FREE: Miden's BlakeG "keeps Poseidon2's exact sponge geometry (state 12, rate 8, digest 4), so invocation counts are hash-invariant" — state 12 / rate 8 / digest 4 IS our frozen LFM_HASH contract, so per-2-to-1 figures transfer directly and every field-native candidate shares the ONE P I measured in slice A. Measured anchors (Miden, GOLDILOCKS = our field), per 2-to-1: Poseidon2 256 main + 16 aux (=304 base-equiv); BlakeG 32-row 4,096 + 768 (=6,400) = 13.9x main / 48x aux; + And8Lookup fixed 2^16x10 = 655,360 cells EVERY proof (our RANGE can absorb the role). THE MATRIX at P=192,000, two-term memory (27 B/cell + 190 MB/sub-proof): keccak 11.166B/284 GiB (band 290-350) | RPO 152/perm -> 1.814B = 6.16x, 48 GiB | Poseidon-orig (corpus 1x P2) 304 -> 1.844B = 6.06x, 49 GiB | Poseidon-orig (MY est) 617 -> 1.905B = 5.86x, 50 GiB | Monolith ~850 -> 1.951B = 5.7x, 52 GiB | BlakeG 6,400 -> 3.037B = 3.68x, 79 GiB. ⚠⚠ I WAS WRONG ABOUT BLAKE and correct it in §2.5: I said "the hash decision may not buy the 2.8x the wrap needs". Premise right (blake IS bit-oriented, DOES pay 48x aux), conclusion wrong — keccak-like in MECHANISM is not keccak-like in MAGNITUDE: KECCAK_RND is 1,480 cols x 24 rows vs BlakeG 128 x 32, same mechanism 12x apart. Blake lands 3.68x BETTER than keccak, comfortably inside the box. Lesson: I reasoned mechanism -> cost ratio without multiplying the widths, with the census formula sitting right there; and I built a narrative around the brief's leading hypothesis instead of falsifying it (the review renders NO pick; evidence strength Blake > RPO > Poseidon-orig > Monolith). ★ MY ESTIMATE WAS 2x CONSERVATIVE (608 main vs Miden's measured 256) — kept as the pessimistic bound, not discarded; a row-per-round layout reuses state columns where my unrolled one allocates fresh. ★ 2-to-1 NORMALIZATION, stated not assumed (team lead's ask): our 118,080 splits into 47,742 compression-shaped (Merkle parents + FRI steps, true 2-to-1) + 67,671 WIDE trace-leaf absorbs (no analogue in per-2-to-1 figures) + 2,667 spine. So apples-to-apples we are 6.2x Airbender's 7,685, NOT 15.4x, and nowhere near the 117x the corpus flagged for the old guest verifier at ~900,000 — THE LFM ALREADY RETIRED THE CORPUS'S HEADLINE ANOMALY, which is the main reason blake lands at 3.68x rather than §I.7's ~220%. ★ DONOR INVENTORY the corpus never analyzed (it lists Plonky3 as unscoped, :772) — verified by listing others/Plonky3 @ 4aed8fe4: poseidon1-air, poseidon2-air, blake3-air, monolith-air, keccak-air ALL PRESENT; for RPO only the bare permutation (rescue/src/rpo/goldilocks.rs), NO rescue-air crate. THAT INVERTS THE NAIVE RANKING: RPO is the cheapest predicted column AND the worst donor situation, while buying ~1 GiB of a ~49 GiB wrap; Poseidon-original is within 7% of it and has a DIRECT donor. Since the residue (1.784B, already measured) dominates every algebraic row, choosing among algebraic candidates on predicted wrap size is CHOOSING ON NOISE. REVISED ORDER: Poseidon-original first (direct poseidon1-air donor + in-tree HADES + LOWEST build risk + corpus has ZERO AIR data so it ADDS information) -> blake (calibrates our model against an independent measurement; expensive build, 13->21 files at Miden) -> Monolith (Goldilocks donor, un-analyzed) -> RPO LAST despite being cheapest. GOVERNANCE (team lead's steer, adopted): TWO STAGES so nothing blocks on authorization — stage 1 UNGATED entirely in prover/src/lfm/** = geometry (DONE) x cells-per-perm (measurable by hosting a candidate AIR behind the socket), giving measured-not-projected columns without touching crypto/**; stage 2 GATED on the USER's crypto/** call = a genuinely candidate-hashed inner proof e2e. Inner-prover seam scoped as a PROPOSAL not built (transcript hardcodes PlatformKeccak256, config.rs pins three Merkle aliases, ProofOptions has NO hash field, grinding hardcoded; Case A ~4-file CPU seam but NON-additive in crypto/**). CORRECTIONS LEDGER added as §5, 8 entries: brief's blake-is-probable (no pick), my blake claim, the FALSIFIED one-parameter 33.7 B/cell model (-> 27 B/cell + 190 MB/sub-proof, ceiling is a BAND 290-350 GiB; my earlier 60-68 GiB figures superseded by ~49), the RESUME's "one options change" (ProofOptions has no hash field at all), my own 6dbc5795 misattribution (dated 07-29, the ORIGINAL chunking leg; zero commits past 891f534f — collision real, my artifact inference wrong), the two-sided chunk knob, §I.7's CONTESTED hash-x-batching (guest version falsified ~5,500x, native unmeasured) + its "recursion diverges" reasoned at 900,000 not our 47,742, and arity-4/4-fold-FRI as measured dead ends. +2026-08-04T21:40Z | [hash-w8] slice 1a — Poseidon-original PERMUTATION, oracle-pinned | lfm 212/212 + 5 ignored (209 + 3 new), make lint exit 0 | DONE — prover/src/lfm/poseidon.rs, PoseidonGoldilocks impl LfmHasher. PARAMETERS (condition b): vendored others/Plonky3/goldilocks/src/poseidon1.rs @ 4aed8fe4 — Grain-LFSR per Poseidon paper Appendix E, field_type=1 alpha=7 exp_flag=0 n=64 t=12 R_F=8 R_P=22, generate_constants.py --field goldilocks --width 12; MDS CIRCULANT first row [1,1,2,1,8,9,10,7,5,9,4,10] (goldilocks/src/mds.rs:92). ★ THIS INDEPENDENTLY CONFIRMS MY SLICE-0 ESTIMATE of 8 full + 22 partial, which had been my own uncited domain knowledge (ZisK's shipped PLONKish Poseidon corroborates from a second direction: width-16, 8F/22P). alpha=7 is FORCED: p-1 = 2^32·3·5·17·257·65537 so neither 3 nor 5 is coprime. ⚠⚠ THE BRIEF'S FIRST ORACLE IS UNUSABLE, and this is a real finding: condition (d) asked to differential against the in-tree HADES skeleton with the same parameters, but crypto/crypto/src/hash/poseidon/mod.rs HARDCODES x^3, and x^3 IS NOT A PERMUTATION OVER GOLDILOCKS (3 | p-1) — differentialling against it would have validated my implementation against a non-permutation. Used instead PLONKY3'S OWN KNOWN-ANSWER VECTOR (width 12, input 0..11, test_poseidon_goldilocks_width_12), which nothing in this repo produced; MATCHED ON THE FIRST RUN, with a Python cross-check of the convention beforehand. FALSIFIED THE KAT THREE WAYS, all restored — x^7->x^6 FAILED, circulant MDS transposed ((j-i)->(i-j)) FAILED, partial-round S-box lane 0 -> lane 11 FAILED: the one vector pins exponent + MDS orientation + partial lane simultaneously. Second guard asserts gcd(alpha, p-1)=1 and that 3 and 5 FAIL it — the skeleton's bug encoded as a test. ★ PINNED PREDICTION for the chip (falsify next): degree-3 one-row layout = 16 (IN+capacity) + 8x36 (full: 12 x^2 + 12 x^3 + 12 out) + 22x14 (partial: x^2 + x^3 + 12 out) = m 612 value cols, a 3, base-equiv 621/perm; at P=192,000 that is 121.5M hash cells chunked (1.019) or 162.8M unchunked (pads to 2^18, 1.365) -> epoch verify 1.906-1.947B = 5.73-5.86x smaller than keccak, RSS ~50-51 GiB. ⚠ 612 IS AN UPPER BOUND, knowingly 2x off Miden's measured 304 (16 cols x 16 rows) — a smarter layout roughly halves the hash term and moves the TOTAL only ~3% because the residue dominates, so measure at 612 and do not optimise. STANDING DECISIONS: added rules 8 (a search that ERRORS looks exactly like one that found nothing — quote the --include glob, use a positive control; plus the term-collision converse) and 9 (a donor's parameters are not a donor's correctness — pin the whole primitive against an EXTERNAL KAT and falsify each convention separately; an in-tree impl is not automatically an oracle), both attributed [hash-w8] per team-lead ruling 4. ⚠ INSTRUMENT ERROR I CAUGHT ON MYSELF: `make lint 2>&1 | tail -3; echo $?` reports the exit of TAIL, not make — my first lint run printed "LINT_EXIT=0" while make had actually FAILED (Error 1, a cargo-fmt diff on the generated constant rows). Fixed with cargo fmt; re-ran capturing make's own status (MAKE_LINT_EXIT=0). Same family as the fri-emitter's "parses cargo test -q for FAILED lines" trap. OWED (slice 1b, not built): chip constraint block replacing HashConstraints' TestPermutation round, cols::NUM_COLUMNS 28 -> 612, census array, LFM_REGISTRY regen if digests move, prove+verify measurement (rule 2 — execute-only proves nothing about a chip), and the padding trap as a chunking sibling OR an explicit padding-corrected line. +2026-08-04T22:05Z | [hash-w8] slice 1b — SPEC ONLY, deliberately not built | lfm 212/212 + 5 ignored, make lint exit 0, tree clean | CHECKPOINTED per the standing-decisions coordination rule ("checkpoint and write a handoff file rather than delivering a half-built slice; quality over completion") — context ran thin and a 612-column unfalsified constraint set would be worse than a spec. others/lfm-hash-matrix-scope.md §6.4 now specifies slice 1b EXECUTABLY: column layout (16 + 8x36 + 22x14 = 612 value cols, offsets given), all 601 constraints (4 capacity + 1 mode-boolean + per-lane x2/x3 + per-round MDS output), the degree argument (exactly 3 via x2=a.a, x3=x2.a, x^7=(x3)^2.a — so max_degree stays 3 and the wrap's blowup 2 is unaffected), the padding obligation (the EXISTING rc-scaled-by-mode-sum trick already solves it — m=0 => a=0 => all zero; keep it, it is load-bearing), the trace-generator contract (same association as poseidon.rs::sbox, which is why it was written that way), and a five-part test plan ending in prove+verify because rule 2 says execute-only proves nothing about a chip. ★ REGISTRATION IS MUCH SMALLER THAN A NEW CHIP: LFM_HASH is ALREADY slot-registered, so the 8-site add-a-chip checklist does not apply — only cols::NUM_COLUMNS, the constraint body and the trace filler change, and the census picks the width up automatically since it reads hash::cols::NUM_COLUMNS; PREP_WIDTH stays 11 and the preprocessed group is untouched so the registry root SHOULD NOT move (verify, do not assume; regen is pre-authorized). ⚠⚠ THE HAZARD THAT MAKES THIS NOT A SMALL CHANGE, and the reason I stopped rather than pressed on: proof.rs:52-54 requires execution to use the SAME hasher whose constants the chips bake in, so swapping HashConstraints to Poseidon breaks ~30 existing call sites that execute with TestPermutation (epoch_tests, constraint_tests, epoch_verify_tests, machine_tests, fixture::HostSponge). DO NOT DO THAT SWAP TO GET A CELLS NUMBER — the number needs only the AIR's declared width plus the five tests; making Poseidon the machine's DEFAULT hash is a separate and larger decision that should be taken deliberately, not as a side effect of a measurement. That distinction is the main thing this checkpoint buys. +2026-08-04T22:25Z | [hash-w8] WAVE 8 CLOSED — slice 1a DONE, slice 1b HANDED TO WAVE 9 | lfm 212/212 + 5 ignored, make lint exit 0 (make's OWN status), tree clean, 7 commits off 891f534f, never pushed, feat/lfm-assembly NOT merged | Team-lead ruling: do NOT start 1b (my own "meaningful start but not a green finish plus falsification" + this phase's history of mid-slice deaths). Standing down. DELIVERED THIS WAVE: (0) the scope report others/lfm-hash-matrix-scope.md — TWO sockets, not one, and a candidate goes behind the LFM_HASH chiplet rather than keccak's hosted-AIR socket; (A) the PERMUTATION axis MEASURED with no hash built, 115,413 keccak vs 187,902 candidate at rate 8 = 1.6281x, the keccak side reproducing entry 10's legs figure exactly; (0c) the MATRIX REVISED on corpus data — every candidate fits the 124 GiB box, blake included at 79 GiB, so the hash decision is NOT cost-gated, plus my own blake-is-keccak-class claim reversed on record and the 2-to-1 normalization showing we are 6.2x Airbender rather than the corpus's 117x; (1a) the Poseidon-original permutation, externally pinned to Plonky3's width-12 KAT, that KAT falsified three ways. WAVE 9 STARTS AT §6.4 of the scope doc, which is executable: 612 value columns (16 + 8x36 + 22x14) with offsets, all 601 constraints, the exactly-degree-3 argument (so blowup 2 is unaffected), the padding obligation already discharged by the existing rc-scaled-by-mode-sum trick (load-bearing, do not clean up), Poseidon's ZERO compress_iv so today's MODE_C.IV term vanishes, the trace-generator association requirement, and a 5-part test plan ending in prove+verify per rule 2. FALSIFICATION TARGET (§6.3): 621 base-equiv/perm, 121.5M hash cells chunked / 162.8M unchunked, total 1.906-1.947B = 5.73-5.86x smaller than keccak, RSS ~50-51 GiB — measure at 612, do NOT optimize (a better layout halves the hash term and moves the TOTAL ~3% because the residue dominates; Miden's achievable 304 means 612 is an upper bound by construction). VERIFIED FOR WAVE 9 rather than assumed: airs.rs:174-176 reads hash::cols::NUM_COLUMNS / layout::hash::PREP_WIDTH / hash::bus_interactions().len() in the census per_chip array, and airs.rs:389-395 in build_air — so a width change PROPAGATES AUTOMATICALLY and no census edit is owed. ⚠ THE ONE THING WAVE 9 MUST NOT DO: swap HashConstraints to Poseidon in order to get a cells number. proof.rs:52-54 requires execution to use the same hasher whose constants the chips bake in, so that swap breaks ~30 TestPermutation call sites (epoch_tests, constraint_tests, epoch_verify_tests, machine_tests, fixture::HostSponge). The cells number needs only the declared width plus the five tests; making Poseidon the machine's DEFAULT hash is a separate, larger decision and needs its own ruling. Blake is wave 9+ , gated on the Poseidon report. +2026-08-06T00:00Z | [hash-w9] slice 1b START — clean-tree ground truth + WIP triage | lfm 212 passed / 0 failed / 5 ignored on CLEAN 8c63e1de (verbatim trailing summary) | KEEP the aborted spawn's stash, as a SKETCH not a baseline (it was never compiled). Reason: I recomputed its two load-bearing totals by hand before trusting it and both match the spec independently — 28 shared + 7x36 + 24 + 22x14 = 612 value columns (the same 612 as §6.4's 16 + 8x36 + 22x14, rearranged so the final round's post-MDS output IS the frozen OUT0..11 the bus reads, which keeps bus_interactions() hasher-independent), and 5 + 8x36 + 22x14 = 601 constraints. Its shape is also exactly the team-lead ruling: a HasherKind threaded at construction time to {constraint body, num_columns, trace filler, executor hasher} with Test as the default and every existing call site reaching it through a default-shaped wrapper. Dropping it would have meant rewriting the same design from the same spec. Everything in it is now under test by me; nothing is trusted because it was there. +2026-08-06T01:30Z | [hash-w9] slice 1b DONE — the Poseidon chip is BUILT, PROVED, and the prediction CONFIRMED | lfm 230 passed / 0 failed / 5 ignored, make lint exit 0 (make's OWN status), tree clean, 3 commits off 8c63e1de, never pushed | ★ THE PINNED PREDICTION HELD, NUMBER FOR NUMBER: 612 value columns, 601 constraints, max_degree 3 declared AND 3 measured, 621 base-equivalent cells/permutation (612 + 3x3) — read off the SAME census instrument that produced entry 10's keccak column, so the two columns of the matrix are comparable by construction. Hash cells 121,497,408 chunked / 162,791,424 unchunked (2^18 pad = 1.365x); epoch totals 1,905,694,804 = 5.86x and 1,946,988,820 = 5.74x under keccak's 11.166 B. Across wave 8's whole P interval [190,569, 193,569] the chunked total moves only 1,904.8M -> 1,906.7M, so the result does not depend on P being exactly 192,000. ⚠ PROVENANCE: wave 9 measured exactly ONE number (621); P and the 1,784,197,396 residue are inherited, and the epoch lines are arithmetic over all three. ★ RULE 2 DISCHARGED: the production prover built this AIR and the production verifier accepted it (the_poseidon_chip_proves_and_verifies on trivial_program, which exercises both hash modes plus padding rows). Until that ran, 612 was a declaration, not a measurement. ⚠⚠ ONE CORRECTION TO §6.3, and it is a UNITS error not a cells error: 'RSS ~50-51 GiB' does not reproduce from the stated two-term model (27 B/cell + 190 MB/sub-proof, 24 sub-proofs). Cell term alone = 47.92/48.96 GiB but 51.5/52.6 GB — so '50-51' is the cell term computed in GB, labelled GiB, with the sub-proof term dropped. Correct figure is ~52-53 GiB. Nothing downstream moves (every value is far inside the 124 GiB box, which is the only claim the number carries), but §2.3's other rows were computed the same way and should be re-derived before anyone compares them at that precision. ★ FALSIFIED FOUR WAYS (rule 1), each mutating the CHIP ONLY so it stops agreeing with the KAT-pinned permutation — mutating chip+executor together would prove nothing: F1 x^7->x^5, F2 MDS transposed (i-o)->(o-i), F3 partial-round S-box binds lane 1 not lane 0 — each 5 failed INCLUDING prove+verify; F4 round constant no longer scaled by the mode sum — EXACTLY 2 failed, the padding row and prove+verify, with satisfaction/KAT-output/every rejection test still GREEN. F4's discrimination is the evidence for §6.4's 'the padding trick is load-bearing, not decoration': it isolates to padding exactly as claimed, because on a real row m=1 and the permutation is unchanged. Instrument checked against a known-green control (21 passed) before believing any 'nothing failed', and failures read from the trailing summary block per rule 7's corollary. ★ THE SEAM, per the team-lead ruling: HasherKind is a CONSTRUCTION-TIME choice threaded to {constraint body, num_columns, trace filler, executor hasher}; Test stays the default, every pre-existing call site keeps its signature AND its behaviour (212 -> 230 tests, zero changed). NOTHING WAS FLIPPED — the machine's real hash is the ecosystem decision this measurement feeds. Two things ASSERTED not assumed: (1) no program digest moves with the hasher (PREP_WIDTH 11 in both layouts, preprocessed group untouched -> every root and program_id bit-identical, census rows and aux widths identical, only LFM_HASH's value width moves) so LFM_REGISTRY needed NO regeneration; (2) a proof does not verify under the other hasher, in BOTH directions. The seam turned out to be exactly the size the ruling predicted — LfmAirs::new (3 call sites) and build_traces (10) all reached through default-shaped wrappers — so the 'stop and report if materially more invasive' escape was not needed. ⚠ 612 REMAINS AN UPPER BOUND, ~2x off Miden's measured 304, and the instruction not to optimise it is now QUANTIFIED: halving the hash term moves the epoch total 1.906B -> 1.846B = 3.2%, because the residue is 93.6% of the chunked total. The hash term is no longer the thing worth engineering. ⚠ PREDECESSOR'S WIP: KEPT and now fully under test. It was ~17 min of never-compiled edits; I recomputed its two load-bearing totals by hand before trusting it (612 columns via a DIFFERENT arrangement than §6.4's — it shares OUT with the final round rather than allocating a 30th output block — and 601 constraints), then compiled it (clean first try), linted it (4 real clippy defects it had never been checked against: 3 needless_range_loop + 1 clone_on_copy), and wrote the 15 tests §6.4's five-part plan called for. Both column arrangements are now asserted against each other. OWED / NEXT: blake is the decision-critical column (§0c: it calibrates our model against an independent Miden measurement and lands 3.68x, so it tests whether the 'every candidate fits the box' conclusion survives a bit-oriented hash); the inner-prover crypto/** seam remains stage 2 and USER-gated. +2026-08-06T02:00Z | [hash-w9] WAVE 9 CLOSED — slice 1b accepted in full, standing down | lfm 230 passed / 0 failed / 5 ignored, make lint exit 0 (make's OWN status), tree clean, 3 commits off 8c63e1de (669f7fba seam / 68248451 tests+measurement / 17e70a0f docs), never pushed, nothing merged in | DELIVERED: the hash matrix's FIRST MEASURED ALGEBRAIC COLUMN. Poseidon-original behind LFM_HASH at 612 value columns / 601 constraints / degree 3 / 621 base-equivalent cells per permutation, proved by the production prover and accepted by the production verifier (rule 2). §6.3 confirmed number for number; the only correction is a UNITS error in the RSS line (~52-53 GiB with both terms in GiB, not ~50-51), which the team lead has accepted and will apply to §2.3's other candidate rows at consolidation — nothing decision-bearing moves, since the only claim that number carries is 'far inside the 124 GiB box'. Four chip-only falsifications all fire; F4 (drop the round constant's mode-sum scaling) breaks the padding row and NOTHING ELSE, which is the demonstration that the padding trick is load-bearing. §6.4's do-not-optimise instruction is now RETIRED WITH A NUMBER rather than left as a judgement: halving the hash term to Miden's achievable 304 moves the epoch total 1.906B -> 1.846B = 3.2%, because the residue is 93.6% of the total. ⚠ FOR WAVE 10 (blake), the two things this wave learned that transfer: (1) the seam is DONE and additive — HasherKind is threaded to {constraint body, num_columns, trace filler, executor hasher}, Test is still the default, and a new candidate needs only a new enum variant plus its own poseidon_cols-shaped layout module; num_columns/bus_interactions/the census all pick it up automatically. (2) The five-part test plan is now a reusable template in prover/src/lfm/poseidon_chip_tests.rs — layout injectivity (catches block-arithmetic off-by-ones a correct TOTAL cannot see), degree both ways (<=3 AND something reaching 3), satisfaction, rejection, padding, prove+verify, plus digests-do-not-move and cross-hasher-rejection. A bit-oriented candidate will differ in ONE structural way that matters to the census: it pays AUX for its lookup interactions, where Poseidon's aux is 3 rows flat, so aux_cols must be re-read from bus_interactions() and not assumed to be 3. ⚠ NOT VERIFIED BY ME: the team lead reports a production BLAKE3 6-round chip landed in PR #903 during this wave. I neither saw nor checked it; wave 10 should confirm it exists and what it costs before planning around it. NO FURTHER ACTION TAKEN — worktree is clean and free. +[hash-w10] 2026-08-06 SLICE 1 — the blake column MEASURED on our stack. Vendored PR #903's BLAKE3 6-round compression (head 89aeeb8c) into `prover/src/lfm/blake3.rs` (primitive + the 10 canonical 6-round vectors + 4 negative controls that break one convention at a time), and its chip into `prover/src/lfm/blake3_chip.rs` with the VM-coupled I/O replaced by LfmMem word tokens per the LFM_KECCAK pattern. `blake3_probe.rs` proves+verifies it standalone against the UNCHANGED production BITWISE table plus an LfmMem mirror, with the preprocessed prefix really committed. Suite 244 passed / 0 failed / 5 ignored (was 230/0/5), `make lint` exit 0 (make's own status). +[hash-w10] MEASURED: 3,056 main + 1,259 interactions (630 aux) = 4,946 base-field-equivalent cells per compression, at 769 constraints, max degree 3 measured = 3 declared. #903's syscall variant is 3,219 + 699 aux = 5,316, so hosting saves 370 cells/compression (7.0%) — entirely I/O: the Ecall receiver, the x10 register read, 22 Memw dword ops, 32 OLD_OUT AreBytes, 5 addr checks and 88 pointer IsHalfwords go away, 11 LfmMem tokens arrive. Byte-range coverage of the DATA columns is untouched (m keeps its 32 explicit AreBytes; h/t/len/flags are XOR operands; OUT bytes are XOR results), which is what makes the LfmMem lane recomposition safe. +[hash-w10] ⚠ FINDING THAT MOVES THE MATRIX, not the column: the non-hash residue is **95.8% byteswap gadget**. On the production epoch ([2 x14, 3, 4 x4, 5 x3, 7, 20], inner blowup 8 / 73 queries) the emitter issues 1,122,145 `felt_be_halves` calls = 1,122,145 BitDec + 71,817,280 BALU rows, which is 99.8% of all BALU rows and pads LFM_BALU to 2^27. Padding-aware, that is 1,684,910,080 of the 1,757,982,868 residue. A field-native hash deletes it, so wave 9's "5.86x under keccak" understates the field-native candidates by ~10x. Numbers and the re-derived matrix in the next slice. +[hash-w10] 2026-08-06 SLICE 2 — column derived, residue split, delegation priced, RSS re-derived. Scope doc §8 (and ledger items 9-11). Suite 244/0/5, `make lint` exit 0. +[hash-w10] BLAKE column: 4,946 cells/compression MEASURED x P 192,000 (rate-8 closed form, re-run and asserted against the emitted rate-17 count) = 967,402,978 chunked; + residue 1,757,982,868 + BITWISE 26,214,400 = **2,751,600,246 = 4.06x under keccak, ~71 GiB**. §2.3 predicted 3.68x/79 GiB for BlakeG — the column landed where it was predicted. +[hash-w10] ⚠ THE RESIDUE IS THE BYTESWAP: 1,684,910,080 of 1,757,982,868 (95.84%). So the FIELD-NATIVE rows move ~10x: Poseidon-original 194,536,041 = **57.4x under keccak, ~6.7 GiB** (was "5.86x, 52 GiB"); RPO 108.6x; Monolith 46.7x. §2.3's "all candidates within noise on size" is FALSE across families — blake is ~11x the algebraic ones — while "every candidate fits the 124 GiB box" survives. +[hash-w10] DELEGATION (user request): priced with our own closed form = **net LOSS +66%** (657M cells). Delegation trace 993,617,378 PLUS 132,933 compressions to verify its proof, vs 967,402,978 hosting it in-machine. Structural: Airbender's delegation circuit moves work out of a FIXED-SIZE 2^20-cycle main circuit; the LFM has no fixed-size box, so its multi-AIR proof already IS that pattern. A 3,056-column AIR has a 6,112-felt leaf = 765 compressions/query — wide delegated chips have expensive proofs. +[hash-w10] Two corrections to earlier waves recorded in the ledger: the RSS sub-proof count is 19 at this shape (24 is the INNER epoch's leg count) and is candidate-dependent, dominating the field-native projections; and LFM_BALU pads 71,974,504 -> 2^27 (86% overshoot, 622M cells) with LFM_BITDEC another 161M — ~783M cells (7% of keccak, 28% of blake) recoverable by the chunking policy KECCAK_RND already has. Not attempted. +[hash-w10] Falsification: 3 chip-only mutations (wire-side rotr8 relabel transposed, message schedule transposed, LfmMem read multiplicity ungated) each fail EXACTLY the 2 prove+verify tests, control green before and after. A 4th (ROT_SHIFT_R 9->10) is reported as a NON-falsification: it trips a debug_assert in 0.01s, so it is evidence about the assert, not the constraints. +[hash-w10] 2026-08-06 SLICE 3 + CLOSE — residue reconciled against hash-delegation-eval.md §3.1/§4.1 (scope doc §8.11). Branch `feat/lfm-hash-matrix` @ d6418d90, tree clean, nothing pushed, nothing stashed. Suite 244 passed / 0 failed / 5 ignored, `make lint` exit 0 (make's own status). Wave accepted by the team lead; worktree goes quiet here. +[hash-w10] THE THREE-WAY DISPUTE RESOLVED, and nobody was doing wrong arithmetic: byteswap share of the residue is 95.84% PADDING-AWARE (1,684,910,080) and 51.38% UNPADDED (903,326,725). The eval's instruction-derived 88-93% was near the padded answer; the team lead's ~50% reproduces the unpadded one almost exactly; the whole gap is a 1.865x padding multiplier neither included, because the gadget is what DRIVES LFM_BALU to 2^27 and LFM_BITDEC to 2^21 — deleting it deletes two padded power-of-two tables, not just rows x width. Secondary miss in the "BALU rows are cheap at 10 base-equiv" intuition: LFM_BITDEC is 165 base-equiv/row, 20% of the unpadded cost from 1.6% of the rows. R_native = 73,072,788 (0.073 B), a 24.1x collapse, inside the eval's 0.024-0.20 B band. +[hash-w10] Eval totals: **0.18 B SURVIVES** (Poseidon measured 0.195 B). **1.10 B DOES NOT, as built** (blake measured 2.752 B) — its flagged open question "does an LFM-hosted BLAKE3 chip shed the byteswap? It should" has the measured answer NO: the chip consumes machine words of four u32 lanes (the LFM_KECCAK convention) and u32 halves are exactly what felt_be_halves produces, so the gadget is UPSTREAM of the chip's input format and hosting cannot delete it. But 1.10 B is RECOVERABLE: a felt-absorbing variant with a per-felt canonicity gate (Sum byte_k*256^k = v does NOT pin the byte string — v and v+p both satisfy it — so without a < p argument the prover chooses what gets absorbed and Fiat-Shamir breaks) lands at 1,097,202,674 = 1.097 B / 10.2x / ~30 GiB against the eval's 1.10 B central. **The 2.5x discrepancy IS that unbuilt variant, ~1.65 B.** +[hash-w10] ⚠ NOT BUILT, and the blocker is cryptographic rather than effort: the OUTPUT side. A blake output word is 32 bits, so a felt built from 8 output bytes is a 64-bit value reduced mod p and the map is not injective — how a blake digest becomes felts (truncate to four u32s / reduce / domain-separate) changes the security argument, the digest width and the token count. Same boundary §6.1/§7.7 draw around Poseidon's parameters and compress_iv. This is the highest-value remaining blake experiment and it needs the crypto call FIRST. +[hash-w10] ⚠ RE-READ wave 9's "the hash term is no longer worth engineering": still true for the algebraic family (halving Poseidon's 621 moves a 0.195 B total by ~30%, on a number already 57x under keccak), now FALSE for blake — hash 35% / serialization 61% of its 2.752 B, and the felt-absorbing variant is a 2.5x lever on the whole column. That verdict was reasoned on the inflated residue. +[hash-w10] NAMED FOLLOW-UPS, none started: chunk LFM_BALU (~783M padding, 7% of keccak / 28% of blake, machinery already exists for KECCAK_RND); LFM_BLAKE3 registration (moves all 14 roots -> separate decision); re-measure post-#889 (our keccak column is pre-#889, ~7% stale-high on the keccak share); the RSS coefficients are keccak-calibrated and are the weakest numbers in §8.6, dominating the field-native rows at 27-41% of their projection. +[team-lead] 2026-08-06 post-close correction pass: delegation-eval self-corrected on w10's measurements (its A5 FALSIFIED — hosted chip does not shed the byteswap; prover-cells-alone delegation beats the as-built chip 1.2-1.8x but loses +66% with second-proof verification folded in — decision metric; its verdict stands on w10's no-fixed-box + leaf-term arguments). Eval authorized a bounded correction pass on hash-delegation-eval.md. Scope-doc ledger item 9 base-mixing fixed here: 95.84% is vs the census base 1.758B, 94.44% vs the ledger base 1.784B; the 26.2M inter-base reconciliation gap (fma-vm-analysis.md:191, never closed) is now 36% of R_native = the dominant uncertainty in the field-native residue. Campaign remains CLOSED; this is bookkeeping. +[team-lead] 2026-08-06 CORRECTION to my own a799b938: the 26,214,528 inter-base difference is NOT an unclosed reconciliation gap and NOT an uncertainty in R_native — [hash-w10] identified it exactly as BITWISE (26,214,400) + KECCAK_RC (128), keccak's own lookup tables, structurally absent from any field-native chip set (one BITWISE-bus hit in chips.rs = keccak's absorb XOR). R_native = 73,072,788 STANDS on the census base; ledger-base remainder = R_native + keccak tables, a different quantity. fma-vm-analysis.md:191's gap is CLOSED. Chip-set-change caveat recorded (frozen-14 would carry BITWISE dead). delegation-eval's correction pass amended accordingly before the wrong figure landed. My error to own: I wrote the eval's 'dominant uncertainty' framing into a799b938 without checking the gap's composition against the census — the identification took one multiplication. diff --git a/others/lfm-assembly-obligations.md b/others/lfm-assembly-obligations.md new file mode 100644 index 000000000..894c675d2 --- /dev/null +++ b/others/lfm-assembly-obligations.md @@ -0,0 +1,598 @@ +# Assembly obligations — debts the epoch-verifier assembly must discharge + +Started 2026-07-31. Each entry is a deferral whose safety argument is still +owed (standing-decisions method rule 5). Assembly (RESUME item 5) may not be +called done while any entry is OPEN. Add entries as legs flag them; close an +entry only with the verifying evidence named in it. + +## STATUS AT WAVE 7 (2026-08-04) + +**The ledger is CLOSED.** Entries 1 through 9 were discharged in waves 5 and 6; +entry 10, the wrap run's own reporting rule, is SATISFIED by the wrap run and +carries its table of numbers (each naming its epoch profile). One NEW item the +wrap run surfaced, which is a resource fact rather than a debt: + +- **The production-shaped wrap cannot be PROVED on 124 GiB.** The verifier of an + inner epoch at blowup 8 / 73 queries is 11.17 billion base-field-equivalent + trace cells, i.e. a projected 350.6 GiB of peak prover RSS from a coefficient + measured twice. Whatever makes it provable — a cheaper hash (84% of the cells), + disk spill, or splitting the wrap — is a decision, not an obligation, so it is + recorded in entry 10 rather than opened as entry 11. + +Two things wave 6 could not close and that belong to the USER rather than to a +leg: + +- **A framework ceiling.** The production prover cannot prove any AIR with + `step_size > 1` (entry 9's note). Lifting it looks like a one-line relaxation + in `crypto/**` — an always-stop item. Until then no end-to-end run of the + assembled verifier at `step_size > 1` is possible. +- **PAGE's preprocessed roots MIGRATED rather than closed** (entry 7's note). + They are the GLOBAL proof's GlobalMemory AIRs' commitments, not an epoch's; no + continuation epoch of any guest carries a PAGE sub-proof. The taxonomy is + worked out and the classifier already handles the zero-init (constant) half; + the ELF-data half would hit its panic, which is the intended handover. + +Also unchanged and now more load-bearing than ever: `check_attestation` has ZERO +production call sites, and DECODE's binding rests on it. + +## OPEN + +1. ~~**`reg_fini` felt-width gap**~~ — **DISCHARGED** (assembly-w6, slice 3) by + the entry's own stated default. The entry said "if assembly arrives and the + `no >u32 register column` argument is still unverified, emit the check"; it is + still unverified, and slice 1's derivation is what made the boundary vectors + live arena data in the assembled verifier, so the check is now emitted: + `epoch::assert_u32` on all 134 cells (one `BitDec` plus one recomposition + each). + - Placed at the ASSEMBLY call site, deliberately, NOT inside + `programs::emit_register_commitment`. The isolated derivation's width gap is + pinned by a guard test that asserts the hazard still exists + (`the_derivation_extends_a_non_u32_register_value_demonstrating_hazard`) and + that test is right: an isolated derivation binds nothing. Assembly is where + the width becomes enforceable, so assembly is where it is enforced, and the + guard stays green. + - ⚠ **THE OBVIOUS TEST FOR THIS IS VACUOUS, and it was written before it was + caught.** "Set a boundary word to `2^32` and watch the epoch fail" fails with + the check REMOVED too, because a wide value moves the derived root and + therefore every challenge after Phase A. What is non-vacuous is the pair + `the_register_boundary_is_width_checked` runs: (a) `assert_u32` in isolation + admits the whole `u32` range and rejects everything above it, and (b) a + STRUCTURAL check that every register-arena `Hint` output is the input of a + 32-bit `BitDec`. FALSIFIED: applying the check to a 3-cell prefix — precisely + the defect a value tamper cannot see — fails (b) and nothing else + (16 passed / 1 failed). + - ★ A subtlety the test found, which sizes the gap exactly: an arena word is a + FIELD ELEMENT, so `FE::from(u64::MAX − 1)` is the felt `2^32 − 3`, a perfectly + good `u32`. The widening this entry names is therefore the interval + `[2^32, p)` and nothing beyond; a first draft of the test used + `u64::MAX − 1` as an out-of-range value and reported the check broken when it + was not. + (Original text kept below.) (flagged by reg-tree, slice 1). + Production's `reg_fini` is `Vec` — the TYPE is the entire + enforcement. An LFM arena is untyped felts, so the machine's accepted set + is wider than production's. Guard test + `the_derivation_extends_a_non_u32_register_value_demonstrating_hazard` + asserts the gap still exists. Assembly owes ONE of: + - a 67-per-column range check on the register boundary columns, OR + - the verified argument that no epoch proof can exist over a >u32 + register column (plausible via REG-C2's Memory-bus value word — + currently UNVERIFIED; verifying it means a coherent-forgery analysis + per method rule 4, not an assertion). + Default is the range check: if assembly arrives and the argument is + still unverified, emit the check. + +2. ~~**`start_index` is unbound to the chain**~~ — **DISCHARGED** + (assembly-w6, slice 1, 2c810857). Phase A now CALLS + `programs::emit_register_commitment` on the register-boundary arena the spine + declares plus a new `reg_fini` arena, so the REGISTER preprocessed root the + transcript absorbs is COMPUTED from those cells. That computation is the + binding, and it is the binding production itself uses. + - `start_index` is no longer even a second READ of slot 64: it IS + `reg_init[X254_INDEX]`, the cell the derivation consumed. Before this slice + the epoch declared 67 INIT words and read exactly one; now every word of + both vectors is read, which is why + `the_spine_hints_each_proof_value_once`' positive control tightened from + `declared − 67 + 1` to `declared` exactly. + - The differential is free and total: a wrong derivation moves the absorbed + root, which moves all 111 challenges, so + `the_epoch_challenge_spine_matches_production` covers it. + - FALSIFIED: `the_derivation_binds_every_register_boundary_word` moves ten + words — first and last of INIT and of FINI, and slots away from 64 — and + every one makes the epoch unverifiable. + (Original text kept below.) **HALF DISCHARGED** (assembly, slice 3): the reading is + settled and the CELL is now the right one; the derivation that closes it is + not built. + - Settled by research (`lfm-team-lead-start-index-research.md`): production + has no arithmetic `start + len` check anywhere. It rebuilds epoch N's + REGISTER preprocessed commitment from epoch N−1's FINI vector and rejects + unless the proof's root matches, and `verify_epoch` then simply reads + `register_init[X254_INDEX]` (`continuation.rs:840-851`). Confirmed + first-hand in the assembly fixture: `compute_expected_commit_bus_balance_view` + takes `register_init[register::X254_INDEX] as u64`. + - Done: the assembled spine declares the register-boundary vector as ONE + arena at production's width and takes `start` from slot 64 of it, so the + COMMIT-bus target and the future REGISTER derivation read the same cell + rather than two words. `the_closure_rejects_a_moved_index_or_output` + moves it by 1, 2 and 7 and the bus fails to close each time. + - Left: the derivation itself. `start_index` is bound to the chain only + once Phase A's REGISTER preprocessed root is COMPUTED from that arena + (reg-tree's emitter) instead of hinted — which is entry 7's work, and + the two now close together. + +3. ~~**Assembly must unify the five remaining two-consumer values**~~ — + **DISCHARGED** (assembly-w5, slice 1, a1f32859). The legs now hang off the + spine, so all four staged values have both consumers inside one program and + there is finally something that could disagree — and nothing does, by + construction rather than by agreement: + - the OOD frame values: `epoch_verify::emit_table_verification` rebuilds ONE + grid with `epoch::emit_reconstruct_ood` and hands the constraint fold and + the DEEP fold two VIEWS of it (see the new degenerate-parameter note below + about why they are different views); + - the claimed parts: the same `absorbs.parts` slice reaches `emit_quotient`'s + Horner and `emit_deep_invariants`' `h_sum_zpow`; + - `ζ`: already discharged — it is the `z` the transcript samples, and the + zerofier, the row points and `z^P` all take that cell; + - the trace roots: `GroupCommitment::from_lanes(root.lanes, …)` takes the + lanes Phase A absorbed, so the Merkle compare and the absorb read one + unpack; + - the public output bytes: discharged in wave 4 by `emit_output_bytes`. + The guard is `epoch_verify_tests::the_assembled_verifier_hints_each_proof_ + value_once` — the same ABSOLUTE count as the spine's, but over the program + that HAS both consumers, plus a positive control that the assembled program + declares strictly more arena words than the spine (without it the guard would + pass just as happily over the spine alone, which is what made the wave-4 + version unable to close this entry). 21 tamper vectors over the assembled + program are all rejected. (Original text kept below.) + Original: **PARTIALLY DISCHARGED** (assembly, slice 1+2): the + assembled spine gives each value ONE cell and hands both views out of one + struct, so the unification is now a construction rather than a rule — + `epoch::RootCells` holds a root's two words AND the eight halves the + transcript absorbs, from a single hint and a single `Unpack`, and + `epoch::TableAbsorbs` is the surface every later leg reads its cells from. + `ζ` is fully discharged: it is no longer a value at all, but the `z` the + transcript samples. The other four are STAGED, not closed — their second + consumers (constraint evaluation, the DEEP fold, the Merkle root compare, + the `program_id` fold) are not yet wired onto the spine, so there is + nothing yet to disagree. They close when those legs hang off + `TableAbsorbs`, and the entry stays OPEN until they do. + Original text: each is hinted twice today — not exploitable while the + legs are separate programs, every one a landmine the moment they share an + arena. Unification means deciding the assembled program's arena layout, + which is assembly's call — that is WHY they were not fixed leg-side: + - the OOD frame values (constraint eval vs DEEP invariants, `ood_steps`); + - the claimed composition parts at `z` (constraint quotient vs DEEP + `h_sum_zpow`); + - `ζ` (constraint zerofier vs DEEP `row_points`/`z_pow`); + - the main-trace roots (Phase A absorb vs authentication root compare); + - the public output bytes (attestation `program_id` fold vs COMMIT-bus + target). + +4. ~~**The FRI leg's three per-sub-proof values are arena words and must be + bound at assembly**~~ — **DISCHARGED** (assembly, slice 1+2). + `epoch::emit_table_challenges` samples each `ζ_k` from the transcript and + absorbs layer root `k` immediately after it, draws `ζ_C` only when + `total_folds > 0`, and absorbs every terminal coefficient after the loop — + production's order at `verifier.rs:1461-1489`. Falsified four ways, each + caught by the differential: absorbing the root BEFORE its `ζ`, never + absorbing the roots, skipping the final-fold draw, and never absorbing the + coefficients. Witnessed at `num_committed = 0/1/2/3` on single-table + fixtures and at **12 committed layers** on the real epoch's CPU sub-proof. + The layer roots remain arena cells, which is correct — they are proof data + — and they are now the SAME cells the FRI walk compares against. + (Original text kept below for the record.) + `declare_fri` hints the + folding challenges `ζ₀..ζ_C`, the terminal-polynomial coefficients and + the committed layer roots, exactly as `emit_sub_proof` hints `γ`/`ζ`. + Two different obligations sit here and they are not interchangeable: + - **`ζ_k` are CHALLENGES.** They must come from `TranscriptReplay`, and + production's own order is load-bearing: sample `ζ_k`, THEN absorb root + `k`, per layer, and only then sample the final-fold `ζ_C` — and that + last one only when `total_folds > 0` (`verifier.rs:1461-1483`, + mirroring `fri/mod.rs:86-118`). A prover who chose `ζ` chooses the + fold, so this is entry 4's family and not a convenience. + - **The coefficients and the layer roots are proof DATA** that the + transcript must absorb, because later challenges (the query indices + among them) depend on them: the roots are appended inside the loop + above and every coefficient is appended after it. Absorbing them in + the wrong order, or not at all, does not fail any test in + `fri_tests` — that suite supplies the real values — so assembly owns + this and nothing leg-side can catch it. + +5. ~~**The standalone FRI driver's hinted index is wider than production's**~~ + — **DISCHARGED** (assembly, slice 1+2). The assembled verifier's query + index reaches the legs as `TranscriptReplay::sample_u64_pow2`'s BITS and + never as a felt: `TableChallenges::iota_bits` is the only index the epoch + spine produces, `log2(lde) − 1` of them, which is production's + `sample_u64(lde_length >> 1)` (`verifier.rs:138-141`) and exactly the + Merkle depth the walk consumes. Checked against production's own `iotas` + for every query of every sub-proof of a real epoch. (Original text below.) + (fri-emitter, noted not deferred). `fri_tests::fri_only_program` hints + `iota` as a felt and takes its low `log2(lde) − 1` bits, so `iota` and + `iota + 2^(n−1)` are the same query to the machine, where production's + `terminal_codeword.get(iota >> C)` would reject the second as + out-of-range. This is a property of the ISOLATION driver, not of the + assembled machine: `SpongeVar::squeeze_bits` produces exactly `nbits` + bits, so an assembled verifier's index is in range by construction. + Assembly owes only that the index reaches the query legs as those bits + and never as a hinted felt. + +6. ~~**The challenges guard cited in comments does not exist yet.**~~ — + **DISCHARGED** (assembly, slice 1+2). The guard is no longer a test to + write but a construction: `epoch::emit_table_challenges` DERIVES β, z, γ, + every `ζ_k` and every query index from `TranscriptReplay`, and + `epoch_tests::the_epoch_challenge_spine_matches_production` checks all 111 + of them against production's own `replay_rounds_after_round_1` over a real + 24-sub-proof epoch. The per-slice differential programs still hint their + challenges; that is now a property of the ISOLATION drivers, not of the + assembled verifier, and the assembled path has no `Instr::Hint` for any + challenge because nothing hints one. + +7. ~~**The preprocessed commitments are hinted in the assembled spine.**~~ — + **DISCHARGED** (assembly-w6, slice 1, 2c810857), and the RULING'S OWN + TAXONOMY IS AMENDED for the second time. Each root now comes from the source + its provenance admits, and which source that is comes from a CLASSIFIER + (`epoch_tests::prep_source`) that recomputes production's candidate functions + and matches — never from a sub-proof index. A preprocessed table whose root + matches nothing known PANICS rather than being hinted unbound, which is the + failure mode the entry needed most. + - **options-only ⇒ interned as program text**, absorbed as literal bytes with + no splice arithmetic (`RootCells::constant`, + `statement_replay::PhaseAPreprocessed::Constant`). + - **REGISTER ⇒ derived in-machine**, which closes entry 2 with it. + - **DECODE ⇒ arena cell + the attestation join**: the same cell Phase A + absorbs is the cell `programs::emit_program_id` folds, differentialled + against production's `recursion::program_id_from_digest`. + - The join is denied STRUCTURALLY by a pair of absolute guards, which are + complete for the class together and neither of which suffices alone: a + second READ of a word fails `the_assembled_verifier_hints_each_proof_value_ + once`, and a second WORD fails + `the_assembled_verifier_declares_exactly_the_shape_words` (a closed form + over the epoch's shapes, not an emitter pass). A fold reading some OTHER + existing value publishes an id that is not production's, which the spine + differential catches. + - FALSIFIED as a COHERENT FORGERY, not a count: + `epoch_program_with(split_decode = true)` gives the fold its own arena copy, + and `a_split_decode_cell_forges_the_attestation` shows that program RUNS the + substitution and attests to a DIFFERENT program's id — while publishing the + honest id when the same surplus arena holds the honest root, so the forgery + is a free choice and not a broken proof. On the joined program the + substitution is inexpressible. + - ★ **THE RULING'S CONDITION (b) IS UNSATISFIABLE AND UNNECESSARY, and the + PAGE obligation MIGRATES rather than closing.** The ruling asks for a real + epoch from a guest with private input pages, on the premise that + `num_private_input_pages = 0` is a fixture property. Three readings + overturn it: + * private-input pages are built NON-preprocessed (`lib.rs:800-828`), so they + could never witness a PAGE preprocessed root at all; + * **no continuation epoch of any guest has a PAGE sub-proof.** + `prove_epoch` REJECTS one outright — "continuation epoch must have no PAGE + configs (L2G bookend replaces PAGE)" (`continuation.rs:695-702`) — and + both `build_epoch_airs` call sites pass `page_configs = &[]` + (`continuation.rs:711-714`, `815-818`). The fixture matches PRODUCTION + here; it is not stripped down; + * the ELF-data page genesis roots the attestation folds are the GLOBAL + proof's GlobalMemory AIRs' preprocessed commitments + (`continuation.rs:997-1010`) — a different proof, out of an epoch + verifier's scope. `recursion::program_id_from_digest`'s own doc says + exactly this ("the supplied DECODE / ELF-data-page roots"). + So the epoch taxonomy is **2 constants + 1 derived + 1 ELF-dependent**, + asserted by `the_preprocessed_commitments_of_a_real_epoch` (census 2/1/1, + plus a guard that no sub-proof carries PAGE's preprocessed width). PAGE's + half becomes the GLOBAL-proof verifier's obligation, whose taxonomy is + already worked out and already in the classifier: zero-init pages share + `page::zero_init_preprocessed_commitment(options)` and are therefore + CONSTANTS (a third finding — the ruling put all of PAGE in the ELF-dependent + family), while ELF-data pages are ELF-dependent and would hit the + classifier's panic, since their `PageConfig`-shaped provenance is not in its + candidate list. That panic is the correct behaviour and the handover note. + - The residual risk the ruling named is unchanged and now carries more weight: + `program_id`'s binding is only as strong as the consumer-side + `check_attestation` compare, which has ZERO production call sites. + - MEASURED cost of the whole wiring, min preset: +59,743 instructions + (2.7% of the assembled verifier), +256 permutations and +63 arena words. + The permutation figure is exactly the prediction — 255 for the REGISTER tree + at blowup 2 (`128·blowup − 1`, reg-tree's pinned closed form) plus 1 for the + `program_id` fold's single rate block. The arena figure is exactly + +67 (`reg_fini`) + 2 (`pc_start`) − 6 (three roots that stopped being arena + data). + (Original text kept below for the record.) + ⚠ **THE ENTRY'S OWN TAXONOMY WAS WRONG AND IS CORRECTED HERE** (assembly-w5, + slice 1, by reading `lib.rs`). The split is **2 constants + 2 ELF-dependent + + 1 derived**, not 3 + 1 + 1: + - **BITWISE and KECCAK_RC are genuinely compile-time constants** — + `bitwise::preprocessed_commitment(proof_options)` and + `tables::keccak_rc::preprocessed_commitment(proof_options)` take the proof + options and nothing else (`lib.rs:707-713`, `lib.rs:771-774`). Intern them. + - **DECODE is ELF-DEPENDENT, not a constant.** `VmAirs::new` builds it as + `create_decode_air(opts).with_preprocessed(decode::commitment_from_elf(elf, + opts), …)` (`lib.rs:743-750`). Interning it would make program identity + ELF-dependent — the same always-stop item the entry raised for PAGE alone. + DECODE is in PAGE's family. + - **REGISTER is derived** (reg-tree). `programs::emit_register_commitment` + now exists, extracted from the isolation program so the spine can call it + on the register-boundary cells it already declares (a1f32859). Wiring it + into Phase A is the remaining work and closes entry 2 with it. + - The corroborating evidence was in plain sight and nobody had connected it: + `recursion::program_id_from_digest` folds `elf_digest`, `pc_start`, + `decode_commitment` and every `(page_base, page_commitment)` — precisely + the ELF-dependent roots and none of the options-only ones. + **PROPOSED RESOLUTION (needs the team lead's ruling, because the alternative + touches program identity):** DECODE and PAGE stay ARENA CELLS and are bound + not by program text but by the attestation — the same cell Phase A absorbs is + the cell the `program_id` fold consumes, which is the two-consumer join one + level up and which the machine already has an emitter for + (`machine_tests::program_id_folds_pages_in_the_production_layout`). That keeps + one LFM program per epoch SHAPE rather than one per guest ELF. ⚠ The residual + risk is named honestly: `program_id`'s binding is only as strong as the + consumer-side `check_attestation` compare, which the RESUME already records as + having ZERO production call sites. So this proposal makes PAGE/DECODE exactly + as bound as the existing chain is, and no more. The alternative — deriving + both in-machine from the ELF bytes, REGISTER-style — costs a full in-machine + LDE+tree per page and needs the ELF itself bound, which is the full-ELF keccak + pass sim/8 deliberately removed. + **MEASURED (`epoch_verify_tests::the_preprocessed_commitments_of_a_real_ + epoch`):** only 4 of this epoch's 24 sub-proofs are preprocessed — index 0 + (11 precomputed columns) BITWISE, 1 (5) DECODE, 5 (9) KECCAK_RC, 8 (3) + REGISTER, per `VmAirs::air_refs`' fixed order (`lib.rs:610-625`). ★ There is + **no PAGE sub-proof in this epoch at all** (`num_private_input_pages = 0`), so + the fixture cannot witness PAGE's half — and per the RESUME's premise rule + this is a claim about the FIXTURE, not about production, so the witness is a + differently-configured real epoch (a guest with private input pages), not a + synthetic AIR. + (Original text below.) Production + takes each preprocessed root from the AIR and REJECTS a proof whose copy + disagrees (`verifier.rs:1184-1209`); the root it absorbs is the verifier's, + never the prover's. `epoch_tests::epoch_challenge_program` hints all of + them. Only REGISTER's has a derivation today (reg-tree, from the previous + epoch's `reg_fini`). BITWISE, DECODE and KECCAK_RC are compile-time + constants of the AIR set and could simply be interned — but **PAGE's cannot + be**: it is a function of the inner ELF, which is per-proof arena data, so + baking it would make program identity proof-dependent (an always-stop + item). PAGE therefore needs a derivation of the same family as REGISTER's, + and that derivation does not exist. Assembly owes: intern the three + constants, wire REGISTER's derivation into Phase A, and either build PAGE's + or state why the ELF-digest binding already covers it. + +8. ~~**The OOD absorb ORDER has no production witness**~~ — **DISCHARGED** + (assembly-w6, slice 2, 36bfd727), and it needed no synthetic AIR at all. + `stark::examples::fibonacci_multi_column::FibonacciMultiColumnAIR` + already carries `transition_offsets: vec![0, 1, 2]` and is generic over the + extension field, so at three columns its next-row block is **3 columns × 2 + ROWS** — `num_eval_points − step_size = 2`. That is the phase's first OOD block + where a column-major and a row-major absorb differ. + - The proof is production's (`multi_prove_ram`, accepted by + `multi_verify_views`) and so is the oracle: the machine's + `emit_table_challenges` replay is differentialled against + `replay_rounds_after_round_1` on every challenge. + - FALSIFIED, and the falsification re-proves this entry's own claim as a + by-product: swapping the absorb loop to ROW-major leaves the 24-sub-proof + epoch spine differential, the assembled-verifier run and the single-table + replay ALL GREEN (206 passed) and fails only the new fixture. The failure + MODE is worth recording — the mutation trips the in-program GRINDING check + first (a moved transcript state invalidates the nonce), so the clean + statement of the property is the test's own row-major CONTROL program, which + stops at `γ` and shows it moves against production's `γ`. + - Unexercised and named rather than chased: an OOD grid with more than TWO + blocks. Three offsets at `step_size = 1` still yields two, and nothing in the + machine is shaped by the block count (`emit_reconstruct_ood` takes two + because the proof carries two), so that is a framework property. + (Original text kept below.) Production absorbs each pruned OOD block + column-major (`verifier.rs:1425-1429`). Injecting a ROW-major absorb leaves + BOTH the single-table differential and the 24-sub-proof epoch spine green, + because every OOD block in either is ONE ROW TALL: the current block's + height is `step_size` (`ood.rs:110-114`) and the phase already knows + `step_size = 1` collapses production, while the next block's height is + `num_eval_points − step_size`, which is 1 for any AIR with two transition + offsets — all 24 of the epoch's are. Measured dims are printed by the spine + test. This is a fourth member of the degenerate-parameter family and the + premise check the RESUME asks for was done: it is a claim about + PRODUCTION, not about fixtures on hand. Closing it needs a synthetic AIR + with three transition offsets (or `step_size > 1`), proved by the + production prover so the oracle stays real. + +9. ~~**The constraint leg's FRAME-STEP view of the OOD grid has no production + witness**~~ — **DISCHARGED** (assembly-w6, slice 2, 36bfd727) by an oracle + rather than a proof, because a proof turned out to be impossible (see the + ceiling below). The defect is the machine's grid→frame-step MAPPING, and + production has a pure function for exactly that mapping: + `StarkTableView::into_frame(main_cols, step_size)` + (`proof/view.rs:269-294`), which the real verifier calls at + `verifier.rs:320-321`. It takes a grid and a `step_size` and needs no prover. + - The rule is extracted out of the emitter as + `epoch_verify::frame_step_view` precisely so it can be differentialled, and + `the_frame_step_view_matches_productions_own_frame_assembly` compares it + against `into_frame` at `(offsets, step_size)` of `(2,1) (3,1) (2,2) (3,2) + (2,4)`, over main AND aux columns, on grids of distinct values. + - FALSIFIED: making `frame_step_view` return the whole grid — wave-5's M2 + defect verbatim — fails this test and NOTHING ELSE (206 passed / 1 failed). + - ⚠ **A FRAMEWORK CEILING, and it is why this entry has no end-to-end + witness.** The production prover cannot prove ANY AIR with `step_size > 1`: + its CPU transition evaluator borrows one row per transition offset + (`RowFrame::from_lde`, called at `evaluator.rs:72`) and asserts the shape + outright — `debug_assert_eq!(lde_step_size, blowup_factor, "RowFrame requires + single-row steps (step_size 1)")` — and `lde_step_size = step_size · + blowup_factor`, so the equality IS `step_size == 1`. MEASURED, not read: + `step_size_tests::the_prover_cannot_prove_a_step_size_two_air` is a + `#[should_panic]` on that message, so the ceiling is recorded and + self-updating. Nothing here runs the ASSEMBLED verifier at `step_size > 1`, + and nothing can until it lifts. + - What lifting it would take, FROM READING and not verified by running: the + assert looks over-strict for the access pattern that exists. + `ConstraintBuilder::main(offset, col)` resolves to row 0 of a step + (`builder.rs:719-724`), `RowFrame::from_lde`'s index for step `k` is + `row + offset · lde_step_size` — the same row the general, multi-row-capable + `Frame::read_from_lde` calls `initial_step_row` — and that general path + already handles `step_size > 1` correctly. So it is plausibly a one-line + relaxation in `crypto/**`, which is an always-stop item and therefore the + USER's call, not a leg's. + - The brief's single-AIR plan ("three transition offsets AND `step_size > 1`") + is unbuildable for two independent reasons: `AirWithBuses::new` HARDCODES + `transition_offsets: vec![0, 1]` (`lookup.rs:922`), so three offsets means an + `AIR` impl and every one outside `crypto/**`'s example tree is IN that tree; + and `step_size > 1` is unprovable per the ceiling. Entries 8 and 9 therefore + get two witnesses, each with a production oracle, and neither is a witness of + the other's defect. + (Original text kept below.) `Op::Var{offset, row}` indexes the frame's evaluation STEP, and + production's own interpreter asserts `row == 0` + (`constraint_ir/interp.rs:240-242`) while taking + `frame.get_evaluation_step(offset)`. A frame step is `step_size` grid rows, so + the constraint leg must read every `step_size`-th row of the reconstructed OOD + grid where DEEP reads all of them. `TableVerifyShape::num_frame_steps` now + carries that, and the emitter builds the strided view. + At `step_size = 1` the strided view and the whole grid are the SAME vector, so + nothing in the suite can tell a correct emitter from one that passes the full + grid to both legs — which is what the wave-5 sketch did. Same family as entry + 8 and closed by the same witness: an AIR with `step_size > 1` proved by the + production prover. Recording it separately because it is a different SITE + (entry 8 is the absorb ORDER, this is the constraint leg's frame indexing) and + a synthetic AIR built for entry 8 must exercise both or it closes only one. + +10. ~~**Every per-epoch cost number must name the epoch SHAPE it describes**~~ — + **SATISFIED by the wrap run** (assembly-w7, 2026-08-04). Every number below + carries its epoch profile, and the entry's own prediction is now MEASURED + rather than projected. The wrap run's table, all on the fixture epoch's + profile `[2 x14, 3, 4 x4, 5 x3, 7, 20]` (24 sub-proofs, fibonacci guest, a + 16-cycle INTERMEDIATE epoch): + + | inner proof | instructions | keccak perms | arena words | main cells | aux ext cells | wrap prove | wrap verify | wrap proof | + |---|---|---|---|---|---|---|---|---| + | min preset (blowup 2, 1 query) | 2,248,650 | 2,872 | 16,541 | 220,107,920 | 87,073,068 | 19.5 s | 0.09 s | 30,707,816 B | + | blowup 8, 1 query (geometry) | 2,425,718 | 3,816 | 16,893 | 230,661,264 | 92,350,764 | 23.3 s | 0.09 s | 31,147,664 B | + | blowup 8, 73 queries (production shape) | 76,501,118 | 118,080 | 817,101 | 5,077,422,224 | 2,029,461,548 | not provable — see below | — | — | + + The wrap's OWN options are blowup 2 / 219 queries / grinding 20 in every row; + prove figures are one 11-core laptop and are observations of a box, not + machine invariants. + + - **The entry's prediction landed exactly.** At blowup 8 / 73 queries the + emitted program's opening permutations are **100,959** — the number wave 6 + projected for THIS epoch — and FRI is **14,454**, the pinned per-2^20 + sub-proof figure. The closed form is asserted in the test, so the emitted + program and the shape arithmetic agree by check and not by eye. + - **The 460,000 design target is a different workload, exactly as the entry + says.** This epoch's whole bill is 118,080 permutations (2,667 spine + + 115,413 legs), 3.9x under the target, because 23 of its 24 tables are tiny. + Nothing here revises 460,000 as a model of a production-SIZED epoch. + - **The production-shaped wrap does not fit on any box we have**, and this is + measured rather than argued: 11,165,806,868 base-field-equivalent cells at + the measured 33.7 bytes/cell is a projected 350.6 GiB of peak RSS, 2.8x the + 124 GiB measurement box. The coefficient has two measured points (15.1 GiB + at 481.3M cells, 15.5 GiB at 507.7M cells against a 15.9 GiB projection), so + it is an extrapolation of a validated ratio, not a guess — but it IS a 22x + extrapolation and the report says so. + - **84.0% of the cells are the hash.** `LFM_KECCAK` + `KECCAK_RND` are + 4,364,173,312 main + 1,672,478,720 aux ext of the total; one permutation + costs 36,256 main + 13,912 aux cells. That is the number the hash matrix + exists to move, and it means the matrix's other columns decide the machine's + size, not its structure. (At blowup 8 / ONE query the same split reads 83.6% + permutation chips + 5.2% their fixed tables: the fixed-height tables shrink + as a share when the workload grows, the permutation itself does not.) + - **The SPINE also grows with the query count, so per-query cost may only be + taken from the DIFFERENCE.** Spine permutations: 1,467 at blowup 2 / 1 query, + 2,235 at blowup 8 / 1 query, 2,667 at blowup 8 / 73 queries. The legs' own + per-query cost is then **1,581.0 at one query and 1,581.0 at 73** (115,413 / + 73) — measured at both ends rather than assumed linear, which is what lets + slice 1a's geometry run stand in for the geometry of the 73-query shape. + (The +768 the spine gains from blowup 2 to 8 at one query is unexplained + here; the plausible cause — the tiny tables' FRI final-poly coefficient + count rises when their LDE does — is an INFERENCE, not a measurement.) + - **The wrap costs 12.7x the epoch's own trace cells at the min preset** + (inner epoch 22,036,988 main + 5,248,588 aux ext = 37,782,752 + base-field-equivalents), 13.4x at blowup 8 / 1 query, and 295x at blowup 8 / + 73 queries. ⚠ This ratio is NOT a machine constant and must not be quoted as + one: the denominator is a 16-guest-cycle epoch, the smallest an epoch gets, + while the numerator is a verifier whose cost is set by query count and tree + depth. A production-sized epoch moves the denominator by orders of magnitude + and the numerator hardly at all. The number to carry forward is cells per + verify; the ratio is here only to stop anyone computing it from these two + tables and believing it. + + (Original text kept below.) The phase's composed predictions were computed at a UNIFORM + `log2_trace = 20` across all sub-proofs (`join_tests::join_leg_cost`'s stated + constants). A real INTERMEDIATE epoch is not shaped like that: the fixture + epoch's measured trace lengths (log2) are `[2 x14, 3, 4 x4, 5 x3, 7, 20]` — + ONE large table and 23 tiny ones. Openings fall 1.88x against the uniform + model on this epoch (100,959 against 189,727 at blowup 8 / 73 queries), and + FRI collapses to a single sub-proof's bill because the other 23 have their LDE + already terminal at blowup 8, so zero committed layers. This does NOT falsify + 213,744 as a model of a production-sized (2^24-step) epoch, where most tables + are large — it says the number is a claim about a WORKLOAD, and the two + workloads must never be compared without saying so. Assembly owes: the wrap + run's numbers must state their epoch's trace-length profile alongside them. + +## STATED DEFERRALS (safety argument given and accepted — not open debts) + +- **`coset_offset ≠ 3` is unexercised in the FRI leg** (reg-tree, FRI + slice 0; accepted 2026-07-31). No production config produces another + value and the domain constants are baked into the program, so the + emitter's handling of a different offset has no witness. Safety argument: + a wrong coset offset moves every domain point and therefore every leaf + and every fold — it can only REJECT proofs (honest ones included), never + accept a forgery, in either direction of the error. Residual plumbing + condition: the emitter must derive the baked constants from + `ProofOptions`' offset (or assert its literal against that source of + truth), so the deferral covers test coverage only, not a hardcoded-3 + emitter. + +## WATCH (anomalies assembly should confirm or explain, not obligations) + +- **PRICED** (assembly, slice 3): the fixture epoch's public output is **8 + bytes**, so the COMMIT-bus gadget is 8 inverse chains — negligible here, and + still unpriced for a production epoch, whose output length is workload-shaped. + What assembly added is the JOIN: the bytes are no longer an arena of their + own but are derived from the halves the statement absorbed + (`epoch::emit_output_bytes`, one `BitDec` and one `MulAdd` per half, whose + recomposition assert doubles as the `< 2^32` range check). So the cost line + is "per output half" for the derivation plus "per output byte" for the fold. + Original entry: +- **The COMMIT-bus target is an unbudgeted per-byte cost item**: the + closure's second half is `Σ 1/(z − fingerprint(byte_i))` over public + output BYTES — one inverse chain per byte, scaling with output length + (deep-join, LogUp slice 1). Not in the target-shape budget. Assembly + must price it against the real epoch's public-output length. + First data point (zerorow, 2026-08-03): the fixture epoch's output is + **8 bytes**, so the gadget is nonempty in practice and the empty + short-circuit is not the common case. That is a 16-cycle epoch and says + nothing about a production epoch's output length — it only rules out + "usually zero". +- **HALT's constraint-leg cost line is out of step**: 9,859 instructions + for 22 columns, inconsistent with its neighbours (deep-join, final + report — noticed, not chased). Assembly composes per-AIR numbers; an + unexplained per-AIR outlier is exactly where a composition error would + hide. + +## STANDING (from the RESUME, restated so this file is self-contained) + +- Every per-epoch number so far is a COMPOSITION of per-AIR measurements, + not a run. Assembly is what confirms or falsifies them. +- The arena-value join obligation — WIDENED 2026-07-31 (deep-join, LogUp + scoping): it is NOT about opened values; it is about **any value two legs + consume**. Opened values were merely the first instance. Every such value + must be one arena cell (or derived in-machine from one), never parallel + copies, one per leg. + - Instance 1, DISCHARGED: constraint/DEEP values = authenticated values + (deep-join slice 1, shared cells + bound index). + - Instance 2, found live in a leg considered DONE — now DISCHARGED + (deep-join 6712b814, merged 94a55e17): the constraint leg hinted + `table_offset = L/N` host-side and the machine never saw `L` — a + prover could satisfy every accumulator with truthful `L₁/N` while the + closure sums arbitrary `L₂`, making bus balance vacuous. Fixed by + in-machine derivation (`emit_table_offset`: `L · N⁻¹`, N⁻¹ a program + constant); the closure sums the same `L` cell. Falsified both ways: + 4 forged contributions rejected by the derivation and accepted by a + split control, and a pass-through stub fails exactly the composition + check + join test. + - Instance 3, found by the audit the L gap triggered — DISCHARGED + (deep-join 5e93fe6d, merged 1418e0b7): `alpha_powers` were hinted, one + arena word each, and `Op::AlphaPow{idx}` read them straight. Every + LogUp fingerprint is built from these powers, so a prover supplying + them independently of α chooses the fingerprints — any tuple can match + any other; strictly worse in degree than the L gap. Fixed by chaining + from the one α the challenges carry (`emit_alpha_powers`, one ExtAlu + per power, count = `max_bus_elements`, which is shape). Guarded by an + absolute rule-7-compliant test (`the_derived_uniforms_are_not_arena_words`) + with positive controls, both negative branches falsified independently. + - Lesson for assembly: a hinted arena word that a differential never + catches (because the host packs it truthfully) is exactly where this + class hides. The full audit is done (5e93fe6d): everything else either + discharged or in OPEN entry 3. diff --git a/others/lfm-constraint-lowering-design.md b/others/lfm-constraint-lowering-design.md new file mode 100644 index 000000000..3bf5e1c7f --- /dev/null +++ b/others/lfm-constraint-lowering-design.md @@ -0,0 +1,686 @@ +# Design: lowering a `ConstraintArtifact` to LFM instructions + +Design (α) from `lfm-design.md` §3 — the constraint-evaluation leg of the epoch +verifier. Written by the phase0 agent 2026-07-30 against +`feat/phase0-constraint-ir`. **Design only; no semantics touched.** + +Every number below is measured by `constraint_op_census` in +`prover/src/tests/constraint_artifact_tests.rs`, which is a standing instrument — +run it, do not trust this file's copy of the numbers after the constraints change. + +Cost facts about the machine (fusion parity, `MulBase` parity, free base→ext, +program-wide constant interning, one-instruction-one-row) come from the ISA +inventory of `prover/src/lfm/`, relayed by the team lead; §2.3 marks which of +them I confirmed against the IR myself and which I took on report. + +--- + +## 0. Headline + +**The leg comes in materially UNDER budget.** `lfm-design.md` §5.2 claimed ≈69K +instructions at 25 AIRs, implicitly assuming roughly one instruction per IR node. +Measured at **28** AIRs, with the machine's actual cost model applied: + +``` +upper bound (one instruction per arithmetic node) 66,652 +− MulAdd fusion (9,069 pairs) −9,069 += ESTIMATE 57,583 ~16.5% under the ≈69K claim +``` + +Fusion is not an optimization here. `MulAdd` costs the same single row as `Mul`, +so emitting `Mul` then `Add` where one instruction would do is pure waste — the +node count is an **upper bound**, not an estimate, until it is applied. + +Three corrections to how the number should be read: + +1. **The IR's `dim` tags are the wrong split to budget against** — they describe + the prover, and the machine runs the verifier (§3). Budgeting from declared + dims understates extension traffic by 14×. +2. **Constants are interned program-wide**, so the 655 per-AIR pooled constants + are **315** actual `Const` rows (§4.2). +3. **57,583 is per distinct AIR.** For the shape we actually recurse — a + CONTINUATION EPOCH — the leg is **63,393 instructions over 24 sub-proofs** at + the minimum epoch and **64,035 over 26 at 2^20 cycles**, both measured + (§8.2.2). Doubling the epoch past CPU's chunk bound costs 642 instructions, so + the leg is ≈63–65K across any plausible epoch size. §8.2.1 corrects an earlier + claim of mine that the leg is workload-shaped — it is not, the architecture + says so, and that is what collapses the registry ladder to one dimension. + +**Nothing in the IR is structurally inexpressible on a straight-line machine.** +The IR is already in precisely the form the machine's soundness argument demands +— §9, the most reassuring section here. + +--- + +## 1. What the pass consumes and produces + +Input: a `ConstraintArtifact` (the flat POD program, per-constraint metadata, AIR +shape, composition degree multiplier). Output: a straight-line `Vec>` +fragment plus the addresses of the per-AIR quotient contributions. + +The pass runs **at registry-build time on the host**, so it may do arbitrary +host-side work — constant folding, peephole fusion, fanout analysis — none of +which costs machine instructions. What it emits is fixed program text whose +digest the registry pins. `Instr::Const` values live in the `LFM_CONST` +preprocessed columns, so they are program data covered by that same digest: this +is what lets a constraint artifact be embedded without a separate commitment +scheme, and it is the concrete reason design (β) was not needed. + +--- + +## 2. Node → instruction mapping, and it is total + +Eleven IR ops. Six are leaves that resolve to an address and emit nothing; five +are arithmetic. One instruction is exactly one row on exactly one chip. + +| IR op | verify-time value | machine lowering | rows | +|---|---|---|---| +| `Var{main,offset,row,col}` | ext | address in the OOD frame region | 0 | +| `RapChallenge{idx}` | ext | address in the challenge region | 0 | +| `AlphaPow{idx}` | ext | address in the alpha-power region | 0 | +| `TableOffset` | ext | address of the per-proof `L/N` | 0 | +| `ConstBase(idx)` | **base** | `Const{(c,0,0,0)}`, interned program-wide | 1 per distinct word | +| `ConstExt(idx)` | ext | `Const{(c0,c1,c2,0)}`, interned program-wide | 1 per distinct word | +| `Add(a,b)` | ext | `ExtAlu{Add}`, or folded into `MulAdd` (§5) | 1 or 0 | +| `Sub(a,b)` | ext | `ExtAlu{Sub}` | 1 | +| `Mul(a,b)` | ext | `ExtAlu{Mul}`, `MulBase` if one operand is base, `MulAdd` if fused | 1 | +| `Neg(a)` | ext | **`ExtAlu{Sub, a: ZERO, b: a}`** — §2.1 | 1 | +| `Embed(a)` | ext | **nothing** — §2.2 | 0 | + +### 2.1 `Op::Neg` has no instruction + +`ExtOp` is `Add | Sub | Mul | Div | MulAdd | MulBase`. There is no unary negate. +`Neg(a)` lowers to `Sub` from a pooled zero, which every program already has +(`IrBuilder` reserves node id 0 as the base-field zero, and zero is interned once +program-wide anyway). The mapping is total, but only via that identity — worth +writing down rather than rediscovering. + +### 2.2 `Op::Embed` is free, and this is a payoff of the word model + +Base→ext conversion costs **no instruction at all**: a base word IS a valid +extension word, the distinction being only which lanes are zero, and those zero +lanes are pinned by constant expressions in the bus tuple rather than by columns +(`SOUNDNESS.md` §4). So `Embed` is a pure address alias — the emitter records +that node `i` refers to node `a`'s address and emits nothing. + +The converse is not free: ext→base costs 1 `LANES` row (`Unpack`). **The +constraint leg never needs it.** Nothing in the IR narrows an extension value to +a base one — `Dim` only ever widens through `binop`'s join. That asymmetry is +what makes the all-extension verifier evaluation (§3) affordable despite carrying +20× more extension traffic than the IR's tags suggest. + +**Measured: 0 `Embed` nodes across all 28 production AIRs**, and 0 `ConstExt`. +Both arms are correctness-only today. Keep them: a missing arm is a panic in +`ConstraintArtifact::program` or, worse, a silent wrong answer in the CUDA kernel. + +### 2.3 What I verified vs what I took on report + +Confirmed by me against the IR and the artifact: the eleven-op inventory, +`Op::Neg` having no ISA counterpart, `Embed`/`ConstExt` being unused in +production, the absence of any ext→base narrowing, and every count in §8. + +Taken from the ISA inventory without independent verification: one instruction = +one row; `MulAdd` and `MulBase` costing the same row as `Mul`; base→ext being +free; program-wide constant interning; group heights padding to +`next_power_of_two().max(4)`. If any of those is wrong the instruction counts in +§8 still stand — they are counts of instructions — but the row/cell conclusions +drawn from them do not. + +--- + +## 3. The split that matters: prover dims are not machine dims + +This is the correction I most want on the record. + +`Dim` records what the **prover** computes. Its frame is base-field, so a +trace-only subexpression stays base, and the IR tags 42,137 of 67,103 arithmetic +nodes `Dim::Base`. + +The machine runs the **verifier's** evaluation at the OOD point, where the frame +holds only extension elements — `eval_program_verifier` resolves every `Var` to +`Value::Ext` regardless of `main`, because the verifier has openings, not trace +cells. Propagating that through `interp::binop`'s rule (base only when both +operands are base values *and* the declared dim is base), a node is base at +verify time **only if its entire subtree is constants**. + +``` +arithmetic nodes 67,103 + base by the IR's own dim 42,137 <- prover-side. NOT the machine's split. + base at verify time 2,916 <- constant-only subtrees + extension 59,146 <- 94% of the arithmetic +``` + +**A 14× discrepancy.** Anyone sizing this leg from the IR's `dim` column would +conclude most of the work is cheap base arithmetic. It is not. + +Two consequences. + +**The 2,916 base nodes cost nothing at all.** A constant-only subtree is a +compile-time constant: the emitter folds it during the host-side pass and interns +the result. Zero rows, which is why they are excluded from §0 rather than charged +as `BaseAlu`. + +**5,041 multiplies must be routed through `MulBase`.** An ext×base multiply is +1 `XALU` row through `MulBase`, versus 4+ if lowered by hand as three base +multiplies plus a repack. So this is not a *reduction* against `Mul` — both are +one row — it is a **routing obligation**: the emitter must recognise the case, or +it pays 4× for it. The eligible operand must be a genuine base cell, because +`LFM_XALU` constrains its shared B-columns to zero on `MulBase` rows so the +received token matches a base writer's (`SOUNDNESS.md` §4); at verify time that +means a folded constant, which is exactly the 5,041 the census counts. + +Note the count would be 9,413 if one used the prover dims — nearly double, and +wrong. + +--- + +## 4. Where operands come from + +### 4.1 Four regions, and the distinction is a soundness boundary + +| region | source | authentication | +|---|---|---| +| OOD frame values (`Var`) | arena, hint-fed | the DEEP/opening leg — the machine hashes them into the openings it checks | +| challenges (`RapChallenge`) | transcript replay | computed in-machine by `LFM_HASH` rows; **never** hinted | +| alpha powers (`AlphaPow`) | derived from α | computed in-machine, once per proof | +| table offset (`TableOffset`) | derived `L/N` | computed in-machine, once per proof | +| constants | the program's own pool | program text; registry-digest-pinned | + +The arena rule (`SOUNDNESS.md` §5) says an arena value is unconstrained by the +reading chip and must be transitively authenticated by a hash the machine +performs. OOD frame values satisfy it because the DEEP leg absorbs them; +challenges must never come from an arena and do not. + +**The constraint leg pays nothing marginal for any of this.** Measured **5,964 +leaf nodes**, all addresses of values other legs already materialized. That is +the single biggest reason the leg is ~1% of the program despite 73,722 nodes. + +### 4.2 Constants are interned program-wide + +Each distinct 4-lane word is one `Const` row regardless of how many nodes, or how +many AIRs, reference it. Summing per-AIR pools overcounts badly, because small +structural constants (0, 1, 2, `2^8`, `2^16`, `2^24`) recur in every table. + +``` +per-AIR pools, summed 655 +interned program-wide 315 <- the actual Const row count +``` + +More than half the apparent constant cost is duplication across tables. + +### 4.3 Address assignment and `mult` + +Addresses are dense and compiler-assigned in emission order: the emitter walks +the node list in index order, assigning address = base + i and skipping folded, +aliased and fused nodes. + +`mult(a)` — the statically known read count every write carries — is the node's +fanout in the IR DAG, plus one if the node is a constraint root (the quotient +recombination reads it). **Measured max fanout 1,632**, so the multiplicity +column holds values into the low thousands; it is a field element so this is +comfortable, but it is not the "small" value one might assume when sizing a range +check on it. + +**Measured: 3 dead nodes (fanout 0) across all 28 AIRs.** Tiny, but the emitter +must DCE them rather than emit zero-multiplicity writes — the registrar's (M) +check is mult-equality. + +--- + +## 5. `MulAdd` fusion is mandatory, not an optimization + +`ExtAlu` carries `MulAdd` as a first-class op **at the same one-row cost as +`Mul`**. The IR has no `MulAdd` node — `CaptureBuilder` emits `Mul` then `Add` — +so an unfused emitter pays two rows where one would do, every time. + +**The fusion is valid only when the `Mul` has exactly one consumer.** Hash-consing +means a shared `Mul` feeds several `Add`s; fusing it into each would recompute it +per consumer. (Fusing into just one of several consumers is cost-neutral, not a +saving: the `Mul` row still has to exist for the others.) A node that is a +constraint root also counts as a consumer — fusing it away would delete the value +§6 needs. + +**Measured: 9,069 fusable `Add` nodes**, 13.6% of the leg. This is the difference +between the upper bound and the estimate in §0. + +Worth naming as a near-miss: the hash-consing that makes the IR compact is the +same property that makes naive fusion unsound. Shared subexpressions are an asset +for program length and a hazard for peepholes — any future fusion needs the same +single-consumer guard. + +--- + +## 6. Zerofier and quotient recombination + +The composition quotient is `H = Σ_c β^c · C_c / Z_c`. + +### 6.1 Uniform zerofiers make this cheap, and the saving is large + +**All 28 production AIRs emit through `RowDomain::ALL`** — measured; nothing under +`prover/src` calls `RowDomain::except_last`. So `end_exemptions = 0` everywhere, +and every constraint of an AIR shares one zerofier `Z = ζ^N − 1`, depending only +on the sub-proof's trace length. + +Per sub-proof: + +``` +ζ^N repeated squaring log2(N) × ExtAlu{Mul} ≈ 20–24 rows +ζ^N − 1 one Sub against the interned 1 1 row +1/Z one ExtAlu{Div} 1 row + ──────────── + ≈ 22–26 rows +``` + +Because `Z` is shared, the division factors out of the sum: +`H_air = (Σ_c β^c · C_c) / Z` — **one division per AIR, not per constraint**. The +sum is a Horner fold, one `ExtAlu{MulAdd}` per constraint, 2,150 total. + +The saving is the entire value of the uniform-zerofier finding, so it is worth a +number. The naive shape — what `main` does today, recomputing `ζ^N` and a full +extension inversion once per constraint (`lfm-design.md` §5.2 hygiene item 1) — +costs `2,150 × ~24 ≈ 51,600` rows. Once per AIR costs `28 × ~24 ≈ 672`. +**≈50,900 rows saved, comparable to the entire rest of the leg.** The GPU path's +uniform-zerofier precondition holding in fact rather than by luck is the same +fact, cashed differently. + +Total recombination ≈ 2,150 `MulAdd` + 28 `Div` + ~672 zerofier ≈ **2,850 rows**, +of which §0 counts the 2,150 β-folds and folds the rest into per-sub-proof +overhead. + +### 6.2 The final comparison + +Comparing `H` against the claimed composition parts is `assert_eq`, which is not +an instruction: it is 2 ALU rows plus an interned constant, via the +division-by-zero mechanism (`div` is constrained `B·OUT = A`, so `B = 0` forces +`A = 0`). A handful of rows per sub-proof; negligible against the above. + +### 6.3 If a constraint ever grows an exemption + +The zerofier gains factors and the emitter must evaluate one **per distinct +`end_exemptions` value per AIR**, not per constraint. The artifact's +`ConstraintMeta` carries exactly what is needed to group them, and +`transition_zerofier_evaluations_grouped` already keys its dedup on that field +host-side. Cost scales with the number of distinct values, currently one. + +--- + +## 7. Boundary constraint and the next-row read + +**Boundary.** Every VM AIR uses `NullBoundaryConstraintBuilder`, so the only +boundary constraint is the framework's `acc[0] = 0` per chip. At ζ that is +`(P(ζ) − 0)/(ζ − 1)`: one Sub for the denominator, one Div, numerator is the +opened value itself. **≈3 rows per sub-proof.** The accumulator's circularity +needs no boundary constraint of its own — it rides the plain `ζ^N − 1` zerofier. + +**The next-row read.** The machine has no rows, so "next row" is not a concept it +needs: `Op::Var{offset: 1, col}` is simply a different address, and the DEEP leg +supplies the `g·ζ` opening alongside the `ζ` ones. Zero extra rows. + +What makes this cheap is a shape fact worth re-verifying rather than assuming: +**every AIR declares exactly one next-row column** (the LogUp accumulator), or +none. That is not folklore — `ood_window_ir_tests` derives the true next-row read +set from the captured IR and asserts equality with the declaration for all 28 +AIRs, and that check is what stands between a correct verifier and one that +silently reconstructs an omitted `g·ζ` column as ZERO. It now covers the three +continuation AIRs, which it did not before this phase. + +--- + +## 8. Measured counts + +### 8.1 Per AIR (28 tables; the artifact is blowup- and trace-length-invariant) + +`instr` = extension ALU + MulBase, before fusion and before program-wide constant +interning (both of which are global, so they cannot be attributed per row). +Leaves are free; constant-only subtrees fold at build time. + +| table | nodes | leaves | fold | ext | mulbase | **instr** | +|---|---:|---:|---:|---:|---:|---:| +| CPU | 600 | 75 | 4 | 417 | 72 | **489** | +| BITWISE | 158 | 33 | 3 | 106 | 6 | **112** | +| LT | 160 | 32 | 2 | 106 | 10 | **116** | +| SHIFT | 393 | 48 | 5 | 299 | 22 | **321** | +| EQ | 124 | 25 | 2 | 77 | 11 | **88** | +| BYTEWISE | 185 | 41 | 0 | 120 | 18 | **138** | +| STORE | 201 | 40 | 2 | 136 | 13 | **149** | +| CPU32 | 516 | 77 | 3 | 356 | 58 | **414** | +| MEMW | 552 | 89 | 3 | 429 | 19 | **448** | +| MEMW_A | 392 | 66 | 3 | 303 | 8 | **311** | +| MEMW_R | 202 | 41 | 2 | 129 | 24 | **153** | +| LOAD | 225 | 48 | 1 | 144 | 18 | **162** | +| DECODE | 35 | 15 | 0 | 18 | 0 | **18** | +| MUL | 388 | 48 | 2 | 276 | 44 | **320** | +| DVRM | 511 | 61 | 7 | 362 | 61 | **423** | +| BRANCH | 147 | 29 | 2 | 96 | 12 | **108** | +| HALT | 825 | 49 | 37 | 600 | 101 | **701** | +| COMMIT | 438 | 55 | 8 | 313 | 46 | **359** | +| PAGE | 63 | 16 | 2 | 34 | 7 | **41** | +| REGISTER | 49 | 15 | 2 | 23 | 6 | **29** | +| KECCAK | 3,997 | 784 | 30 | 2,960 | 186 | **3,146** | +| KECCAK_RND | 16,317 | 2,262 | 17 | 12,677 | 1,339 | **14,016** | +| KECCAK_RC | 51 | 23 | 0 | 26 | 0 | **26** | +| ECSM | 22,162 | 1,093 | 1,513 | 17,611 | 1,653 | **19,264** | +| ECDAS | 24,848 | 851 | 1,262 | 21,424 | 1,294 | **22,718** | +| L2G_GLOBAL | 47 | 15 | 1 | 24 | 3 | **27** | +| L2G_MEMORY | 93 | 21 | 1 | 60 | 5 | **65** | +| GLOBAL_MEMORY | 43 | 12 | 2 | 20 | 5 | **25** | +| **TOTAL** | **73,722** | **5,964** | **2,916** | **59,146** | **5,041** | **64,187** | + +Program-wide: + 315 interned `Const` rows + 2,150 β-folds = **66,652 upper +bound**; − 9,069 fused = **57,583 estimate**. + +### 8.2 The per-epoch multiplier, MEASURED + +The §8.1 total is per distinct AIR. An epoch evaluates the leg once per +SUB-PROOF, and the 14 split-table families are chunked — +`chunks = ceil(rows / max_rows[table])`, with `max_rows` sized per table so each +chunk costs about the same memory (`tables/mod.rs::max_rows`). So + +``` +constraint rows per epoch = Σ over sub-proofs instr(that sub-proof's AIR) +``` + +Measured by `epoch_chunk_multiplier`, which builds real traces so the chunk +counts are the prover's own splitting rather than a reconstruction of it: + +| fixture | cycles | chunked sub-proofs | chunked | fixed | pages | **epoch total** | **multiplier** | +|---|---:|---:|---:|---:|---:|---:|---:| +| `fib_iterative_1M` | 1.0M | 16 | 4,282 | 60,389 | 41 | **64,712** | **1.01×** | +| `fib_iterative_2M` | 2.0M | 20 | 5,566 | 60,389 | 41 | **65,996** | **1.03×** | +| `array_multipass_20M` | 20.4M | 123 | 34,938 | 60,389 | 205 | **95,532** | **1.49×** | + +**The multiplier is small, and the reason is structural**: chunking multiplies +the CHEAP AIRs. CPU is 489 instructions, MEMW_R 153; even 40 CPU chunks at 20M +cycles adds only 19,560. The expensive AIRs — ECSM 19,264, ECDAS 22,718, +KECCAK_RND 14,016 — are never chunked, contributing exactly one sub-proof each. + +So the leg runs **≈65K per epoch at 1–2M cycles, ≈96K at 20M**. `lfm-design.md` +§5.2's ≈69K was closer to right than my earlier warning implied; the correction +is a modest growth term in epoch size, not a multiplier on the whole figure. + +### 8.2.1 CORRECTION — my "workload-shaped" claim was wrong + +An earlier version of this document said the leg was workload-shaped: that +ECDAS + ECSM + KECCAK_RND are 87% of the per-AIR total, so an epoch doing no +elliptic-curve work would drop 65%. **That is false, and the architecture says +so plainly.** + +`FIXED_TABLE_COUNT = 10` is documented as "tables that always contribute exactly +one sub-proof, **regardless of `TableCounts`**: bitwise, decode, halt, commit, +keccak, keccak_rnd, keccak_rc, register, ecsm, ecdas" (`prover/src/lib.rs`). +ECSM, ECDAS and the keccak tables are present in **every** epoch whether the +workload touches them or not — a zero-row table still needs its sub-proof, since +dropping it would remove its constraints from verification. + +The measurement above confirms it: the `fib_iterative` fixtures use no +elliptic-curve and no keccak work, and still carry the full 60,389-instruction +fixed block. + +The correct statement is the opposite of what I wrote: **the constraint leg is +essentially workload-INDEPENDENT.** ~94% of it is the always-present fixed block; +what varies is the chunked remainder, which tracks epoch size rather than +instruction mix. + +#### Why this matters more than an erratum: it collapses the registry ladder + +This lands directly on the open profile-ladder question — *how many distinct +programs must the registry carry?* + +A constraint leg that were workload-shaped would make the emitted program vary +with workload class, and the registry would have to carry a **cross-product**: +workload classes × epoch shapes. That is the feared outcome, and it is the shape +that makes registry entries hard to enumerate. + +Because the leg is ~94% fixed, the emitted program barely varies with what the +workload computes. What remains is the chunked term, which tracks **epoch SIZE** +— the 1.01× → 1.49× growth measured in §8.2. So the ladder is +**one-dimensional**: a short list of epoch shapes, not a cross-product. Each rung +is an epoch size, and every workload of that size shares a program. + +That is the most consequential consequence of the measurement, and it is the +opposite of what my erratum-version claimed. It also composes with the +`page_base` uniform promotion, which removes the *other* source of +workload-dependence (§0.1 of the uniform proposal): with both, the emitted +program's identity depends on epoch shape alone. + +#### The generalizable lesson + +The node census cannot see how sub-proofs are **assembled**. It reads captured +IR, one AIR at a time; nothing in it knows that `FIXED_TABLE_COUNT` forces a +sub-proof for a zero-row table. Any inference about workload sensitivity, epoch +composition, or sub-proof count is therefore outside what that instrument can +support, however tempting the per-AIR table makes it. I asserted one anyway. + +### 8.2.2 The continuation epoch — the shape we actually recurse + +The table above is the monolithic shape, which is the wrong one for the target. +A continuation epoch differs in three ways: + +- **PAGE does not appear.** Epochs pass `page_configs = &[]` + (`continuation.rs:693`, `:797`, enforced prover-side at `:677-681`), so + `create_page_air` is never called and the per-page term vanishes. +- **One L2G_MEMORY sub-proof** per epoch: +65 instructions. +- **Intermediate epochs drop HALT** (9 fixed tables, not 10): −701. + +Composition and totals, computed by `continuation_epoch_constraint_leg`: + +``` +14 split families (>= 1 chunk each) 3,640 + 9 fixed, no HALT 59,688 + 1 L2G_MEMORY 65 +INTERMEDIATE epoch 63,393 instr over 24 sub-proofs +FINAL epoch (+HALT) 64,094 instr over 25 sub-proofs +``` + +**The 24/25 sub-proof count is independently measured** on the LFM fibonacci +epoch fixture, and the test asserts that this composition reproduces it — so the +shape is pinned rather than inferred. If the epoch shape changes, the arithmetic +stops matching and the test fails. + +Those 24/25 are the **minimum**: one chunk per family, i.e. an epoch of ≤2^19 +cycles. `continuation_epoch_chunk_counts_measured` drives the real continuation +path — `Executor::resume_with_limit` for one epoch's cycles, then +`Traces::from_image_and_logs` — to measure a larger epoch first-hand: + +| epoch | cycles | chunked sub-proofs | **total sub-proofs** | **instr** | +|---|---:|---:|---:|---:| +| minimum | ≤2^19 | 14 | **24** | **63,393** | +| measured | 2^20 | 16 (CPU ×2, MEMW_R ×2) | **26** | **64,035** | + +Doubling the epoch past CPU's 2^19 chunk bound costs **642 instructions** — one +extra CPU chunk (489) and one extra MEMW_R (153). That is the whole growth term, +and it is why §8.2's monolithic 1.49× at 20M cycles is an over-estimate for an +epoch: an epoch never gets that large, because it is capped at `epoch_size`. + +Two things fell out of running it that are worth more than the numbers: + +- **`fib_iterative_2M` and `array_multipass_20M` produce IDENTICAL chunk counts** + for their first 2^20 cycles — two quite different workloads, same 16 sub-proofs + and same 4,282 instructions. Workload-independence, visible directly rather + than argued from `FIXED_TABLE_COUNT`. +- **The test asserts `traces.page_configs.is_empty()`**, so "a continuation epoch + never builds PAGE" is now pinned by a run rather than read off a comment. + +**94% of it is the fixed block**, which is the sharpest statement of §8.2.1: the +constraint leg for a continuation epoch is ≈63K instructions essentially +regardless of what the workload does, growing only with epoch size as cheap AIRs +chunk. + +The global proof carries one L2G_GLOBAL per epoch (27 each) plus one +GLOBAL_MEMORY per touched page (25 each) — negligible at any plausible page +count, which is what settles the page-base question as an identity problem rather +than a size one. + +### 8.3 Against the design doc's claim + +| | design doc (25 AIRs) | measured (28 AIRs) | +|---|---:|---:| +| IR nodes | 73,539 | 73,722 | +| arithmetic ops | 66,982 | 67,103 | +| constraint-leg instr | ≈69K | 66,652 upper bound | +| with mandatory fusion | — | **57,583** | + +The ≈69K claim assumed roughly 1:1 with nodes. Fusion is common — 9,069 pairs — +so the per-distinct-AIR figure lands **16.5% under**. Applying §8.2's measured +per-epoch multiplier (1.01–1.49×) puts a real epoch at **≈58K–86K instructions**, +which brackets the ≈69K claim rather than contradicting it. + +### 8.4 On converting rows to cells + +One instruction is one row on one chip, but group heights pad to +`next_power_of_two().max(4)`, so marginal row cost is zero until a boundary is +crossed and the meaningful metric is per-chip padded height × value width. +`airs::lfm_cell_counts` is the instrument for that. Everything above is in +INSTRUCTIONS/rows; the padded-cell figure needs the per-chip distribution, which +depends on how this leg's rows interleave with the rest of the program's — not +something the leg can be costed for in isolation. + +--- + +## 9. Structural expressibility: nothing blocks + +There is nothing the machine cannot express, and the reason is stronger than "it +happens to work". + +- **The IR is a pure DAG with no control flow.** No branches, no loops, no + data-dependent addressing. `ConstraintProgram` is a topologically ordered node + list, which is what a straight-line program *is*. +- **`nodes[i]` references only `< i`.** The IR's own documented invariant, + enforced by `ConstraintArtifact::validate_self`. It is *identical* to the + machine's acyclicity premise (A) — "operand address < destination address" + (`SOUNDNESS.md` §2). Dense address assignment in node order satisfies (A) **by + construction**, with no reordering pass and no verification burden beyond the + check the artifact already runs. +- **Fanout is statically known**, so `mult` comes straight off the DAG. The + write-once model needs exactly this and the IR already has it. +- **No division in the constraint algebra.** `Op` has no `Div`. Division enters + only at the zerofier/quotient step (§6), a handful of rows per sub-proof. +- **No ext→base narrowing anywhere**, so the one conversion that costs a row + (`Unpack`) is never needed by this leg. + +The only genuine mismatch is trivial: `Op::Neg` has no instruction and lowers to +a subtract from zero (§2.1). A lowering detail, not a structural obstacle. + +--- + +## 10. What I did not verify + +- Nothing remaining on the epoch numbers. §8.2.2 is now first-hand for the + continuation path at both the minimum epoch and 2^20 cycles; §8.2's monolithic + table is retained only as the whole-execution comparison, and is explicitly + the wrong shape for the target. +- **The machine-side cost facts listed in §2.3**, taken from the ISA inventory + rather than read by me. The instruction counts survive if any is wrong; the row + and cell conclusions do not. +- **Padded-cell cost**, which needs the whole program's per-chip distribution + (§8.4), not this leg alone. + +--- + +# Corrections from building it — the emitter agent, 2026-07-30 + +Added by the agent that implemented this design as `prover/src/lfm/constraints.rs` +(branch `feat/lfm-constraint-emitter`). The original text above is left as its +author wrote it; everything below is measured by +`lfm::constraint_tests::constraint_leg_instruction_census` and +`..::continuation_epoch_constraint_leg_cost`, both of which fail if the numbers +move. Where a correction is a judgement rather than a measurement, it says so. + +## What reproduced exactly + +- **§8.1's whole per-AIR `instr` column**, all 28 tables, total **64,187**. The + census test asserts it table by table and fails loudly if any entry drifts. +- **§8.2.2's `63,393` intermediate-epoch budget**, rebuilt from those counts by + the same 14-families / 9-fixed / 1-L2G_MEMORY composition. +- **§4.3's "3 dead nodes (fanout 0)"** — but see below for what they are. +- **§2.2's claim that production has no `Embed` and no `ConstExt`**, and §9's + claim that dense address assignment in node order satisfies acyclicity by + construction. The emitter needed no reordering pass. + +## What the emitter actually costs + +``` +per distinct AIR (28) 64,187 unfused → 55,147 emitted +per INTERMEDIATE epoch 54,358 leg + 2,894 recombination = 57,252 over 24 sub-proofs +per FINAL epoch 55,058 leg + 2,944 recombination = 58,002 over 25 sub-proofs +``` + +**9.7% under the 63,393 budget**, with the recombination included — which §8.2.2 +does not count. + +## Three corrections + +### 1. `MulBase` is cost-neutral, not a 4× routing obligation (§3) + +§3 says an emitter that fails to detect the ext×base case "pays 4× for it", +comparing against a hand-lowering as three base multiplies plus a repack. +**That comparison has no basis.** Read against `chips::xalu`: `Mul` and +`MulBase` are selectors on the SAME chip at the same width, so both are one row; +and the `B` operand is received through `ext_token` on every selector, so a +base-valued word `(c, 0, 0, 0)` is already a legal `Mul` operand yielding the +same product. Nobody would lower an ext×base multiply by hand when `ExtOp::Mul` +exists, so the 4× alternative is not a lowering anyone would reach for. + +Detection is therefore **optional, not obligatory, and worth zero rows**. The +emitter does it anyway — it states the intent, and the chip's constraints 18–19 +pin the operand's high lanes to zero on those rows — but a reader sizing this leg +should not expect a saving, and 5,041 is a count of `MulBase` rows rather than of +rows avoided. + +### 2. Fusion saves 9,040, not 9,069 (§5) + +Measured by construction: the emitter writes exactly 9,040 `MulAdd` rows. + +**9,113** `(Add, Mul)` operand pairs individually satisfy the single-consumer +guard, but an `Add` carries ONE multiply, so a sum whose two operands are both +single-consumer products can absorb only one of them. There are 73 such sums. +The achievable saving is bounded by the pairs, not equal to them — 9,069 sits +between the two counts and I could not reproduce it under either rule. + +The consequence for §0's arithmetic is small (29 rows on 57,583) but the shape of +the claim matters: a candidate count is an upper bound on a fusion saving, never +the saving itself. + +### 3. The "3 dead nodes" cost no rows, and a reachability count is not comparable + +§4.3's three fanout-0 nodes reproduce exactly under its own local measure — but +**none of them is arithmetic**. Across all 28 AIRs there are **zero** arithmetic +nodes with local fanout 0, so dead-code elimination saves **zero rows** on any +production artifact. The emitter does DCE anyway, and its test has to INJECT an +unreachable node to exercise the path, because the capture front-end does not +produce one. + +Separately, **2,376 nodes are unreachable from any root** once one notices that a +folded constant does not keep its operands alive. Every one of them is itself a +constant, and the census already counts them under `fold` — ECSM 1,186 of its +1,513, ECDAS 1,190 of its 1,262. **Anyone adding §8.1's `fold` column to a +reachability-based dead count will double-count exactly those 2,376.** The +emitter's report keeps them in a separate `unreached_const` field for this +reason; it was the one place the implementation and the design first disagreed, +and the disagreement was in the bookkeeping, not in the program. + +## Two deliberate departures from the spec + +- **Two extra rows per sub-proof for reciprocal guards.** §6 divides the β-fold + by `Z` and §7 divides the boundary numerator by `ζ − p`. Under the machine's + `0/0 = 1` convention a direct divide silently returns 1 when the denominator + AND numerator vanish, so a `ζ` on the trace domain would be accepted. The + emitter inverts against the interned one instead (`1/0` has no satisfying + assignment) and multiplies. §6.1's ≈22–26 row zerofier block measures at ≈24–28. + The out-of-domain sampler already excludes such a `ζ`, but the sampler is not + in this leg, and per method rule 5 a deferral's safety argument is itself a + claim — the guard costs two rows and removes the need for one. +- **Boundary constraints are an explicit shape parameter**, not read from the + artifact, because `AIR::boundary_constraints` is a function of the public + inputs and the artifact deliberately excludes it. §7 is right that production + has exactly one per interacting AIR (`acc[0] = 0` on the last aux column, at + `g^0`), and the emitter carries the general `{col, point, value}` form anyway. + +## What the implementation cannot tell you + +The differential runs every one of the 28 AIRs against `eval_program_verifier`, +and the composition check runs against a real proof of L2G_MEMORY — but only +L2G_MEMORY is checked against a real proof, and only its trace length, part count +and single boundary constraint are exercised end to end. Nothing here says a +27-AIR epoch assembles correctly; that is the assembly leg's question, and the +per-epoch figures above are compositions of per-AIR measurements, not a run. diff --git a/others/lfm-fri-leg-state.md b/others/lfm-fri-leg-state.md new file mode 100644 index 000000000..0105ea1b6 --- /dev/null +++ b/others/lfm-fri-leg-state.md @@ -0,0 +1,67 @@ +# FRI leg — state at reg-tree's retirement (2026-07-31) + +Written by team-lead from reg-tree's slice reports. The agent hit its session +limit BEFORE writing its own handoff, so this file substitutes: it records +what the agent reported in messages but never committed as prose. Code state +is fully committed and green; nothing here contradicts the tree. + +## Where the leg stands + +DONE (all on feat/lfm, slice 1 = 85f99c81): +- `FriShape` mirrors production's `FriFoldLayout` (`crypto/stark/src/fri/ + terminal.rs:45-54`), every parameter taken from `ProofOptions` including + the coset offset (asserted: `shape.coset_offset == opts.coset_offset` on + the real proof — no hardcoded 3; the ledger's coset deferral is + coverage-only, as accepted). +- Host-side unit tests over `k ∈ {0, 6, 7, 63}` and the clamp regime + (`trace_bits ≤ 7`), expectations hand-derived from the spec, not the + module. This is where the spec §7 dead-branch requirement is discharged + (ruling: shape is compile-time in LFM, so those branches are emitter + arithmetic, not emitted control flow). +- The fixture-blindness result, demonstrated at the worst constant: + deleting `saturating_sub(1)` from `num_committed` fails both synthetic + tests and PASSES the real-proof differential (fixture has + `total_folds = 0`). The only real proof available cannot witness the + fold mechanism at all. +- The sizing prediction committed AS A TEST (`the_fri_sizing_prediction`) + before any measurement exists: 174/186/198 perms per query, + 38,106 / 20,460 / 14,454 total at blowup 2/4/8. Blowup-2 row reproduces + the spec §8 worked example. +- The query-index bits join point: `emit_query_with_bits` / + `emit_sub_proof_with_bits` return `QueryOutput { deep, bits }` + (8b8e55bf, fully additive). Guarded by an absolute rule-7 test (every + returned bit consumed by some `Select`). + +NOT STARTED: the emitter itself — per-layer walk + fold + terminal check. + +## What the successor needs to know beyond the committed spec + +1. **Implementation spec** = `others/lfm-fri-verify-spec.md` INCLUDING its + addendum (reg-tree's first-hand findings folded in at 85f99c81). The + emission checklist at the end lists the ten things that silently break + bit-exactness. Read the whole file before emitting anything. +2. **The parity/Select detail** (verify-side, easy to miss from the + prover-side reading): leaf ordering is parity-dependent + (`verifier.rs:637-641` — `if iota % 2 == 1 { [sym, v] } else + { [v, sym] }`), so the machine must Select on the LOW index bit per + layer. The FOLD needs no parity branch (spec §3's sign-cancellation + result); parity matters ONLY for leaf byte order. +3. **⚠ OWED CHECK, never completed:** reg-tree hypothesised that + `sub_proof::emit_leaf_hash` at `GroupShape { num_columns: 1, is_ext: + true }` is byte-identical to production's FRI leaf + (`FieldElementPairBackend::hash_data`, 48 bytes, components 0,1,2, 8B + big-endian each) and said it would verify byte-for-byte RATHER THAN + ASSUME. No confirmation ever arrived. The successor must do that check + before reusing the gadget — treat it as unverified. +4. **Join obligation**: fold values and walk leaves through the SAME cells, + index bits from `QueryOutput.bits` (never a fresh decomposition), and + the first fold consumes the DEEP leg's `p₀(υ)/p₀(−υ)` cells — the seam + the ledger's STANDING clause covers. The zero-layer shape + (`num_committed = 0`) is a first-class emitted shape, pinned by + `the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold`. +5. **Primary instrument** (approved plan): synthetic codewords driven + through production's OWN `commit_phase_from_evaluations` + `query_phase`, + differentialled against the verifier's own check, sweeping + `num_committed` over 0/1/2/3+. Only the input is synthetic. +6. Measure against the pinned prediction test; a miss means the shape is + not what we think — investigate, never fudge. diff --git a/others/lfm-fri-verify-spec.md b/others/lfm-fri-verify-spec.md new file mode 100644 index 000000000..e4d702644 --- /dev/null +++ b/others/lfm-fri-verify-spec.md @@ -0,0 +1,675 @@ + + +# Production FRI VERIFY path — implementation spec for LFM emission + +Worktree: `/private/tmp/claude-501/-Users-maurofab-workspace-lambda-vm-3/0cdd934d-c82f-4724-bc05-01b1924f85f0/scratchpad/wt-reg-tree` + +Note: this worktree contains **only the unbatched FRI**. `grep -rl batched crypto/stark/src/` returns no FRI-verify file — the batched-FRI verifier (#768) is not on this branch. Everything below is the single production verify path. + +--- + +## 1. The verify-side query loop ✓ VERIFIED + +**Entry:** `step_3_verify_fri`, `crypto/stark/src/verifier.rs:387-483`. +**Per-query core:** `verify_query_and_sym_openings`, `crypto/stark/src/verifier.rs:660-748`. + +Driver (`verifier.rs:469-482`) — one call per query index, no cross-query state: + +```rust +(0..challenges.iotas.len()) + .zip(evaluation_point_inverse) + .all(|(i, eval)| { + Self::verify_query_and_sym_openings( + proof, &challenges.zetas, challenges.iotas[i], proof.query(i), + eval, &deep_poly_evaluations[i], &deep_poly_evaluations_sym[i], + &terminal_codeword, + ) + }) +``` + +**Exact sequence for ONE query** (`iota`): + +1. Take `p₀(υ)`, `p₀(−υ)` from the DEEP reconstruction (these are *not* Merkle-checked here; step 4 authenticates the underlying trace/composition leaves). +2. **Fold 0** (unauthenticated, no layer): `v ← (p₀+p₀ˢ) + υ⁻¹·ζ₀·(p₀−p₀ˢ)`. `index ← iota`. +3. **For i = 0 .. num_committed−1**: authenticate the leaf `{v, evaluation_sym[i]}` against `fri_layers_merkle_roots[i]` at Merkle position `index>>1`; then fold `v ← (v+sym) + υ^(−2^(i+1))·ζ_{i+1}·(v−sym)`; then `index >>= 1`. +4. **Terminal**: `terminal_codeword[index] == v`. + +So per query: `num_committed` Merkle authentications, `num_committed + 1` folds, 1 array lookup + equality. Note the **asymmetry**: folds = layers + 1, because the first fold consumes DEEP values rather than a committed layer. + +Verbatim core (`verifier.rs:692-747`): + +```rust + let evaluation_point_vec: Vec> = + core::iter::successors(Some(evaluation_point_inv.square()), |evaluation_point| { + Some(evaluation_point.square()) + }) + .take(fri_layers_merkle_roots.len()) + .collect(); + + // Reconstruct p₁(𝜐²) + let mut v = + (p0_eval + p0_eval_sym) + evaluation_point_inv * &zetas[0] * (p0_eval - p0_eval_sym); + let mut index = iota; + + let openings_ok = fri_layers_merkle_roots + .iter() + .zip(fri_decommitment.layers_evaluations_sym()) + .zip(evaluation_point_vec) + .enumerate() + .fold( + true, + |result, (i, ((merkle_root, evaluation_sym), evaluation_point_inv))| { + let openings_ok = Self::verify_fri_layer_openings( + merkle_root, + fri_decommitment.layer_auth_path(i), + &v, + evaluation_sym, + index, + ); + + // Update `v` with next value pᵢ₊₁(𝜐^(2ⁱ⁺¹)). + v = (&v + evaluation_sym) + + evaluation_point_inv * &zetas[i + 1] * (&v - evaluation_sym); + + index >>= 1; + + result & openings_ok + }, + ); + + let terminal_ok = terminal_codeword.get(index).is_some_and(|t| &v == t); + openings_ok & terminal_ok +``` + +**Degenerate branch you must emit** (`verifier.rs:683-690`): when `zetas.is_empty()` (`total_folds == 0`, clamp case) the terminal codeword *is* p₀, and the check is `terminal[2·iota] == p₀ ∧ terminal[2·iota+1] == p₀ˢ`. Not reachable under production presets (see §7) but present. + +**Structural pre-checks that must precede the loop** (`verifier.rs:426-448`) — all three are soundness-critical and reject rather than panic: +- `fri_layers_merkle_roots().len() == num_committed` +- `fri_final_poly_coeffs().len() == 1 << effective_k` +- every query's `layers_auth_paths_len() == num_committed` **and** `layers_evaluations_sym().len() == num_committed`. The comment at 434-441 is explicit: these vecs are *not* Fiat-Shamir-bound, so this length check is the only thing pinning them. + +--- + +## 2. Layer commitments and the stop condition ✓ VERIFIED + +Single source of truth: `FriFoldLayout::new`, `crypto/stark/src/fri/terminal.rs:45-54`: + +```rust + pub(crate) fn new(lde_log: u32, blowup_log: u32, k: u32) -> Self { + let terminal_log = (blowup_log + k).min(lde_log); + let total_folds = lde_log - terminal_log; + Self { + total_folds, + num_committed: total_folds.saturating_sub(1) as usize, + terminal_len: 1usize << terminal_log, + effective_k: terminal_log - blowup_log, + } + } +``` + +Verifier binding (`verifier.rs:375-382`): `k = air.options().fri_final_poly_log_degree`, `blowup_log = (lde_length/trace_length).trailing_zeros()`, `lde_log = lde_length.trailing_zeros()`. + +With `n = log₂(lde_length)`, `b = log₂(blowup)`, `k = 7`: + +| quantity | value | +|---|---| +| `terminal_log` | `min(b+k, n)` | +| `total_folds` | `n − b − k` | +| `num_committed` (= Merkle roots = auth paths per query) | `n − b − k − 1` | +| `terminal_len` | `2^(b+k)` | +| `effective_k` | `k` (unclamped) | +| `zetas.len()` | `num_committed + 1` | + +**Yes, there is an "early stop at k=7", and it is universal.** `DEFAULT_FRI_FINAL_POLY_LOG_DEGREE: u8 = 7` (`crypto/stark/src/proof/options.rs:93`) is written into every constructor: `default_test_options` (:72), `GoldilocksCubicProofOptions::with_params` (:132), and `MIN_PROOF_OPTIONS` (`prover/src/recursion.rs:44`). Folding stops at codeword length `2^(b+7)` and the prover ships `2^7 = 128` coefficients instead of folding to a constant. + +`.min(lde_log)` is the tiny-trace clamp: only when `n ≤ b+7`, i.e. trace_bits ≤ 7. Then `effective_k = n − b < k`, `total_folds = 0`, no zetas, no layers. + +Prover/verifier symmetry: `commit_phase_from_evaluations` (`crypto/stark/src/fri/mod.rs:76-118`) runs `num_committed` commit iterations then **one extra unconditional final fold** if `total_folds > 0` — that final fold is never Merkle-committed. The verifier mirrors this in the transcript replay (`verifier.rs:1463-1483`): one zeta per root, then `if total_folds > 0 { zetas.push(sample) }`. + +--- + +## 3. The fold ✓ VERIFIED + +`crypto/stark/src/fri/fri_functions.rs:8-59`, in full: + +```rust +/// Evaluation-form FRI fold: given evaluations in bit-reversed order where +/// consecutive pairs (2j, 2j+1) are conjugates (p(x_j), p(-x_j)), compute +/// the folded evaluations: (lo + hi) + inv_twiddle[j] * zeta * (lo - hi) +/// = 2 * (p_even(x_j²) + zeta * p_odd(x_j²)) +pub(crate) fn fold_evaluations_in_place, E: IsField>( + evals: &mut Vec>, + zeta: &FieldElement, + inv_twiddles: &[FieldElement], +) { + let half = evals.len() / 2; + for j in 0..half { + let lo = &evals[2 * j]; + let hi = &evals[2 * j + 1]; + let sum = lo + hi; + let diff = lo - hi; + evals[j] = &sum + &(&inv_twiddles[j] * &(zeta * &diff)); + } + evals.truncate(half); +} + +pub(crate) fn compute_coset_twiddles_inv( + coset_offset: &FieldElement, + domain_size: usize, +) -> Vec> { + let half = domain_size / 2; + let order = domain_size.trailing_zeros() as u64; + let mut points = get_powers_of_primitive_root_coset(order, half, coset_offset).unwrap(); + in_place_bit_reverse_permute(&mut points); + FieldElement::inplace_batch_inverse(&mut points).unwrap(); + points +} + +pub(crate) fn update_twiddles_in_place(twiddles: &mut Vec>) { + let new_len = twiddles.len() / 2; + for j in 0..new_len { + twiddles[j] = twiddles[2 * j].square(); + } + twiddles.truncate(new_len); +} +``` + +**Formula: `f(j) = (lo + hi) + x⁻¹·ζ·(lo − hi)`.** + +- **UNNORMALIZED.** No division by 2. The result is `2·(p_even(x²) + ζ·p_odd(x²))` — the factor 2^i accumulates across layers and is absorbed identically on both sides. Do not "fix" this; the terminal comparison is against the prover's own accumulated scaling. +- The point enters as its **inverse**, multiplied into the odd part. Association in the verifier is `(x⁻¹ · ζ) · diff` (`verifier.rs:701, 729`) — a base×ext mul followed by an ext×ext mul. Match this exactly if you care about bit-exactness of intermediate representations; the field result is associative but your chip decomposition may not be. +- Verifier form is `v ← (v+sym) + x⁻¹·ζ·(v−sym)` — same shape, with `v`/`sym` in place of `lo`/`hi`. + +### ⚠ The parity/sign compensation — critical, and non-obvious ✓ VERIFIED + +The prover's `inv_twiddles[j]` is the inverse of the point at the **even** slot `2j`. The verifier's `evaluation_point_vec[i] = υ^(−2^(i+1))` is the inverse of the point at the **query's own** position `iota>>(i+1)`, which is the odd slot whenever the relevant index bit is 1. + +I traced this. With `x_j = offset·ω_N^{br_m(j)}` (`m = log₂N − 1`) and `br_m(2j) = br_{m−1}(j)`, the two differ by exactly `(−1)^{bit}`. But when the query sits in the odd slot, `(v, sym) = (hi, lo)`, so `(v − sym) = −(lo − hi)`. The two sign flips cancel: + +``` +(hi + lo) + (−x⁻¹)·ζ·(hi − lo) = (lo + hi) + x⁻¹·ζ·(lo − hi) +``` + +**Consequence for LFM: the fold arithmetic requires NO parity branch.** You derive `υ⁻¹` once and square repeatedly. Parity is consulted *only* for leaf ordering in the Merkle check (§4). + +--- + +## 4. Per-layer Merkle authentication ✓ VERIFIED + +`verify_fri_layer_openings`, `crypto/stark/src/verifier.rs:626-649`: + +```rust + let evaluations = if iota % 2 == 1 { + vec![evaluation_sym.clone(), evaluation.clone()] + } else { + vec![evaluation.clone(), evaluation_sym.clone()] + }; + + verify_merkle_path::>( + auth_path_sym, + merkle_root, + iota >> 1, + &evaluations, + ) +``` + +- **Leaf** = the conjugate pair `{p_i(υ^(2^i)), p_i(−υ^(2^i))}` ordered so the **even codeword slot comes first**. `iota` here is the running `index`, not the original query challenge. +- **Byte layout** = 48 bytes: two `Degree3GoldilocksExtensionField` elements, each 24 bytes = three Goldilocks limbs in **component order 0,1,2**, each 8 bytes **big-endian** from `canonical_u64()`. Cited: `crypto/math/src/field/extensions_goldilocks.rs:497-503` (`write_bytes_be`), `:567-571` (`stream_bytes` → same 24 bytes), `crypto/math/src/field/goldilocks.rs:493-495`. One keccak-256 absorb of 48 bytes = **1 permutation** (rate 136). +- **Index** = `index >> 1`; **evolution** = `index >>= 1` after each layer (`verifier.rs:735`), starting at `index = iota`. +- **Tree** = one independent tree per layer, `2^(n−i−2)` leaves, root at `fri_layers_merkle_roots[i]`. + +### Which backend — the answer is "both, and they are byte-identical" + +This is the sharp edge you flagged, and the two sides genuinely use **different types**: + +| side | type | citation | +|---|---|---| +| prover commit | `FriLayerMerkleTree = MerkleTree>` | `crypto/stark/src/config.rs:23-24`, used at `crypto/stark/src/fri/mod.rs:100` | +| verifier | `BatchedMerkleTreeBackend` = `BatchKeccak256Backend` = `FieldElementVectorBackend` | `crypto/stark/src/config.rs:19-20`, used at `verifier.rs:643` | + +They agree because both leaf hashes stream the same bytes into one fresh keccak: + +- `FieldElementPairBackend::hash_data` (`crypto/crypto/src/merkle_tree/backends/field_element_vector.rs:122-127`) streams `input[0]` then `input[1]`. +- `FieldElementVectorBackend::hash_data` (`:193-198`) delegates to `hash_data_from_slices(input, &[])` (`:173-179`), which streams every element of `a` then `b`. +- `hash_new_parent` is literally the same function in both (`:129-131` and `:200-202` both call `hash_new_parent_bytes`). + +Both are `FieldElement*Backend` (`crypto/crypto/src/merkle_tree/backends/types.rs:12,15`). + +**It is NOT the trace's `commit_bit_reversed` + `ROWS_PER_LEAF=2` scheme.** The distinction is real and you must emit them differently: + +- Trace/composition leaves (`crypto/stark/src/commitment.rs:81-91`) apply `reverse_index(rows_per_leaf*leaf_idx + k, num_rows)` **inside** the leaf builder, and concatenate **column-by-column across all columns** for two rows. Leaf size = `2 · num_cols · byte_len`. +- FRI layer leaves (`crypto/stark/src/fri/mod.rs:96-99`) take `evals.chunks_exact(2)` of an **already bit-reversed single codeword** — no permutation applied at commit time, exactly one column. Leaf size = 48 bytes, always. + +Path verification fold is shared (`crypto/crypto/src/merkle_tree/proof.rs:31-51`): `index % 2 == 0 ? H(acc‖sib) : H(sib‖acc)`, `index >>= 1`, compare to root. Path length = `log₂(num_leaves)` — no length field, no domain separation, no leaf-index in the hash. + +--- + +## 5. The terminal polynomial ✓ VERIFIED + +`crypto/stark/src/fri/terminal.rs` has both directions. The **verify** side is `terminal_codeword_from_coeffs` (`:125-156`), called once per proof at `verifier.rs:450-456`: + +```rust + let terminal_offset = domain.coset_offset.pow(1u64 << layout.total_folds); + let terminal_codeword = + crate::fri::terminal::terminal_codeword_from_coeffs::( + proof.fri_final_poly_coeffs(), + &terminal_offset, + layout.terminal_len, + ); +``` + +and its body (`terminal.rs:134-155`): + +```rust + assert!( + !coeffs.is_empty() + && coeffs.len().is_power_of_two() + && codeword_len.is_power_of_two() + && coeffs.len() <= codeword_len + && codeword_len.is_multiple_of(coeffs.len()), + ... + ); + + let poly = Polynomial::new(coeffs); + let blowup = codeword_len / coeffs.len(); + + // Step 1: coset FFT to get natural-order evaluations. + let mut natural = + Polynomial::evaluate_offset_fft::(&poly, blowup, Some(coeffs.len()), terminal_offset) + .expect("terminal coset size must be a power of two within the field's two-adicity"); + + // Step 2: convert natural order to bit-reversed (FRI) order. + in_place_bit_reverse_permute(&mut natural); + natural +``` + +**It is an FFT, not a per-point polynomial evaluation and not a coefficient comparison.** The final check is `terminal_codeword.get(index).is_some_and(|t| &v == t)` (`verifier.rs:746`) — a single array lookup and extension-field equality, done once per query against a codeword materialized once per proof. + +Cost breakdown: `evaluate_offset_fft` = `poly.scale(offset)` then `evaluate_fft` (`crypto/math/src/polynomial.rs:325-326`), i.e. `2^k` ext×base scalings plus a `terminal_len`-point extension-field FFT, plus a `terminal_len` bit-reverse permute. `terminal_offset` is a base-field `pow` with exponent `2^total_folds` ≈ `total_folds` squarings (square-and-multiply, `crypto/math/src/field/traits.rs:122-142`). + +⚠ Design note for LFM: this is the point where my earlier sim/24 measurement applies — replacing this FFT with per-point Horner **regressed +20M cycles**. Emit the FFT. + +The assert at `:134` is unreachable in the verifier flow because `verifier.rs:431` length-checks `coeffs` first — but if your emitter reorders those, you convert a rejection into a panic. + +--- + +## 6. The evaluation point per layer ✓ VERIFIED + +`query_challenge_to_evaluation_point`, `verifier.rs:489-496`: + +```rust + let raw = iota * 2 + if sym { 1 } else { 0 }; + domain.lde_coset_element(reverse_index(raw, domain.lde_length as u64)) +``` + +with `lde_coset_element(i) = coset_offset · lde_primitive_root^i` (`crypto/stark/src/domain.rs:116-118`) and `reverse_index(i, size) = i.reverse_bits() >> (usize::BITS − size.trailing_zeros())` (`crypto/math/src/fft/bit_reversing.rs:15-21`). + +**Yes — this is exactly the `υ = offset · g^{br(2·iota)}` convention documented in `prover/src/lfm/sub_proof.rs:47-52`.** Identical function, identical bit-reversal width (`lde_length`). Your `pow_bits` construction is faithful to production. + +The symmetric point: `br(2·iota+1) = br(2·iota) + L/2` and `g^{L/2} = −1`, so `−υ`. Confirmed by `sym: bool` selecting `raw = 2·iota+1`, and used with `sym=true` on the DEEP side only. + +**How it changes across layers — this is the part that saves you work.** The verifier never re-derives a point. It computes `υ⁻¹` once (batch-inverted across all queries, `verifier.rs:459-467`) and then produces the whole chain by repeated squaring (`verifier.rs:692-697`): + +``` +evaluation_point_vec[i] = υ^(−2^(i+1)), i = 0..num_committed−1 +``` + +So layer `i`'s point is `υ^(2^(i+1))` — **no bit-reversal, no domain lookup, no coset offset, past the first point**. One base-field squaring per layer. Combined with §3's sign result, the entire per-layer point derivation is: one squaring, and nothing else. + +The base-field batch inverse (`verifier.rs:459-467`) is over all `Q` queries at once and **fails closed**: `if inplace_batch_inverse(...).is_err() { return false }` — a zero evaluation point (malformed index) rejects rather than panics. + +--- + +## 7. Degenerate parameters — what is actually constant ✓ VERIFIED + +`ProofOptions` fields consumed by the FRI verify path (`crypto/stark/src/proof/options.rs:52-61`): + +| parameter | production values | constant? | +|---|---|---| +| `fri_final_poly_log_degree` (k) | **7** — always | ✅ **CONSTANT across every config in the repo** | +| `coset_offset` | **3** — always | ✅ **CONSTANT** | +| `blowup_factor` | 2, 4, 8 (min uses 2) | ❌ varies (3 values) | +| `fri_number_of_queries` | 219 / 110 / 73 (min: 1) | ❌ varies, but **fully determined by blowup** | +| `grinding_factor` | 20 (min: 1) | ✅ effectively constant at 20 in all secure presets | +| layer count `num_committed` | `n − b − 8` | ❌ varies with trace size | + +Sources: `MIN_PROOF_OPTIONS` (`prover/src/recursion.rs:39-45`), `Preset` (`prover/src/recursion.rs:53-86`), `GoldilocksCubicProofOptions::with_params` (`crypto/stark/src/proof/options.rs:106-134`), `DEFAULT_FRI_FINAL_POLY_LOG_DEGREE` (`:93`), `DEFAULT_GRINDING = 20` (`:96`). + +Query counts are computed, not stored — I recomputed the JBR formula (`:121-125`) and it reproduces the doc comments exactly: blowup 2 → 219, blowup 4 → 110, blowup 8 → 73. + +Derived per-preset FRI shape: + +| preset | b | terminal_log | terminal_len | coeffs | queries | num_committed | +|---|---|---|---|---|---|---| +| Min | 1 | 8 | 256 | 128 | 1 | trace_bits − 8 | +| Blowup2 | 1 | 8 | 256 | 128 | 219 | trace_bits − 8 | +| Blowup4 | 2 | 9 | 512 | 128 | 110 | trace_bits − 8 | +| Blowup8 | 3 | 10 | 1024 | 128 | 73 | trace_bits − 8 | + +Note the invariant: **`num_committed = trace_bits − 8` for every preset**, since `n = trace_bits + b` cancels `b`. + +### ⚠ What a differential over real proofs CANNOT distinguish + +This is the answer you actually need. Because `k = 7` and `coset_offset = 3` are **hardcoded constants with no production variation**, a differential test over real proofs is blind to: + +1. **Any k-dependent logic.** An implementation that hardcodes `terminal_log = b + 7`, hardcodes 128 coefficients, or hardcodes `terminal_len ∈ {256,512,1024}` is indistinguishable from one that reads `k` from the AIR. Only `crypto/stark/src/tests/small_trace_tests.rs:177` (k=0), `:215` (k=63), `:720` (k=6) exercise other values. +2. **The clamp path** (`.min(lde_log)`, `terminal.rs:46`). Requires trace_bits ≤ 7. Never reached in production. +3. **The `zetas.is_empty()` no-fold branch** (`verifier.rs:683-690`). Same condition. Dead in production. +4. **`effective_k ≠ k`.** Only occurs under the clamp. Production always has `effective_k == 7`, so an implementation that conflates the two passes everything. +5. **Any `coset_offset ≠ 3` handling**, including the `evaluate_offset_fft` offset path and `terminal_offset = 3^(2^total_folds)`. +6. **Grinding-factor variation.** Only 20 and 1 appear. + +Recommendation: build the differential over **synthetic proofs at k ∈ {0, 6, 7, 63} and trace_bits ≤ 7**, using the fixtures already in `small_trace_tests.rs`, or accept that those branches are unexercised and pin them with structural assertions instead. + +--- + +## 8. Counts for sizing — DERIVED ✓ VERIFIED + +Let `n = log₂(lde_length)`, `b = log₂(blowup)`, `k = 7`, `Q` = query count, `C = num_committed = n − b − k − 1`. + +### Merkle path steps + +Layer `i` codeword length = `2^(n−i−1)`; leaves = `2^(n−i−2)`. Path length = `log₂(leaves)`: + +$$\text{pathlen}(i) = n - i - 2$$ + +Derived from `build_merkle_path` (`crypto/crypto/src/merkle_tree/merkle.rs:271-288`) walking `pos → parent_index(pos)` until `ROOT`, over a tree with `2·leaves − 1` nodes (`:199-200`); the leaf count is already a power of two so `complete_until_power_of_two` (`:194`) is a no-op. + +**Per query, total path steps:** + +$$\sum_{i=0}^{C-1}(n-i-2) \;=\; C(n-2) \;-\; \frac{C(C-1)}{2}$$ + +Last layer's path length is `n − C − 1 = b + k` — consistent with its `2^(b+k)` leaves. ✓ + +**Keccak permutations per query** = `C` leaf hashes (48 B → 1 perm each) + path parents (64 B → 1 perm each): + +$$\text{perms/query} \;=\; C \;+\; C(n-2) - \tfrac{C(C-1)}{2}$$ + +Worked example, trace_bits = 20, Blowup2 (`n=21, C=12`): path steps = `12·19 − 66 = 162`; perms/query = `174`; × 219 queries = **38,106 keccak permutations** for the FRI leg alone. + +### Field operations + +**Per query, per fold** (`verifier.rs:701` and `:729-730`, identical shape): +- 2 ext additions, 1 ext subtraction +- 1 base×ext multiplication (`x⁻¹ · ζ`) +- 1 ext×ext multiplication + +There are `C + 1` folds. Total per query: + +$$(C+1)\times(2\ \text{ext-add} + 1\ \text{ext-sub} + 1\ \text{base}\!\times\!\text{ext} + 1\ \text{ext}\!\times\!\text{ext})$$ + +**Per query, point chain** (`verifier.rs:692-697`): `C` base-field squarings. + +**Per query, initial point** (`verifier.rs:462, 495`): one `reverse_index` (bit ops), one base-field `pow` with an `n`-bit exponent ≈ `n` squarings + ≤ `n` muls (square-and-multiply, `crypto/math/src/field/traits.rs:134-142`), one base mul by `coset_offset`. + +**Amortized per query** (`verifier.rs:465`, `crypto/math/src/field/element.rs:90-108`): batch inverse over `Q` base elements = `3(Q−1)` muls + 1 inversion → ~3 base muls/query. + +**Once per proof:** `terminal_offset` pow ≈ `total_folds` base squarings; `Polynomial::new` + `scale` = `2^k` ext×base muls + `2^k` base muls for the geometric offset powers; one `terminal_len`-point extension FFT ≈ `(terminal_len/2)·log₂(terminal_len)` butterflies; one `terminal_len` bit-reverse permute. For Blowup2: 1024 butterflies over Ext3. For Blowup4: 2304. + +**Terminal check per query:** 1 bounds-checked index + 1 ext equality (3 base comparisons). + +--- + +## Emission checklist (things that will silently break bit-exactness) + +1. Fold is **unnormalized** — no `/2`. §3. +2. Mul association is `(x⁻¹ · ζ) · diff`, base×ext then ext×ext. §3. +3. **No parity branch in the fold**; parity branch **only** in leaf ordering. §3, §4. +4. Leaf = 48 bytes, Ext3 components 0,1,2, each 8 B big-endian, even codeword slot first. §4. +5. FRI leaves are pairs of an **already-bit-reversed** codeword — do not re-apply `reverse_index` the way the trace commitment does. §4. +6. Terminal is an **FFT**, not Horner. §5. +7. `index` starts at `iota` (not `2·iota`) and the Merkle position is `index >> 1`. §1, §4. +8. Folds = layers **+ 1**; the first fold has no Merkle check. §1. +9. The three structural length checks must run **before** the query loop. §1. +10. `zetas[i+1]` in the loop, `zetas[0]` for the first fold — off-by-one here verifies nothing. §1. + +--- + +# Addendum — the fri leg's own measurements and decisions + +Appended 2026-07-31 by the fri leg. The spec above is the scout's; this section +is FIRST-HAND from this worktree and is where the two disagree or the spec is +silent. + +## ★ The fixture folds NOTHING — the leg's instrument problem + +**Measured, not inferred**, off the real proof: `fri_layers_merkle_roots = 0`, +`fri_final_poly_coeffs = 4`, 219 query decommitments. Pinned by +`join_tests::the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold`. + +The fixture is the `min` preset over a `2^4`-step epoch, so its sub-proof has +`log2(lde) = 3`; §2's arithmetic gives `terminal_log = min(1+7, 3) = 3`, +`total_folds = 0`, `num_committed = 0`, and `query_phase` takes its +empty-decommitment branch. + +This is §7's blindness taken one step further. §7 says a differential over real +proofs cannot distinguish implementations that differ only off `k = 7` / +`coset_offset = 3`. On the fixture specifically it is worse: **the production +instance exercises none of the mechanism at all** — no fold, no walk, no +terminal lookup. An emitter differentialled only against it would fold nothing +and pass everything. + +Consequence: the primary instrument is synthetic codewords driven through +production's own `commit_phase_from_evaluations` + `query_phase`, differentialled +against the verifier's own check, with `num_committed` swept. Only the INPUT is +synthetic; it remains a differential against production code. + +## Correction to §4 — and to what I first reported + +I told the team lead the FRI layer leaf is committed under +`PairKeccak256Backend` and **"not the trace's `BatchedMerkleTreeBackend`"**. +That is true prover-side and **wrong as a statement about the verify path**, +which is what the machine emits. §4's "both, and they are byte-identical" is the +correct account: `verify_fri_layer_openings` (`verifier.rs:643`) calls +`verify_merkle_path::>` over a +two-element vector. The emitted leaf bytes are unaffected — both stream the two +elements into one fresh keccak — but the claim as I stated it was wrong. + +The verify-side reading also surfaces something the prover-side reading hides: +**the leaf ordering is parity-dependent** (`verifier.rs:637-641`, +`if iota % 2 == 1 { [sym, v] } else { [v, sym] }`). The verifier holds the +folded `v` and receives `sym`, so the machine must SELECT the order on the low +index bit. Reading only the prover's `chunks_exact(2)` would have missed it. + +## Predictions, written BEFORE measuring + +Per §8, with `n = trace_bits + b`, `C = num_committed = trace_bits − 8` at +`k = 7`, `steps = C(n−2) − C(C−1)/2`, `perms/query = C + steps`: + +| blowup | b | n | C | Q | steps/q | perms/q | FRI perms/epoch-table | +|--------|---|----|----|-----|---------|---------|-----------------------| +| 2 | 1 | 21 | 12 | 219 | 162 | 174 | **38,106** | +| 4 | 2 | 22 | 12 | 110 | 174 | 186 | **20,460** | +| 8 | 3 | 23 | 12 | 73 | 186 | 198 | **14,454** | + +at `trace_bits = 20`. The blowup-2 row reproduces §8's worked example exactly, +which is the check that the formula is being read as written rather than +re-derived by guess. + +⚠ This also corrects an arithmetic slip in my first report to the team lead: I +quoted "180 path steps, ≈192 perms/query" for blowup 8. The correct figures are +**186 and 198** — I mis-summed `Σ_{i=0}^{11}(21−i)`. + +**Standalone result worth carrying out of this leg:** FRI is **2.6× cheaper at +blowup 8 than at blowup 2** (14,454 vs 38,106), because the query count falls +3× while per-query cost rises only 14%. The blowup-8 decision was made on DEEP +and on the keccak bill; this is an independent third leg pointing the same way. + +## Decision on §7's dead branches — deferral with its argument + +§7 lists the clamp path, the `zetas.is_empty()` no-fold branch, and +`effective_k != k` as unreachable under production presets. The team-lead +charter requires either synthetic coverage or a structural pin. **Neither +option is quite right for this machine, and the reason is worth stating.** + +In LFM, shape is COMPILE-TIME. `num_committed`, `terminal_len` and +`effective_k` are program constants, so none of these is emitted control flow — +there is no branch in the program to leave dead. What exists instead is: + +1. **Host-side shape arithmetic** in the emitter (the `FriFoldLayout` + computation, clamp included). This is ordinary Rust running at program-build + time, so it is covered by ordinary unit tests over synthetic + `(trace_bits, blowup, k)` — including `k ∈ {0, 6, 63}` and `trace_bits ≤ 7` — + at no proving cost. This is where §7's requirement is discharged. +2. **Two program SHAPES**: `num_committed = 0` and `num_committed > 0`. Both are + emitted and both are differentialled. The zero case is not a dead branch to + pin — it is the fixture's own shape and a real production path for small + tables. + +So: no dead program text is emitted, and nothing is left unexercised. The one +thing genuinely NOT covered is a proof whose `coset_offset ≠ 3`, because no +production configuration produces one and the LDE domain constants are baked +into the program; that is a deferral, and its safety argument is that a wrong +coset offset changes every domain point and therefore every leaf, so it cannot +produce a passing proof — it can only fail. Stated rather than assumed. + +--- + +# Addendum 2 — the emitter, and the correction the emitter forced + +Appended 2026-08-03 by the fri-emitter agent. FIRST-HAND from this worktree +unless marked otherwise. Where this and the sections above disagree, this is +later and was measured. + +## ★ The blindness premise above is FALSE, and cheaply so + +§7 and Addendum 1 conclude that no real proof can witness the fold mechanism, +and that synthetic codewords through production's commit phase are the only +instrument. That is true of the fixture AS WRITTEN and false of the fixture as +available. The L2G trace is `boundaries.len().next_power_of_two()` +(`prover/src/tables/local_to_global.rs:269`) and `num_committed = trace_bits − 8`, +so asking `real_fixture`'s own construction for more boundaries produces real +production proofs that fold: + +| boundaries | n | total_folds | num_committed | coeffs | zetas | queries | +|-----------:|---:|------------:|--------------:|-------:|------:|--------:| +| 4 | 3 | 0 | 0 | 4 | 0 | 219 | +| 512 | 10 | 2 | 1 | 128 | 2 | 219 | +| 1024 | 11 | 3 | 2 | 128 | 3 | 219 | +| 2048 | 12 | 4 | 3 | 128 | 4 | 219 | + +Real committed roots, real authentication paths at real depths, real terminal +coefficients, and the folding challenges out of production's own +`replay_rounds_after_round_1`. All four prove in well under a second. +`fri_tests` uses these throughout; **nothing in the FRI leg is synthetic.** + +The consequence that matters: the `num_committed = total_folds.saturating_sub(1)` +off-by-one — Addendum 1's headline example of a soundness-relevant constant +invisible to real data — is now caught by SEVEN tests on real proofs, including +an executed walk that cannot reach a committed root. Addendum 1's conclusion was +right about the 4-row fixture and wrong about the prover. + +`join_tests::the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold` +stays as written: it is still true of that fixture and still worth announcing if +it changes. Only its reasoning about what real data *could* do is superseded. + +## Deviations from the emission checklist, with derivations + +**Checklist item 6 — "Terminal is an FFT, not Horner" — OVERRIDDEN.** The +emitter evaluates the terminal polynomial at `υ^(2^total_folds)` per query and +never materializes the codeword. The sim/24 measurement behind item 6 is sound +and does not transfer, because the two machines disagree about the price of an +array index: production's `terminal_codeword.get(index)` is one load, while a +straight-line machine with no addressable memory needs a `Select` tree +`terminal_len − 1` wide. At blowup 8 that is 1,023 selects per query (74,679 at +73 queries), against which the FFT — 5,120 butterflies at ~3 rows — is the +smaller half of the bill. Evaluating costs `total_folds` squarings plus +`2^effective_k − 1` ext `MulAdd`s: 140 rows per query, ~10,220 at 73 queries, +against ~90,000. The direction reverses because the guest amortizes one FFT +across queries and pays nothing per lookup, and this machine pays nothing for +the FFT it does not run and everything for the lookup it cannot do. + +Equivalence, and it is exact rather than approximate: the terminal codeword is +`P` over the terminal coset in bit-reversed order, so position `index` holds +`P(terminal_offset · ω_T^{br(index)})`, and with `index = iota >> C`, +`terminal_offset = coset_offset^(2^total_folds)` and `ω_T = g^(2^total_folds)` +that point is exactly `υ^(2^total_folds)` — the bits of `iota` the shift keeps +are the bits `br` puts inside `ω_T`'s order. Checked against production's own +`evaluate_offset_fft` + `in_place_bit_reverse_permute` at all 219 indices of all +four shapes (`the_terminal_point_is_the_query_point_folded`). + +A bonus the FFT form does not have: **the two branches unify.** At +`total_folds = 0` the exponent is 1, the two positions `2·iota` and `2·iota+1` +are `υ` and `−υ`, and production's `zetas.is_empty()` branch becomes "evaluate +`P` twice" rather than a separate code path. And the terminal point is BOUND to +the query point by construction instead of by a second derivation. + +**Checklist item 2 — mul association — DEVIATED, deliberately.** The emitter +reuses `edsl::fri_fold`, which computes `(ζ·diff)·x⁻¹` where production writes +`(x⁻¹·ζ)·diff`. Same operation count (one `Mul`, one `MulBase`), same field +element — Fp3 multiplication in the chip is exact, so associativity is exact — +and reusing the tested primitive avoids re-deriving a registry digest. The +real-proof differential is what says the values agree. + +**Checklist item 9 — "the three structural length checks must run before the +query loop" — DISCHARGED BY CONSTRUCTION, not by emitted checks.** `declare_fri` +fixes each arena's length from the shape and the executor refuses any other +length before an instruction runs. The attack production's comment names — send +the per-query vectors EMPTY so the fold loop runs zero iterations and accepts +vacuously — has no encoding here. Demonstrated in +`the_shape_pins_the_lengths_production_must_check_at_runtime`. + +Items 1, 3, 4, 5, 7, 8, 10 are emitted as written and each was falsified +individually (see the leg's report). + +## Measured against Addendum 1's pinned prediction — exactly + +| blowup | perms/query | predicted | per sub-proof | predicted | +|-------:|------------:|----------:|--------------:|----------:| +| 2 | 174 | 174 | 38,106 | 38,106 | +| 4 | 186 | 186 | 20,460 | 20,460 | +| 8 | 198 | 198 | 14,454 | 14,454 | + +At `trace_bits = 20`, counted as emitted `LFM_KECCAK` rows, marginal per query. +The same formula on EXECUTED programs at n = 10/11/12 gives 1,971 / 4,161 / 6,570 +permutations for 219 queries, also exactly. + +Where the leg's rows go depends on the currency, and the two currencies point +OPPOSITE ways — the trap `edsl::keccak_leaf_hash`'s own doc comment warns about, +met again here. At blowup 8, `trace_bits = 20`, per query: 8,682 instructions, +198 permutations, 73 byteswaps (`1 + 6 · num_committed`: two extension values, +three components each, per layer leaf, plus the index). + +- **In INSTRUCTIONS the byteswaps dominate.** Each is one `LFM_BITDEC` row plus + 64 `LFM_BALU` rows, so 73 of them are ~4,745 of the 8,682 — about 55%. +- **In main-trace CELLS the hashing dominates overwhelmingly.** A permutation is + 36,256 cells and a byteswap 322, so 198 permutations are 7.18M cells against + 23.5K — hashing is ~305× the swap bill. + +Neither number is the leg's cost on its own; which one binds depends on which +chip is the constraint. The rows-of-different-chips-are-not-comparable rule in +`others/lfm-target-shape.md` is the reason both are reported. What is not +avoidable either way: the same values are consumed as field elements by the fold +and as bytes by the hash, so something must connect the two representations. + +Addendum 1's 2.6× blowup-8 advantage is unaffected and confirmed. + +## ★ A new instance of method rule 7: a subtraction of two emissions IS a differential + +The leg's structural guard — "the FRI join adds no second point derivation" — +was written as `selects(joined) − selects(trace_only) == C + 2·path_steps`, on +the reasoning that a second derivation would add `index_bits`. That guard is +VACUOUS, and injecting the exact defect it denies (a `QueryOutput` handing out a +freshly derived point) left it green. The defect lives in +`emit_sub_proof_with_bits`, which both sides of the subtraction call, so both +gained `index_bits` selects and the difference never moved. + +Rule 7 says a relative test dies when its two sides unify. The instance worth +adding is that **a difference of two counts taken from our own emitter is still +a relative test**, however much it looks like a count — the marginal-cost idiom +this phase uses everywhere (`marginal()` in `join_tests`, `marginal_fri` here) is +safe only when the result is compared against a number that did not come from +the emitter. The fix was to write the count as a closed form over the SHAPES +(`index_bits + 2·merkle_depth·num_groups + num_committed + 2·path_steps`) and +compare against that; it then fails with a surplus of exactly `index_bits`. + +Related trap, hit in the same session: the falsification harness itself reported +all seven deliberate breakages as "nothing failed", because `cargo test -q` +names failures only in its trailing summary block and the parser was looking for +per-test `FAILED` lines. Rule 3 applies to instruments, not just to oracles — +and "my breakage changed nothing" is the reading that should always be checked +against the tool before it is believed. diff --git a/others/lfm-hash-matrix-scope.md b/others/lfm-hash-matrix-scope.md new file mode 100644 index 000000000..a9a2001aa --- /dev/null +++ b/others/lfm-hash-matrix-scope.md @@ -0,0 +1,1480 @@ +# The hash matrix — slice 0 scoping report + +Wave 8 (`[hash-w8]`), 2026-08-04, branch `feat/lfm-hash-matrix` off `feat/lfm-assembly` +@ 891f534f. Written BEFORE any hash was built, which is the point: it exists to be +falsified by wave 9's measurement rather than to stand in for it. + +Provenance is marked throughout (standing-decisions method rule 6). **VERIFIED** +means I read the code and cite it. **DERIVED** means arithmetic over verified +numbers, shown so it can be rechecked. **INHERITED** means it comes from another +agent's report or the team lead's notes and I did not re-establish it. + +--- + +## 0. Headline + +**The machine already has a hash swap surface, and it is NOT the socket keccak is +plugged into.** `prover/src/lfm/hash.rs` is titled "The LFM hash interface — the +machine's swap surface" and freezes a contract — the `LfmHasher` trait, a 12-felt +sponge state, a 4-felt digest, and the `LFM_HASH` bus tuples and opcode numbers — +behind which sits `TestPermutation`, explicitly labelled **NOT cryptographic** +(`hash.rs:1-12`, `hash.rs:46-53`). Its own doc comment names the candidate set: +"Poseidon2 is broken; candidates are Poseidon-original, RPO/XHash, Monolith and +reduced-round Blake2s" (`hash.rs:3-5`). VERIFIED. + +That reframes the brief's phrase "behind the same socket". There are **two** +sockets, and they are documented as not interchangeable: + +| | socket 1 — keccak hosting | socket 2 — the `LFM_HASH` chiplet | +|---|---|---| +| what fills it | production `KECCAK_RND`/`KECCAK_RC`/`BITWISE` AIRs, unchanged, via `keccak_adapter` | one degree-3 chip, `chips::hash`, 28 value columns, 1 row per permutation | +| gadget | `edsl::keccak_merkle_walk`, `keccak256` | `edsl::merkle_walk`, `edsl::SpongeVar` | +| digest | 2 machine cells (8 felts, 32 bytes) | 1 machine cell (4 felts) | +| status | measured: 11.17 B cells / epoch verify | placeholder permutation only | + +`edsl.rs:137-143` states it outright: "the two are not interchangeable: +`merkle_walk` compresses with `LFM_HASH`/`TestPermutation`, the deliberately +non-cryptographic Milestone-C placeholder, so it can only ever authenticate the +Milestone-C fixture tree. Production trees are keccak throughout." VERIFIED. + +So a candidate column is **not** a variation on the keccak column. It is socket 2 +carrying the epoch verifier's real workload for the first time, which is why the +matrix's other columns "ARE the number" rather than a refinement of it. + +**★ THE RESULT THAT MATTERS MOST, and it is not the one I expected: every candidate +fits the 124 GiB box.** Keccak's wrap needs 290–350 GiB; the predicted candidates +span ~48 GiB (RPO) to ~79 GiB (Blake, the only one with a real in-AIR measurement). +**The hash decision is therefore not cost-gated** — it is very nearly a purely +cryptographic choice, which is a far better position than my first pass described. +Matrix in §2.3, my reversal on blake in §2.5. + +**Recommendation, in one line: build Poseidon-original first — not because it is +cheapest, but because it has a direct `poseidon1-air` donor in the vendored Plonky3 +tree, an in-tree HADES skeleton, and zero AIR data anywhere in the corpus, so its +column is the one that adds information rather than confirming it.** Reasons in §4. + +--- + +## 1. The cost model, validated against the measured keccak column + +The census formula, read from source (VERIFIED, `airs.rs:122-128`, `airs.rs:246-249`, +`airs.rs:234-236`): + +``` +main_cells = padded_rows × (NUM_COLUMNS − PREP_WIDTH) +aux_cells = padded_rows × ceil(bus_interactions / 2) +base-field-equivalent = main_cells + 3 × aux_cells (aux are cubic-extension elements) +``` + +`airs.rs:270-278` confirms `LfmChipCells.main_cols` is "the AIR's width less its +preprocessed prefix" and `aux_cols` is "one per pair of bus interactions". + +Writing the hash bill as `P × (m + 3a) × padding`, where `P` = permutations per +verify, `m` = main cells per permutation, `a` = aux cells per permutation: + +| quantity | keccak, production shape | source | +|---|---|---| +| `P` | 118,080 | INHERITED (ledger entry 10) | +| `m` | 36,256 = 736 (`LFM_KECCAK`, 1 row) + 24 × 1,480 (`KECCAK_RND`) | INHERITED, widths pinned by a test (status log 2026-07-30T15:40Z) | +| `a` | 13,912 | INHERITED (ledger entry 10) | +| `m + 3a` | **77,992** base-field-equivalent cells per permutation | DERIVED | +| padding | 1.01871 | DERIVED (see below) | + +DERIVED check — the model reproduces the ledger's measured numbers exactly: + +``` +total base-equiv = 5,077,422,224 + 3 × 2,029,461,548 = 11,165,806,868 ✓ ledger +hash base-equiv = 4,364,173,312 + 3 × 1,672,478,720 = 9,381,609,472 ✓ 84.02 % of total +non-hash residue = 11,165,806,868 − 9,381,609,472 = 1,784,197,396 (15.98 %) +P × (m + 3a) = 118,080 × 77,992 = 9,209,295,360 +implied padding = 9,381,609,472 / 9,209,295,360 = 1.01871 +``` + +The 1.87 % padding agrees with the ledger's independent statement that default +chunking wastes 1.7 % of round rows at the production shape — close but not +identical, the residue presumably being `LFM_KECCAK`'s own power-of-two padding. I +did not resolve that split and it does not move any conclusion. + +**What this model cannot see** (rule 6): it prices only the hash chips. It assumes +the non-hash residue of 1,784,197,396 cells is candidate-independent, which is +FALSE in a direction that favours every candidate — part of that residue is +byte-serialization work (`felt_be_halves`, `Unpack`s, `LFM_BITDEC`/`LFM_BALU`) that +a field-native hash deletes outright. Treating the residue as fixed therefore makes +my candidate predictions **conservative** (too big), not optimistic. It also cannot +see prover wall-time or RSS, only cells. + +### 1.1 `LFM_HASH` as it stands today + +VERIFIED from `layout.rs` (`pub mod hash`) and `chips.rs:479-523`: + +- `PREP_WIDTH = 11` — `IN_ADDR0..2`, `OUT_ADDR0..2`, `MODE_C`, `MODE_P`, `MULT0..2`. +- `NUM_COLUMNS = PREP_WIDTH + 28`; the 28 value columns are `IN0..IN11` (12), + `S8..S11` (4 materialized capacity columns), `OUT0..OUT11` (12). +- 6 bus interactions, all `BusId::LfmMem` (3 receivers, 3 senders) → `aux_cols = 3`. +- `max_degree() = 3` (`chips.rs:532-534`); one row per permutation. + +So `TestPermutation` costs `m = 28`, `a = 3`, i.e. **37 base-field-equivalent cells +per permutation** — 2,108× cheaper than keccak's 77,992. That number is a floor +with no cryptographic content whatsoever: it is ONE degree-3 round. Any real +candidate multiplies it by its round count and S-box overhead. Quoting 37 as a +candidate cost would be the single worst error available in this leg. + +### 1.2 The rate penalty — the axis that moves the WRONG way + +VERIFIED: keccak's rate is 136 bytes (`layout.rs:116`), and the machine serializes +each felt as 8 bytes (two 4-byte halves, `keccak_host.rs:15`, +`edsl.rs:128-134`). So keccak absorbs **17 felts per permutation**. + +VERIFIED: the `LFM_HASH` sponge is "state = 3 cells (rate 2, capacity 1)" +(`edsl.rs:16-17`), i.e. **8 felts per permutation**. + +DERIVED: on absorption-bound work a candidate behind socket 2 pays **2.125×** as +many permutations as keccak. Not a rounding detail — it directly offsets the +cells-per-permutation win, and it is a consequence of the FROZEN +`HASH_STATE_FELTS = 12`. + +Where it bites, and where it does not (VERIFIED against `fri.rs:139-144` and +`edsl.rs:149-155`): + +- **Merkle parent step** — keccak hashes 64 bytes = 8 felts, "64 bytes sits inside + one 136-byte rate block, so a level is exactly ONE permutation". A candidate + absorbs 8 felts into a rate of 8 → also exactly one permutation. **1:1, no penalty.** +- **FRI layer leaf** — a 48-byte pair = 6 felts, one rate block either way. **1:1.** +- **Trace leaf hash** — a row PAIR column-major over `c` columns is `2c` felts. + keccak: `ceil(16c / 136)` permutations. Candidate: `ceil(2c / 8)`. At `c = 10` + that is 2 vs 3; at `c = 1480`, 175 vs 370. **Here the candidate is up to 2.125× + worse.** +- **Transcript/spine** — absorption-bound, so ~2.125×. + +This is why `P` must be measured rather than assumed, and why I recommend pinning it +first (§4). + +--- + +## 2. Predictions + +These are the falsifiable content of this report. Every candidate row is an +ESTIMATE; the derivations are given so wave 9 can kill them with a measurement. + +### 2.1 Where each candidate's `P` comes from — and why it is measurable now + +`epoch_verify::query_permutations` is already a **closed form over shapes** +(VERIFIED, `epoch_verify.rs:414-434`): + +``` +per_query = leaf_permutations(sub) + groups × sub.merkle_depth + fri.permutations_per_query() +leaf_permutations = Σ_groups num_blocks(g.leaf_bytes()) num_blocks(n) = n/136 + 1 +``` + +Its doc comment carries the warning that matters here: "A leaf is NOT one +permutation. It covers `ROWS_PER_LEAF · num_columns` elements at 8 or 24 bytes +each … so the epoch's widest table (2,056 OOD columns) has a leaf worth hundreds of +permutations while a FRI layer's one-column leaf is worth one. Predicting the leg's +bill as 'one leaf plus one per level' undercounts it by the whole width of the +trace" (`epoch_verify.rs:404-412`). VERIFIED. + +**Consequence, and the single most useful thing in this report:** a candidate's `P` +is a function of the SHAPES only — not of any permutation's internals. Swapping +`num_blocks(bytes) = bytes/136 + 1` for `ceil(felts/8)` yields the candidate's `P` +**with no hash implemented at all**. That is a pure-arithmetic, additive, +differentially-testable slice, and it should come first (§4). + +Only the leaf and spine terms move; parent steps and FRI leaves are 1:1 (§1.2). So + +``` +P_candidate = 2.125 × (absorption-bound part of P) + 1.0 × (path-bound part of P) +``` + +and the split is exactly what the closed form computes. + +**UPDATE — slice A ran, so this axis is now MEASURED, not bounded.** I built the +rate-parameterised closed form and the numbers below come out of the suite +(`epoch_verify_tests::the_assembled_epoch_verifier_runs`, blowup 8 / 73 queries, +real trace lengths): + +``` +keccak rate 17 felts/perm: 115,413 permutations (67,671 leaves + 47,742 paths/FRI) +LFM_HASH rate 8 felts/perm: 187,902 permutations (140,160 leaves + 47,742 paths/FRI) +candidate/keccak = 1.6281x (leaf term alone 2.0712x) +absorption-bound share of the keccak bill: 58.6% widest leaf: 3,456 felts +``` + +**The keccak side reproduces the ledger exactly: 115,413 is entry 10's own legs +figure** (118,080 = 2,667 spine + 115,413 legs). That is the corroboration that +makes the candidate side trustworthy — the same function, at a different rate. + +So the interval collapses to a point: the candidate pays **1.63×** the permutations, +not the 2.125× ceiling, because 41.4 % of the keccak bill is path/FRI work that is +1:1 at any rate. Adding the spine (2,667, absorption-bound, so bounded between 1.0× +and 2.125×) gives + +``` +P_candidate ∈ [190,569 , 193,569] = 1.614x – 1.639x keccak's 118,080 +``` + +— a ±0.8 % spread, so the spine's uncertainty is immaterial and I use ~192,000. + +### 2.2 Cells per permutation + +**REVISED after the corpus extraction landed.** My first pass estimated round counts +from my own domain knowledge; the corpus supplies a MEASURED anchor that is better +than my estimate and, critically, one that normalizes onto our socket exactly. + +**The normalization is free, and this is the single luckiest fact in the leg.** The +corpus's primary hash artifact (Miden's BlakeG addendum) states: *"BlakeG keeps +Poseidon2's exact sponge geometry (state 12, rate 8, digest 4), so invocation counts +are hash-invariant and the whole price is per-invocation trace cost."* **State 12, +rate 8, digest 4 is exactly our frozen `LFM_HASH` contract** +(`HASH_STATE_FELTS = 12`, rate 2 cells = 8 felts, `HASH_DIGEST_FELTS = 4`). So the +corpus's per-2-to-1 cell figures transfer to our socket directly, and every +field-native candidate shares ONE permutation count — the `P ≈ 192,000` measured in +§2.1. + +Measured anchors from the corpus, per 2-to-1 compression (Miden, **Goldilocks — our +field**): + +| | main cells | aux (EF) | base-equiv `m+3a` | provenance | +|---|---|---|---|---| +| Poseidon2 (the dead baseline) | 256 | 16 | 304 | MEASURED, 16-col AIR × 16 rows/perm | +| BlakeG 32-row | 4,096 | 768 | **6,400** | MEASURED — 13.9× main, 48× aux | +| BlakeG 64-row (1st gen) | 5,120 | 768 | 7,424 | MEASURED | + +Plus, for any Blake-class candidate, an `And8Lookup` table AIR at a **fixed 2¹⁶ × 10 += 655,360 cells in every proof regardless of workload** (the corpus notes our `RANGE` +chip can absorb that role, so it need not add a chip). + +⚠ **My own estimate was 2× CONSERVATIVE, and I am keeping it as the pessimistic +bound rather than discarding it.** I derived ~608 main cells for Poseidon-original +in a one-row layout; Miden's measured Poseidon2 at t=12 over Goldilocks is 256 main. +Their 16 columns × 16 rows beats my unrolled 608 because a row-per-round layout +reuses the state columns instead of allocating fresh ones per round. Since the +corpus rates Poseidon-original at **≈1× Poseidon2 in-AIR** ("AIR cost is +S-box-dominated; round counts match"), 304 base-equivalent is the central case and +617 is my conservative bound. Both are in the matrix. + +⚠ **PROVENANCE of each candidate's figure, per the corpus's own marking** — this is +the part that decides how much weight each column carries: + +- **Blake: the ONLY candidate with a real in-AIR measurement.** 13.9× main / 48× aux + vs Poseidon2, read off Miden's BlakeG branches. Also the only one with a shipped + production existence proof (Airbender runs blake2s-7 as its only hash), **caveat: + at an 80-bit target; ours is 100/128-bit**, which raises query counts and the bill + proportionally. +- **RPO: a ROW-COUNT ratio, not a benchmark.** 0.5× is `HASH_CYCLE_LEN` 8 vs 16 read + off Miden source at an *assumed-equal column count*. The corpus does not check + whether an RPO AIR needs the same 16 columns, and RPO's inverse S-box typically + needs its own witness per lane. **Read 0.5× as rows, with columns unverified.** +- **Poseidon-original: reasoned, with ZERO AIR data anywhere in the corpus.** The + ≈1× is inferred from S-box dominance and matching round counts. What IS measured + is its *migration* bill: ZisK's upstream PR = 181 files, +49,324/−13,097. +- **Monolith: the weakest-evidenced row** — "few×", priced by analogy to Miden's + `And8Lookup`, with the native claim coming from the designers' own design goal. + The corpus's own verdict: "Watch, don't bet the protocol yet." + +The layout constraint remains real and VERIFIED: `max_degree() = 3` for the +`LFM_HASH` chip (`chips.rs:532-534`), and over-declaration is safe while +under-declaration is not +(`prover/src/tests/constraint_set_tests_a.rs:66-74`). An `x⁷` S-box at degree 3 +needs two intermediate columns (`x²`, `x³`, then `x⁷ = (x³)²·x`). + +The layout constraint is real and VERIFIED: `max_degree() = 3` for the `LFM_HASH` +chip (`chips.rs:532-534`), and `max_degree` "is what the engine uses as the +composition-poly degree bound … over-declaration is safe, under-declaration is not" +(`prover/src/tests/constraint_set_tests_a.rs:66-74`). Raising it is possible but +the wrap runs at blowup 2, so a higher-degree composition polynomial costs LDE +cells — self-defeating for a memory play. **So every candidate must express its +S-box in degree ≤ 3, which for `x^7` means two intermediate columns per S-box** +(`x²`, `x³`, then `x⁷ = (x³)² · x`, degree 3 over columns). + +⚠ **Correction to my own first pass, kept because it decides layout.** I initially +carried `a = 3` into every layout. Wrong: `aux_cells = rows × ceil(interactions/2)` +scales with ROWS, so a 30-rows-per-permutation layout pays 90 aux cells (270 +base-equivalent), not 3. This is also why Miden's 16-row Poseidon2 shows 16 aux and +not 1. Aux count triple, so row count is not free even for a lookup-free hash. + +For a purely algebraic candidate the aux bill is otherwise just the chip's existing +6 `LfmMem` interactions (`aux_cols = 3` per row); row-to-row state wiring is +transition constraints, not buses, so it adds none. **The collapse from keccak's +13,912 aux per permutation is structural** — that number is `KECCAK_RND`'s BITWISE +lookups, which ARE bus interactions, and an algebraic hash has none. + +**A Blake-class candidate does NOT get that collapse**, and this is where I have to +correct myself hardest (see §2.5): its 768 aux per compression is 48× Poseidon2's, +for exactly the reason keccak's is large. What I got wrong was the conclusion I drew +from it. + +### 2.3 The predicted matrix, assembled + +One `P` for every field-native candidate (they share the socket's rate-8 sponge, and +the corpus confirms invocation counts are hash-invariant at this geometry), so the +matrix is a single multiplication per row. Memory uses the **two-term model** wave 7 +established after falsifying the one-parameter 33.7 B/cell figure: **≈27 B/cell plus +≈190 MB per sub-proof** (peak RSS carries a per-sub-proof term). + +| candidate | base-equiv per perm | `P` | hash cells | **total cells** | vs keccak | projected RSS | +|---|---|---|---|---|---|---| +| **keccak — MEASURED, ours** | 77,992 | 118,080 | 9,381.6 M | **11.166 B** | 1.00× | 284 GiB (band 290–350) | +| RPO (0.5× P2 rows, cols unverified) | 152 | 192,000 | 29.7 M | **1.814 B** | **6.16×** | **48 GiB** | +| Poseidon-original (corpus ≈1× P2) | 304 | 192,000 | 59.5 M | **1.844 B** | **6.06×** | **49 GiB** | +| Poseidon-original (MY conservative est.) | 617 | 192,000 | 120.7 M | 1.905 B | 5.86× | 50 GiB | +| Monolith (few×, band — weakest evidence) | ~850 | 192,000 | 166.9 M | ~1.951 B | ~5.7× | ~52 GiB | +| **BlakeG 32-row — MEASURED (Miden)** | 6,400 | 192,000 | 1,252.4 M | **3.037 B** | **3.68×** | **79 GiB** | + +Blake's row includes the fixed 655,360-cell `And8Lookup` table (negligible at this +scale, and our `RANGE` chip can absorb the role rather than adding a chip). + +**★ THE HEADLINE, AND IT REVERSES WHAT I TOLD YOU FIRST: every candidate fits the +124 GiB box, blake included.** The keccak wrap needs 290–350 GiB; the cheapest +candidate needs ~48 GiB and the most expensive ~79 GiB. The hash choice is therefore +**not** gated on cost — all four make the production wrap provable on hardware we +have. That reframes the decision as almost purely cryptographic, which is a much +better position than the one my first pass described. + +Two robustness notes: +- **The residue dominates every candidate row.** Once the hash is cheap, 1.784 B of + a ~1.85 B total is the already-measured non-hash verifier. So the algebraic rows + are insensitive to their (weakly-evidenced) cell estimates: RPO vs Poseidon vs + Monolith differ by 7 % in total cells while their per-permutation estimates differ + by 5.6×. **Choosing among the algebraic candidates on predicted wrap size is + choosing on noise.** +- Blake is the one row where the hash still matters — 1.25 B of its 3.04 B — so it + is also the only row whose estimate is worth refining, and it is the row that is + already measured. + +### 2.4 The 2-to-1 normalization, stated rather than assumed + +The corpus's cross-system figures are per **2-to-1 compression**; ours are keccak +**permutations** at a 136-byte rate. Comparing them directly would be wrong, so the +split at the production shape (DERIVED from §2.1's measured decomposition): + +| | permutations | is it a 2-to-1 compression? | +|---|---|---| +| Merkle parents + FRI layer steps | 47,742 | **yes** — 64 bytes / 8 felts, one block either way | +| wide trace-leaf absorbs | 67,671 | **no** — up to 3,456 felts, a sponge run | +| spine (transcript) | 2,667 | no — absorption | + +**So our apples-to-apples compression count is ≈47,742, i.e. 6.2× Airbender's 7,685 +— not the 15.4× a naive 118,080/7,685 gives, and nowhere near the 117× the corpus +flagged for the old guest verifier at ~900,000.** The LFM has already retired most +of the corpus's headline anomaly; that is worth recording, because §I.7's +"recursion diverges at a Blake-class hash" conclusion was reasoned at ~900,000 +compressions and does not transfer to this machine at 47,742. It is the main reason +blake lands at 3.68× rather than the corpus's ~220 %. + +⚠ Our leaf term is 59 % of the bill and has **no analogue** in the per-2-to-1 +figures. It is also the term the frozen rate-8 state penalises (§1.2). Any +cross-system comparison that omits it understates us by 1.6×. + +All numbers name their epoch shape: fixture epoch, profile +`[2 ×14, 3, 4 ×4, 5 ×3, 7, 20]`, 24 sub-proofs, fibonacci guest, 16-cycle +INTERMEDIATE epoch, inner blowup 8 / 73 queries; wrap options blowup 2 / 219 +queries / grinding 20 (entry 10's rule). + +--- + +### 2.5 Where I was wrong about blake, and why + +My first pass said: *"Blake2s is ARX on 32-bit words, so it needs the same +bit-decomposition mechanism that makes keccak's aux 53.5 % of its cost. So blake's +in-AIR character is keccak-like, not Poseidon-like … the hash decision may not buy +the 2.8× memory relief the wrap needs at all."* + +**The premise was right and the conclusion was wrong.** Blake IS bit-oriented, and it +does pay a 48× aux penalty against Poseidon2 — that part survives contact with the +corpus's measurement. What I inferred from it does not, for a reason I had no excuse +to miss: **keccak-like in mechanism is not keccak-like in magnitude.** Our keccak +costs 77,992 base-equivalent cells per permutation because `KECCAK_RND` is 1,480 +columns over 24 rows; BlakeG is 128 columns over 32 rows. Same mechanism, **12× +apart**. Blake lands at 3.68× better than keccak and ~79 GiB — comfortably inside the +box, not outside it. + +Two lessons I would keep: +1. I reasoned from a *mechanism* to a *cost ratio* without ever multiplying the + widths, which the census formula in §1 was sitting right there to do. A ratio + claim needs the arithmetic even when the qualitative story is correct. +2. The brief told me blake was the probable ship choice, and I built a narrative + ("the decision-critical column") that made my analysis load-bearing for it. The + corpus **renders no pick at all**. Being handed a leading hypothesis is a reason + for more falsification, not less. + +Also corrected: the brief's framing that blake is "the most probable final ship +choice" is not what the review says. Ranked by *evidence strength* rather than +preference: **Blake** (only real in-AIR measurement, plus a shipped 80-bit production +system) > **RPO** (a rows-only ratio, columns unverified) > **Poseidon-original** +(reasoned ≈1×, zero AIR data, but a measured 181-file migration bill) > **Monolith** +("few×", priced by analogy). All four columns are scoped here; none is privileged. + +--- + +## 2.6 What already exists in-tree — searched structurally, and it changes §4's risks + +I ran this myself after the dispatched inventory leg failed to report. Method note +worth recording because it nearly cost me a false claim: my first pass used +`grep -r --include=*.rs` **unquoted**, which the shell tried to glob and failed with +"no matches found" — indistinguishable from grep finding nothing. Two of my +"nothing exists" readings were shell errors, not evidence. Re-run quoted. + +Searched: `find` over every `.rs`/`.toml` in the repo for +`blake|poseidon|rescue|rpo|monolith|griffin|sha2|sha256|anemoi|reinforced`; then +`grep -rn --include='*.rs'` (quoted) for the same terms plus `hades_permutation`, +`PermutationParameters`; then read the files found. + +**FOUND — a Poseidon-original skeleton (VERIFIED):** `crypto/crypto/src/hash/poseidon/` +(96 + 45 lines). A `Poseidon` trait over `PermutationParameters` whose +`hades_permutation` is `N_FULL_ROUNDS/2` full rounds → `N_PARTIAL_ROUNDS` partial → +`N_FULL_ROUNDS/2` full (`mod.rs:28-41`). **That is Poseidon-original's HADES +structure, and it independently confirms the round SHAPE my §2.2 estimate assumed.** +The trait carries `RATE`, `CAPACITY`, `ALPHA`, `N_FULL_ROUNDS`, `N_PARTIAL_ROUNDS`, +`MDS_MATRIX`, `ROUND_CONSTANTS` (`parameters.rs:11-27`), with a default `mix`. + +**But it has NO concrete instance.** `grep` for `PermutationParameters for` / +`impl PermutationParameters` across every `.rs` in the repo returns nothing, so +there is no parameter set, no round constants, no MDS matrix and no field binding +anywhere in-tree. The permutation is a generic skeleton, not a usable hash. + +**FOUND — Poseidon Merkle backends (VERIFIED):** `TreePoseidon` +(`crypto/crypto/src/merkle_tree/backends/field_element.rs:50-71`) and +`BatchPoseidonTree

` (`field_element_vector.rs:206`) both implement +`IsMerkleTreeBackend` with `Node = Data = FieldElement` — i.e. a +**field-element** tree, next to the byte-oriented `Digest`-generic backend in the +same file. So the commitment layer is already a trait with a field-native Poseidon +implementation behind it. UNVERIFIED, and important: whether the *prover* is generic +over that trait or pins a concrete backend. I did not establish it. + +**FOUND — sha256 AIR SPECS (VERIFIED as files, not as an AIR):** `spec/src/sha256.toml`, +`sha256round.toml`, `sha256msgsched.toml`, `sha256consts.toml` — 749 lines. I found +no generated Rust AIR for them in `prover/src` or `crypto`. Relevance: sha256 is +bit-oriented like blake, so this is the closest in-tree precedent for what a blake +AIR's shape and effort look like — worth reading before costing blake. + +**ABSENT — blake, Rescue/RPO, Monolith, Griffin, Anemoi: no AIR and no software +implementation, in any spelling.** One trap resolved: `monolith` matches 26 times +across `prover/src` (`statement.rs`, `paged_mem.rs`, `page.rs`, `lib.rs`, +`recursion.rs`), and every occurrence is the *monolithic proof* concept, nothing to +do with the Monolith hash. A term-only search would have reported a Monolith +implementation that does not exist. + +### 2.6.1 Donor AIRs — the vendored Plonky3 tree, which the corpus never analyzed + +VERIFIED by listing the tree myself: the main checkout's `others/Plonky3` (@ 4aed8fe4) +carries **`poseidon1-air/`, `poseidon2-air/`, `blake3-air/`, `monolith-air/`, +`keccak-air/`** as crates, alongside bare permutations in `poseidon1/`, `monolith/`, +`rescue/`. The corpus explicitly lists Plonky3 as unscoped +(`recursion_architectures.md:772`: "Candidates still unscoped: `others/leanVM-b`, +`others/Plonky3`"), so none of this is in the review. + +This **materially changes the per-candidate build estimates**, and it reorders them: + +| candidate | donor AIR | build risk | +|---|---|---| +| **Poseidon-original** | **`poseidon1-air`** — a direct donor for exactly this hash | **lowest** | +| Monolith | `monolith-air` (+ `monolith/` = Monolith-64 Goldilocks, width 8/16) | low, and un-analyzed by the corpus | +| Blake | `blake3-air`, plus Miden's BlakeG branches (unmerged, 13→21 files / 3.8→5.1 k lines) | high — bit-oriented, needs the lookup table | +| **RPO** | **NONE.** Only the permutation (`rescue/src/rpo/goldilocks.rs`, 394 lines); **no `rescue-air` crate exists**, and the corpus's only AIR pointer is Miden git history | **highest** | + +⚠ **That inverts the naive ranking.** RPO is the cheapest predicted column (~48 GiB, +6.16×) and has the *worst* donor situation — its 0.5× is a rows-only ratio with +unverified columns AND there is no AIR to copy. Poseidon-original is within 7 % of +RPO on predicted total cells (§2.3's residue argument) and has a direct donor. **So +the cheapest-looking column is the expensive one to build, and the difference it +would buy is inside the noise.** + +⚠ **A donor warning that transfers, VERIFIED in the corpus** +(`openvm-port-study-brief.md:214-221`): *"The hash is the wall, and it is worse than +'swap constants.' `Poseidon2SubAir` is a single-variant enum locked to +`BabyBearPoseidon2LinearLayers` … Round constants convert to any `F` by type but +produce numerically meaningless values … A Goldilocks Poseidon2/RPO chip is a **new +chip of the same shape**, not a parameter change."* Expect a donor to supply +structure, not code. + +**Consequence for §4's worst risk — it shrinks but does not vanish.** The oracle +problem is no longer "write a Poseidon from nothing and check it against itself". +The HADES structure is in-tree and reviewed, and the remaining input is a +**parameter set** (α, round counts, MDS, round constants for the chosen field and +`t = 12`), which must come from a published reviewed source. Route that keeps it +additive: implement `PermutationParameters` for a LOCAL type inside +`prover/src/lfm/` — a foreign trait on a local type needs no `crypto/**` edit, +where adding a parameter set WOULD be an always-stop change. + +--- + +## 3. Build inventory + +Split by the standing-decisions boundary. **Nothing on the critical path for a +CELLS measurement touches `crypto/**` or `prover/src/tables/**`.** + +### 3.1 Additive LFM work (pre-authorized) + +| piece | what | oracle for differential testing | +|---|---|---| +| ~~A. candidate-`P` closed form~~ **DONE** | `blocks_at_rate`/`leaf_permutations_at_rate`/`query_permutations_at_rate` in `epoch_verify.rs`, rate as a parameter | **the rate-17 case reproduces `query_permutations` exactly** — a real differential, because the new function is written through FELTS and the old through BYTES and `keccak_host::num_blocks`, and NEITHER delegates to the other (rule 7's trap avoided deliberately; making one delegate would have made the test vacuous). The existing assert ties `query_permutations` to the EMITTED count, so the chain reaches the emitter | +| B. `LfmHasher` impl for the candidate | `permute([FE;12]) -> [FE;12]` + `compress_iv` | a reference Poseidon implementation over Goldilocks with the same round constants / MDS; test vectors. **This is the piece with a real oracle problem — see §4 risks** | +| C. the chip's constraint block | replace `chips::hash::HashConstraints`' `t_i` block with the candidate round function at degree ≤ 3 | `constraint_set_tests_a`-style degree check (`measured <= max_degree`), plus prove+verify: rule 2 says execute-only tests prove nothing about chips | +| D. gadgets on socket 2 | a candidate `merkle_walk` / sponge already exist (`edsl::merkle_walk`, `SpongeVar`) and are hash-agnostic by construction | they are already exercised against `fixture::HostSponge`, which mirrors the trait — so B's correctness carries them | +| E. re-emit + census | emit the epoch verifier with socket-2 gadgets, run `report_census` | `the_census_agrees_with_the_traces_the_prover_builds` (exists, green) | + +### 3.2 Always-stop / out of scope for a cells measurement + +- **An inner prover under the candidate hash — ALWAYS-STOP, and it needs the USER, + not the team lead.** The inner-prover trace (INHERITED from the dispatched leg via + the team lead; I did not verify it myself) is: the transcript **hardcodes** + `PlatformKeccak256`; `config.rs` **pins three Merkle aliases**; `ProofOptions` has + **no hash field at all**; grinding is hardcoded. The cheapest seam ("Case A") is + about **4 files on the CPU path but is non-additive inside `crypto/**`**; the + general version ("Case B") additionally breaks a trait and the proof format. + + **Scope it as a proposal, do not build it.** The proposal: a `config.rs` + feature-flag seam, a defaulted `D` type parameter on `DefaultTranscript`, and + `D`-generic grinding — with the CUDA path and the pinned static commitments + costed, since both are affected. That is a `crypto/**` decision and therefore the + user's call. + + Note §2.6 found `TreePoseidon`/`BatchPoseidonTree` already implementing + `IsMerkleTreeBackend` over field elements, so the Merkle half has an + implementation waiting. **Do not read "additive" into that** — whether the prover + is generic over the trait or pins a concrete backend is exactly what `config.rs` + pinning three aliases suggests it is not. +- **Widening `HASH_STATE_FELTS`** past 12 to cut the 2.125× rate penalty (§1.2). + The contract is frozen and the bus tuples/opcodes are pinned; this is a team-lead + decision, and it is the single cleanest lever on the candidate's `P`. +- **Raising `max_degree` above 3** to shorten the S-box. Interacts with the wrap's + blowup 2; almost certainly a net loss, but it is a framework-ceiling question and + rule "report a ceiling rather than working around it" applies. + +### 3.3 TWO STAGES — and stage 1 must not block on stage 2's authorization + +This is the governance shape the matrix should be built in, so that no column waits +on a `crypto/**` decision: + +**Stage 1 — UNGATED, entirely inside `prover/src/lfm/**` (mine to build).** +A candidate's column factorises into two independently obtainable numbers: + +``` +column = permutations-per-verify × cells-per-permutation + └─ GEOMETRY: derived from the wave-7 census once rate/digest is + normalized (§2.1 DONE, §2.4 normalization stated) + └─ MEASURABLE: host a candidate AIR behind the socket and + differential it against a reference implementation +``` + +Both halves are additive LFM work. **That yields measured-not-projected columns for +the whole matrix without touching `crypto/**` at all** — which is the point, because +it means the hash decision gets real numbers before anyone has to authorize anything. + +**Stage 2 — GATED on the user's `crypto/**` call.** A genuinely candidate-hashed +inner proof, verified end to end. This validates stage 1's columns against reality +and is the only thing that makes the column a cryptographic claim rather than a +geometric one. It needs §3.2's proposal authorized first. + +### 3.4 The measurement this buys, and what it does NOT buy + +Slices A–E produce a **geometry** measurement: the true cell cost of an epoch +verifier that hashes with the candidate, at the real production shape. It is the +matrix column the phase asked for. + +It does **not** verify a real candidate-hashed proof, because no such proof can be +produced without §3.2's inner-prover work. Stating that limit precisely is a rule-6 +obligation: the column is *"cells to verify an epoch of this shape, hashing with +H"*, and its cryptographic content is the same as the placeholder's until an inner +proof under `H` exists. That is a fair trade for the decision the matrix feeds +(size), and a bad trade for any soundness claim. + +--- + +## 4. Order of work, and the risks + +**Recommended order:** + +1. ~~**Slice A first — the `P` predictor.**~~ **DONE, in this session** (§2.1). + Measured 1.63×, and it answered the question it was built to answer: the epoch + is 58.6 % absorption-bound, so the frozen 12-felt state IS costing real + permutations — but 1.63×, not the 2.125× ceiling. Widening the state is worth + raising (§3.2) and is NOT urgent: it would recover at most 1.63 → 1.0, i.e. + ~0.07 B cells of a ~1.9 B total (4 %), because the residue dominates once the + hash is cheap. **That is a decision this measurement retires** rather than + escalates. +2. **Then Poseidon-original** (slices B, C, E) — **but for a different reason than + my first pass gave.** Not "cheapest column": §2.3 shows the algebraic candidates + are within 7 % of each other on total cells, so cheapness is not a + discriminator. The reasons that survive are: a **direct `poseidon1-air` donor** + (§2.6.1), an in-tree HADES skeleton whose round structure is already confirmed, + the **lowest build risk of any candidate**, and — decisively — the corpus has + **zero AIR data** for it, so measuring it *adds* information instead of + re-confirming a number Miden already published. It also validates socket 2 under + real load for the first time. +3. **Then blake.** Its column is already measured externally (13.9×/48×), so + building it mainly **calibrates our cost model against an independent + measurement** — worth real money for trusting every other column. Budget it as + the expensive build: bit-oriented, needs the lookup table (our `RANGE` can absorb + the `And8Lookup` role), and Miden's own effort was 13→21 files / 3.8→5.1 k lines. +4. **Then Monolith** — `monolith-air` is a Goldilocks donor and the corpus never + analyzed it, so this is the second-highest information-per-effort column. +5. **RPO last, despite being the cheapest predicted column.** It has no `rescue-air` + donor anywhere (only Miden git history), its 0.5× is rows-only with unverified + columns, and what it would buy over Poseidon-original is ~1 GiB of a ~49 GiB wrap. + **Highest build risk for the smallest real difference.** + +**A cheap cross-check available before any of this:** blake's measured 13.9×/48× +against Poseidon2 can be run through §1's census formula *today*, at our `P`. I did +exactly that in §2.3 and it is what produced the reversal in §2.5. Any candidate the +corpus has numbers for should get this treatment before it gets a build. + +**Risks, worst first:** + +- **The oracle problem for slice B — DOWNGRADED by §2.4, not eliminated.** A HADES + permutation with Poseidon-original's exact round structure is already in-tree + (`crypto/crypto/src/hash/poseidon/`), so the chip can be differentialled against + a reviewed software reference rather than against itself. What is missing is a + concrete `PermutationParameters` — and that is a *cryptographic* input, not an + engineering one (next risk). Rule 7 still applies at the end: once the chip and + the reference share a code path, the differential dies and must be replaced by an + absolute property of the output. +- **Parameter selection is the real remaining risk, and it is not mine to make.** + α, round counts, the MDS matrix and the round constants for the chosen field at + `t = 12` must come from a published, reviewed source. Picking them ad hoc yields + a measured column for a hash nobody would ship — decision-irrelevant, exactly the + failure mode the Poseidon2 ban exists to avoid. Note the in-tree skeleton is + field-generic (`type F: IsPrimeField`), so **which field the candidate is over is + itself an open input** I did not resolve. +- **My round-count estimates are partly corroborated, not verified** (§2.2, §2.4). + The HADES *structure* (R_F/2 · R_P · R_F/2) is confirmed from in-tree source; the + specific 8-full/22-partial counts are still my own domain knowledge and set `m`. + Cheap to fix: read the corpus, which I could not (below). +- **Four research legs never reported.** I dispatched agents for the inner-prover + hash blast radius, the in-tree AIR inventory, the corpus's Part I.7 candidate + data, and a full socket spec; none had returned when I closed. I covered the + inventory myself (§2.4) and the socket myself (§0, §1.1) — the **corpus data and + the inner-prover blast radius are the two genuine gaps in this report**, and both + are cheap for wave 9 to close. +- **Parameter selection is a cryptographic act.** Round counts, MDS matrix and round + constants for Poseidon-original over Goldilocks at t=12 must come from a + published, reviewed source, not from me. Picking them ad hoc would produce a + measured column for a hash nobody would ship — decision-irrelevant, exactly the + failure mode the Poseidon2 ban exists to avoid. +- **The residue is not actually candidate-independent** (§1). It makes my numbers + conservative, so it is a soundness-of-argument risk rather than a wrong-direction + one, but a candidate column that quietly keeps keccak's byte-serialization + gadgets in the residue would understate the win. +- **`TestPermutation`'s 37 cells/permutation is a trap.** It is one non-cryptographic + degree-3 round. Anyone reading the census after slice A/E without reading §1.1 + could report a 2,108× win. Guard: the report and any test that prints it should + carry the "NOT cryptographic" label the source does. + + +--- + +## 5. Corrections ledger — claims elsewhere that this document supersedes + +Recorded so they stop propagating. + +1. **The brief's "blake is the most probable final ship choice."** The review renders + NO pick (§2.5). Ranked by evidence strength: Blake > RPO > Poseidon-original > + Monolith. Scope all four; privilege none. + +2. **My own "the hash decision may not buy the 2.8× the wrap needs."** Wrong — every + candidate buys it, blake included (§2.3, §2.5). Right premise, unmultiplied + arithmetic. + +3. **The one-parameter 33.7 B/cell memory model — FALSIFIED** by wave 7's follow-up. + Use the two-term fit: **≈27 B/cell + ≈190 MB/sub-proof**, and the keccak wrap + ceiling is a **band, 290–350 GiB (2.3–2.8× the box)**, not a point. Every RSS + figure in this document uses the two-term model. My earlier 59.8–67.8 GiB + Poseidon figures were computed on the falsified coefficient and are superseded by + §2.3's ~49 GiB. + +4. **The RESUME's wave-7 line "one options change plus the hash swap."** Wrong on the + options half: `ProofOptions` has **no hash field at all** (§3.2), so there is no + options change to make — the swap is a `crypto/**` seam, always-stop, user's call. + +5. **My own status-log implication that `chunking.rs`'s commit 6dbc5795 was the live + agent's new work.** It is dated 2026-07-29 — the ORIGINAL chunking leg. At the + time I looked there were **zero commits past 891f534f** on `feat/lfm-assembly`; + the only new material was the uncommitted edit. The collision was real, my + inference about which artifacts evidenced it was not. + +6. **The KECCAK_RND chunk knob is TWO-SIDED and cannot buy the memory** (wave-7 + follow-up, measured by proving): retuning cut min-preset RSS 15.1 → 10.1 GiB but + grew the proof +78 % (30.7 → 54.6 MB) and verify 2.4×; at the production shape + padding waste is already 1.7 %. **A cheaper hash is the only large memory lever**, + which is this document's motivation. + +7. **§I.7's "hash choice decides batching" is CONTESTED in-corpus** — the guest-model + version is falsified (the crossover constant is off ~5,500×, so unbatched wins on + both axes at any hash price) and the native version is unmeasured. Do not import + it. Relatedly, §I.7's "recursion diverges at a Blake-class hash" was reasoned at + ~900,000 compressions; **this machine does ≈47,742** (§2.4), so it does not + transfer. + +8. **Arity-4 Merkle and layer-0 4-fold FRI are measured dead ends** on our economics + (+7M net, and "DEEP doubling loses everywhere"). Arity trades permutations for + bytes and so wins only when permutations are expensive — which the candidates make + *less* true, not more. + +9. **§2.3's whole candidate table — SUPERSEDED by §8.6** (`[hash-w10]`, MEASURED). + Every candidate row held the 1,784,197,396 residue fixed, and that residue is + overwhelmingly the `felt_be_halves` byteswap gadget, which the field-native + candidates delete. ⚠ Base discipline (team-lead, post-eval catch; corrected by + `[hash-w10]` before it could propagate wrong): the measured **95.84 %** is + against the CHIP-HEIGHT CENSUS base of 1,757,982,868 (keccak permutation chips + AND their lookup tables excluded; byteswap + R_native sum to it exactly); + against the LEDGER base of 1,784,197,396 it is **94.44 %**. The 26,214,528 + between the bases is NOT an open reconciliation gap — it is exactly + `BITWISE` (1,048,576 × (10 + 3×5) = 26,214,400) + `KECCAK_RC` (32 × 4 = 128), + keccak's own lookup tables, which the ledger base includes. This CLOSES the + gap `fma-vm-analysis.md:191` flagged. **R_native stays 73,072,788**: a + field-native machine has no chip sending to any BITWISE-served bus + (structurally checked — one hit in chips.rs, keccak's absorb XOR, LfmMem as + positive control), so folding keccak's tables into its residue is a category + error. Caveat: that costing assumes a CHIP-SET change; under the frozen-14-chip + principle a field-native hash in today's set still carries BITWISE as a dead + 26.2 M table — a design decision, not a measurement. Do not quote + "95.8 % of 1.784 B"; pick a base and name it. §2.3's "48–79 GiB, so the choice is not cost-gated" splits into ~71 GiB + (blake, which keeps the gadget AND `BITWISE`) and ~4–8 GiB (algebraic). The + "every candidate fits the 124 GiB box" half survives; the "choosing on predicted + wrap size is choosing on noise" half does not, across families. + +10. **The two-term RSS model's sub-proof count is 19 at this shape, not 24** — + and it is candidate-dependent (§8.6). 24 is the INNER epoch's leg count; the + per-sub-proof term is about the WRAP being produced, which carries 13 chip + classes plus 6 `KECCAK_RND` chunks. Immaterial for keccak, but it is 27–41 % of + the projection for the field-native rows, where it becomes the dominant term. + +11. **"Delegation" is not an available lever for this machine** (§8.7, priced). + A separate blake circuit plus verifying its proof costs +66 % over hosting the + chip in the epoch verifier's own multi-proof. Airbender's delegation circuit + exists to move hash work out of a FIXED-SIZE main circuit; the LFM has none, + so its multi-AIR proof already is that pattern. + +--- + +## 6. Slice 1a DONE — the permutation, and the pinned prediction for the chip + +**Landed:** `prover/src/lfm/poseidon.rs` — `PoseidonGoldilocks` implementing +`LfmHasher`, with parameters and an external oracle. + +### 6.1 Parameter provenance (condition (b), discharged) + +From the vendored `others/Plonky3/goldilocks/src/poseidon1.rs` @ 4aed8fe4, which +documents them as Grain-LFSR generated per the Poseidon paper Appendix E with +`field_type=1, alpha=7 (exp_flag=0), n=64, t=12, R_F=8, R_P=22`, via +`poseidon/generate_constants.py --field goldilocks --width 12`. MDS is CIRCULANT +with first row `[1,1,2,1,8,9,10,7,5,9,4,10]` (`goldilocks/src/mds.rs:92`). + +**This independently confirms my slice-0 estimate of 8 full + 22 partial**, which +was my own domain knowledge and is now cited. The corpus corroborates from a second +direction: ZisK's shipped PLONKish Poseidon is width-16, 8 full + 22 partial. + +⚠ **Ship-grade parameter selection remains a separate cryptographic decision for the +ecosystem, NOT settled by this measurement.** Cells depend on round counts and S-box +degree, not on the constants' numeric values, so the measurement is valid; what to +ship is not ours. Likewise `compress_iv` is ZERO capacity — plain sponge +compression — and domain separation is deliberately not invented here. + +### 6.2 The oracle (condition (d), discharged — but NOT as specified) + +⚠ **The brief's first oracle is unusable and this is a real finding.** Condition (d) +asked for "the in-tree HADES skeleton instantiated with the same parameters". That +skeleton (`crypto/crypto/src/hash/poseidon/mod.rs`) hardcodes an `x^3` S-box, and +**`x^3` is not a permutation over Goldilocks**: `p - 1 = 2^32 · 3 · 5 · 17 · 257 · +65537`, so 3 is not coprime to the group order. Differentialling against it would +have validated my implementation against a non-permutation. + +Used instead: **Plonky3's own known-answer vector** for width 12 (input `0..11`), +which nothing in this repository produced. `the_permutation_matches_the_plonky3_ +known_answer_vector` matched **on the first run**, with a Python cross-check of the +same convention beforehand. + +**Falsified three ways (rule 1), each restored** — the KAT pins every convention it +needs to: +| mutation | result | +|---|---| +| `x^7 → x^6` (wrong exponent) | FAILED, correctly | +| circulant MDS transposed (`(j−i)` → `(i−j)`) | FAILED, correctly | +| partial-round S-box lane 0 → lane 11 | FAILED, correctly | + +A second test asserts `gcd(α, p−1) = 1` and that 3 and 5 fail it — the skeleton's +bug, encoded as a guard. + +### 6.3 PINNED PREDICTION for the chip — falsify this next + +My degree-3 layout, one row per permutation (`x⁷ = (x³)²·x` needs `x²`,`x³` as +columns; the MDS is linear so it costs no columns): + +``` +IN0..11 + S8..11 = 16 +8 full rounds × (12·x² + 12·x³ + 12 out) = 288 +22 partial × (x² + x³ + 12 out) = 308 + m = 612 value columns, 1 row + a = 3 (the chip's 6 LfmMem interactions) +base-equiv per permutation = 612 + 3·3 = 621 +``` + +At the measured `P ≈ 192,000`: hash cells **121.5 M** with a chunking sibling +(1.019 padding) or **162.8 M** unchunked (pads to 2¹⁸, 1.365) — so the epoch verify +totals **1.906–1.947 B cells = 5.73–5.86× smaller than keccak**, RSS **≈50–51 GiB**. + +⚠ **612 is an UPPER BOUND, and knowingly 2× off a known-achievable layout.** Miden's +measured Poseidon2 at the same width is 256 main + 16 aux = 304 base-equivalent, via +16 columns × 16 rows. A smarter layout could roughly halve my hash term — which +moves the TOTAL by ~3 %, because the residue dominates (§2.3). So the column is +worth measuring at 612 and not worth optimising. + +### 6.4 Slice 1b — the chip, specified to be executable (NOT built) + +**Why this is a spec and not code:** my context ran thin, and +`lfm-standing-decisions.md`'s coordination rule is explicit — "checkpoint and write +a handoff file rather than delivering a half-built slice. Quality over completion." +A 612-column constraint set that compiles but is unfalsified would be worse than +this document. Everything below is derived, not guessed; the arithmetic is checked +against §6.3. + +#### Column layout (value section, after `PREP_WIDTH = 11`) + +| block | columns | offset | +|---|---|---| +| `IN0..IN11` | 12 | 0 | +| `S8..S11` (capacity materialization) | 4 | 12 | +| per FULL round (×8): `x2[0..12]`, `x3[0..12]`, `out[0..12]` | 36 each | 16 + … | +| per PARTIAL round (×22): `x2`, `x3` (lane 0 only), `out[0..12]` | 14 each | … | +| **total value columns** | **612** | = 16 + 8·36 + 22·14 | + +#### Constraints (601 total: 4 + 1 + 8·36 + 22·14) + +Let `m = MODE_C + MODE_P` (the existing mode-sum column pair), and per round `r` +let `a_i = state_i + rc[r][i] · m` — an EXPRESSION, degree 1, where `state` is +`IN`/`S` on round 0 and the previous round's `out` afterwards. + +1. **Capacity copy** (4): `S_i − MODE_P · IN_{8+i} = 0`. Degree 2. Note Poseidon's + `compress_iv` is ZERO, so the `MODE_C · IV_i` term of the `TestPermutation` + version vanishes — do not carry it over. +2. **Mode boolean** (1): `m · (1 − m) = 0`, unchanged from today. +3. **Per active lane**: `x2_i − a_i · a_i = 0` (degree 2) and + `x3_i − x2_i · a_i = 0` (degree 2, since `x2_i` is a column). +4. **Per round output** (12 each): `out_j − Σ_i M[j][i] · f_i = 0` where + `f_i = (x3_i)² · a_i` for S-boxed lanes (degree 3) and `f_i = a_i` otherwise + (degree 1). `M[j][i] = MDS_CIRC_ROW[(i − j) mod 12]`, matching + `poseidon::PoseidonGoldilocks::mds`. + +**Degree is exactly 3**, so `max_degree()` stays 3 and the wrap's blowup 2 is +unaffected — the whole reason the S-box is decomposed rather than written `a^7`. + +**Padding obligation, and it is already solved by the existing trick:** scaling the +round constant by `m` (as `chips.rs:548-553` does today) makes an all-zero row +satisfy everything — `m = 0 ⇒ a = 0 ⇒ x2 = x3 = out = 0` — WITHOUT a degree-4 gate. +Keep it; it is load-bearing, not decoration. + +#### Trace generator contract + +Mirror `poseidon::PoseidonGoldilocks::permute` but RECORD `x2`, `x3` and the +post-MDS state per round. It must use the **same association** — +`x2 = a·a`, `x3 = x2·a`, `x⁷ = (x3)²·a` — which is why `poseidon.rs::sbox` is +already written that way. Any other association gives the same field element and a +different trace, and the constraints would reject it. + +#### Test plan (all five needed before the number is real) + +1. `max_degree` measured ≤ declared, via the `CaptureBuilder` route + `prover/src/tests/constraint_set_tests_a.rs:75-94` uses. +2. **Satisfaction**: a real Poseidon row (from the generator) makes every one of + the 601 constraints evaluate to zero. +3. **Rejection**: perturb ONE column — one `x2`, one `x3`, one `out`, and one + capacity cell, separately — and assert a constraint fires each time. Rule 1. +4. **Padding**: an all-zero row satisfies everything. +5. **Prove+verify** — rule 2: execute-only tests prove nothing about a chip, so the + column is not MEASURED until the production prover runs this AIR. This is the + step that makes §6.3's prediction a measurement. + +#### Registration — much smaller than a new chip + +`LFM_HASH` is **already** slot-registered (`airs.rs` `LFM_CHIP_NAMES`), so the +8-site checklist for ADDING a chip does not apply. What changes: `cols::NUM_COLUMNS` +(28 → 612 value columns), the constraint set body, the trace filler, and the census +picks the new width up automatically because it reads `hash::cols::NUM_COLUMNS`. +`PREP_WIDTH` stays 11 and the preprocessed group is untouched, so **the registry +root for this chip should NOT move** — verify that rather than assume it, and +regenerate `LFM_REGISTRY` if any digest shifts (pre-authorized). + +⚠ **The one genuine hazard, and it is why this is not a small change:** the chips +bake the hasher's constants into their constraints, so `proof.rs:52-54` requires +execution to use the SAME hasher. Swapping `HashConstraints` to Poseidon therefore +breaks every existing call site that executes with `TestPermutation` (~30 across +`epoch_tests`, `constraint_tests`, `epoch_verify_tests`, `machine_tests`, and +`fixture::HostSponge`). **Do not do that swap to get a cells number.** The cells +number needs only the AIR's declared width plus tests 1-5 above; the global hasher +swap is a separate, larger decision about what the machine's default hash IS, and it +should be taken deliberately rather than as a side effect of a measurement. + +### 6.5 What slice 1 still owes (superseded by 6.4 — kept for the index) + +Not built: the chip's constraint block (replace `chips::hash::HashConstraints`' +`TestPermutation` round with the 30-round chain), `cols::NUM_COLUMNS` 28 → 612, the +census array, `LFM_REGISTRY` regeneration if any digest moves, and the prove+verify +measurement (rule 2: execute-only tests prove nothing about a chip). The padding +trap (condition (c)) must be handled as a chunking sibling OR an explicit +padding-corrected line beside the raw one — both numbers are pinned above so the +first measurement cannot silently read 36 % high. + +--- + +## 7. Slice 1b DONE — the chip is BUILT, PROVED, and the prediction CONFIRMED + +**Landed** (`[hash-w9]`): the Poseidon-original `LFM_HASH` chip behind a +construction-time hasher choice, plus 15 tests. `lfm` suite **230 passed / 0 +failed / 5 ignored**, `make lint` exit 0 (make's own status). + +### 7.1 The measurement, number by number against §6.3 + +Every figure below is measured through the SAME census instrument that produced +entry 10's keccak column (`main + 3·aux`), so the two columns of the matrix are +comparable by construction rather than by argument. + +| §6.3 pinned | measured | verdict | +|---|---|---| +| 612 value columns | **612** | CONFIRMED | +| 601 constraints | **601** | CONFIRMED | +| `max_degree` 3 | **3** declared, **3** measured max | CONFIRMED | +| 621 base-equiv cells/permutation | **621** (612 + 3·3) | CONFIRMED | +| hash cells 121.5 M chunked | **121,497,408** | CONFIRMED | +| hash cells 162.8 M unchunked | **162,791,424** (2^18 pad = 1.365×) | CONFIRMED | +| epoch total 1.906 B chunked | **1,905,694,804** = 5.86× under keccak | CONFIRMED | +| epoch total 1.947 B unchunked | **1,946,988,820** = 5.74× under keccak | CONFIRMED | + +⚠ **Provenance, because only one of these inputs is new.** Wave 9 measured +exactly one number: **621 cells per permutation**, off an AIR the production +prover built and the production verifier accepted. `P` (wave 8's closed form) +and the 1,784,197,396 residue (entry 10) are inherited measurements; the epoch +lines are arithmetic over all three. Across wave 8's whole `P` interval +[190,569, 193,569] the chunked total moves only 1,904.8 M → 1,906.7 M +(5.856–5.862×), so the conclusion does not depend on `P` being exactly 192,000. + +### 7.2 ⚠ CORRECTION — the RSS figure, and it is a units error, not a cells error + +§6.3's "RSS ≈50–51 GiB" does **not** reproduce from the stated two-term model +(27 B/cell + 190 MB/sub-proof) over this epoch's 24 sub-proofs: + +- cell term alone: 47.92 GiB chunked / 48.96 GiB unchunked — **but 51.5 / 52.6 + GB**, which is almost certainly where "50–51" came from: the cell term + computed in GB and labelled GiB, with the sub-proof term dropped. +- both terms, in GiB: **52.2 GiB chunked / 53.2 GiB unchunked**. + +Use **≈52–53 GiB**. Nothing downstream changes — every figure in the band is +far inside the 124 GiB box, which is the only claim the number carries — but +the ~4% understatement is recorded so the matrix's other rows (which were +computed the same way in §2.3) get re-derived before anyone compares them at +that precision. + +### 7.3 The prediction that was NOT confirmed, and it was never a cells claim + +§6.3 called 612 "an UPPER BOUND, knowingly 2× off Miden's measured 304". That +is untouched by this measurement: 612 is what MY layout costs, and a row-per- +round layout reusing state columns would still be roughly half. The instruction +not to optimise it stands for the reason given — halving the hash term moves the +epoch total from 1.906 B to 1.846 B, i.e. **3.2%**, because the residue +dominates at 93.6% of the chunked total. The hash term is no longer the thing +worth engineering. + +### 7.4 What the chip actually is + +- **Layout.** The frozen `IN`/`S`/`OUT` prefix keeps its offsets and the final + round's post-MDS output IS `OUT`, so `bus_interactions()` is + hasher-independent and the `LFM_HASH` tuple contract stays literally frozen: + `28 + 7·36 + 24 + 22·14 = 612`, the same 612 as §6.4's `16 + 8·36 + 22·14` + arranged differently. Both totals are asserted, against each other and against + the built width. +- **Degree exactly 3**, via `x⁷ = (x³)²·x` over witnessed `x²`/`x³`. Asserted + both ways: nothing exceeds 3, and something reaches it — a decomposition that + quietly went quadratic would mean the S-box had stopped being computed. +- **The padding trick is load-bearing, now demonstrated rather than asserted.** + See F4 below: removing the round constant's mode-sum scaling breaks the + padding row and NOTHING ELSE, because on a real row `m = 1` and the + permutation is unchanged. That is the cleanest possible evidence for §6.4's + "keep it; it is load-bearing, not decoration". + +### 7.5 Falsification (rule 1) — four mutations, each of the CHIP only + +Mutating chip *and* executor together proves nothing: they would move as one. +Each mutation below changes only the constraint body, so the chip stops agreeing +with the permutation the external Plonky3 KAT pins. Instrument checked against a +known-green control first, and failures read from the trailing summary block +(per rule 7's corollary, per-test lines do not name failures). + +| mutation | result | +|---|---| +| F1 `x⁷ → x⁵` in the chip | 5 failed, incl. prove+verify | +| F2 circulant MDS transposed `(i−o) → (o−i)` | 5 failed, incl. prove+verify | +| F3 partial-round S-box binds lane 1, not lane 0 | 5 failed, incl. prove+verify | +| F4 round constant no longer scaled by the mode sum | **exactly 2 failed**: the padding row and prove+verify | +| CONTROL (unmutated) | 21 passed, 0 failed | + +F4's *discrimination* is the interesting one — satisfaction, the KAT-output +check and every rejection test stay green, so the padding trick is isolated to +padding exactly as §6.4 claimed. + +### 7.6 The seam, and the two things asserted rather than assumed + +Per the team lead's ruling the hasher is a **construction-time** choice +(`HasherKind`) threaded to the constraint body, the width, the trace filler and +the executor. `Test` remains the default; every pre-existing call site keeps its +signature and its behaviour. **Nothing was flipped** — the machine's real hash is +the ecosystem decision this measurement feeds. + +1. **No program digest moves with the hasher.** `PREP_WIDTH` is 11 in both + layouts and the preprocessed group is untouched, so every root and every + program id is bit-identical, and the census's row counts and aux widths are + too — only `LFM_HASH`'s value width moves. `LFM_REGISTRY` did not need + regenerating. Asserted, because a hash experiment silently reassigning + program identity is exactly the failure that must not pass quietly. +2. **A proof does not verify under the other hasher**, in both directions. + +### 7.7 What this leg does NOT settle + +It does not choose a hash, and it is not evidence that Poseidon-original should +be the machine's default. Parameters are published ones adequate to measure an +AIR's *shape*; `compress_iv` is zero because domain separation is a +cryptographic decision deliberately not invented here. Per §0c the decision is +not cost-gated anyway — every candidate fits the box — so this column adds +information (the corpus had zero Poseidon AIR data) without rendering a pick. + +--- + +## 8. Slice 2 DONE — the blake column MEASURED, and the residue turns out to BE the byteswap + +Wave 10 (`[hash-w10]`), 2026-08-06. Donor: PR #903 `feat(prover,executor): BLAKE3 +6-round compression accelerator`, head **`89aeeb8c2b0389e9d21a861c9e3a10a7b1b5704e`**. + +**Landed:** `prover/src/lfm/blake3.rs` (the primitive + the 10 canonical 6-round +vectors + negative controls), `prover/src/lfm/blake3_chip.rs` (the chip, hosted +on `LfmMem`), `prover/src/lfm/blake3_probe.rs` (prove+verify, falsification, and +two `#[ignore]`d measurement instruments). `lfm` suite **244 passed / 0 failed / +5 ignored** (was 230/0/5), `make lint` exit 0 (make's own status). + +### 8.1 The headline, and it is not the column + +The blake column came out where §2.3 predicted (3.7–4.1× under keccak against a +predicted 3.68×). **The finding that matters is the one the column was measured +against: the 1,784,197,396-cell "non-hash residue" that every candidate row in +§2.3 sits on is 95.8 % a single gadget** — `felt_be_halves`, the felt → +big-endian-u32-halves serializer, at 1,684,910,080 cells. + +§1's warning was right and an order of magnitude too quiet. It said "part of that +residue is byte-serialization work … a field-native hash deletes outright" and +concluded the candidate predictions were therefore conservative. They were +conservative by **~10×**, not by a few percent, and §2.3's robustness note — +"choosing among the algebraic candidates on predicted wrap size is choosing on +noise" — is now **false in the one comparison the decision turns on**: blake and +the field-native candidates are ~11× apart, not 1.6×. + +### 8.2 The measurement: cells per compression, on our stack + +MEASURED, prove+verify (rule 2), `blake3_probe::the_hosted_chip_proves_and_verifies`: +the chip is built with its real 1,259 interactions and its real 769 constraints, +its preprocessed prefix is committed for real via `commit_columns`, and both its +buses are closed — `ByteAlu`/`AreBytes` against the UNCHANGED production +`BITWISE` table, `LfmMem` against a mirror AIR. + +| | #903, syscall variant | hosted here | basis | +|---|---|---|---| +| value columns | 3,219 | **3,056** | MEASURED | +| bus interactions | 1,397 | **1,259** | MEASURED | +| aux columns (`⌈i/2⌉`) | 699 | **630** | DERIVED | +| **base-equiv `m + 3a`** | **5,316** | **4,946** | **MEASURED** | +| constraints | 814 | **769** | MEASURED | +| max degree | 3 | **3** declared, **3** measured | MEASURED | + +5,316 reproduces #903's own stated figure exactly, which is the corroboration +that the two are being counted the same way. + +**The 370-cell (7.0 %) saving is entirely I/O.** Dropped: the `Ecall` receiver, +the x10 register read, 22 `Memw` dword ops, the 32 `OLD_OUT` `AreBytes`, 4 addr +`AreBytes` + the alignment `AND`, and 88 pointer `IsHalfword`s (149 interactions); +and the `TIMESTAMP`/`ADDR`/`PTR`/`OLD_OUT` columns (162), plus `MU` moving into +the preprocessed prefix. Added: 11 `LfmMem` word tokens (7 reads of the 28 input +`u32`s, 4 writes of the 16 output `u32`s), the machine word being four `u32` +lanes exactly as `LFM_KECCAK` defines it. + +⚠ **Dropping those range checks is sound, not merely cheaper, and the argument is +worth keeping.** Each guarded something that no longer exists: `OLD_OUT` is the +previous memory content in a `Memw` write's `old` field and an `LfmMem` write has +none; the address checks guard a prover-witnessed pointer read out of x10, where +here every address is a preprocessed column the admission validator vouches for. +The byte-range coverage of the DATA columns is untouched — `m`'s 64 bytes keep +their 32 explicit `AreBytes` (they are never XOR-consumed), `h` is an operand of +the feed-forward XOR, `t_lo`/`t_hi`/`block_len`/`flags` are `v[12..16]` and hence +`vd` operands of round-0 `G`s, and all 64 `OUT` bytes are XOR *results*. So every +byte a token recomposes is range-checked before the recomposition, which is the +same transitive argument `chips::keccak` records for its 400 state bytes. + +**Basis label: hosted-measured, not registered.** `LFM_BLAKE3` is not in the +fixed AIR set — `airs.rs` still names 14 chips — so what is proved is the chip +under our AIR framework with its buses closed by a synthetic memory, not an +epoch verifier that hashes with blake. Registration would move every program +digest and is a separate decision. What the probe therefore *cannot* see is +listed in its module doc: whether an LFM program can drive the chip, whether the +validator accepts the address assignment, and anything cryptographic about A6R. + +### 8.3 The geometry: blake's `P` is the rate-8 count, and no extra compressions + +VERIFIED against #903's ABI. A 2-to-1 Merkle compression is `compress(h = IV, +m = left‖right)` — 64 bytes of message, ONE compression, 1:1 with keccak's +64-bytes-inside-a-136-byte-rate parent. An absorb of `N` felts is `⌈N/8⌉` +compressions at 8 bytes per felt. The counter `t`, `block_len` and `flags` live +in `v[12..16]`, i.e. in the *state*, not in message space, so **the message-mode +framing forces no extra compressions**. Blake therefore shares the field-native +candidates' `P`, re-derived on this run rather than quoted: + +``` +legs @ rate 17 (keccak) 115,413 = the EMITTED count, exactly (assert) +legs @ rate 8 (blake and field-native) 187,902 +spine 2,667 absorption-bound, so 1.0x–2.125x +P at rate 8 in [190,569 , 193,570] — §2.1's interval, reproduced +``` + +⚠ One conservatism carried deliberately: `blocks_at_rate` uses keccak's +`⌊n/rate⌋ + 1` padding convention, which always spends a trailing block. BLAKE3 +signals length in `block_len` and needs none, so blake's true count is between +`⌈N/8⌉` and this. Using the same convention on both sides is what makes the +rate-17 case reproduce the emitted count exactly, so it is kept and the +direction recorded: **blake's `P` here is an upper bound.** + +### 8.4 The residue, split — MEASURED + +Instrument: `blake3_probe::the_blake_column_and_the_residue_split` (`#[ignore]`d; +proves a real inner epoch at blowup 8 and emits ~2.25M instructions). Epoch +`[2 ×14, 3, 4 ×4, 5 ×3, 7, 20]`, inner blowup 8 / 73 queries, **19 sub-proofs**. + +``` +LFM_BALU 134,217,728 rows × 4 main / 2 aux 1,342,177,280 12.02% +LFM_BITDEC 2,097,152 rows × 66 main / 33 aux 346,030,080 3.10% +LFM_LANES 2,097,152 rows × 4 main / 5 aux 39,845,888 0.36% +KECCAK_RND 2,883,584 rows ×1480 main /576 aux 9,250,537,472 82.85% +LFM_KECCAK 131,072 rows ×736 main / 88 aux 131,072,000 1.17% +BITWISE 1,048,576 rows × 10 main / 5 aux 26,214,400 0.23% +(+ 8 more chips, 0.63% between them) ------------ +TOTAL 11,165,806,868 +``` + +The keccak permutation chips come to **9,381,609,600**, which reconciles §1's +9,381,609,472 to within `KECCAK_RC`'s 128 cells — so §1's residue of +1,784,197,396 was `total − LFM_KECCAK − KECCAK_RND` and **included the `BITWISE` +table**. Stated cleanly: + +| | cells | basis | +|---|---|---| +| keccak permutation chips (`LFM_KECCAK`+`KECCAK_RND`+`KECCAK_RC`) | 9,381,609,600 | MEASURED | +| `BITWISE`, fixed 2²⁰ (blake keeps it, field-native deletes it) | 26,214,400 | MEASURED | +| residue | 1,757,982,868 | MEASURED | +| — of which the byteswap gadget | **1,684,910,080 (95.84 %)** | MEASURED | +| **residue, byte-oriented** (blake: gadget + `BITWISE` kept) | **1,757,982,868** | MEASURED | +| **residue, field-native** (both deleted) | **73,072,788** | DERIVED | + +**How the byteswap share is counted, and why it is exact rather than attributed.** +`felt_be_halves` is one `BitDec(64)` plus 64 `BALU` rows per felt +(`machine_tests::felt_be_halves_cost` pins that). Every other production +`bit_dec` site passes 32 bits or a Merkle depth — `sample_u64_pow2` *asserts* +`nbits ≤ 32` — so a 64-bit decomposition in this program IS the gadget. The +instrument prints the whole width histogram so a future 64-bit caller shows up +instead of being silently folded in: + +``` +BitDec widths: {4: 1022, 5: 73, 6: 292, 7: 219, 9: 73, 22: 73, 32: 398, 64: 1,122,145} +``` + +1,122,145 gadget calls ⇒ 71,817,280 `BALU` rows, **99.78 % of all `LFM_BALU` +rows**. Padding-aware: the two chips cost 1,688,207,360 with the gadget and +3,297,280 without, hence the 1,684,910,080. + +⚠ **The field-native line is DERIVED by subtraction from a keccak-shaped +emission**, not measured on a re-emitted field-native verifier, and it is an +**upper** bound: a field-native absorb also deletes Pack/Unpack traffic around +the gadget, and `LFM_LANES` still costs 39,845,888 here (55 % of the whole +field-native residue). + +⚠ **The gadget's cost is structural given the current ISA, not an endianness +accident.** Big-endian order is what forces the 32-term weighted recombination, +but *any* felt → two-`u32`-halves split needs a range-checked decomposition, and +the LFM has no 32-bit range-check instruction (`LFM_RANGE` is a 2¹⁶ table that +`chips::range`'s own comment calls "idle in v0"). Wiring one would be the lever; +it is unbuilt and unmeasured and is NOT assumed anywhere above. + +### 8.5 ★ A cheap lever nobody has pulled: chunk `LFM_BALU` + +`LFM_BALU` has 71,974,504 real rows and pads to 2²⁷ = 134,217,728 — an **86 % +overshoot**, 622 M cells of pure padding. `LFM_BITDEC` pads 1,124,295 → 2,097,152 +for another 161 M. Together **≈783 M cells, 7.0 % of the keccak total and 28 % of +blake's**, recoverable by the chunking policy `KECCAK_RND` already has +(`airs.rs`'s chunk machinery is generic; nothing about it is keccak-specific). +DERIVED from this run's census; not attempted, and it does not move the +field-native rows, whose `BALU` is tiny. + +### 8.6 The matrix, RE-DERIVED — and §2.3's rows are superseded + +`P = 192,000`; hash cells are `rows × cells-per-permutation` with rows either +chunked (≈1.9 % waste, the `KECCAK_RND` policy) or padded to the next power of +two (36 % waste); RSS is the two-term model with the **candidate's own sub-proof +count** (see the correction below). + +| candidate | basis of cells/perm | hash cells | **total cells** | vs keccak | subs | **RSS GiB** | +|---|---|---|---|---|---|---| +| **keccak — MEASURED, ours** | 77,992 | 9,407,824,000 | **11,165,806,868** | 1.00× | 19 | **284.1** | +| **BLAKE3-6r, chunked** | **4,946 MEASURED** | 967,402,978 | **2,751,600,246** | **4.06×** | 12 | **71.3** | +| BLAKE3-6r, padded | 4,946 MEASURED | 1,296,564,224 | 3,080,761,492 | 3.62× | 12 | 79.6 | +| **Poseidon-original, chunked** | **621 MEASURED (w9)** | 121,463,253 | **194,536,041** | **57.4×** | 10 | **6.7** | +| Poseidon-original, padded | 621 MEASURED (w9) | 162,791,424 | 235,864,212 | 47.3× | 10 | 7.7 | +| RPO, chunked | 152 INHERITED est. | 29,730,136 | 102,802,924 | 108.6× | 10 | 4.4 | +| Monolith, chunked | ~850 INHERITED est. | 166,254,050 | 239,326,838 | 46.7× | 10 | 7.8 | + +**What changed and why.** §2.3 put every candidate in a 48–79 GiB band and +concluded the choice was not cost-gated. The first half survives — **every +candidate still fits the 124 GiB box** — but the band was an artefact of holding +the byteswap gadget fixed across rows that delete it. The real spread is +**4.4 GiB to 71 GiB, and blake is ~11× the field-native candidates**, so wrap +size *is* a discriminator between the byte-oriented and the algebraic families +(though still not among the algebraic ones, where §2.3's noise argument holds: +RPO, Poseidon and Monolith differ by 2.4× on cells-per-permutation and land +within 1.8× on total). + +⚠ **Correction to the sub-proof count, which wave 9 and §2.3 both got wrong.** +The two-term model's per-sub-proof term is about the proof being *produced* — the +wrap. At this shape the wrap has **19** sub-proofs (13 chip classes + 6 +`KECCAK_RND` chunks), not 24; 24 is the INNER epoch's leg count. It is also +candidate-dependent: blake drops three keccak-family chips and adds one (≈12), +field-native drops four including `BITWISE` (≈10). At keccak's scale this moves +nothing, but for the field-native rows **the sub-proof term is 27–41 % of the +projection** — it is the dominant term there, which makes those the weakest RSS +numbers in the table. + +⚠ **Both RSS coefficients were calibrated on keccak-shaped runs** — a machine +whose largest tables are a 1,480-column round chip and a 2²⁰-row lookup table. +Nothing has checked that 27 B/cell survives a machine whose widest table is a +3,056-column single-row chip, still less one with no lookup table at all. Every +GiB figure above is a projection carrying that caveat. + +For the record, §2.3's own rows recomputed with BOTH terms in GiB at its stated +24 sub-proofs (the wave-9 erratum discharged — the gap is the dropped sub-proof +term against a GB-labelled-GiB cell term, which happened to cancel to ~2 %): + +| §2.3 row | its cells | cell term GiB | + sub-proof term | §2.3 printed | +|---|---|---|---|---| +| RPO | 1.814 B | 45.6 | **49.9** | 48 | +| Poseidon-original | 1.844 B | 46.4 | **50.6** | 49 | +| Poseidon (conservative) | 1.905 B | 47.9 | **52.1** | 50 | +| Monolith | 1.951 B | 49.1 | **53.3** | ~52 | +| BlakeG 32-row | 3.037 B | 76.4 | **80.6** | 79 | +| keccak | 11.166 B | 280.8 | **285.0** | 284 | + +These are corrected in place but **superseded** by the table above: their cell +totals all carry the byteswap gadget. + +### 8.7 The delegation topology — priced, and it is a net LOSS here + +User request: price blake3 in a SEPARATE specialized circuit (Airbender's +pattern — their blake2s delegation circuit does ~19 proofs' Merkle work in one +2²⁰ instance) against in-trace hosting. Instrument: +`blake3_probe::the_delegation_topology_priced_against_in_machine_hosting`, +arithmetic over the same closed form the epoch's own permutation count comes +from. Inputs INHERITED from the epoch's 2²⁰ leg: 2 composition parts, 73 queries, +198 FRI compressions per query, blowup 8. + +``` +IN-MACHINE LFM_BLAKE3 as one more AIR of the epoch verifier's multi-proof: + 192,000 rows x 4,946 = 967,402,978 cells. Nothing else changes. + +DELEGATED (a) the delegation proof's own trace (LFM_BLAKE3 + its BITWISE) + 993,617,378 cells + (b) verifying that proof inside the epoch verifier: + LFM_BLAKE3 AIR (2^18 rows, 3,056 main + 630 aux) + 1,523/query x 73 = 111,179 compressions + BITWISE AIR (2^20 rows, 10 main + 5 aux) + 298/query x 73 = 21,754 compressions + = 132,933 compressions = 657,486,618 extra cells, ON TOP of (a) +``` + +**Verdict: delegation costs (a) + (b) where in-machine costs (a) alone — a net +loss of 657 M cells, +66 %.** The reason is structural rather than a tuning +accident. What Airbender's delegation circuit buys *them* is moving hash work out +of a **fixed-size** main circuit (a 2²⁰-cycle RISC-V trace) whose cycles the +hashing would otherwise consume. **The LFM has no fixed-size box**: every chip's +height is program shape, and the proof is already a multi-AIR proof over +independently-sized tables connected by a bus. Our architecture *is* the +delegation pattern; a second proof only adds a verification. + +The term that makes (b) expensive is the leaf term, the same one §2.4 flagged as +having no analogue in cross-system 2-to-1 figures: a 3,056-column AIR has a +6,112-felt main leaf, which is 765 compressions to absorb, 73 times per query. +**A delegation circuit is wide by construction, and wide traces have expensive +leaves** — so the wider and more efficient you make the delegated chip, the worse +its proof is to verify. + +Two variants considered and priced the same way. *Batching K epochs' compressions +into one instance* (the literal Airbender shape) saves the fixed `BITWISE` table +K−1 times — 26.2 M cells each, 468 M at K = 19 — but still pays (b) once, so it +is a loss until K ≳ 25 and it gives up one-proof-per-epoch. *Padding +amortisation* buys nothing: 192,000 pads to 2¹⁸ and 384,000 to 2¹⁹, the same +36 % either way, and chunking already fixes it (§8.5). + +⚠ This prices CELLS only. It cannot see prover wall time, proof size on the +wire, or the engineering cost of a second circuit and its glue — and those are +where a delegation argument would have to be made if anyone wants to remake it. + +### 8.8 Falsification (rule 1) — chip-only mutations, control-validated + +Mutating the chip *and* the primitive together would prove nothing, so each +mutation below changes only `blake3_chip.rs`, leaving `blake3.rs` — pinned by the +canonical vectors — intact. That works because the probe's mirror AIR computes +its `LfmMem` words from `blake3::blake3_compress_6round`, which is an +INDEPENDENT implementation of the compression: the bus is a genuine differential +between the primitive and the chip's own dataflow, and neither delegates to the +other (rule 7's trap avoided deliberately). + +Failures read from the trailing summary block, and a green control was run first +and again after restoring (rule 7's corollary). + +| mutation | result | +|---|---| +| CONTROL (unmutated) | 14 passed, 0 failed | +| F1 `rotr8`'s free byte relabel transposed in the WIRE interpretation only | **exactly 2 failed**, both prove+verify | +| F2 message schedule transposed in the chip's `run_flow` (`sched[p] = prev[i]`) | **exactly 2 failed**, both prove+verify | +| F3 the `LfmMem` read multiplicity ungated (`Column(MU)` → `One`), so padding rows read | **exactly 2 failed**, both prove+verify | +| CONTROL again (restored) | 14 passed, 0 failed | + +The *discrimination* is the useful part: in all three the only casualties are +`the_hosted_chip_proves_and_verifies` and the control, while the six trace-tamper +tests and the layout/degree tests stay green — so the mutations are isolated to +what the proof sees, which is what a chip-only falsification is supposed to show. + +⚠ **A fourth mutation is recorded because it FAILED to be a falsification.** +Changing `ROT_SHIFT_R` from `[4, 9]` to `[4, 10]` made 8 tests fail in 0.01 s — +`ValueFlow::rot_shift`'s `debug_assert_eq!` panics before any proof is built. The +mutation *is* caught, but by an assert, not by prove+verify, so it is evidence +about the debug assert and not about the constraint set. Reported rather than +quietly replaced: a falsification harness that counts a panic as a constraint +rejection would be exactly the "my mutation changed nothing" instrument bug rule +7's corollary warns about, inverted. + +Six trace-tamper tests back the chip-only set: a flipped OUT byte, a flipped +message byte (the one that would go green if the 32 message `AreBytes` were ever +dropped as redundant), a flipped add3 carry bit, a padding row turned real, a +bumped read multiplicity, and the all-zero-padding assertion. + +### 8.9 Provenance of the primitive, and why rule 9 is discharged differently + +Rule 9 wants an EXTERNAL known-answer vector that nothing in this repository +produced. **That is impossible in the usual form here**: the 6-round variant is +not standard BLAKE3, so no published vector and no crate exposes it. The chain +#903 supplies, recorded rather than waved at: + +1. a z3-proved model of the compression dataflow (`z3_blake_verify.py`); +2. a Python oracle (`blake3_ref.py`) whose **7-round** instantiation is pinned + against the official `blake3` crate's published vectors — so the G-function, + message schedule, counter split and feed-forward are externally validated and + only the round count varies; +3. that oracle at `rounds = 6` emitting the 10 canonical vectors this port is + pinned against. + +The external anchor is therefore one step removed. To check that the vectors +nevertheless *discriminate* rather than merely being reachable, +`breaking_one_convention_at_a_time_breaks_the_vectors` runs a parameterised +control at four broken conventions — rotr12→rotr13, rotr16↔rotr8 (the cheapest +possible error, since both are free byte relabels in the chip), the message +schedule transposed, and 7 rounds instead of 6 — and each stops reproducing +vector 0. A fifth test shows the counter's two halves are not interchangeable. + +⚠ Security assumption **A6R** (6-round collision resistance) is named and +unratified. Nothing here ratifies it; this leg prices the AIR. + +### 8.10 What this leg does NOT settle + +- It does not register `LFM_BLAKE3`, so no epoch verifier has ever hashed with + blake. The column is "cells to verify an epoch of this shape if it hashed with + H", the same limit §3.4 states for every column in this matrix. +- The field-native residue is a subtraction, not a re-emission (§8.4). +- It does not choose a hash. It does sharpen the choice: the decision is no + longer between candidates that are all within 1.6× on size, but between a + byte-oriented family at ~71 GiB and an algebraic family at ~4–8 GiB — with the + byte-oriented family holding the only real in-AIR measurements and a shipped + production existence proof, and the algebraic family holding the size. + +### 8.11 ★ Reconciliation against `hash-delegation-eval.md` §3.1 / §4.1 + +Requested by the team lead, who put the byteswap's CELL share near ~50% against +the eval's instruction-derived 88–93%. **Both figures are right, about different +quantities, and the whole gap is a padding term neither included.** + +``` +byteswap share of the residue, UNPADDED closed form 903,326,725 = 51.38% <- the ~50% estimate +byteswap share of the residue, PADDING-AWARE 1,684,910,080 = 95.84% <- what the machine pays +padding multiplier on the gadget 1.865x +``` + +The gadget is what DRIVES `LFM_BALU` to 2²⁷ (71,974,504 real rows) and +`LFM_BITDEC` to 2²¹ (1,124,295 real rows), so removing it does not remove +`rows × width` — it removes two padded power-of-two tables. The team lead's +"`BALU` rows are cheap at 10 base-equiv" is correct and is exactly why the +unpadded number is ~50 %; what it misses is (a) the gadget's other half, +`LFM_BITDEC`, at **165** base-equiv per row — 20 % of the unpadded cost from 1.6 % +of the rows — and (b) the padding. **The measurement is the padding-aware one: a +machine that does not byteswap does not build those tables at all.** + +Item by item against the eval's own numbers: + +| eval claim | MEASURED | verdict | +|---|---|---| +| §3.1 byteswap = 88–93 % of residue | **95.84 %** (cells) | eval LOW by 3–8 pts, right conclusion | +| §3.1 `R_native` = 0.024–0.20 B, central 0.059 B | **0.073 B** (73,072,788) | **inside the band**, 24 % above central | +| §3.1 "residue collapses ~10–70×, central ~30×" | **24.1×** | inside the band | +| §4.1 Poseidon in-trace + `R_native` = 0.145–0.32 B, c. **0.18 B** | **0.195 B** | **✅ SURVIVES**, 8 % above central | +| §4.1 BLAKE3-6 in-trace + `R_native` = 1.06–1.24 B, c. **1.10 B** | **2.752 B** | **❌ DOES NOT SURVIVE as built — 2.5× out** | +| §4.1 hosted chip "should land near ~5,150 (≈3 % lower)" | **4,946 (7.0 % lower)** | direction right, magnitude 2.3× understated | +| §4.1 BLAKE3 hash cells 1,040,064,768 | **967,402,978** | −7.0 %, follows from the line above | +| §3.1 open: "does an LFM-hosted BLAKE3 chip shed the byteswap? It should" | **NO** | ❌ **the unfavourable answer** | + +**Why blake does not shed it, and it is not an implementation choice I made.** +The chip consumes machine words of four `u32` lanes — the `LFM_KECCAK` convention +— and `u32` halves are precisely what `felt_be_halves` produces. The gadget is +UPSTREAM of the chip's input format, so hosting the chip cannot delete it. Blake +therefore pays the byte-oriented residue AND keeps `BITWISE`. + +**But the eval's 1.10 B is recoverable, and the convergence is exact.** A variant +that receives full 64-bit felts and decomposes them to bytes inside its own +constraints does shed the gadget. It needs one thing the eval's sketch omits: a +**canonicity gate per absorbed felt**. `Σ byteₖ·256ᵏ = v` over the field does NOT +pin the byte string — `v` and `v + p` both satisfy it — so without a `< p` +argument the prover chooses what gets absorbed and Fiat–Shamir breaks. (That is +why `felt_be_halves` routes through `bit_dec`, whose doc says outright: "`bit_dec` +also enforces canonicity (`< p`) … production renders `canonical_u64()`" — +VERIFIED, `transcript_replay.rs:735-736`.) A borrow-chain `< p` gate at degree 3 +is small against a 3,056-column chip; at my ESTIMATE of ~156 extra base-equiv per +compression (8 absorbed felts × ~20): + +``` +felt-absorbing BLAKE3 (ESTIMATE, UNBUILT): 5,102/compression + R_native + BITWISE + = 1,097,202,674 = 1.097 B, 10.2x, ~30 GiB +eval §4.1 central = 1.10 B +``` + +**So the 2.5× discrepancy is not an arithmetic disagreement — it is precisely the +value of the unbuilt felt-absorbing variant, ≈1.65 B.** The eval priced a design; +I measured the one that exists. Both numbers should be carried, labelled. + +⚠ **I did not build it, and the reason is a cryptographic decision, not effort.** +The input side is engineering; the OUTPUT side is not. A blake output word is 32 +bits, so a felt built from 8 output bytes is a 64-bit value reduced mod `p` and +the map is not injective. How a blake digest becomes felts — truncate to four +`u32`s, reduce, domain-separate — changes the security argument, the digest +width, and the token count. That is the ecosystem's call, the same boundary +§6.1/§7.7 draw around Poseidon's parameters and `compress_iv`. + +**Net effect on the eval's verdict.** Its §4.1 conclusion strengthens rather than +weakens: the field VM's fully-delegated floor is `R_native` + the stub, measured +at **0.073 B + ~7.7 M ≈ 0.081 B** against its predicted 0.067 B central — and its +"delegation's field-side win over a field-native algebraic hash is 0.113 B / 63 %" +becomes **0.195 − 0.081 = 0.114 B / 58 %**, i.e. essentially unchanged. What +changes is the in-trace blake row it is competing against, and only for the +variant nobody has built. + +⚠ And one finding of mine that bears directly on the eval's scheme (§8.7): +**delegation as a SEPARATE PROOF is a net loss of +66 % on this machine.** The LFM +has no fixed-size main circuit to escape — every chip's height is program shape — +so its multi-AIR proof already IS the delegation pattern, and a second proof only +adds a verification whose leaf term is large precisely because a delegated hash +chip is wide. diff --git a/others/lfm-logup-handoff.md b/others/lfm-logup-handoff.md new file mode 100644 index 000000000..83557546b --- /dev/null +++ b/others/lfm-logup-handoff.md @@ -0,0 +1,158 @@ +# LogUp closure — handoff + +Written 2026-07-31 by deep-join, at the end of its context. The leg is +COMPLETE and green; this file exists so the next agent does not have to +re-derive what took the longest to establish. Everything below is first-hand +unless marked otherwise. + +**State: `cargo test -p lambda-vm-prover --lib lfm` green (171+), `make lint` +clean, all committed on `feat/lfm-deep-join`.** Files: `prover/src/lfm/logup.rs` +(emitter), `prover/src/lfm/logup_tests.rs` (8 tests), plus +`constraints::emit_table_offset` / `emit_alpha_powers`. + +--- + +## 1. What the closure IS — the part that took longest to establish + +Production's check is one block, `crypto/stark/src/verifier.rs:1303-1334`: + +``` +if needs_lookup_challenges { + total = Σ over (air, proof) where air.has_trace_interaction() + && proof.bus_table_contribution().is_some() + if total != *expected_bus_balance { return false } +} +``` + +Two things about it are not guessable from the name: + +**The target is not zero and not a constant.** It is +`compute_expected_commit_bus_balance_view` → `compute_commit_bus_offset` +(`prover/src/lib.rs:909`): + +``` +expected = Σ_i 1 / (z − (BusId::Commit + (start+i)·α + byte_i·α²)) +``` + +over the public output BYTES. The COMMIT output bus has a receiver the verifier +computes rather than proves, and this is that missing remainder. So half the leg +is a per-byte inverse gadget whose length scales with public output — an +unbudgeted cost item, now on the ledger's WATCH list. + +**`L` is NOT an opened value.** It is a proof-carried scalar, absorbed into the +per-table transcript fork (`verifier.rs:1274`). The original charter assumed the +join was to the authentication leg; it is not. What binds `L` to a trace is the +constraint leg — traced end to end: + +- `verifier.rs:306` computes `logup_table_offset = L / N`, feeds it to the + transition context at `:326`; +- `lookup.rs:2260` (`emit_logup_accumulated`) enforces + `acc_next − acc_curr − Σterms + L/N = 0`; +- `lookup.rs:1346` pins `acc[0] = 0` — note its `_bus_public_inputs` parameter + is UNUSED, so the boundary is *not* where `L` enters; +- together the accumulator wraps to zero after `N` rows iff `L` is the table's + true total. + +## 2. The gap this leg closed, and the one the audit found after + +**Instance 2 (`L`).** `constraint_tests.rs` computed `table_offset = L/N` +host-side and hinted it; the machine never saw `L`. Adding a closure that hinted +`L` would have given the prover two independent words — truthful `L₁/N` so every +accumulator wraps, arbitrary `L₂` so the sum hits the target. Fixed by +`emit_table_offset` (`L · N⁻¹`, `N⁻¹` a program constant since `N` is shape). + +**Instance 3 (`alpha_powers`), worse in degree.** They were hinted one word +each, and `Op::AlphaPow{idx}` read them straight. Every LogUp FINGERPRINT is +`z − Σ vⱼ·αʲ`, so a prover choosing the powers chooses the fingerprints and any +tuple can be made to match any other. Fixed by `emit_alpha_powers`, chaining +from the one α. + +Both are recorded in `lfm-assembly-obligations.md`. The generalised rule — +**any value TWO legs consume must be one cell** — came out of this leg. + +## 3. What is WITNESSED, and how + +- COMMIT-bus target vs production's `compute_commit_bus_offset`, 28 + (length, start) combinations including the empty short-circuit. Lengths + matter: `start` advances *inside* the gadget, so a reset-or-reversed formula + agrees only at length 1. +- A deliberate fingerprint COLLISION is unprovable (`1/0`), matching + production's `.ok()?`; the neighbouring non-colliding `z` still folds, so the + rejection is the collision and not the shape. **This test is mandatory, not + optional** — the machine's `0/0 = 1` convention means a term written as a + direct divide would accept exactly what production rejects. Same trap the DEEP + denominators hit. +- The closure over a real sender/receiver pair whose bus genuinely closes, with + `multi_verify` at target zero as the oracle (checked FIRST — a fixture whose + bus did not close would make agreement meaningless). Every single-lane move of + either contribution rejected. +- The `L` join: a split-arena control accepts 4 coherent forgeries (truthful + offset so accumulators wrap, forged `L` + matching target so the closure + balances); the derived shape rejects all 4. +- An ABSOLUTE structural guard (method rule 7): + `the_derived_uniforms_are_not_arena_words` asserts which cells are + `Instr::Hint` outputs and which are computed, with positive controls. Immune + to variant unification. Both negative branches falsified independently — they + are ordered, so the first masks the second; break them one at a time. +- **Per-CHUNK accumulation** (the degenerate parameter the team lead + prioritised). `VmAirs::new` builds one AIR per chunk + (`lib.rs:702`: `(0..table_counts.cpu).map(|i| … CPU[i])`), so a family of `k` + chunks is `k` sub-proofs and the closure sums per chunk. Witnessed with a + 3-table fixture — one sender, TWO receiver chunks of one family, one lookup + each. Both halves: the 3-term sum closes, and all three 2-term readings are + nonzero, plus a closure compiled for 2 tables rejects both chunk drops. On any + 1-chunk-per-family fixture the two readings agree, which is why this needed + building. +- `has_trace_interaction()` is shape, and production checks the proof's presence + against it in BOTH directions (`verifier.rs:1238` and `:1244`), so the two can + never disagree in a proof that verifies. `num_contributing_tables` is + therefore a program constant; a short arena is rejected. + +## 4. What is NOT witnessed — precise statements + +- ~~**Zero-row fixed tables.**~~ — **SETTLED BY MEASUREMENT** 2026-08-03 + (zerorow, `logup_tests::a_zero_row_fixed_table_carries_some_zero_not_none`). + The answer is **`Some(zero)`**, and the inference held exactly. A real + intermediate epoch (epoch 0 of the fibonacci fixture, proved over the + production epoch AIR set and ACCEPTED by `multi_verify_views`) carries **five** + zero-row fixed tables — KECCAK, KECCAK_RND, KECCAK_RC, ECSM, ECDAS — each with + `has_bus_public_inputs() == true` and `L` exactly zero. `num_contributing_tables` + is therefore safe as a program constant. Two things the closure did not know: + * The other half of the old argument is now a run too, not a deduction: + stripping `bus_public_inputs` from any of those five sub-proofs makes the same + proof FAIL to verify. So `Some` is forced, not merely observed. + * **A zero-row table is not a blank one.** KECCAK_RND's and ECSM's traces are + literally all zero, but KECCAK pads with `state_ptr[lane] = 8·lane`, KECCAK_RC + is a preprocessed constant table, and ECDAS pads likewise — those three have + NO rows on any bus (every multiplicity column is zero) over a trace that is + not blank. Any future emitter that treats "unused table" as "blank trace" + would be wrong on three of the five. +- ~~**Real epoch table-set length.**~~ — closed by the same test, which runs the + closure over all **24** contributions of a real epoch (8 output bytes, target + from production's own `compute_expected_commit_bus_balance_view`), and rejects + all 72 single-lane moves. The two- and three-table fixtures remain the ones + that isolate per-chunk accumulation. +- **`start_index`** is unbound to the chain — ledger OPEN entry 2. Do not invent + a binding; read how production carries it across epochs first. +- **The five remaining two-consumer values** — ledger OPEN entry 3. Deliberately + not fixed leg-side: unifying them means deciding the assembled program's arena + layout, which is assembly's call. + +## 5. Traps for whoever continues + +- `open_sub_proof` (constraint_tests) handles the SINGLE-table case only — it + transcribes `multi_verify_views` without the per-table domain separator. A + multi-table fixture cannot go through it. That is why the closure fixtures + read `bus_table_contribution()` off the proof directly rather than replaying. +- `EmptyConstraints` leaves ONE coefficient in the transition run, and + `open_sub_proof` recovers `beta` from the second (`constraint_tests.rs:972`). + Any synthetic AIR you want to push through it needs at least one real + transition constraint. The preprocessed fixture's `CopiedColumn` exists purely + for this. +- `test_utils::production_airs` builds BITWISE, DECODE, KECCAK_RC, REGISTER and + PAGE WITHOUT their preprocessed commitments, so `is_preprocessed()` reads + false on five tables that are preprocessed in a real epoch. Any census taken + off those objects silently drops an opening group each. +- A doc comment citing a guard test is not evidence the guard exists — + `challenges_are_not_an_arena_in_the_assembled_verifier` was cited at + `constraint_tests.rs:165` and had never been written. Ledger OPEN entry 4. diff --git a/others/lfm-migration-riders.md b/others/lfm-migration-riders.md new file mode 100644 index 000000000..dfb0438af --- /dev/null +++ b/others/lfm-migration-riders.md @@ -0,0 +1,67 @@ +# Riders to carry into the ecosystem hash migration + +Things that are cheap-to-free if they ride the transcript/hash rebuild (which +is already proof-breaking and already owed), and not worth a proof-breaking +change on their own. Each entry: what, why it helps, what it costs today. + +## 1. Constant-consumption challenge sampling + +**What:** make the production `sample_field_element` consume a FIXED number of +candidates per draw instead of looping on rejection. + +**Why:** a straight-line machine cannot follow a data-dependent consumption +schedule, so the LFM transcript replay encodes the no-rejection schedule and is +unprovable for a transcript that ever rejects (`SOUNDNESS.md` §6.3). With +constant consumption the restriction disappears for every future machine. + +**Cost today:** completeness only, bounded `< 10^-6` per proof at production +draw counts. Acceptable — hence a rider, not a fix. + +## 2. One-byte pad at the end of the statement encoding + +**What:** pad the continuation-epoch statement so its length is `≡ 0 (mod 4)`. + +**Why:** the encoding is `207 + L + 16R` bytes (not 223 — an arithmetic slip in +the first report, now machine-checked by +`epoch_statement_cursor_is_three_plus_output_len`). Every subsequent absorb +inherits the resulting cursor — including all of Phase A, whose roots are +individually 32-byte-aligned but land misaligned because they inherit the +statement's cursor. (Alignment is a property of the CURSOR, not of the field: +this was initially mis-analysed as "Phase A needs no splice", which is true in +isolation and false in context.) + +**⚠ Second correction — the shift is NOT unconditionally 3.** It is +`(3 + L) mod 4`, where `L = |public_output|`. The earlier claim of "≡ 3" quietly +assumed `L ≡ 0 (mod 4)`, which is false in general: `public_output` is collected +one byte per COMMIT operation (`trace_builder`), so `L` is whatever the workload +produced. Consequences: the Phase-A splice cost is WORKLOAD-DEPENDENT, and it is +**zero** whenever `L ≡ 1 (mod 4)` — roughly one workload in four pays nothing at +all. A pad that fixes the cursor would make the cost zero and, more usefully, +*predictable*, which is the stronger argument for the rider. + +**⚠ Third correction (2026-07-30, measured on a real fixture) — the `16R` term +is not live.** `runtime_page_ranges` is ALWAYS EMPTY for continuation epochs +(PAGE tables are skipped; the struct comment says so). So the real encoding is +`207 + L`, with `R = 0`, and the shift is `(3 + L) mod 4` full stop. R1e's +synthetic test shape uses `R = 2`, which is a legitimate test shape but means +any arithmetic above quoting `16R` is computed over a term the real statement +does not have. The rider's conclusion is unchanged — the shift still depends on +`L`, and a pad still makes it predictable — but do not read `16R` as live. + +**Cost today:** 2 roots × 8 halves × T tables spliced, whenever the inherited +shift is nonzero — at T = 24 that is 384 `BitDec` + ~13k `BALU` rows per proof, +and zero for the ~1-in-4 workloads whose output length lands the cursor on a +boundary. Against a ≈7.3M-instruction +epoch verify that is ~0.2% of instructions; the `BitDec` rows are wide, so +call it low single-digit percent of the machine's fixed trace floor. Real, but +nowhere near worth a proof-breaking change by itself. + +**Note:** the encoding is already versioned by its domain tag +(`LAMBDAVM_CONTINUATION_EPOCH_V2`), so a pad is a tag bump — exactly the kind of +change a migration absorbs for free. + +## Rule for adding to this list + +An entry belongs here if (a) it costs the machine real work today, (b) fixing it +requires a proof-breaking or production-semantics change, and (c) the migration +has to touch that code anyway. If (c) is false it is a normal PR, not a rider. diff --git a/others/lfm-page-base-uniform-proposal.md b/others/lfm-page-base-uniform-proposal.md new file mode 100644 index 000000000..dcb696a39 --- /dev/null +++ b/others/lfm-page-base-uniform-proposal.md @@ -0,0 +1,426 @@ +# Proposal: promote `page_base` (and `epoch_label`) to runtime uniforms + +Status: **GATE CLEARED 2026-07-30** by an independent read-only trace; proposal +revised accordingly. Still proposal only — no semantics touched. + +Three things the gate trace changed, all of which made the proposal *safer* and +one of which retargets it: + +1. **The constant was never a binding.** I argued the uniform would be sound + *because* `page_base` is already bound by the preprocessed commitment. That + premise was wrong — it is not bound by anything (§4). The conclusion survives + and is stronger: there is nothing to break. +2. **On the continuation path the AIR that matters is GLOBAL_MEMORY, not PAGE** + (§0.1). PAGE is never constructed for a continuation epoch. +3. **`epoch_label` is materially safer than `page_base`**, so my recommendation + to move them together as equal-risk was wrong (§4.2). + +## The problem, stated precisely + +Four production AIRs fold a workload-dependent value into their captured +constraint IR as a literal constant: + +| table | parameter | where it enters | +|---|---|---| +| `PAGE` | `page_base` | `tables/page.rs:533,539` — `LinearTerm::Constant(page_base_lo)`, `BusValue::constant(page_base_hi)` | +| `GLOBAL_MEMORY` | `page_base` | `continuation.rs:228` → `global_memory::bus_interactions(config.page_base)` | +| `L2G_GLOBAL` | `epoch_label` | `tables/local_to_global.rs:360` — `BusValue::constant(epoch_label)` | +| `L2G_MEMORY` | `epoch_label` | `tables/local_to_global.rs:447` — `LinearTerm::Constant(epoch_label as i64 - 1)` | + +`BusValue::constant` / `LinearTerm::Constant` lower through +`ConstraintBuilder::const_base`, so the value becomes an `Op::ConstBase` leaf in +the captured program. A different parameter value is a different program. + +### 0.0 PRIORITY — `epoch_label` is on the critical path; `page_base` is not + +This reordering follows from the epoch composition measured in the lowering +design, and I did not draw it myself: + +``` +epoch proof = 14 split families + 9 or 10 fixed + 1 L2G_MEMORY +``` + +No PAGE (`page_configs = &[]`). No GLOBAL_MEMORY — that lives in the *global* +proof. So **the only parameterized AIR in an epoch proof is `L2G_MEMORY`, and its +parameter is `epoch_label`.** + +`epoch_label` is `index + 1`. Unpromoted, the registry therefore needs **one +distinct program per epoch index**, and the ladder grows **linearly with epoch +count** — which is precisely the workload-dependence the constraint leg was just +shown NOT to have (a ~94%-fixed leg collapses the ladder to one dimension in +epoch size). Winning that structurally and then losing it to a bus constant would +be a poor trade. + +`page_base` reaches the machine only through GLOBAL_MEMORY, i.e. only when the +GLOBAL proof comes into scope — a later leg, and one where size was never the +issue (25 instructions per touched page against a ~63K leg). + +**Order: `epoch_label` first (§4.3), then `page_base`/GLOBAL_MEMORY.** For +`epoch_label` the framing is ladder-collapsing, not low-risk-warm-up; it is both, +but the first is why it goes first. + +### 0.1 SCOPE — on the continuation path, PAGE is never built + +Continuation epochs pass `page_configs = &[]` (`continuation.rs:693`, `:797`, +enforced prover-side at `:677-681`), so `create_page_air` is **not called** for +an epoch proof. The page-base-as-constant AIR on the critical path is +**`GLOBAL_MEMORY`** — `global_memory::bus_interactions(page_base)` +(`tables/global_memory.rs:172-214`) via `global_memory_air` +(`continuation.rs:220`). + +We recurse continuation epochs, so **GLOBAL_MEMORY is the target**; PAGE matters +only for monolithic proofs. The mechanism below is identical for both — the two +tables differ only in which constants they fold — but the priority is not, and +an implementation that fixed PAGE alone would leave the target path untouched. + +**Why this is an identity problem, not a size problem.** Size is negligible +(measured below). The blocker is that LFM program identity is a registry-pinned +digest over the emitted program. If the program embeds constraint evaluation and +the constraints vary with the workload's page set, then registry entries become +workload-dependent — and page bases are arbitrary addresses, not a small +enumerable ladder. That breaks the premise the registry exists to uphold. + +### Measured, at blowup 2 (from `constraint_artifact_tests`) + +| table | nodes | bytes | two parameter values differ by | +|---|---|---|---| +| `PAGE` | 63 | 1,240 | 1 constant value; node count and roots stable | +| `GLOBAL_MEMORY` | 43 | 904 | 1 constant value; node count and roots stable | +| `L2G_GLOBAL` | 47 / 48 | 968 | +1 constant, +1 node, **roots move** | +| `L2G_MEMORY` | 93 / 95 | 1,768 | +1 constant, +2 nodes, **roots move** | + +Two things worth pulling out of that table. + +First, the variation is **not** confined to constant values, which is what one +would naively assume. The builder interns constants by value, so a parameter +whose value is already in the table costs no new node while a fresh one appends +— shifting every later node id and therefore the constraint ROOTS. `L2G_GLOBAL` +at `epoch_label = 1` reuses the existing `1`; at `epoch_label = 7` it appends. +**This kills the cheap patch.** "Emit one program and swap a constant per page" +is not available, because the programs are not even the same length. + +Second, what IS invariant is the algebra: shape, metadata, `num_base`, and +constraint count are identical across parameter values (asserted by +`parameterized_airs_vary_per_parameter_value`). That invariance is exactly what +makes the uniform promotion viable — the parameter is genuinely a value, not a +structural choice. + +### Scale + +- Page size `DEFAULT_PAGE_SIZE = 1 << 18` = 256 KiB (`tables/page.rs:50`) — + VERIFIED. +- `local_to_global::MAX_EPOCHS = 1 << 20` (`tables/local_to_global.rs:83`), a + hard cap from the `IsB20` range — VERIFIED. Real epoch counts are far smaller + (a small ethrex block is 1–2 epochs). +- **11 distinct ELF page bases** for the committed ethrex ELF, derived statically + from its `PT_LOAD` headers (not file size, which overcounts). All carry + `init_values`, so a monolithic ethrex `program_id` folds exactly 11 pairs. + Plus 1 private-input page for every committed ethrex fixture. — from the gate + trace. +- **Continuation touched-set size: NOT MEASURED and not statically derivable.** + It is recorded nowhere. The design comments imply tens rather than thousands; + that is INFERENCE, not measurement, and is labelled as such wherever it is used. + +Either way this confirms size was never the issue: at 25 instructions per +GLOBAL_MEMORY sub-proof, even a four-figure page count is noise against a ~65K +constraint leg. The problem was only ever identity. + +## Proposed mechanism + +### 1. A new IR leaf: base-field runtime uniform + +```rust +// crypto/stark/src/constraint_ir/ir.rs +Op::BaseUniform { idx: u16 }, // Dim::Base +``` + +with device tag `OP_BASE_UNIFORM = 11` (the next free value; tags 0..10 keep +their meanings, so **every already-serialized artifact stays valid** and the +16-byte `DeviceNode` layout is untouched). + +**It must be a BASE-field uniform, and that is the whole design constraint.** +Every uniform the IR has today — `RapChallenge`, `AlphaPow`, `TableOffset` — is +`Dim::Ext`. Reusing that machinery would be the obvious move and it is wrong: +`binop` promotes to the extension whenever either operand is `Ext`, so +`page_base_lo + OFFSET_column` would become an extension add. The values would +still agree (embedding is a ring homomorphism) but every downstream node's dim +flips, and `eval_program` would then hit `as_base()` on an extension value for a +base-rooted constraint — a panic, not a wrong answer. It would also silently move +the prover's hot path from base to extension arithmetic. So: a new leaf, base +dim, resolved against a `&[FieldElement]`. + +Degree is 0, same as a constant, so `max_degree` and the composition bound are +untouched — **no proof-format change**. + +### 2. Cost to the two DeviceProgram consumers + +The parity requirement (CUDA kernel and CPU walker consume `DeviceProgram` +bit-identically) is preserved by construction: the change is one additional tag, +handled the same way in both. + +**CPU walker** (`eval_device_program`) — one match arm, structurally identical to +the existing `OP_RAP_CHALLENGE` arm but reading a `u64` table instead of a +`[u64;3]` one, plus one new `&[u64]` parameter: + +```rust +OP_BASE_UNIFORM => Value::Base(FpE::from_raw(base_uniforms[node.a as usize])), +``` + +**CUDA kernel** — one `case` in the `switch (op)`, one extra `const uint64_t*` +kernel parameter, one small device allocation (a handful of `u64`s, uploaded +once per proof alongside the existing uniform buffers). No layout change, no new +divergence class beyond one more case in a switch that already has eleven. + +This is the cheapest extension the IR admits. Anything that instead tried to +patch constants per-instance would require re-uploading the constant table per +page, which is strictly worse on the device. + +### 3. Plumbing: on the AIR, NOT on the context + +This is the part that needs a decision, because the obvious route is wrong. + +The existing uniforms arrive via `TransitionEvaluationContext`, which is built +once per proof and shared across AIRs. `page_base` is **per-AIR** — a multi-proof +contains many PAGE AIRs with different bases — so it cannot ride that path +without being wrong. + +Proposed instead: + +```rust +// crypto/stark/src/traits.rs +fn base_uniforms(&self) -> &[FieldElement] { &[] } +``` + +`AirWithBuses` stores the slice it was constructed with; +`compute_transition_prover` / `compute_transition` already have `&self`, so they +can hand it to the folder at construction. No signature change reaches the +prover or verifier driver. + +Bus layer (the largest chunk of actual work, and the only semantics-adjacent +part): `BusValue::Uniform(idx)` and `LinearTerm::Uniform { coefficient, idx }` +alongside the existing `Constant` variants, lowering to `b.base_uniform(idx)`. +Then four call sites change — `page.rs`, `global_memory.rs`, and two in +`local_to_global.rs`. + +### 4. SOUNDNESS — gate cleared, and my premise was wrong in my favour + +I argued the uniform would be sound *because* `page_base` is already bound by the +preprocessed commitment. **That premise is false.** The gate trace established: + +- `page::compute_precomputed_commitment` covers only OFFSET and INIT. + `page.rs:380-383` says the commitment "depends only on the blowup factor — not + on page_base", pinned by `static_commitments_tests.rs:82`. +- `page_base` is **not absorbed into the transcript** — the verifier absorbs only + preprocessed and trace roots. +- It reaches `program_id` only for ELF-backed data pages. + +So the compile-time constant is a **verifier-side local, not a commitment**. +Removing it costs nothing, because it was never buying anything. The conclusion +survives and is stronger than the argument I made for it — but I had the reason +backwards, and a proposal resting on a false premise is one edit away from +resting on nothing. + +#### 4.1 THE LOAD-BEARING INVARIANT + +> **The uniform MUST be populated from the same verifier-side sources that +> produce the constant today: `page_configs` / `canonical_page_bases( +> bundle.touched_page_bases)`. It must NEVER be sourced from the proof or from +> the trace.** + +This is not a note. It is the entire soundness content of the change, and it is +*more* critical precisely because §4 found no binding: if a prover-chosen base +ever reached this uniform, **nothing downstream would catch it**. No preprocessed +root covers it. No transcript absorb covers it. `program_id` is not a safety net +(it folds page bases only for ELF-backed data pages). The value would be +unconstrained, and the failure would be silent. + +The rule is the one `trace_ood_next_row_columns` already states: computed +identically by prover and verifier, never read from the prover-controlled proof. + +#### 4.2 `epoch_label` is NOT symmetric with `page_base` + +I recommended moving them together as the same mechanism at the same risk. The +mechanism is the same; **the risk is not**, and the proposal should not have +flattened them. + +`epoch_label` is **verifier-derived by construction**: it comes from the +verifier's own `enumerate()` position (`continuation.rs:1293-1295`, +`local_to_global::epoch_label(index) = index + 1`) and is never read from the +bundle. Prover and verifier compute it identically because neither has a choice — +it is a loop counter. There is no supply route to get wrong. + +`page_base` has a real supply route (`bundle.touched_page_bases` → +`canonical_page_bases`), which is exactly where §4.1's invariant has to hold. + +So `epoch_label` has no supply route to get wrong, while `page_base` does. That +makes it the safer promotion — but "safer" is not "free", and the threat if the +invariant is broken is SHARPER here, not softer. §4.3. + +### 4.3 THE `epoch_label` THREAT MODEL — prover-chosen POSITION + +`epoch_label` is not an incidental constant. **It is what pins an epoch's +position in the chain**, in two places: + +- `L2G_MEMORY` (`local_to_global.rs:447`): `IsB20[epoch_label − 1 − init_epoch]`. + This is the cross-epoch ORDERING check — a cell's originating epoch must + precede its finalizing epoch. The range check is what forces + `init_epoch < epoch_label`. +- `L2G_GLOBAL` (`:360`): `BusValue::constant(epoch_label)` is the `fini_epoch` + carried by the token the next epoch consumes. It is the chain link itself. + +Today the constant is compiled into the AIR, and **the verifier builds that AIR +from its own `enumerate()` index** — so the verifier's AIR encodes the position +it expects, and a prover cannot assert a different one. Promotion moves that +value out of program text. If it were ever sourced from the bundle: + +> **Threat: a prover-chosen POSITION.** Inflating `epoch_label` relaxes +> `IsB20[label − 1 − init_epoch]`, admitting `init_epoch` values the ordering +> check exists to reject. Choosing labels freely lets two epochs claim the same +> position (**replay**) or claim positions out of order (**reorder**). + +This is sharper than the `page_base` case. There the risk is a wrong *address*; +here it is the integrity of the epoch chain — the property continuation +soundness rests on. + +So the invariant has the same shape as §4.1 and a different reason: + +> **The `epoch_label` uniform MUST be derived positionally from the verifier's +> own `enumerate()` (`continuation.rs:1293-1295`, +> `local_to_global::epoch_label(index) = index + 1`). It must NEVER be read from +> the bundle.** + +Note this is *easier* to honour than §4.1's, because the value is a loop counter +the verifier already computes — there is no plausible implementation that reads +it from the proof unless someone deliberately adds one. The invariant is written +down so that nobody does. + +#### Acceptance criteria for the `epoch_label` promotion + +1. **`parameterized_airs_vary_per_parameter_value` must become deletable** for + the two L2G tables — and deleted only after being shown to fail *for the right + reason* (artifacts now equal across labels), not merely to fail. +2. **`test_split_verify_rejects_reordered_epochs` and + `test_split_verify_rejects_dropped_last_epoch` must still pass, unchanged.** + These are the existing falsifiers for the ordering property, and they are the + real acceptance test: if promotion weakened the chain, they are what should + catch it. A promotion that required editing them is a promotion that broke + something. +3. A new negative test: supplying a `epoch_label` uniform that disagrees with the + verifier's positional derivation must be rejected. If it cannot be rejected — + because nothing checks it — that is the finding, and it means the invariant + needs a mechanism rather than a review rule. + +### 5. Effect on the artifact format + +Small and additive: + +- `AirShape` gains `num_base_uniforms: u32` — the COUNT, never the values. +- `validate_against` gains that one field comparison. +- `ConstraintArtifact::program()` gains the `OP_BASE_UNIFORM` decode arm. +- Values are **not** stored — they are supplied at verify time. That is the point. + +### 5.1 DESIGN REFINEMENT — uniforms ride in the program, not in every signature + +My first sketch put a `&[F]` uniform slice on every evaluation entry point: +`eval_program`, `eval_program_verifier`, `eval_device_program`, and the interp +`run` helper. That is a lot of signature churn across the interpreter, the device +walker, the CUDA kernel's host side, and every test that calls them — for a value +that behaves exactly like a constant at evaluation time. + +**Better: resolve the uniforms into the program struct, alongside the constants.** + +```rust +ConstraintProgram { …, base_uniforms: Vec> } // resolved values +DeviceProgram { …, base_uniforms: Vec } // raw limbs +ConstraintArtifact{ …, shape.num_base_uniforms: u32 } // COUNT ONLY +``` + +`OP_BASE_UNIFORM`'s `a` operand indexes `base_uniforms` exactly as +`OP_CONST_BASE`'s indexes `base_consts`. Consequences: + +- **No evaluation signature changes at all.** Both walkers read the table off the + program they were already handed. The CUDA kernel gains one buffer, uploaded + the same way `base_consts` already is — not a new parameter threaded through + the host API. +- The AIR fills the table at CONSTRUCTION time from its verifier-derived value + (§4.3), which is the natural place for it: the AIR already knows its own + `epoch_label`. +- `ConstraintArtifact::program()` needs the values to produce a runnable program, + so it becomes `program_with_uniforms(&[FieldElement])`, with `program()` + retained for the `num_base_uniforms == 0` case and erroring otherwise. That + error is useful: it makes "you forgot to supply the uniform" a loud failure + rather than a silent zero. + +**The hazard this creates, and it must be documented at the field.** +`ConstraintProgram` becomes a hybrid: `base_consts` is program identity, +`base_uniforms` is per-instance. If anything ever hashed a `ConstraintProgram` +including its uniforms, the digest would go back to varying per epoch — the exact +bug being fixed, reintroduced one layer down. + +Today nothing hashes a `ConstraintProgram` (the artifact is the serialized, +registry-pinned object, and it stores only the count), so the hazard is latent +rather than live. It should be closed by construction if cheap — e.g. the field +carries a `#[doc]` warning and the artifact codec has no path that reads it — and +called out in review either way. + +**This refinement is a design decision, not an implementation detail**, which is +why it is written here rather than made unilaterally in code. + +Payoff, in the artifact's own terms: the four parameterized tables collapse from +"one artifact per parameter value" to one artifact each, and the node-count / +root-id instability measured above disappears (PAGE stays 63 nodes for every +base; `L2G_GLOBAL` stops oscillating between 47 and 48). + +## Costs and risks, honestly + +- **One extra runtime op for `L2G_MEMORY`.** `epoch_label - 1` is folded at + capture time today; as a uniform it becomes a runtime subtraction on the + prover's per-row path. Trivially avoidable by supplying `epoch_label - 1` as + the uniform instead of `epoch_label` — mentioning it because it is the kind of + detail that turns into a surprise regression otherwise. +- **No prover-hot-path regression from the zero-skip.** + `ProverEvalFolder::fold_fingerprint_term` skips the multiply when the value is + zero; that test is on the runtime `FieldElement`, so it behaves identically + whether the value came from a constant or a uniform. (`page_base_hi` is 0 for + every address below 2^32, so this was worth checking rather than assuming.) +- **`crypto/**` blast radius.** New `Op` variant, new device tag, new + `ConstraintBuilder` method, two new bus-layer variants. All additive, but the + new case has to be added in six places that must agree: `interp::run` and + `DeviceProgram::lower` match `Op` exhaustively (so those two are compiler- + enforced), while `eval_device_program`, `ConstraintArtifact::program`, + `ConstraintArtifact::validate_self` and the CUDA kernel match the numeric tag + and are **not** — a missing arm there is a runtime panic or, in the kernel, a + silent wrong answer. The existing differential suites (28 AIRs × both folders × + the flat blob) are what would catch it, and they already exist; the CUDA side + is covered only by `gpu_constraint_interp*` under the `cuda` feature. +- **Not in scope here:** whether the machine wants the uniform as a program + constant per shape (registry ladder) or as an authenticated arena read. That + is the shape-static question from the target-shape doc and it is the lead's + call, not mine. + +## What I recommend (revised twice) + +**`epoch_label` first** — it is the only parameterized AIR in an epoch proof, and +leaving it unpromoted makes the registry ladder grow linearly with epoch count +(§0.0). `page_base`/GLOBAL_MEMORY follows when the global-proof leg comes into +scope. PAGE last: monolithic-only, and it gets the fix for free once the +mechanism exists. + +Sequence: + +1. IR leaf (`Op::BaseUniform`, tag 11) + both `DeviceProgram` consumers + + `AirShape::num_base_uniforms`, with the existing 28-AIR differential suites as + the safety net. Falsify the walker parity by breaking each side + independently — a suite that has never been shown to catch a divergence is not + yet a safety net. +2. Bus-layer `BusValue::Uniform` / `LinearTerm::Uniform`. +3. **`L2G_MEMORY` and `L2G_GLOBAL`** (`epoch_label`), with §4.3's invariant + enforced at the supply point. Acceptance is §4.3's three criteria — in + particular the two existing epoch-ordering rejection tests must pass + unchanged. +4. Then `GLOBAL_MEMORY` (`page_base`) with §4.1's invariant; then PAGE. +5. Re-measure: each promoted table collapses to one artifact, and + `parameterized_airs_vary_per_parameter_value` becomes deletable for it. + +The acceptance test is a test that must **stop** passing — a sharper contract +than one that must keep passing, since it cannot be satisfied by doing nothing. diff --git a/others/lfm-phase0-handoff.md b/others/lfm-phase0-handoff.md new file mode 100644 index 000000000..35e8817a3 --- /dev/null +++ b/others/lfm-phase0-handoff.md @@ -0,0 +1,106 @@ +# Phase 0 handoff — constraint artifact track + +Written 2026-07-30 by the phase0 agent. Branch `feat/phase0-constraint-ir`, +worktree `.../scratchpad/wt-phase0`, off `origin/main e0add1d5`. **Never pushed.** + +Five commits, all green (`make lint` 0, stark 216, prover 530 lib tests): + +``` +b36f15fa ConstraintArtifact + rkyv codec + the scoped verify-path unban +2a6f9036 all 28 production AIRs (3 continuation tables were in NO enumeration) +d2fb95c9 constraint-lowering design + the op census instrument +ef7587fd design revised against the machine's real cost model +058ba5ef per-epoch multiplier + the workload-shaped self-correction +1414d726 real continuation-epoch chunk counts, first-hand +69b3b348 uniform promotion reordered — epoch_label is the critical path +``` + +## State: what is done + +**Phase 0 proper is complete.** Constraints serialize at build time +(`ConstraintArtifact` = flat program + zerofier metadata + AIR shape + degree +multiplier), round-trip bit-exactly against both folders on all 28 AIRs, and the +verify-path prohibition is scoped to CAPTURE with a guest-safe +`precaptured_constraint_program()` alongside. Nothing is wired into the +production verify path, as instructed. + +**The lowering design is written and measured** +(`others/lfm-constraint-lowering-design.md`). Continuation epoch leg: 63,393 +instructions at the minimum shape, 64,035 at a 2^20 epoch, 63–65K across any +plausible epoch size. + +## State: what is NEXT, and it is not started + +**The uniform promotion.** Fully specified in +`others/lfm-page-base-uniform-proposal.md`; **no code written**. Order, set by +the team lead and derived from the epoch composition: + +1. **`epoch_label`** — the two L2G tables. FIRST, because an epoch proof's only + parameterized AIR is `L2G_MEMORY`, so unpromoted the registry needs one + program per epoch index and the ladder grows linearly with epoch count. +2. `page_base` — GLOBAL_MEMORY, when the global-proof leg comes into scope. +3. PAGE last (monolithic-only; gets the fix free once the mechanism exists). + +### Read these three sections before writing anything + +- **§4.3 — the `epoch_label` threat model.** The invariant is that the uniform is + derived positionally from the verifier's own `enumerate()`, never from the + bundle. Failure mode is epoch **replay or reorder**, not a wrong address. +- **§5.1 — the design refinement.** Uniforms resolve into + `ConstraintProgram.base_uniforms` / `DeviceProgram.base_uniforms` alongside the + constants, rather than being threaded as a new parameter through every + evaluation entry point. Avoids churn across both walkers, the CUDA host side + and every caller. **Carries a hazard**: `ConstraintProgram` becomes a hybrid of + program identity and per-instance values; nothing must ever hash it including + the uniforms. Latent today (only the artifact is hashed, and it stores the + count only). **This is a design decision awaiting the lead's agreement, not a + settled implementation detail.** +- **§4.3's three acceptance criteria**, of which the second is the real one: + `test_split_verify_rejects_reordered_epochs` and + `..._dropped_last_epoch` (`continuation.rs:1711`, `:1693`) must pass + **unchanged**. They pop and swap epochs in a genuinely proved bundle. A + promotion that required editing them broke something. + +### Falsifications to run (not optional) + +- Break the CPU walker and the CUDA walker **independently** and confirm the + differential suites catch each. A suite never shown to catch a divergence is + not yet a safety net — this repo's suites have now been shown to catch two + distinct classes (structural wire change, and an evaluation-only change on the + path with no structural check), so the bar is set. +- Delete `parameterized_airs_vary_per_parameter_value` only after showing it + fails **for the right reason** (artifacts equal across labels), not merely that + it fails. + +## Instruments left behind (use them; do not re-derive) + +All in `prover/src/tests/constraint_artifact_tests.rs`: + +| test | answers | +|---|---| +| `constraint_op_census` | per-AIR instruction counts. **Read its "WHAT THIS INSTRUMENT CANNOT SEE" note first** — it cannot see how sub-proofs are assembled, and a census-only inference from it was the one thing this track got wrong. | +| `epoch_chunk_multiplier` | monolithic per-proof totals via real traces | +| `continuation_epoch_constraint_leg` | epoch composition, asserts the measured 24/25 sub-proof count | +| `continuation_epoch_chunk_counts_measured` | a real epoch's chunk counts, first-hand, no proving needed | +| `parameterized_airs_vary_per_parameter_value` | characterizes the four parameterized AIRs; becomes the promotion's falsifier | + +Plus `prover/src/bin/compute_constraint_artifacts.rs` (generator; emits ONE +REPRESENTATIVE per parameterized table, not the full set — see its header) and +`crypto/stark/src/constraint_ir/artifact_tests.rs` (17 unit tests incl. the +rejection paths and nonzero `end_exemptions`, which no production AIR exercises). + +## Things a successor would otherwise rediscover + +- **`test_utils::production_airs()` is the single 28-AIR list**, and every suite + asserts `NUM_PRODUCTION_AIRS`. That assert exists because three hand-copied + lists all shared the same blind spot. Add tables there, once. +- **The IR's `dim` tags are prover-side.** The machine runs the verifier, where + the frame is all-extension: 42,137 declared base, 2,916 actually base. Do not + size anything from the declared dims. +- **Production zerofiers are uniform** (every AIR emits `RowDomain::ALL`), worth + ≈50,900 instructions and the GPU path's precondition holding in fact. +- **Hash-consing makes peepholes unsound** without a single-consumer guard. This + is documented on `ConstraintArtifact` itself, not just in the design doc. +- **Open, not mine to decide**: whether to check in generated artifacts (ruled + no — generate at build time, pin by digest); and the `check_attestation` + production gap, which is a real finding but not this track's. diff --git a/others/lfm-r1f-handoff.md b/others/lfm-r1f-handoff.md new file mode 100644 index 000000000..b6995d5ea --- /dev/null +++ b/others/lfm-r1f-handoff.md @@ -0,0 +1,164 @@ +# R1f handoff — keccak-emitter → successor + +Written 2026-07-30. R1f is PARTIAL: (b) and half of (a) are done and committed; +(c) and (d) are not started. Handing off on context, per the standing decisions' +"quality over completion". + +**State: `cargo test -p lambda-vm-prover --lib lfm` green, `make lint` 0, +everything committed** — `feat/lfm` at `2d4aa350` plus the doc/verification +slice. Only `others/` is untracked. R1a–R1e are all closed and green. + +--- + +## 1. What is DONE + +### 1b. Real proof bytes — `lfm/proof_fixture.rs` +A two-epoch continuation proof in the guest's wire format, produced by the SAME +encoder the guest's blob comes from (`prove_continuation` → +`encode_continuation_guest_input`, both already `pub`). No new format, no new +visibility. + +The existing dump path (`test_dump_recursion_input`) is `#[ignore]`d, driven by +five env vars, and writes a fixed `/tmp` path — unusable from a deterministic +test — so only its two encoder calls were reused. + +**Epoch size is measured, not guessed**: the `fibonacci` guest gives ONE epoch at +`log2` 6/8/10 and TWO at 4, so it runs 17–64 cycles. `FIXTURE_EPOCH_LOG2 = 4`, +preset `min`. Blob: 310,212 B at one epoch, 587,188 B at two. The cache lives in +`temp_dir`, NOT the repo — a checked-in binary drifts from the encoder silently, +so the GENERATION path is what a cold run exercises. + +Test: `continuation_fixture_generates_two_epochs`. + +### 1a (half). Arena filler — `lfm/proof_arena.rs` +Reads an epoch's main-trace Merkle roots out of the archived blob in place and +packs them into arena halves. Tests: +`arena_filler_reads_real_committed_roots`, +`supplied_preprocessed_roots_are_embedded_in_the_blob`. + +**Measured on the real proof**: epoch 0 = 24 sub-proofs / 8-byte public output; +epoch 1 = 25 / 0-byte. Confirms `T_epoch = counts + (10 final | 9 intermediate) ++ pages + 1`, and confirms `T = 24` for SOUNDNESS §6.3 (now marked measured +rather than assumed). + +**NOT done**: openings and sibling-path extraction. The API is located — +`query_list_len()`, `query(i) -> FriDecommitmentView`, `deep_poly_openings_len()` +at `crypto/stark/src/proof/view.rs:409-423` — so this is mechanical, not +exploratory. + +--- + +## 2. What REMAINS — (c) and (d) + +### ★ `edsl::merkle_walk` CANNOT be used. Build `keccak_merkle_walk`. +The existing walk calls `LfmBuilder::compress` → the `LFM_HASH` chiplet running +`TestPermutation`, the deliberately non-cryptographic Milestone-C placeholder. It +authenticates the Milestone-C fixture tree because that tree used the same +placeholder. **Production trees are keccak throughout**, so no amount of correct +path-walking reproduces a production root. This was the leg's original spec +instruction and it is wrong; both prerequisites for the replacement already exist +(R1c/R1d keccak256 over byte streams, R1e slice a big-endian rendering). + +### The conventions, read from source +**Leaf** (`crypto/stark/src/commitment.rs`, `ROWS_PER_LEAF = 2`, line 42): + +``` +leaf(i) = keccak( col_0[br(2i)] ‖ col_1[br(2i)] ‖ … ‖ col_0[br(2i+1)] ‖ … ) +``` + +Every element via `write_bytes_be` (8 bytes base, 24 ext). `br` is a bit-reversal +of the row index — a host-side arena-filler concern, not the machine's. One path +authenticates a value and its symmetric counterpart, which is why the pair is the +leaf. + +**Parent** (`crypto/crypto/src/merkle_tree/backends/field_element.rs:41`): +`keccak(left ‖ right)`, 64 bytes, **no domain separation, no ordering flag**. + +### Shape and cost +- Per level: TWO `select`s (a digest is two machine words and both must swap on + the same bit), then `keccak256` over 16 halves. 64 bytes fits inside one + 136-byte rate block ⇒ one permutation per level. +- The LEAF is the expensive part and **byteswapping dominates it, not hashing**: + `2 · cols` elements each needing `felt_be_halves` (1 `BitDec` + 64 `BALU`). For + a 50-column table ≈ 100 `BitDec` + 6.4k `BALU` against only ~6 permutations. + **Measure this — it is the input to whether a byteswap chiplet is worth + proposing.** It is not avoidable by pre-swapping in the arena: opened values are + consumed as field elements by the FRI algebra AND as bytes by the leaf hash, so + something must connect the two representations. +- Root comparison: `assert_word_eq_lanes` with the root's unpack hoisted, as + `fri_toy_program` already does per query. + +### (d) Tamper vectors +Wrong sibling, wrong index bits, wrong leaf → all must reject. + +--- + +## 3. Non-obvious decisions and WHY + +- **★ Archived accessors are METHODS, not relaxed field visibility.** rkyv mirrors + the source field's visibility onto the archived struct, so making + `ContinuationProof::epochs` `pub(crate)` would have opened the OWNED type at the + same time — silently becoming the route the team lead had explicitly rejected. + `impl ArchivedContinuationProof { pub(crate) fn num_epochs / epoch_proof / + epoch_public_output }` exposes only the path `verify_continuation_archived` + already traverses. **If you need anything else off an epoch, add a method there; + do not touch the field.** Visibility on the OWNED type is a different question + and needs a ruling. +- **The fixture is BYTES because that is what production is.** The guest never + holds a `ContinuationProof`; it reads a blob zero-copy. A reader over bytes is + the direct analogue, and divergence between the two is a meaningful signal. +- **Pack each field into its OWN halves.** An arena is a vector of words, not a + byte stream. Concatenating fields then packing lets any field of + non-multiple-of-four length shift everything behind it — silently, since the + halves count still comes out right. This cost real debugging time in R1e. +- **Shape-static values are program CONSTANTS, never arena reads** (table counts, + page-range list, `num_private_input_pages`). A program reading them from an + arena claims to verify a shape it was not compiled for. + +## 4. Preprocessed roots — the ruling, and what it still owes + +Team lead's ruling: static roots (BITWISE, KECCAK_RC) are shape-static ⇒ program +constants, which is already how `LfmAirs` treats them. Supplied roots come from +the blob. + +**Verified, partially.** `ContinuationGuestInput` carries `decode_commitment` and +`page_commitments` as `pub` fields, and the fixture's DECODE root is present and +nonzero. **Caveat: this fixture has ZERO page commitments** (fibonacci touches no +data pages), so the page path is present-but-unexercised — do not treat it as +tested. + +**Refinement the ruling did not cover: REGISTER is DERIVED, not supplied.** +`EpochProof` (`continuation.rs:394`) carries `reg_fini: Vec` — the register +FILE — and the verifier derives the next epoch's REGISTER root from it. The data +is in the blob, but a derivation step sits between it and the root. Budget for it. + +**Also from `EpochProof`: `runtime_page_ranges` is ALWAYS EMPTY for continuation +epochs** (PAGE tables are skipped; the comment at line 401 says so). R1e's +`epoch_statement_shape()` uses two ranges, which is fine for a synthetic shape but +means the REAL statement has `R = 0`, so its length is `207 + L` and the Phase-A +shift is `(3 + L) mod 4` with no `16R` term. + +## 5. Method rules (non-negotiable — these caught every real bug this phase) + +1. **Falsify every new mechanism.** Break it, watch the RIGHT test fail, revert. + If nothing fails, or the wrong thing fails, the TEST is wrong. +2. **Execute-only tests prove nothing about chips.** Only prove+verify sees them. +3. **Scrutinise the oracle** as hard as the thing under test. Here the best oracle + is the real proof's own committed root — use it rather than recomputing a leaf + host-side and comparing against yourself. +4. **Soundness claims need coherent forgeries**, not trace tampering. +5. **A deferral's safety argument is itself a claim needing evidence.** Twice this + phase a "surely it's fine" premise was false: a mask that looked cosmetic was + pinning arena bytes past a length prefix, and a remembered public-output length + was simply wrong. + +## 6. Process + +- Append one line to `others/lfm-agent-status.log` per slice; commit each green + slice yourself (`git -c user.name="Mauro Toscano" -c + user.email="maurotoscano2@gmail.com"`), no AI attribution, never commit red. +- `others/lfm-standing-decisions.md` lists what is pre-authorised — read it before + stopping to ask. +- `others/lfm-target-shape.md` has the epoch composition and the chaining + obligations that come next (R1g). +- `make lint` from the repo root is the gate; `cargo fmt --check` is not enough. diff --git a/others/lfm-standing-decisions.md b/others/lfm-standing-decisions.md new file mode 100644 index 000000000..01dead7b6 --- /dev/null +++ b/others/lfm-standing-decisions.md @@ -0,0 +1,144 @@ +# Standing decisions — Phase R agents + +Read this before stopping to ask. If your question is answered here, proceed. +Last updated 2026-08-03 by team-lead (rule-7 refinement from the FRI leg). + +## Pre-authorized — do NOT ask + +- **Merging `origin/main` into your branch** when your premise depends on + upstream state, using: stash tracked-dirty files → `git merge --ff-only` + (or a real merge if the branch has commits) → pop → full suite + drift + tests → regenerate the registry if digests moved. Report what happened. +- **Regenerating `LFM_REGISTRY`** via `cargo run --bin compute_lfm_registry + --release` and pasting the block, whenever a program or a layout changes. + Always re-run the drift tests after, and report moved-vs-survived. +- **Adding tests beyond the spec**, including tests that assert a hazard + still exists (see the guard-test map in `lfm-agent-handoff.md` §8). +- **Refactoring for correctness or clarity inside `prover/src/lfm/`** — + extracting shared helpers, renaming, splitting files — provided the full + suite stays green and the public surface other slices depend on is either + unchanged or reported. +- **Overriding my spec when it is wrong.** You have done this three times + and been right three times. Implement the correct thing, flag it loudly in + the report as a deviation with the derivation. Do not implement something + you believe is wrong because I wrote it. +- **Committing your own slice** to the branch you were given, signed + (`git -c user.name="Mauro Toscano" -c user.email="maurotoscano2@gmail.com"`), + no AI attribution or co-author trailers, once the suite is green and lint + is clean. Never commit red. Never force-push. Never rewrite history. +- **Deciding the internal design** of anything the spec describes by + behaviour rather than by construction. + +## Always stop and ask + +- Anything touching `crypto/**` or `prover/src/tables/**` beyond additive + `BusId` variants — those are production paths shared with the VM. +- Pushing to a remote, opening or merging a PR, or any GitHub write. +- Deleting or rewriting another agent's work, or reverting a guard test. +- A framework ceiling (interaction counts, capture limits, aux widths): + report it as a finding rather than working around it silently. +- Anything that would make program identity proof-dependent, add a runtime + off-switch to the registry check, or weaken a soundness obligation to make + a test pass. + +## Method (non-negotiable — these caught every real bug this phase) + +1. **Falsify every new mechanism.** Break it deliberately, watch the right + test fail, revert. If nothing fails, the TEST is wrong, not the mechanism. +2. **Execute-only tests prove nothing about chips.** Where the executor + mirrors a computation the chip also does, only a prove+verify test sees + the chip. +3. **Scrutinise the oracle** as hard as the thing under test. A wrong oracle + looks exactly like a wrong implementation. +4. **Soundness claims need coherent forgeries**, not trace tampering — build + the attack so every bus balances and every claimed value is consistent, + then show the one constraint that rejects it. +5. **A deferral's safety argument is itself a claim needing evidence.** + Deferring work behind a loud assert is fine. Deferring it because you + believe it is cosmetic, without checking, is not — the check is what tells + you whether the thing you postponed was a convenience or a soundness + obligation. Two instances this phase: a trailing-half mask that looked + cosmetic actually pinned arena bytes past a length prefix (without it a + prover rewrites the absorbed string while the prefix claims otherwise), and + a "public output is surely 4-byte aligned" recollection that was simply + false — public output is one byte per COMMIT op with no alignment + guarantee. Verify the premise, then defer. +6. **Mark provenance; never assert past your evidence.** In any document, + separate what you verified first-hand from what you took from someone + else's report — and give every instrument a "what this cannot see" note + naming the questions it is structurally unable to answer. Not bookkeeping: + the one claim this phase that was flat wrong ("the constraint leg is + workload-shaped, a no-EC epoch drops 65%") was the single sentence its + author wrote without marking provenance. It was asserted from a node + census, which cannot see how sub-proofs are ASSEMBLED — and ten tables + contribute a sub-proof each regardless of whether they have any rows. It + reached the team lead's durable notes before the author's own + re-measurement pulled it back. Both halves of this rule answer the same + question: *how do I stop myself asserting past my evidence?* + +7. **A relative test dies the moment its two sides unify.** A differential + between two code paths is worthless once one is implemented in terms of + the other — the refactor that makes an API additive-by-delegation is + exactly what kills any test comparing the two forms, silently, at that + moment. When you delegate, replace the comparison with an ABSOLUTE + property of the output. Demonstrated (reg-tree, bits exposure): the + guard comparing `emit_sub_proof` against `emit_sub_proof_with_bits` + stayed green with the exact denied defect injected, because both sides + were the same program by construction; the replacement asserts every + returned bit is consumed by some `Select`, which the walk's own bits + satisfy and a fresh copy cannot. This is rule 1 applied to a guard: + falsify the guard itself, especially right after a delegation refactor. + + REFINEMENT (fri-emitter, 2026-08-03): **a difference of two counts taken + from our own emitter is still a relative test**, however much it looks + like an absolute count. `selects(joined) − selects(trace_only)` stayed + green with the exact denied defect injected, because the defect lived in + the function BOTH sides call and the difference never moved. The + marginal-cost idiom (`marginal()`, `marginal_fri`) is safe only when the + result is compared against a number that did not come from the emitter — + a pinned prediction, or a closed form over the shapes. Corollary from the + same session, rule 3 applied to instruments: a falsification harness that + parses `cargo test -q` for per-test FAILED lines reports every breakage + as "nothing failed" (failures are named only in the trailing summary + block) — check the instrument against a known breakage before believing + "my mutation changed nothing". + +8. **A search that ERRORS looks exactly like a search that found nothing** + (`[hash-w8]`, 2026-08-04). `grep -r --include=*.rs pattern .` with the glob + UNQUOTED makes the shell try to expand `*.rs` in the cwd and fail with + "no matches found" — printing nothing, exiting non-zero, and reading + identically to "the pattern is absent". Two of my absence readings this wave + were shell errors, not evidence. **Quote the glob** (`--include='*.rs'`), and + treat an empty search result as a claim needing a positive control: run the + same search for a term you KNOW is present and confirm it prints. + + Corollary, same wave: **term collisions manufacture false positives in the + other direction.** "monolith" matches 26 times across `prover/src` and every + occurrence is the *monolithic proof* concept, nothing to do with the Monolith + hash — a term-only search would have reported an implementation that does not + exist. An existence claim needs the match READ, not counted. + +9. **A donor's parameters are not a donor's correctness** (`[hash-w8]`, + 2026-08-04). Copying published constants gives you a shape, not a working + primitive: the round order, the MDS orientation, the S-box exponent and which + lane a partial round touches are all independent ways to be wrong while every + constant is right. Pin the whole primitive against an EXTERNAL known-answer + vector that nothing in this repository produced, then falsify that vector's + test by breaking each convention separately — if flipping the MDS + orientation still passes, the vector is not pinning what you think. + Demonstrated: three falsifications (wrong exponent, transposed circulant, + partial S-box on lane 11) each fail the one KAT. + + ⚠ And a concrete trap this found: `crypto/crypto/src/hash/poseidon/`'s HADES + skeleton hardcodes `x^3`, which is **not a permutation over Goldilocks** + (`3 | p-1`). An in-tree implementation is not automatically an oracle — check + that it is CORRECT before differentialling against it. + +## Coordination + +- Append one line to `others/lfm-agent-status.log` at every slice boundary. +- Report via SendMessage to `main`; plain text output does not reach me. +- Answers to blocking questions arrive as `others/lfm-team-lead-*.md` files + as well as mailbox messages — check the directory if a reply seems overdue. +- If your context runs thin, checkpoint and write a handoff file rather than + delivering a half-built slice. Quality over completion. diff --git a/others/lfm-target-shape.md b/others/lfm-target-shape.md new file mode 100644 index 000000000..5cfac611d --- /dev/null +++ b/others/lfm-target-shape.md @@ -0,0 +1,282 @@ +# The target: recursing CONTINUATION EPOCHS + +User-confirmed 2026-07-29. Every Phase R track should build against this shape, +not against a monolithic proof. + +## What the machine must verify + +**One continuation EPOCH proof**, and later the **global proof** that ties +epochs together. Not `prove()`/`verify()`'s monolithic shape — that path exists +but is not the target, and building for it would silently miss AIRs (see below). + +Sub-proof count per epoch (`prover/src/continuation.rs`, verified earlier this +phase): + +``` +T_epoch = table_counts.total() # 14 split-table families, chunked + + (10 if final_epoch else 9) # FIXED_TABLE_COUNT, minus HALT + # on intermediate epochs + + page_configs.len() # one PAGE AIR per touched page + + 1 # the epoch-local L2G table +``` + +The global proof carries one L2G sub-proof per epoch plus GLOBAL_MEMORY. + +## Consequence 1 — there are 28 AIRs, not 25 + +`l2g_global_air`, `l2g_memory_air`, `global_memory_air` are private fns in +`continuation.rs` and appear in NONE of the four hand-maintained 25-item +enumerations, none of which asserts a count. They are exactly the AIRs a +continuation proof adds. Any artifact, coverage claim or constraint-evaluation +leg scoped to "the 25" is complete for monolithic proofs and quietly incomplete +for the target. `l2g_memory_air` is the one with real constraints +(`L2gMemoryConstraints`); the other two are `EmptyConstraints` but still need +shape + meta + max_degree. + +## Consequence 2 — the statement is the ContinuationEpoch variant + +`absorb_statement(StatementKind::ContinuationEpoch { epoch_label })` with +`CONTINUATION_EPOCH_TAG = b"LAMBDAVM_CONTINUATION_EPOCH_V2"` (30 bytes, ≡ 2 mod +4 — one of the two misalignment points R1e handles), and the trailing +`epoch_label` u64 the monolithic variant lacks. R1e is already building this +variant; do not "simplify" it to the monolithic tag. + +## Consequence 3 — chaining is part of the statement, not an extra + +Verifying epochs in isolation is not verifying a continuation. The chaining +obligations, all of which the RV64 guest already performs and the machine will +have to emit: + +- epoch *i*'s `reg_fini` feeds epoch *i+1*'s REGISTER root — a DERIVATION, not a + comparison, and nothing is supplied (see below); +- L2G root equality between each epoch's own L2G commitment and the + corresponding sub-proof in the global proof; +- the attestation fold `program_id ‖ concatenated public_output`. + +### ★ The REGISTER derivation IS the binding — decided, not a TODO (R1g) + +The first obligation is often written as "compare `reg_fini` against the next +epoch's supplied REGISTER root". There is no supplied root and no comparison. +The chaining loop carries `register_init = epoch.reg_fini()` forward and +`build_epoch_airs` (`continuation.rs:636`) *constructs* the next epoch's +preprocessed commitment from it via +`register::compute_precomputed_commitment_with_fini`. Lie about `reg_fini` and +the constructed commitment no longer matches the one the proof was made against, +so the proof fails. The binding is structural. + +This settles the long-carried Phase-0 item **"wire the REGISTER verify-side +supply route"**. `VmAirs::new` does have a `register_preprocessed: +Option<(Commitment, usize)>` parameter that every verify caller passes `None` +to, so the plumbing looks like an unfinished route. It is not unfinished — it +must stay unwired. **Computing the commitment from `reg_fini` is what ties the +VALUES to the commitment.** Supply the root instead and `reg_fini` has no +remaining role, so a prover can offer a root consistent with a `reg_fini` it +never honoured, and the cross-epoch chain that `reg_fini` carries goes +unenforced. The in-guest RV64 verifier's per-epoch recomputation is therefore +load-bearing, not wasteful. + +Consequence for the machine: it must EMIT that derivation — 3 columns × 128 +rows, an inverse FFT and an LDE FFT each, then a full Merkle tree build. Its +output is exactly the preprocessed root Phase A absorbs, which is why Phase A +cannot be replayed over a real epoch without it. Cost is negligible: 255 +permutations at blowup 2 against ~1.4M for the epoch verify (~0.02%), 1023 +against ~460k at blowup 8 (~0.2%). See the sizing note below before quoting any +ratio. + +### ★ The derivation, BUILT and MEASURED (R1g(i)) + +First-hand, from `machine_tests::register_derivation_cost` and +`the_register_derivation_matches_production` on branch +`feat/lfm-register-derivation`. Everything in this subsection is measured or +asserted by a test unless marked otherwise. + +**The permutation prediction was exactly right.** 255 / 511 / 1023 at blowup +2 / 4 / 8, matching `2·leaves − 1` with `leaves = 128·blowup / ROWS_PER_LEAF`, +i.e. `128·blowup − 1`. So was the noise figure: **0.0182%** of an epoch's +hashing at blowup 2 and **0.2224%** at blowup 8, against the ~1.4M / ~460k +baselines above. A leaf is 48 bytes (three columns × a row pair) and a parent +64, so each is exactly one rate block and the permutation count is the node +count with nothing else in it. + +**The FFT half of the prediction was wrong in an interesting direction.** The +text above says "3 columns × 128 rows, an inverse FFT and an LDE FFT each". +Measured, the transform is **8.5% of the derivation's own arithmetic at blowup +8** (18,176 `LFM_BALU` rows of 214,784) and the remaining **91.5% is byte +swapping** — turning each extended field element into the big-endian bytes the +leaf hash absorbs, 64 rows per value via `felt_be_halves`. Anyone budgeting this +leg from "it is two FFTs" is budgeting the small half. Two corrections feed +that: + +- **Only two of the three columns need a transform.** OFFSET holds the register + word addresses, which are fixed, so its extension is a program CONSTANT — the + shape-static principle paying out for once. It is computed at program-build + time by production's own `interpolate_fft`/`evaluate_polynomial_on_lde_domain` + and interned, which also means the three columns reach one tree by two + different routes and a matching root pins the emitter against the function it + emits. +- **The constant column is not free at leaf time.** Its values are byte-swapped + into the leaf like any other, one bit decomposition and 64 ALU rows each. + Pre-computing those halves as constants would drop 32,768 rows at blowup 8 — + and save **zero committed cells**, because `layout::padded_rows` rounds the + group to a power of two and 214,784 and 182,016 both land on 2^18. Measured, + not estimated; the same holds at blowup 2 and 4. Left undone deliberately. + +**Answer to the gadget question: no second hashing gadget.** The tree build +needs no gadget `keccak_merkle_walk` does not already contain. What it needed +was for the PARENT step to stop being welded to the walk's `Select`: +`edsl::keccak_hash_pair` is now that step, `keccak_merkle_walk` calls it after +its select, and `keccak_merkle_tree_root` calls it with no select at all — a +tree knows every child's side when the program is emitted. The leaf gadget is +`keccak_leaf_hash`, reused unchanged. Applying the sizing rule to the +alternative: a `Select` is **17 main cells against a permutation's 36,256**, so +the two selects a walk step carries would add 0.09% to each parent (0.03% over +the whole tree) — the case against routing the tree through the walk is +structural, not economic. A walk visits one node per level; a tree visits `2^k`. + +For the FRI leg the shape of the answer transfers but the answer does not, and +this leg did not check which: a verifier RECEIVES layer roots from the proof and +authenticates openings against them, so FRI plausibly wants the WALK with a +different leaf width rather than a build — and `keccak_leaf_hash` is already +parameterized by leaf width. Unverified; the FRI leg should settle it rather +than inherit this sentence. + +**What this leg does not establish.** The two arenas are unbound. In the +assembled verifier `R_{i+1}` is the vector the next epoch reads as its INIT and +the published root is what that epoch's Phase A absorbs; until those joins exist +a prover may supply any pair and get the honestly-derived root for it. Same +standing caveat as the L2G binding. + +**A fourth member of the degenerate-parameter family, demonstrated.** A real +register file is mostly zeros — the fixture's epoch-0 boundary is **3/67 nonzero +INIT, 10/67 nonzero FINI, 9 rows differing**. Deliberately dropping row 40 from +the emitted columns PASSED the differential against the real fixture and was +caught only by a synthetic file with all 67 rows distinct. The falsification is +recorded because it is the rule's clearest instance so far: the real data is not +merely a weak witness here, it is blind over 57 of 67 rows. + +### Sizing rule — compare against the WHOLE leg, never a sample of it + +Two gadget-sizing errors this phase produced ratios that were arithmetically +correct and pointed the wrong way: + +1. **Rows are not a cost unit across chips.** An `LFM_BALU` row is 4 + non-preprocessed columns; one keccak permutation expands into 24 + `KECCAK_RND` rounds of 1480. Compare CELLS. (This killed the byteswap-chiplet + proposal: 322 cells vs 36,256, no crossover at any table width.) +2. **A sample of a leg is not the leg.** The REGISTER tree was first sized + against R1f's opening program — 22 permutations — giving "12–46× the opening + leg" and the conclusion that it was expensive. But R1f was ONE query on ONE + table, roughly `1/(queries × tables)` of the epoch's opening work. Against the + whole epoch verify the same gadget is ~0.02%. Same number, opposite decision. + +Baseline to size against, from *Scale* below: **~1.4M keccak permutations per +epoch verify at blowup 2 / 219 queries, ~460k at blowup 8 / 73 queries.** + +## Consequence 4 — page-parameterized constraints are an identity risk + +PAGE's captured constraint program folds `page_base` into IR constants, so it is +not one static blob. Size is negligible; IDENTITY is not — if constraint +artifacts vary with the workload's page set, a machine program embedding +constraint evaluation would too, and page bases are arbitrary addresses rather +than a small ladder. Registry entries must not become workload-dependent +(SOUNDNESS §2). The eventual fix is promoting page base to a runtime uniform — +it is already authenticated via supplied page roots. Tracked as a machine-side +design item; Phase 0 measures and reports it. + +## The shape-static principle (R1e, and it generalises) + +**Shape-static values are program CONSTANTS, never arena reads.** The table +counts, the page-range list and `num_private_input_pages` determine how many +sub-proofs Phase A absorbs and what the AIR layout is. A program that read them +from an arena would be claiming to verify a shape it was not compiled for — the +prover would choose the shape, which is exactly the property the registry exists +to deny. Only genuinely per-proof data (ELF digest, public output, epoch label, +roots, openings) comes from arenas. + +Corollary, and the tension to watch: every shape-static constant is part of +program identity, so each distinct shape is a distinct registry entry. That is +correct and cheap for a small ladder of shapes; it is what makes Consequence 4 +(page bases folded into constraint constants) a real problem rather than a +theoretical one, since page bases are not a small ladder. When the constant set +stops being enumerable, the answer is the runtime-uniform promotion — a value +supplied at verify time and authenticated, not a constant. + +## Next-row PRUNING is program text, not arena data (constraint leg) + +Same class as the shape-static principle, and it bites the other way round. + +The verifier opens every trace column at `z` but prunes the `g·z` block down to +each AIR's DECLARED `trace_ood_next_row_columns`, reconstructing **ZERO** for +every column outside that set (`ood::OodLayout::reconstruct_full`). So a machine +that hinted a value into an undeclared next-row slot would evaluate constraints +over a frame **no verifier can produce** — accepting proofs the real verifier +rejects. The declared set is AIR shape, it is carried in `AirShape`, and the +zeros belong in the emitted program as the pooled zero constant. + +`lfm::constraints::hint_ood_frame` does this, and +`pruned_next_row_columns_are_program_zeros` pins it by asserting, column by +column on CPU's real shape, that a slot is the pooled zero cell **iff** the AIR +does not declare it. The cheap consequence is that a frame costs +`width + (steps − 1)·|next_row_columns|` arena words instead of `steps · width`; +the expensive consequence is the one above. + +The generalisation for any leg that touches openings: **when the verifier +reconstructs a value rather than reading it, the machine must reconstruct it +too — from program text.** Reading it from an arena hands the prover a degree of +freedom the protocol does not give them, and it is invisible in a differential +test that feeds both sides the same frame. + +## A degenerate parameter hides implementations from every real test + +Third member of the same family, and the most general. + +**When every production instance shares one value of a parameter, a differential +over production data cannot distinguish implementations that differ only off +that value.** The real data does not exercise the difference, so the wrong +implementation passes — not by luck, but structurally. The synthetic case is the +only witness there can be. + +Two instances so far, both in the constraint/DEEP legs: + +- **`step_size = 1`.** `build_pruned_trace_term_coeffs` walks column-major, so + along a fixed row the γ exponent advances by the block's ROW COUNT, not by one. + Every one of the 28 production AIRs has `step_size = 1` and a single next row, + collapsing both strides to one. A per-row Horner in plain γ therefore passes + the real-proof differential against the production reconstruction, the census, + and every other test in the suite — and is wrong for the first AIR that widens + a step. Witnessed by `the_coefficient_exponent_formula_holds_at_a_wider_step`, + which builds a `step_size = 2` layout through the verifier's own + `build_pruned_trace_term_coeffs`. +- **`end_exemptions = 0`.** Every production constraint emits through + `RowDomain::ALL`, so a consumer that ignored the field entirely would pass the + whole production sweep. `crypto/stark`'s `ExemptConstraints` exists for exactly + this reason, and its author says so: "so that *always zero in production* + cannot decay into *never tested*". + +**The falsification needs two halves.** Showing the synthetic case passes proves +nothing on its own — a test that only checks the right answer passes against +both implementations if the wrong one happens to agree there too. It must also +assert that the DEGENERATE reading gives a DIFFERENT answer at the synthetic +value. Without that second half the test passes against the wrong emitter, which +is the failure mode it exists to prevent. + +How to find these: for each parameter a leg consumes, ask what values production +actually takes. If the answer is "one", that parameter needs a synthetic witness +before the leg can be called tested. + +## Alignment is a property of the cursor, not the field (R1e) + +A 32-byte root is self-aligned and still lands misaligned if it inherits an odd +byte cursor. The epoch statement is `207 + |public_output| + 16·ranges` bytes, +`≡ 3 (mod 4)`, so EVERY subsequent absorb — all of Phase A included — is spliced +at shift 3. Machine-checked by +`epoch_statement_ends_three_bytes_past_a_boundary`. Anyone reasoning about a +field's alignment in isolation will get this wrong. + +## Scale, for sizing decisions + +A small ethrex block is 1–2 epochs plus the global proof, so the target is one +epoch verify (≈7.3M machine instructions unfused) plus chaining — not a fleet. +Keccak permutations per epoch verify: ~1.4M at blowup 2 / 219 queries, ~460k at +blowup 8 / 73 queries — which is what makes chunking mandatory and the +blowup/topology choice worth revisiting before the wrap run. diff --git a/others/lfm-team-lead-call.md b/others/lfm-team-lead-call.md new file mode 100644 index 000000000..d80c49320 --- /dev/null +++ b/others/lfm-team-lead-call.md @@ -0,0 +1,39 @@ +# Team-lead call for keccak-probe — R1d blocker resolution + +Written 2026-07-29 ~19:30Z as a filesystem fallback because two mailbox +authorizations apparently did not reach you (msgs 3007f24b, db4f6d79). +This file is the operative instruction; it answers your (A)/(B) question. + +## THE CALL: (A) — MERGE. Authorized. + +Protocol (stops are hard stops — report and wait): + +1. `git stash push prover/src/lib.rs prover/src/tables/types.rs` — the only + dirty TRACKED files. NOTE: `git stash list` already shows an unrelated + pre-existing stash (`bench-keccak-vs-leanvm WIP`) — leave it alone; your + push becomes stash@{0}, pop takes it back off, the old one stays put. + Verify `git status` shows a clean tracked tree (untracked lfm/ + bin + + others/ remain, untouched by merge mechanics). +2. `git merge origin/main` — feat/lfm has zero local commits, so this must + be a clean FAST-FORWARD to 5fd961a0. Anything else: STOP. +3. `git stash pop` (your stash@{0}) — our lib.rs hunk is ~line 21, theirs + ~209–270; types.rs BusId arms are ours alone. Conflict: STOP. +4. Full lfm suite + drift tests. If digests moved (crypto/math changes could + perturb the commit pipeline): regenerate via + `cargo run --bin compute_lfm_registry --release`, paste, re-run, and + report MOVED vs SURVIVED as a finding either way. +5. Confirm `out_buf`/`out_pos` exist in + crypto/crypto/src/fiat_shamir/default_transcript.rs, then finish R1d in + one pass per the original spec, with your corrections folded in: + ext3 = 3 independently rejection-sampled candidates (≈3× per-draw + completeness loss — state the per-proof bound at real draw counts), and + sample_u64 at pow-2 bounds = low nbits of the BE u64, no rejection. + +Your interim work is ACCEPTED: the sample() replay + reversed-coefficient +Linear design, the vacuous-test catch (keep both tests; the lesson is noted +and your R1b/R1c audit of it is appreciated), and building the +version-independent half without waiting was the right judgment call. + +Going forward: check this directory for `lfm-team-lead-*.md` whenever a +blocker answer seems overdue — I will use files for anything +authorization-shaped from now on, with mailbox pings as notification. diff --git a/others/lfm-team-lead-decode-page-ruling.md b/others/lfm-team-lead-decode-page-ruling.md new file mode 100644 index 000000000..d34e33c01 --- /dev/null +++ b/others/lfm-team-lead-decode-page-ruling.md @@ -0,0 +1,97 @@ +# Team-lead ruling: DECODE and PAGE preprocessed commitments (ledger entry 7) + +Ruled 2026-08-04, on wave 5's proposal. Subject to USER veto — flagged in the +session report the day it was made. If vetoed, the fallback is the in-machine +derivation and this file must record the reversal. + +## The ruling + +**ACCEPTED as proposed.** The five preprocessed commitments split by what they +are a function of, and each kind gets the binding that kind admits: + +| Commitment | Function of | Binding | +|---|---|---| +| BITWISE | proof options only | INTERN as program constant | +| KECCAK_RC | proof options only | INTERN as program constant | +| REGISTER | previous epoch's `reg_fini` | DERIVE in-machine (Phase A calls `programs::emit_register_commitment` on the register-boundary arena — this is also what closes entry 2) | +| DECODE | the inner ELF | ARENA CELL, bound by the attestation join | +| PAGE | the inner ELF | ARENA CELL, bound by the attestation join | + +"Attestation join" means: the SAME arena cell Phase A absorbs is the cell the +`program_id` fold consumes — the two-consumer join one level up, for which the +machine already has an emitter +(`machine_tests::program_id_folds_pages_in_the_production_layout`). + +## Why (three legs, none of them new judgment) + +1. **The alternative trips the always-stop item; the proposal doesn't.** + Interning an ELF-dependent root makes LFM program identity a function of + the guest ELF — one registry entry per guest program instead of one per + epoch shape. That is the exact clause on the standing always-stop list, + and it contradicts the phase's pin-SHAPE-not-values rule (nothing derived + from per-proof data may be a program constant). +2. **It mirrors production's own layering.** `recursion::program_id_from_digest` + folds `elf_digest`, `pc_start`, `decode_commitment` and every + `(page_base, page_commitment)` — precisely the ELF-dependent roots and none + of the options-only ones. Production already draws the line this ruling + draws; LFM is copying an existing boundary, not inventing one. +3. **The in-machine derivation relitigates a measured decision.** Deriving + DECODE/PAGE from ELF bytes costs a full in-machine LDE+tree per page and + requires the ELF itself to be bound in-guest — the full-ELF keccak pass + that sim/8 (`program_id` v2) deliberately removed, with the savings + measured. Re-adding it needs new evidence, not a default. + +## The residual risk, named plainly + +`program_id`'s binding is only as strong as the consumer-side +`check_attestation` compare, which has ZERO production call sites (RESUME, +"Open items needing the USER"; PoC at +`prover/src/tests/recursion_soundness_gap_poc.rs`). This ruling makes +DECODE/PAGE **exactly as bound as `elf_digest` and `pc_start` already are — and +no more**. It adds no new weakness, but it does add two more values whose +ultimate binding rests on a ritual nothing in production performs. The +check_attestation gap therefore gets MORE load-bearing with this ruling, and +the case for the user deciding to wire it into the CLI gets stronger. That +decision stays with the user; it is not part of this ruling. + +## Conditions attached (wave 6 must satisfy both) + +1. **The join must be structural, not a copy** — one cell with two consumers, + per the two-consumer rule that closed three soundness gaps this phase. A + guard must assert it, and the guard must be falsified (run the split-cell + forgery and watch it fail for the right reason). +2. ~~**PAGE's half is UNWITNESSED in the current fixture** + (`num_private_input_pages = 0` — a fixture property, not a production one). + The witness is a differently-configured real epoch (a guest with private + input pages). Wave 6 should build that epoch and run the assembled + verifier against it if it is cheap; if it is not cheap, the ledger keeps an + explicit OPEN entry saying PAGE's join is design-complete but unwitnessed.~~ + +## Amendment (2026-08-04, after wave 6): condition (b) REVERSED + +Condition (b) asked for a witness that CANNOT EXIST, and the premise behind it +("fixture property, not a production one") was wrong — wave 6 established this +by reading, three ways: + +1. Private-input pages are built NON-preprocessed (`lib.rs:800-828`), so a + guest with private input pages could never witness a PAGE preprocessed root. +2. **No continuation epoch of any guest has a PAGE sub-proof.** `prove_epoch` + rejects page configs outright — "continuation epoch must have no PAGE + configs (L2G bookend replaces PAGE)" (`continuation.rs:695-702`) — and both + `build_epoch_airs` call sites pass `page_configs = &[]`. The fixture was + matching production, not stripped down. +3. The ELF-data page roots the attestation folds are the GLOBAL proof's + GlobalMemory AIR commitments (`continuation.rs:997-1010`), per + `program_id_from_digest`'s own doc. + +Consequence: **PAGE migrates out of the epoch verifier's scope** to the +global-proof verifier rather than closing here. The provenance classifier +panics on any preprocessed root it cannot attribute, which is the intended +handover to that future work. + +Taxonomy correction to the table above: PAGE's ZERO-INIT root is options-only +(`page::zero_init_preprocessed_commitment`) and belongs in the CONSTANT +family; only the ELF-DATA page roots are ELF-dependent, and those live in the +global proof. The DECODE half of the ruling stands unchanged, conditions +satisfied (structural join + coherent-forgery falsification, wave 6 @ +2c810857). diff --git a/others/lfm-team-lead-fixture-ruling.md b/others/lfm-team-lead-fixture-ruling.md new file mode 100644 index 000000000..909e3e26d --- /dev/null +++ b/others/lfm-team-lead-fixture-ruling.md @@ -0,0 +1,64 @@ +# Team-lead ruling: R1f(b) fixture route + +For keccak-emitter, 2026-07-30. Answers the blocker in +`lfm-r1f-handoff.md` §3b. Supersedes any earlier phrasing. + +## RULING: route 2 — the machine consumes BYTES. But for a better reason than +## the one you gave. + +You argued route 2 on collision-avoidance and caching. Both true, both +secondary. The decisive argument is **fidelity**: bytes are how recursion +actually receives a proof in production. + +The RV64 recursion guest does not get a `ContinuationProof`. It gets a blob in +private input and reads it zero-copy through rkyv (`StarkProofView`'s +`Owned | Archived` split exists precisely for that, and the continuation path +verifies in place from archived bytes). So an LFM arena filler whose input is a +byte blob is the direct analogue of the guest's reader, and any divergence +between them is a *meaningful* signal. An arena filler that consumed an +in-memory `ContinuationProof` would be testing a path production does not have. + +That also disposes of route 1 on the merits, not just on collision risk: adding +a `pub(crate)` accessor would let `lfm` reach into a structure the real +recursion path never sees. + +## Use the EXISTING blob, do not invent a format + +Do not design a fixture encoding. The repo already produces exactly the bytes +the recursion guest consumes, and there is already a dump path for capturing +them to a file. Look for: + +- the continuation guest-input encoder (grep for `encode_continuation_guest_input` + or the encoder used by `prover/src/recursion.rs` to build the guest's private + input — it embeds the supplied roots, which you will need); +- the dump test that writes such a blob to disk (grep `test_dump_recursion_input` + and the `RECURSION_DUMP_PRESET` / `INNER_ELF` / `INNER_INPUT` / `EPOCH_LOG2` + environment knobs; earlier campaign work used exactly this to produce fixed + blobs for measurement). + +If that machinery exists and is reachable, your fixture is a captured blob plus +a small checked-in note recording the knobs used to produce it. If it turns out +to be `#[ignore]`d, env-gated, or otherwise not directly usable, say so and +propose the smallest thing that reuses the same ENCODER rather than a new one — +the encoder is what must not drift, the harness around it is incidental. + +Keep the blob small: a two-epoch continuation over an existing tiny test ELF +(`fibonacci`/`empty`), not ethrex. You need real structure, not real workload. +If the smallest honest blob is still large enough to be awkward in git, put it +under the scratchpad and have the test regenerate-or-load, with the regeneration +path exercised rather than the checked-in bytes. + +## Consequence for slice (a) + +The arena filler's input type is therefore `&[u8]` (or the archived view over +it), not `MultiProofView`. Its job is: read the archived blob, pull out the +roots and the openings for one query, and lay them out as arena words for the +emitter. Where the guest's reader and your filler disagree about layout, the +guest is right. + +## Standing note + +You have now stopped twice on decisions that were genuinely mine, and both times +the stop was correct. Do not let this ruling make you more reluctant to decide +things yourself — the standing decisions still say implement-and-flag when the +call is yours. This one was not. diff --git a/others/lfm-team-lead-partial-half-ruling.md b/others/lfm-team-lead-partial-half-ruling.md new file mode 100644 index 000000000..992b92b57 --- /dev/null +++ b/others/lfm-team-lead-partial-half-ruling.md @@ -0,0 +1,48 @@ +# Team-lead ruling: partial-half appends (handoff §10 last item) + +For keccak-emitter, before starting §2 item 1. Written 2026-07-29 ~21:05Z. + +## The reframe that dissolves most of it + +Append boundaries are NOT machine-visible. The production transcript's hasher +sees only the CONCATENATED byte stream between two finalize points (samples); +`append_bytes` boundaries have no representation in the digest input. All +lengths are shape-static in a straight-line program. Therefore the emitter's +correct unit of packing is the SEGMENT (sample-to-sample), not the append: +concatenate every absorbed byte rendering in the segment at emit time, then +chunk the whole segment into halves. "A partial half in the middle of a +segment that the NEXT APPEND must continue into" cannot occur — the emitter +already holds the next append's bytes when it packs. + +Segment prefixes are safe by construction: each segment after the first +starts with the 32-byte reversed digest (a multiple of 4), and the reversed- +digest words already exist on the bus. + +## What genuinely remains, and the ruling + +The residual problem is CONSTANT/DYNAMIC MISALIGNMENT: a constant of length +≢ 0 (mod 4) — e.g. the 27-byte `LAMBDAVM_STARK_STATEMENT_V3` tag — shifts a +following DYNAMIC value (a root word, a count) so one half mixes constant +bytes with dynamic bytes. The `stream_half + pad_const` trick covers this +only when the dynamic side's overlapping bytes are known-zero; in general it +needs a byte-level splice of the dynamic value at a constant offset +(re-aligning u32 halves by s∈{1,2,3} bytes ⇒ byte extraction, BitDec-32 per +affected half or a byte-table route; constant volume, and it only occurs in +the STATEMENT-ABSORB leg — a few dozen halves per proof, not in FRI/Merkle +traffic). + +RULING — the predecessor's recommendation, adopted with the reframe: +1. R1d NOW: `append` accepts whole halves only, loud assert, plus the test + that the assert fires (pin the limitation, don't leave it latent). This is + sufficient for the entire FRI-verifier scope: digests, field elements and + u64 renderings are all 4-byte multiples. +2. Document IN THE EMITTER (doc comment): the segment-level concatenation + argument above, so nobody reintroduces per-append packing; and that the + statement-absorb leg will add a `splice_misaligned(constant_prefix_len, + dynamic_halves)` helper (BitDec-based, constant offset, tiny volume) when + that leg is built — an extension point, not a redesign. +3. Do NOT build the splice now. Scope discipline: no verifier leg needs it + yet, and its design should be reviewed against the real statement stream. + +If anything in §2 contradicts this ruling, the ruling wins; report the +contradiction. diff --git a/others/lfm-team-lead-shared-commitment-ruling.md b/others/lfm-team-lead-shared-commitment-ruling.md new file mode 100644 index 000000000..910927ee5 --- /dev/null +++ b/others/lfm-team-lead-shared-commitment-ruling.md @@ -0,0 +1,62 @@ +# Team-lead ruling — shared/batched commitment lever + +Written 2026-07-31, in answer to deep-join's slice-1 finding. Binding for the +rest of Phase R unless the user overrules. + +## The finding being ruled on + +deep-join measured the joined DEEP/authentication leg: authentication is +99.0% of its instructions, and the walk is charged PER MATRIX — a narrow +table pays ~88 of its ~92 permutations walking FOUR per-matrix trees to the +SAME index. A shared/batched commitment across a sub-proof's matrices would +collapse ~3/4 of that, the biggest single lever anywhere in the leg. Ruling +was requested before the FRI leg adds a fifth tree per query. + +## Ruling + +1. **PARKED for the Phase R e2e.** The charter is a keccak e2e with ZERO + inner-prover changes. ✓ VERIFIED on feat/lfm: the shared-MMCS / + batched-FRI restructuring (PR #768 line of work) is NOT on this branch's + base — `BatchedMerkleTree` in `crypto/stark/src/config.rs:19` is merely + the leaf-hash backend's name (`BatchKeccak256Backend`), and the + commitment layer still builds one tree per committed matrix. The lever + therefore requires landing inner-prover commitment restructuring (it + exists unmerged on `feat/batched-fri-per-epoch`), which is exactly the + class of change keccak-first exists to avoid. No mid-phase shape change. + +2. **The FRI leg targets the CURRENT unbatched shape.** Warning for whoever + writes that brief: the batched path does not only share trees — it also + restructures folding (fold-to-scalar terminal, no early stop), so "add + the shared commitment later" is not a tree-only edit; it changes the FRI + leg's own shape. Building both shapes now means building FRI twice before + any e2e exists. One shape, e2e first. + +3. **RECORDED as a first-class input to the hash/batching decision.** + Provenance: prior campaign measurements (sim/4, sim/36 — RV32 guest-side, + NOT LFM) had batching cut permutations ~5× and left the verdict "gated on + hash cost in the verifier." LFM's cost model is permutation-dominated + (deep-join: hashing outweighs its own byteswapping 21.6×; authentication + is 99% of the joined leg). This is the strongest evidence yet that the + FINAL shape wants the batched inner proof. The hash-matrix phase after + e2e must therefore include a batched-shape cell, measured, not argued. + +4. **Prediction — CORRECTED 2026-07-31 after measurement** (deep-join, + b728043c). My original pin (~3/4 collapse, 213,744 → 55–70k at blowup 8) + was too optimistic by ~1.7×. The measured figure is **111,471 — a 48% + collapse**. The reasoning was right about walks and wrong about their + share: walks DO collapse 69% (1,958 → 616 permutations per query), but + they are only two thirds of the bill; the other third is leaf absorbs, + which sharing barely touches (absorbs scale with total bytes, walks with + tree count, and only the tree count collapses). This is arithmetic over + the shape — `ceil(leaf_bytes/136)` absorbs plus one permutation per + level — under one assumption: one tree per sub-proof, leaf = the + matrices' row pairs concatenated in matrix order. + +## What this ruling could not see — CLOSED + +The original version flagged leaf WIDENING under a shared tree as +unmeasured. Measured (deep-join, b728043c): widening is a small SAVING, +not a cost — absorbs go 970 → 911 per query, structurally, because total +leaf bytes do not change when matrices share a leaf; the only bytes that +move are the padding of the leaves that vanish. The sim/4 guest-side ++266M analogue does not transfer to LFM's permutation-count model. diff --git a/others/lfm-team-lead-start-index-research.md b/others/lfm-team-lead-start-index-research.md new file mode 100644 index 000000000..3808f5d30 --- /dev/null +++ b/others/lfm-team-lead-start-index-research.md @@ -0,0 +1,71 @@ +# start_index binding — production reading (ledger entry 2) + +This file replaces a raw 518 KB JSONL session transcript that was committed + +under this name at e105dea2 (recoverable from git history if the full session + +is ever needed). What follows is the research agent's final report, extracted + +verbatim from that transcript. Findings are also summarised in + +`lfm-assembly-obligations.md` entry 2. + +--- + +## Answer: production carries `start_index` as a register value (x254), and binds it by *reconstructing the AIR from the previous epoch's FINI* — never by an arithmetic cross-epoch check + +### 1. Where it lives (✓ VERIFIED, all read) + +- **Executor: no notion of it.** `Memory::commit_public_output` only appends bytes to `self.public_output` (`executor/src/vm/memory.rs:208-215`); its own doc says "the COMMIT AIR concatenates calls via the running `x254` index" (`memory.rs:41-43`, `:204-207`). No offset/counter is exported. +- **Prover trace side:** the counter is a *synthetic single-word register x254 at word-address 508*, held in `RegisterState.index_register` (`prover/src/tables/trace_builder.rs:159-160`), **seeded from `init[X254_INDEX]`** (`trace_builder.rs:194`), advanced by `count` per COMMIT ecall (`trace_builder.rs:1254-1274`, `:605-612` with a sync debug-assert), and emitted as the REGISTER final state at address 508 (`trace_builder.rs:266-273`). `register::X254_INDEX = 64` (`prover/src/tables/register.rs:134`). +- **Verifier side:** `compute_commit_bus_offset(public_output, start_index, z, alpha)` builds the COMMIT-bus receiver terms at `global_index = start_index + i` (`prover/src/lib.rs:909-946`, esp. `:929`), reached via `compute_expected_commit_bus_balance_view` (`lib.rs:969-978`). + +### 2. Is it a public/absorbed value? **No — not directly.** (✓ VERIFIED) + +`absorb_statement` absorbs only: domain tag, ELF digest, length-prefixed `public_output`, the 14 `table_counts`, `num_private_input_pages` (hard-coded 0 for epochs, `continuation.rs:103-104`), `fri_final_poly_log_degree`, `runtime_page_ranges`, and `epoch_label` (`prover/src/statement.rs:80-152`). **No `start_index`, no register vectors.** The `EpochProof` bundle fields are `proof, public_output, table_counts, runtime_page_ranges, reg_fini, l2g_root` (`continuation.rs:393-411`) — `start_index` is not a field; it is *derived* from `reg_fini` of the predecessor. + +### 3. What the verifier actually checks between epochs (✓ VERIFIED — read the body) + +`verify_continuation_view` loops epochs and does exactly this (`continuation.rs:1310-1338`): + +```rust +let mut register_init = register::register_init_from_entry_point(elf.entry_point); // :1310 +... +public_output.extend_from_slice(epoch_public_output); // :1334 +register_init = epoch.reg_fini()?; // :1337 +``` + +and `verify_epoch` derives (`continuation.rs:840-851`): + +```rust +let commit_start_index = register_init.get(register::X254_INDEX).copied().unwrap_or(0) as u64; +``` + +**There is NO explicit `epoch[i].start_index == epoch[i-1].start_index + epoch[i-1].output_len` check.** I confirmed structurally, not by term search: the only `len()` uses in `continuation.rs` are proof/table/epoch counts and `reg_fini_len()` (grep of all `.len()` sites, lines 335–2337; the only length validations are `expected_proof_count != proof.len()` at `:806-809` and `reg_fini_len() != NUM_REGISTER_ADDRESSES` at `:1302-1307`). Output bytes are simply concatenated in order (`:1334`) and returned (`:1399`). + +The binding is **structural**, in three composed locks: + +1. **Preprocessed REGISTER (OFFSET, INIT, FINI).** Each epoch's AIR is rebuilt by the verifier with `compute_precomputed_commitment_with_fini(opts, register_init, reg_fini)` and `NUM_PREPROCESSED_COLS_WITH_FINI = 3` (`continuation.rs:656-659`; `register.rs:67`, `:302-322`). The STARK verifier **rejects unless the proof's preprocessed root equals the AIR-recomputed one**, then absorbs it (`crypto/stark/src/verifier.rs:1184-1209`). So trace INIT/FINI are locked to the verifier's u32 vectors. +2. **REG-C2 on the epoch-local Memory bus** sends `(1, address, timestamp, FINI)`, matching MEMW's last receive (`register.rs:406-434`), so FINI = real last write to x254. +3. **The verifier reuses the *same* vector** as epoch i's FINI and epoch i+1's INIT (`continuation.rs:1337` feeding `:820`), so `init(i+1) == fini(i)` holds by construction — documented at `register.rs:59-67` and `docs/continuations_design.md:445-470` ("two locks"). + +Epoch 0 is anchored: `init_value_for_address(508, _) => 0` (`register.rs:150-158`), so `start_index = 0` at genesis, likewise for monolithic proofs. + +**The `start = prev_start + prev_len` relation is a derived consequence, not a check:** the COMMIT chip's MEMW token asserts `old x254 == INDEX` and writes `INDEX + count` (`commit.rs:592-652`, esp. the `value` linear form `INDEX + count_0 + 2^32·count_1` at `:616-629`); the CommitNextByte chain forces `index+1, address+1, count-1` per row (`commit.rs:262-332`) with ZERO-bus end detection (`:399-430`); and the COMMIT-bus sender `(INDEX, VALUE)` with mult `mu-end` (`commit.rs:708-722`) must balance against the verifier's receiver terms over the claimed `public_output` at `start_index + i`. So the epoch's byte count *equals* its x254 delta, and a lying `public_output` length imbalances the bus. + +### 4. Inside the AIR? **No boundary constraint exists at all.** (✓ VERIFIED) + +`CommitConstraints` emits only 8 transition constraints — three IS_BIT, `(first+end)(1−mu)`, and two ADD pairs (`commit.rs:730-771`). There is no row-0/last-row anchor on `INDEX`. Structurally confirmed: every VM AIR is `AirWithBuses` (`prover/src/test_utils.rs:122`, COMMIT at `:844-853`), and `NullBoundaryConstraintBuilder`'s impl body is **empty** (`crypto/stark/src/lookup.rs:1647-1653`); a repo-wide grep for `boundary_constraints|BoundaryConstraint` under `prover/src/tables/` returns nothing. So `start_index` is bound *only* by the MEMW-token → REGISTER-preprocessed-commitment path, plus the verifier-side offset. + +### 5. Register file: yes, x254 is reg slot 64 (✓ VERIFIED) + +`NUM_REGISTER_ADDRESSES = 67` (x0–x31 → 0..63, x254 → 508, x255 → 510/511) (`register.rs:43-52`, `:116-128`); `fini_from_trace` reads FINI on those 67 rows (`register.rs:265-269`). Enforcement of `reg_init[254](N) == reg_fini[254](N−1)` = the three locks in §3 — the verifier literally passes the same `Vec` to both sides, so equality cannot be violated; the *untrusted* part is `reg_fini` itself, pinned by lock 1 + lock 2. + +### Notes / caveats + +- Because FINI is committed as `FE::from(u32)` (`register.rs:318`) and the root is verifier-recomputed, `start_index < 2^32` is forced on the production continuation path (relevant to LFM ledger OPEN entry 1, `others/lfm-assembly-obligations.md:15-23`). +- The recursion attestation carries only `program_id || public_output` (`prover/src/recursion.rs:286-294`, `:311-314`) — **no start_index, no per-epoch offsets**, so nothing at the attestation layer binds position either. +- ✗ UNVERIFIED (did not trace): that no ordinary RV instruction can address word 508 (5-bit rd/rs fields ⇒ ≤ addr 62). If LFM ever re-derives x254 writes, that bound deserves its own check. +- Regression test that exercises the carry: `test_commit_across_epochs_verifies` (`continuation.rs:1457-1498`), which forces a commit into an epoch where x254 is already 2. + +**For LFM:** production's binding is *"rebuild epoch N's REGISTER preprocessed commitment from epoch N−1's FINI vector"* — an AIR-reconstruction binding, not an in-machine arithmetic one. The in-machine analogue is to make the arena's `start_index` word the same word the REGISTER preprocessed-commitment recompute consumes (i.e. bind it to `reg_fini[64]` of the previous epoch, which the reg leg already handles), rather than inventing a `start + len` equation the production verifier does not have. \ No newline at end of file diff --git a/prover/Cargo.toml b/prover/Cargo.toml index d4ebdeb0d..5bc24ce39 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -15,6 +15,18 @@ instruments = ["stark/instruments"] nvtx = ["cuda", "instruments", "stark/nvtx"] profile-markers = ["stark/profile-markers"] disk-spill = ["stark/disk-spill"] +# The `LFM_HASH` BLAKE3 arm's round count. Off = 7 rounds (standard BLAKE3, +# externally anchored, the A6R-free default); on = the 6-round A6R variant. +# It is a compile-time knob rather than a parameter because the chip's column +# layout is `8 · rounds` G-blocks wide and `hash::num_columns` is a `const fn`. +# +# It FORWARDS to `crypto`, where the primitive now lives, rather than declaring a +# second knob: the chip and the commitment backends must be at the same round +# count or a build commits under one hash and prices another. `crypto`'s is the +# only definition. The remaining way to desync them is enabling `crypto`'s alone, +# which the `SOCKET_ROUNDS == BLAKE3_ROUNDS` assertion in `lfm::blake3_socket` +# catches at compile time. +blake3-6round = ["crypto/blake3-6round"] [dependencies] stark = { path = "../crypto/stark" } @@ -36,6 +48,11 @@ criterion = { version = "0.5", default-features = false } tikv-jemallocator = "0.6" tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] } tiny-keccak = { version = "2.0", features = ["keccak"] } +# The external anchor for `lfm::blake3` at 7 rounds and for the `LFM_HASH` +# BLAKE3 socket (`blake3::hash(a ‖ b ‖ "LFMC")`). Test-only on purpose: the +# machine never calls it, so it can never become a second implementation that +# the chip is silently checked against instead of the vectors. +blake3 = { version = "1.8.5", default-features = false, features = ["std", "pure"] } # Enable stark's test-utils so cross-crate tests can reach # `compute_precomputed_commitment_for_testing`. Only active under cargo test/bench. stark = { path = "../crypto/stark", features = ["test-utils"] } diff --git a/prover/src/auto_storage.rs b/prover/src/auto_storage.rs index 6b5ed8a5d..4d59d351c 100644 --- a/prover/src/auto_storage.rs +++ b/prover/src/auto_storage.rs @@ -228,6 +228,23 @@ pub fn decide(lengths: &TableLengths, blowup_factor: u8) -> StorageMode { mode } +/// The LFM wrap's `StorageMode`: `Disk` when `FORCE_DISK_SPILL` is set, else +/// `Ram`. +/// +/// There is deliberately no estimate here. [`decide`]'s is keyed off the RV64 +/// executor's [`TableLengths`], and the wrap has no analogue of one — its table +/// set is program shape, fixed before execution, and its dominant family +/// (`KECCAK_RND`, one table per chunk) has a column profile the model was never +/// calibrated against. Guessing a mode from it would decide the wrap's storage +/// on an uncalibrated number; the explicit knob decides it on the operator's. +pub fn decide_lfm() -> StorageMode { + if std::env::var("FORCE_DISK_SPILL").is_ok() { + log::info!("lfm storage_mode: Disk (forced via FORCE_DISK_SPILL)"); + return StorageMode::Disk; + } + StorageMode::Ram +} + /// Peak RAM estimate in bytes for a proof whose trace shape matches `lengths`. /// /// `table_parallelism` is the prover's `k` (`stark::prover::table_parallelism`), diff --git a/prover/src/bin/compute_constraint_artifacts.rs b/prover/src/bin/compute_constraint_artifacts.rs new file mode 100644 index 000000000..21ec7b0e5 --- /dev/null +++ b/prover/src/bin/compute_constraint_artifacts.rs @@ -0,0 +1,109 @@ +//! Serializes every production table's constraint artifact to disk, so the +//! constraints exist as DATA rather than only as compiled code. +//! +//! Run with: +//! cargo run --bin compute_constraint_artifacts --release -- +//! +//! Writes `/.bin` per table plus a `MANIFEST.txt` recording each +//! artifact's size, and prints the size table (the recursion machine's +//! program-length budget). +//! +//! Capture is a build-time operation: it hash-conses the whole constraint body, +//! which is exactly what a guest must not do. That is the point of writing the +//! result down — see `stark::constraint_ir::artifact`. +//! +//! The artifacts are NOT proof-options or trace-length dependent (pinned by +//! `artifacts_are_invariant_across_proof_options` and +//! `artifacts_are_invariant_across_trace_length`), so a table's file covers every +//! blowup factor and every epoch size. +//! +//! ⚠️ But four tables are PARAMETERIZED, so this emits ONE REPRESENTATIVE, not +//! the complete set: `PAGE` and `GLOBAL_MEMORY` fold a page base into constant +//! bus terms, and both `L2G` tables fold an epoch label. Their files are the +//! artifact at `test_utils::PAGE_TEST_BASE` / `EPOCH_TEST_LABEL` only. A real +//! continuation proof needs one artifact per distinct page base and per distinct +//! epoch label — see +//! `constraint_artifact_tests::parameterized_airs_vary_per_parameter_value`. +//! Treating this directory as "the constraint artifacts" would be wrong for +//! exactly the tables a continuation proof cares most about. +//! +//! ⚠️ These bytes are not an oracle. Nothing about a serialized artifact proves +//! it matches the compiled folder — only +//! `prover/src/tests/constraint_artifact_tests.rs` does, by evaluating the +//! deserialized artifact against the folders on random frames. Regenerating +//! these files does not bless a constraint change. + +use std::path::PathBuf; + +use lambda_vm_prover::test_utils::production_airs; +use stark::constraint_ir::ConstraintArtifact; +use stark::proof::options::GoldilocksCubicProofOptions; + +fn main() { + let out_dir: PathBuf = std::env::args() + .nth(1) + .unwrap_or_else(|| "constraint_artifacts".to_string()) + .into(); + + let options = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid"); + + std::fs::create_dir_all(&out_dir) + .unwrap_or_else(|e| panic!("cannot create {}: {e}", out_dir.display())); + + let mut manifest = String::from( + "# Per-table constraint artifacts. Sizes in bytes.\n\ + # table constraints nodes base_consts ext_consts bytes\n", + ); + let mut total_bytes = 0usize; + let mut total_nodes = 0usize; + + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + "table", "constr", "nodes", "bconst", "econst", "bytes" + ); + + for (label, air) in production_airs(&options) { + let artifact = ConstraintArtifact::capture(&*air); + artifact + .validate_against(&*air) + .unwrap_or_else(|e| panic!("[{label}] artifact rejected against its own AIR: {e}")); + let bytes = artifact + .to_bytes() + .unwrap_or_else(|e| panic!("[{label}] serialize failed: {e}")); + + let path = out_dir.join(format!("{label}.bin")); + std::fs::write(&path, &bytes) + .unwrap_or_else(|e| panic!("cannot write {}: {e}", path.display())); + + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + label, + artifact.roots.len(), + artifact.nodes.len(), + artifact.base_consts.len(), + artifact.ext_consts.len(), + bytes.len() + ); + manifest.push_str(&format!( + "{label} {} {} {} {} {}\n", + artifact.roots.len(), + artifact.nodes.len(), + artifact.base_consts.len(), + artifact.ext_consts.len(), + bytes.len() + )); + total_bytes += bytes.len(); + total_nodes += artifact.nodes.len(); + } + + manifest.push_str(&format!("TOTAL - {total_nodes} - - {total_bytes}\n")); + let manifest_path = out_dir.join("MANIFEST.txt"); + std::fs::write(&manifest_path, &manifest) + .unwrap_or_else(|e| panic!("cannot write {}: {e}", manifest_path.display())); + + println!( + "\ntotal: {total_nodes} nodes, {total_bytes} bytes ({:.1} KiB)\nwritten to {}", + total_bytes as f64 / 1024.0, + out_dir.display() + ); +} diff --git a/prover/src/bin/compute_lfm_registry.rs b/prover/src/bin/compute_lfm_registry.rs new file mode 100644 index 000000000..4662ca440 --- /dev/null +++ b/prover/src/bin/compute_lfm_registry.rs @@ -0,0 +1,80 @@ +//! Regenerates the `LFM_REGISTRY` constant table. +//! +//! Usage: `cargo run --bin compute_lfm_registry --release`, then paste the +//! output over the generated block in `prover/src/lfm/registry.rs`. Drift +//! tests recompute and compare on every PR; a drift failure is investigated, +//! never re-blessed (the `compute_static_commitments` policy). + +use lambda_vm_prover::GoldilocksCubicProofOptions; +use lambda_vm_prover::lfm::hash::HasherKind; +use lambda_vm_prover::lfm::programs::{ + KECCAK_SPONGE_LEN, fri_toy_program, keccak_chain_program, keccak_sponge_program, + statement_replay_program, transcript_replay_program, trivial_program, +}; +use lambda_vm_prover::lfm::registry::build_artifacts_with_hasher; +use lambda_vm_prover::lfm::validate; + +/// Blowups registered in v0 (extend alongside `STATIC_BLOWUP_FACTORS` when +/// other presets come online). +const REGISTRY_BLOWUP_FACTORS: &[u8] = &[2]; + +/// The `LFM_HASH` permutation the v0 registry is generated under. +/// +/// Bound into every digest below, so changing it here is a re-blessing of the +/// whole table, not a re-run. A second hasher becomes additional rows, never a +/// silent replacement of these. +const REGISTRY_HASHER: HasherKind = HasherKind::Test; + +fn fmt_bytes(bytes: &[u8; 32]) -> String { + let inner = bytes + .iter() + .map(|b| format!("{b:#04x}")) + .collect::>() + .join(", "); + format!("[{inner}]") +} + +fn main() { + let programs = [ + ("TrivialV0", trivial_program()), + ("FriToyV0", fri_toy_program()), + ("KeccakChainV0", keccak_chain_program()), + ("KeccakSpongeV0", keccak_sponge_program(KECCAK_SPONGE_LEN)), + ("TranscriptReplayV0", transcript_replay_program()), + ("StatementReplayV0", statement_replay_program()), + ]; + println!("pub static LFM_REGISTRY: &[LfmRegistryEntry] = &["); + for (kind, program) in &programs { + // A program digest enters the registry only after admission passes — + // the gate `validator.rs` declares, wired here rather than left to the + // convention that every kind also has a hand-written admissibility test. + validate(program).unwrap_or_else(|v| panic!("{kind} is not admissible: {v:?}")); + for &blowup in REGISTRY_BLOWUP_FACTORS { + let options = GoldilocksCubicProofOptions::with_blowup(blowup).expect("proof options"); + let artifacts = build_artifacts_with_hasher(program, &options, REGISTRY_HASHER); + println!(" LfmRegistryEntry {{"); + println!(" kind: LfmProgramKind::{kind},"); + println!(" blowup_factor: {blowup},"); + println!(" roots: ["); + for root in &artifacts.roots { + println!(" {},", fmt_bytes(root)); + } + println!(" ],"); + let heights = artifacts + .log_heights + .iter() + .map(u8::to_string) + .collect::>() + .join(", "); + println!(" log_heights: [{heights}],"); + println!( + " keccak_rnd_chunks: {},", + artifacts.keccak_rnd_chunks + ); + println!(" hasher: HasherKind::{:?},", artifacts.hasher); + println!(" program_id: {},", fmt_bytes(&artifacts.program_id)); + println!(" }},"); + } + } + println!("];"); +} diff --git a/prover/src/continuation.rs b/prover/src/continuation.rs index 85f2d6223..04f0ff7f6 100644 --- a/prover/src/continuation.rs +++ b/prover/src/continuation.rs @@ -143,7 +143,7 @@ fn global_transcript( /// The L2G epoch-local table's single transition constraint: `MU ∈ {0,1}` /// (`MU·(1−MU) = 0`) at constraint index 0. #[derive(Clone, Copy)] -struct L2gMemoryConstraints; +pub(crate) struct L2gMemoryConstraints; impl ConstraintSet for L2gMemoryConstraints { fn eval>(&self, b: &mut B) { @@ -163,7 +163,7 @@ impl ConstraintSet for L2gMemoryConstraints { /// committed trace (equal Merkle roots). So under collision resistance the trace the /// global bus runs over already satisfies all those constraints — do not add them /// here (it would be redundant, not a missing check). -fn l2g_global_air( +pub(crate) fn l2g_global_air( opts: &ProofOptions, epoch_label: u64, ) -> AirWithBuses { @@ -184,7 +184,7 @@ fn l2g_global_air( /// check too: this proof has the BITWISE provider, and the global proof commits /// the identical trace (the commitment binding compares roots), so checking here /// covers both. `epoch_label` is the `fini_epoch` constant used by both. -fn l2g_memory_air( +pub(crate) fn l2g_memory_air( opts: &ProofOptions, epoch_label: u64, ) -> AirWithBuses { @@ -223,7 +223,7 @@ fn l2g_memory_air( /// genesis commitment from `config.init_values` — the recursion guest's /// supplied roots skip the in-VM FFT + Merkle build (see `verify_global`). /// `None` recomputes from `config` as before. -fn global_memory_air( +pub(crate) fn global_memory_air( opts: &ProofOptions, config: &PageConfig, preprocessed: Option, @@ -492,6 +492,55 @@ impl ContinuationProof { } } +/// Zero-copy readers over an ARCHIVED bundle, for the LFM arena filler. +/// +/// Deliberately on the archived type only. The recursion guest never holds a +/// `ContinuationProof` — it reads a blob from private input and verifies in +/// place ([`verify_continuation_archived`]) — so these expose a path production +/// actually traverses. The equivalent on the owned type would expose a structure +/// the real recursion path never sees, which is a weaker proposition. +/// +/// Methods rather than relaxed field visibility because rkyv mirrors the source +/// field's visibility onto the archived struct: opening `epochs` would open the +/// owned type at the same time. +impl ArchivedContinuationProof { + pub(crate) fn num_epochs(&self) -> usize { + self.epochs.len() + } + + /// Epoch `i`'s STARK proof (its tables, epoch-local L2G sub-table last), as + /// the same view the verifier reads in place. + pub(crate) fn epoch_proof(&self, i: usize) -> MultiProofView<'_, F, E, ()> { + MultiProofView::Archived(&self.epochs[i].proof) + } + + /// Bytes epoch `i` committed. + pub(crate) fn epoch_public_output(&self, i: usize) -> &[u8] { + self.epochs[i].public_output.as_slice() + } + + /// Epoch `i`'s own committed L2G table root — the left-hand side of the + /// cross-epoch binding [`crate::verify_l2g_commitment_binding_view`] checks + /// against the global proof's `i`-th sub-proof. + pub(crate) fn epoch_l2g_root(&self, i: usize) -> Commitment { + self.epochs[i].l2g_root + } + + /// Epoch `i`'s final register file `R_{i+1}`, the vector + /// [`build_epoch_airs`] preprocesses as FINI and the chaining loop carries + /// forward as epoch `i+1`'s INIT. + pub(crate) fn epoch_reg_fini(&self, i: usize) -> Result, Error> { + EpochProofView::Archived(&self.epochs[i]).reg_fini() + } + + /// The one cross-epoch global-memory proof, as the same view the verifier + /// reads in place. Its first `num_epochs()` sub-proofs are the per-epoch L2G + /// tables the binding ties to. + pub(crate) fn global_proof(&self) -> MultiProofView<'_, F, E, ()> { + MultiProofView::Archived(&self.global) + } +} + /// Borrowed view over an [`EpochProof`] (owned or archived-in-place). Lets /// `verify_epoch` take a single argument again instead of the field-by-field /// parameter list the owned/archived split used to force on every caller: @@ -744,6 +793,7 @@ fn prove_epoch( &mut seed(), #[cfg(feature = "disk-spill")] stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| Error::Prover(format!("{e:?}")))?; @@ -941,6 +991,7 @@ fn prove_global( ), #[cfg(feature = "disk-spill")] stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| Error::Prover(format!("{e:?}"))) } diff --git a/prover/src/lfm/SOUNDNESS.md b/prover/src/lfm/SOUNDNESS.md new file mode 100644 index 000000000..4285b7e50 --- /dev/null +++ b/prover/src/lfm/SOUNDNESS.md @@ -0,0 +1,249 @@ +# LFM write-once memory: the soundness argument (Phase 0b) + +Status: **for review**. This is the document a reviewer is invited to reject. If the argument +below does not convince, the fallback is the VM's timestamped memory argument (known-sound, +priced: 37% of MEMW's columns are timestamps plus 8 `<`-lookups per row) or in-circuit +well-formedness checks (uniqueness, per-selector booleanity, mult bounds — a material repricing). + +## 1. Setting + +An LFM **program** is a set of *instruction column groups*: per-chip preprocessed matrices holding +addresses, opcode selectors and multiplicities. They are committed once (interpolate → LDE → +row-pair Merkle) and their roots are pinned in `LFM_REGISTRY`, a drift-tested Rust constant table. +At verify time the roots are resolved from the registry and the framework rejects any proof whose +preprocessed commitment differs (`verifier.rs` equality check; on the prover side a mismatching +trace fails with `PrecomputedCommitmentMismatch`). The **main** (witness) columns carry values +only. + +Memory is not a table. A cell is a word `(v0..v3)` at an address `a`; the producing instruction's +chip **sends** the token `(a, v0, v1, v2, v3)` on the `LfmMem` bus with multiplicity `mult(a)` — a +*preprocessed* column — and each consuming instruction's chip **receives** the same token once, +with multiplicity gated by its (also preprocessed) `is_real`/selector columns. The LogUp argument +the whole prover already runs enforces, per bus, with soundness error `O(D/|E|)` over the +challenges `z, α` (`E` = the degree-3 extension, `D` = total interaction count): + +> **(B) Balance.** The multiset of sent tokens with multiplicity equals the multiset of received +> tokens with multiplicity. + +### 1.1 The framework premises the machine inherits + +(B) is not a fact about the LFM prover; it is an assumption about the *outer* verifier that checks +the LFM proof (`lfm/proof.rs` → `Verifier::multi_verify_views`), which is ordinary, unmodified +framework code. Four of its checks are load-bearing here and the machine defends none of them +independently: **per-column opening-width pinning** (`trace_opening_widths_well_formed`), which +pins each query opening's precomputed/main/aux split against the AIR rather than only their sum; +**`ood_blocks_well_formed`**, which derives the OOD table's shape from the AIR; **the +precomputed-root equality check** cited above, which is what delivers §2's registry premise; and +**the composition part-count check**, which fixes `num_composition_parts` from the AIR's degree +bound instead of reading it off the proof. + +The width pin is the one worth spelling out, because it is what makes (B) hold with error +`O(D/|E|)` rather than not at all. Every LFM chip is `with_preprocessed` with a non-empty aux trace +whose root is absorbed *after* the LogUp challenges `z, α`. If only the sum of the opening widths is +pinned, a prover may declare one value column into the aux group and choose it with the challenges +in hand; the bus check is a single aggregate scalar equality per proof, so one free extension +element solves it for an arbitrary perturbation — and (B) is gone while §2's whole custody chain +still passes, because that chain binds the *addressing* columns and the break is in the value +columns' commitment timing. **The machine's soundness is therefore only as good as the base tree's +version of these checks**: on a base predating one of them the theorem of §3 has no (B) to consume +and a wrap proof certifies nothing, no matter how much of §2–§6 holds. Item 8 of §7 is the +reviewer's version of this. + +## 2. What is vouched, and by whom + +| obligation | enforced by | mechanism | +|---|---|---| +| per-op algebra (`out = a·b + c`, …) | AIR | transition constraints on value columns | +| bit booleanity where a witness bit exists | AIR | degree-2 constraint | +| selector sum-boolean per ALU row | AIR | degree-2 constraint (belt over suspenders) | +| token balance (B) | AIR | LogUp, framework-emitted | +| **(U) uniqueness** — no address written twice | registrar | admission validator check 1 | +| **(A) acyclicity** — operand addr < destination addr | registrar | check 2 (dense emission order gives it by construction; re-checked) | +| **(M) mult-equality** — `mult(a)` = number of emitted reads of `a` | registrar | check 3 | +| **(S) selector one-hot-ness** | registrar | check 4 | +| **(P) padding rows all-zero** (mult = 0, is_real = 0) | registrar | check 5 | +| arena discipline | registrar | check 6 | + +"Registrar" means: the release-mode validator (`lfm/validator.rs`) ran on this exact program +before its digest entered `LFM_REGISTRY`, and the proof's preprocessed roots equal the registry's. +The chain of custody is: validator ⇒ digest ⇒ registry constant ⇒ drift tests on every PR ⇒ +root-equality check at prove and verify time. **There is no runtime off-switch and there must +never be one; the registry check *is* the soundness argument's first premise.** + +This is the industry-standard trust shape for this machine class: SP1 v4 relies on the same +premise but checks it only in dev builds (its validator omits the double-write check entirely); +Risc0 makes it structural (write destinations are program text, so uniqueness is syntactic). We +run the full checklist, in release, at admission — strictly more than either reference. + +## 3. The claim + +> **Theorem.** Assume (U), (A), (M), (S), (P) hold for the program (registrar) and (B) holds for +> the proof (LogUp). Then in any accepted execution, every read of address `a` observes the unique +> value written at `a`. + +**Argument.** By (U) each address has at most one producing instruction, so "the write at `a`" is +well-defined; let `W(a) = (a, w0..w3)` be its token, sent with multiplicity `mult(a)` (the +preprocessed column — the prover cannot vary it). By (M), `mult(a)` equals the number of program +reads of `a`. By (P), padding rows contribute no tokens (their multiplicities are preprocessed +zeros). + +Consider the multiset equation (B) on `LfmMem`. Every receive token is generated by some real +instruction row whose address operand columns are preprocessed, so the *addresses and counts* of +all receives are program text; only the value lanes are witness. Fix an address `a`. The sends at +address `a` are exactly `mult(a)` copies of `W(a)` (one writer, (U)). The receives at address `a` +are exactly the program's reads of `a` — `mult(a)` of them, by (M) — each carrying the value lanes +the reading chip's row exhibits. Balance of the full multiset then forces the sub-multisets at +each address to match (tokens include the address, and the fingerprint separates distinct tuples +except with the LogUp soundness error), so each of the `mult(a)` receive tokens equals `W(a)`: +every read observes the written value. + +Two degenerate cases are closed by the remaining premises. If a value could feed its own +producing row (`a := f(a)`), the send and receive would cancel *within* the row for **any** value +— balance holds vacuously and the value is unconstrained. (A) excludes this: every operand +address is strictly below its destination, so the read-token's address refers to an +earlier-produced cell, and the dataflow relation is a DAG; induction over addresses in ascending +order grounds every value in constants, hints, or hash outputs. If selectors were not one-hot, one +row could emit tokens under two op semantics at once; (S) excludes it beyond the in-AIR +sum-boolean. + +## 4. What the AIR must still get right (per-chip obligations) + +The argument above reduces chip soundness to: *each chip's constraints must force the value lanes +of every token it sends to be the correct function of the value lanes of the tokens it receives, +on every row where its (preprocessed) multiplicities are nonzero.* Concretely: base ops constrain +lane 0 and send `(a, out, 0, 0, 0)` with constant-zero high lanes in the tuple (a base cell +cannot smuggle extension lanes); ext ops likewise pin lane 3 ≡ 0; `MulBase` additionally +constrains the shared B-columns to zero on its rows so the received token matches a base writer's; +`BitDec`'s canonicity gadget (`G/Z/GINV`) forces the 64 bit columns to recompose to the *canonical* +representative — without it, bits summing to `v + p` would satisfy the linear recomposition and +two distinct bit-vectors could both "be" `v`. + +## 5. Arenas + +`Hint` rows send unconstrained words into memory (their chip has no constraints by design). The +**arena rule** restores soundness at the program level: every arena-sourced value must be +transitively authenticated by a hash the machine itself performs (Merkle openings are absorbed +into hashed paths; anything transcript-derived is never hinted). This is a *program-review* +obligation, enforced at emitter review, exactly like the reference systems' hint discipline — the +machine-level theorem above is indifferent to hint values; it only guarantees reads see what was +hinted. + +## 6. Transcript replay (R1d) + +`edsl::TranscriptReplay` reproduces the production `DefaultTranscript` inside the machine. Three +things about it are worth a reviewer's attention. + +### 6.1 Absorbed data is hinted, and that is correct + +The replay absorbs arena-supplied words. That is not a breach of the arena rule (§5). The rule bans +*hinting a challenge*; it does not ban hinting the data a challenge is derived FROM — in +Fiat–Shamir that data is precisely the untrusted proof material, and binding it is the entire +point. Every challenge the machine uses is computed by `LFM_KECCAK` rows from the absorbed +segment, never read from an arena. The obligation that remains is the ordinary one: whatever is +absorbed must also be the thing the rest of the program checks against. + +### 6.2 The canonicity guard is a constraint, not a witness + +`sample_field_element` must reject candidates ≥ `p`. Since `p = (2^32 − 1)·2^32 + 1`, a candidate +`hi·2^32 + lo` with canonical `u32` halves is out of range **iff** `hi = 2^32 − 1 ∧ lo ≠ 0` — the +same predicate `BitDec` already uses for 64-bit canonicity (§4), over the same split. + +The guard emits one instruction: `div(lo, (2^32 − 1) − hi)`. `LFM_BALU` constrains division as +`SEL_DIV·(B·OUT − A) = 0`, so with `B = 0` it reads `A = 0` and leaves `OUT` free. The division is +therefore provable exactly when `hi ≠ 2^32 − 1` or `lo = 0`. Nothing is hinted and nothing needs +verifying — it is the same assert-via-division mechanism `assert_eq` is built from. (An earlier +plan used an `is_zero` gadget with a hinted-and-verified inverse; the division subsumes it.) + +`machine_tests::canonicity_guard_rejects_an_out_of_range_candidate_in_the_proof` exhibits a +coherent forgery at candidate `p` — every bus balances and the mul-add's own constraint is +satisfied — and confirms it is rejected; neutralising `emit_base(3, …)` makes that forgery +ACCEPTED, which is what pins the guard on this one constraint. + +### 6.3 Zero rejection: a completeness restriction, and why it is not a parameter + +The production sampler *loops* on an out-of-range candidate. The number of candidates a draw +consumes is therefore data-dependent, and so is every later draw's position in the output buffer. +A straight-line machine has exactly one shape, so it cannot follow that. The emitted program +encodes the **no-rejection schedule** and is unprovable for any transcript that ever rejects. + +This costs completeness only, never soundness. The emitted relation is a strict subset of the real +one: challenge values are pinned by constraints to the no-rejection schedule, so a transcript that +would have rejected yields *no* LFM proof rather than a wrong one. An honest prover sees it as a +loud `LfmExecError::DivByZero`, not a silent divergence. + +The bound. A candidate is uniform over `2^64` values and `2^64 − p = 2^32 − 1` of them are out of +range, so `q = (2^32 − 1)/2^64 ≈ 2^−32` per candidate. Only `sample_field_element` draws are +exposed: `sample_u64` at a power-of-two bound has `threshold = 0`, so it accepts its first +candidate unconditionally and contributes nothing. Every verifier challenge is a cubic-extension +element, i.e. three independent base draws, so with `E` extension draws the union bound gives + +> `P[the program cannot prove this proof] ≤ 3E · (2^32 − 1)/2^64` + +(`reject_probability_per_proof` in `transcript_replay.rs`, which takes the BASE draw count `3E`). + +The verified per-proof draw schedule, for a multi-proof over `T` tables, is + +> `E = 2` (LogUp `z, α` — shared transcript, drawn before the per-table forks) +> ` + 2` (bus-balance replay, on a forked transcript) +> ` + Σ_t (3 + L_t)` — per table `β`, `z_OOD`, `γ`, then `L_t` FRI fold challenges, +> with `L_t = max(log2(trace_length_t) − 7, 0)`, **independent of the blowup factor**. + +`β` and `γ` are one draw each no matter how many terms they batch (both expand to powers), which +is what keeps `E` small. At `T = 24` with tables at their row cap (`L_t = 12`), `E = 364`, so +`3E = 1,092` base candidates and `P ≈ 2.5·10^−7`. At a larger `T ≈ 60`, `E = 904` and +`P ≈ 6.3·10^−7`. + +`T = 24` is **measured, not assumed**: reading a real two-epoch continuation proof +(`machine_tests::arena_filler_reads_real_committed_roots`) gives 24 sub-proofs for an +intermediate epoch and 25 for the final one, the extra being HALT. It was an honest hedge when +this section was written; it no longer needs to be. + +**State it as `< 10^−6` per proof at production shapes**, growing by `≈ 1.05·10^−8` per additional +table — each table contributes `3 + L_t ≈ 15` extension draws, so the per-table increment is 15× +the `≈ 7·10^−10` an individual extension draw costs. (Do not quote the per-draw figure as the +per-table one; the two differ by that factor of 15.) + +Headroom is large: 1% failure needs `≈ 4.3·10^7` base candidates (`≈ 2^25.4`) and 50% needs `2^31`, +four-plus orders of magnitude beyond any realistic verifier. Every figure above is pinned by +`machine_tests::zero_rejection_completeness_bound` rather than merely asserted here. + +**One host/machine divergence worth recording.** The verifier does not check that the +prover-supplied `trace_length` is a power of two. A malicious proof could therefore hand +`sample_u64` a non-power-of-two bound, making `threshold` nonzero and putting even a query draw on +the rejection path — the one circumstance in which a `u64` draw could matter to this bound. Such a +proof is simply unprovable in the machine, which is the safe direction (unprovable = rejected), but +it is a case where the host transcript and the emitted program diverge rather than agree. + +**Do not record this as "k-rejection is an emitter parameter later".** It is not. Supporting even +one rejection requires the downstream schedule to branch, which in a straight-line machine means +either a program per rejection pattern (`2^draws` of them, and program identity would become +proof-dependent, breaking the registry premise in §2) or a production transcript change to +constant-consumption sampling. The realistic route, if the bound ever stops being acceptable, is +the latter — make the production sampler consume a fixed number of candidates per draw — and that +is a change to `crypto`, not to this emitter. + +Timing note for whoever picks that up: the ecosystem hash migration already has to rebuild the +transcript (a field-native sponge replaces the keccak chain), and constant-consumption sampling is +a design constraint to carry into that rebuild rather than a separate migration. Fixing it there +costs nothing extra and removes this restriction for every future machine; retrofitting it onto the +current transcript would be a second proof-breaking change for no other benefit. + +## 7. Reviewer checklist (reject if any fails) + +1. Is the validator actually on the only path into `LFM_REGISTRY`, in release builds, with no + env-var or feature bypass? +2. Do the drift tests pin the registry on every PR (not merge-queue-only)? +3. Does every chip keep the one sign convention (writes = senders `Column(mult)`, reads = + receivers `Column(is_real)`, no `Negated` forms)? +4. Are all address/selector/mult columns actually in the preprocessed group of every chip + (no witness-supplied addressing anywhere)? +5. Does the BitDec canonicity constraint cover the full 64-bit range for `p = 2^64 − 2^32 + 1` + (top-32-all-ones ⇒ bottom-32-zero)? +6. Is the LogUp soundness error budget (`O(D/|E|)`, `|E| ≈ 2^192`) acceptable at the machine's + interaction counts (≤ 2^25 per epoch)? +7. Does every `TranscriptReplay` challenge reach the program through machine keccak rows rather + than an arena, and is the zero-rejection completeness bound (§6.3) acceptable at this + program's actual draw count? +8. Is the base tree at or past every framework verifier fix the inherited premises of §1.1 name — + currently per-column opening-width pinning (`trace_opening_widths_well_formed`, #909 / + `6949ceb9`)? On an older base (B) is not delivered, and nothing below §1.1 can recover it. diff --git a/prover/src/lfm/airs.rs b/prover/src/lfm/airs.rs new file mode 100644 index 000000000..1b78aef76 --- /dev/null +++ b/prover/src/lfm/airs.rs @@ -0,0 +1,584 @@ +//! `LfmAirs` — the machine's fixed 14-chip AIR set, a sibling of `VmAirs`. +//! +//! The chip set never varies; only heights do (per program). Programs are +//! supplied preprocessed roots (resolved from `LFM_REGISTRY` at verify time), +//! so constructing the verify-side AIR set costs nothing — there is no keygen +//! in this framework. Proved and verified by the same generic +//! `multi_prove` / `multi_verify_views` machinery as the RV64 VM; **zero +//! `VmAirs` edits** — the sibling-AIR-set property, preserved deliberately. + +use stark::config::Commitment; +use stark::constraints::builder::{ConstraintSet, EmptyConstraints}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, NullBoundaryConstraintBuilder, +}; +use stark::proof::options::ProofOptions; +use stark::trace::TraceTable; +use stark::traits::AIR; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; + +use super::chips::{balu, bitdec, const_, hash, hint, keccak, lanes, public, range, select, xalu}; +use super::hash::HasherKind; +use super::layout; +use super::trace::LfmTraces; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +pub type LfmAir = AirWithBuses; +pub type DynLfmAir<'a> = &'a dyn AIR; + +/// The frozen chip order — everywhere: roots, digests, traces, proofs. +/// +/// Slots 11–13 are the production keccak family, hosted unchanged. They belong +/// to the *fixed* machine, so **every** LFM proof carries them — including the +/// 2^20-row BITWISE table, which costs a few seconds of prove time even for a +/// program containing no keccak at all. That is the deliberate price of the +/// fixed-machine principle: the chip set never varies with the program, only +/// heights do, so a program stays nothing but a vector of preprocessed roots +/// plus a registry entry. Making the set program-dependent would move shape +/// negotiation onto the verify path, which this design refuses. +/// +/// This is the count of chip *classes*, and the width of the roots and +/// log-heights arrays. `KECCAK_RND` (slot 11) may be instantiated more than +/// once — see [`num_lfm_airs`] — but its chunk count is program shape read +/// from the registry, not shape negotiated on the verify path, so the +/// principle above holds. +pub const NUM_LFM_CHIPS: usize = 14; +pub const LFM_CHIP_NAMES: [&str; NUM_LFM_CHIPS] = [ + "LFM_CONST", + "LFM_BALU", + "LFM_XALU", + "LFM_SELECT", + "LFM_BITDEC", + "LFM_HASH", + "LFM_KECCAK", + "LFM_LANES", + "LFM_HINT", + "LFM_PUBLIC", + "LFM_RANGE", + "KECCAK_RND", + "KECCAK_RC", + "BITWISE", +]; + +/// Slot of `KECCAK_RND`, the one AIR in the set with **no** preprocessed +/// columns — it has no root to supply, pin, or bind into the program digest. +/// It is also the one slot that expands into several AIR instances; the +/// chunks sit contiguously at 11.., so `KECCAK_RC` and `BITWISE` follow them +/// in the AIR list while keeping chip-class indices 12 and 13 in the roots +/// and log-heights arrays. +pub const KECCAK_RND_SLOT: usize = 11; + +/// AIR instances (and sub-proofs) in a proof whose `KECCAK_RND` is split into +/// `keccak_rnd_chunks` instances. +pub const fn num_lfm_airs(keccak_rnd_chunks: usize) -> usize { + NUM_LFM_CHIPS - 1 + keccak_rnd_chunks +} + +/// Permutations in each `KECCAK_RND` chunk, in chunk order. +pub fn keccak_rnd_chunk_permutations(program: &super::compiler::LfmProgram) -> Vec { + let total = program.groups.keccak.real_rows; + let per = program.chunking.permutations_per_chunk(); + (0..program.chunking.chunk_count(total)) + .map(|i| total.saturating_sub(i * per).min(per)) + .collect() +} + +/// Each `KECCAK_RND` chunk's trace height: 24 rows per permutation, padded — +/// the same `.next_power_of_two().max(4)` rule `generate_keccak_rnd_trace` +/// applies, now once per chunk. +pub fn keccak_rnd_chunk_rows(program: &super::compiler::LfmProgram) -> Vec { + keccak_rnd_chunk_permutations(program) + .into_iter() + .map(|perms| { + (perms * super::chunking::KECCAK_RND_ROWS_PER_PERMUTATION) + .next_power_of_two() + .max(4) + }) + .collect() +} + +/// One chip instance's trace geometry, as the cell instrument sees it. +/// +/// The per-chip decomposition of [`lfm_cell_counts`] — that function sums +/// exactly these rows, so a census and a total can never disagree. `name` is not +/// unique: every `KECCAK_RND` chunk reports under the same chip name, which is +/// the point (they are the same AIR at different heights). +#[derive(Clone, Copy, Debug)] +pub struct LfmChipCells { + pub name: &'static str, + /// Padded trace rows — the height the prover commits. + pub rows: u64, + /// Value columns: the AIR's width less its preprocessed prefix. + pub main_cols: usize, + /// Aux (LogUp) columns, one per pair of bus interactions. + pub aux_cols: usize, +} + +impl LfmChipCells { + pub fn main_cells(&self) -> u64 { + self.rows * self.main_cols as u64 + } + + pub fn aux_cells(&self) -> u64 { + self.rows * self.aux_cols as u64 + } +} + +/// Per-chip trace geometry for a compiled program, in the frozen AIR order +/// (`KECCAK_RND`'s chunks expanded, so the vector has one entry per sub-proof). +/// +/// Extracted from [`lfm_cell_counts`] rather than written beside it: a second +/// copy of this table is how a census would come to describe a different machine +/// than the one the totals describe. +pub fn lfm_chip_census(program: &super::compiler::LfmProgram) -> Vec { + lfm_chip_census_with_hasher(program, HasherKind::default()) +} + +/// [`lfm_chip_census`] for a program proved under `hasher`. +/// +/// Only `LFM_HASH`'s width moves with the hasher; every other chip is +/// hash-independent, and the preprocessed prefix is the hasher-independent +/// instruction group, so the row counts and the roots do not move either. +pub fn lfm_chip_census_with_hasher( + program: &super::compiler::LfmProgram, + hasher: HasherKind, +) -> Vec { + let range_rows = layout::range::NUM_ROWS as u64; + let g = &program.groups; + // Every chip class except `KECCAK_RND`, which is counted per chunk below. + let per_chip: [(u64, usize, usize, usize); NUM_LFM_CHIPS - 1] = [ + ( + g.const_.padded_rows as u64, + const_::cols::NUM_COLUMNS, + layout::const_::PREP_WIDTH, + const_::bus_interactions().len(), + ), + ( + g.balu.padded_rows as u64, + balu::cols::NUM_COLUMNS, + layout::balu::PREP_WIDTH, + balu::bus_interactions().len(), + ), + ( + g.xalu.padded_rows as u64, + xalu::cols::NUM_COLUMNS, + layout::xalu::PREP_WIDTH, + xalu::bus_interactions().len(), + ), + ( + g.select.padded_rows as u64, + select::cols::NUM_COLUMNS, + layout::select::PREP_WIDTH, + select::bus_interactions().len(), + ), + ( + g.bitdec.padded_rows as u64, + bitdec::cols::NUM_COLUMNS, + layout::bitdec::PREP_WIDTH, + bitdec::bus_interactions().len(), + ), + ( + g.hash.padded_rows as u64, + hash::num_columns(hasher), + layout::hash::PREP_WIDTH, + hash::bus_interactions(hasher).len(), + ), + ( + g.keccak.padded_rows as u64, + keccak::cols::NUM_COLUMNS, + layout::keccak::PREP_WIDTH, + keccak::bus_interactions().len(), + ), + ( + g.lanes.padded_rows as u64, + lanes::cols::NUM_COLUMNS, + layout::lanes::PREP_WIDTH, + lanes::bus_interactions().len(), + ), + ( + g.hint.padded_rows as u64, + hint::cols::NUM_COLUMNS, + layout::hint::PREP_WIDTH, + hint::bus_interactions().len(), + ), + ( + g.public.padded_rows as u64, + public::cols::NUM_COLUMNS, + layout::public::PREP_WIDTH, + public::bus_interactions().len(), + ), + ( + range_rows, + range::cols::NUM_COLUMNS, + layout::range::PREP_WIDTH, + range::bus_interactions().len(), + ), + // The keccak family's two fixed tables. `KECCAK_RND`'s chunks follow. + ( + keccak_rc::NUM_ROWS as u64, + keccak_rc::cols::NUM_COLUMNS, + keccak_rc::NUM_PRECOMPUTED_COLS, + keccak_rc::bus_interactions().len(), + ), + ( + bitwise::NUM_ROWS as u64, + bitwise::cols::NUM_COLUMNS, + bitwise::NUM_PRECOMPUTED_COLS, + bitwise::bus_interactions().len(), + ), + ]; + // The frozen AIR order is `air_refs`': chip classes 0..=10, then every + // `KECCAK_RND` chunk, then `KECCAK_RC` and `BITWISE`. `per_chip` above lists + // the classes with the last two at the end, so the chunks are spliced in + // before them rather than appended. + let rnd_interactions = keccak_rnd::bus_interactions().len(); + let mut census = Vec::with_capacity(per_chip.len() + 1); + for (slot, (rows, num_cols, prep, interactions)) in per_chip.into_iter().enumerate() { + if slot == KECCAK_RND_SLOT { + for rows in keccak_rnd_chunk_rows(program) { + census.push(LfmChipCells { + name: LFM_CHIP_NAMES[KECCAK_RND_SLOT], + rows: rows as u64, + main_cols: keccak_rnd::cols::NUM_COLUMNS, + aux_cols: rnd_interactions.div_ceil(2), + }); + } + } + census.push(LfmChipCells { + // `per_chip`'s last two entries are chip classes 12 and 13, which sit + // at indices 11 and 12 of that array — hence the shift past the + // `KECCAK_RND` slot rather than a plain index. + name: LFM_CHIP_NAMES[if slot >= KECCAK_RND_SLOT { + slot + 1 + } else { + slot + }], + rows, + main_cols: num_cols - prep, + aux_cols: interactions.div_ceil(2), + }); + } + census +} + +/// Trace-cell counts for a compiled program, the LFM analogue of the VM's +/// `total_field_elements` / `total_auxiliary_field_elements` (same +/// semantics: main counts base-field value cells excluding preprocessed +/// columns; aux counts extension-field elements, one per aux column per +/// row). This is the kill-risk-3 instrument: machine cells per verification +/// vs the verified proof's own cells. +pub fn lfm_cell_counts(program: &super::compiler::LfmProgram) -> (u64, u64) { + lfm_cell_counts_with_hasher(program, HasherKind::default()) +} + +/// [`lfm_cell_counts`] for a program proved under `hasher` — the hash matrix's +/// instrument. +pub fn lfm_cell_counts_with_hasher( + program: &super::compiler::LfmProgram, + hasher: HasherKind, +) -> (u64, u64) { + lfm_chip_census_with_hasher(program, hasher) + .iter() + .fold((0u64, 0u64), |(main, aux), c| { + (main + c.main_cells(), aux + c.aux_cells()) + }) +} + +pub struct LfmAirs { + const_: LfmAir, + balu: LfmAir, + xalu: LfmAir, + select: LfmAir, + bitdec: LfmAir, + hash: LfmAir, + keccak: LfmAir, + lanes: LfmAir, + hint: LfmAir, + public: LfmAir, + range: LfmAir, + /// One instance per `KECCAK_RND` chunk. Every instance is the identical + /// AIR — chunking changes only how many rows each one carries — so they + /// are built in a loop rather than named individually. + keccak_rnd: Vec>, + keccak_rc: LfmAir, + bitwise: LfmAir, +} + +/// Builds an AIR with **no** preprocessed columns — `KECCAK_RND` only. +fn build_air_no_prep + 'static>( + num_columns: usize, + interactions: Vec, + options: &ProofOptions, + constraint_set: CS, + name: &'static str, +) -> LfmAir { + AirWithBuses::new( + num_columns, + AuxiliaryTraceBuildData { interactions }, + options, + 1, + constraint_set, + ) + .with_name(name) +} + +#[allow(clippy::too_many_arguments)] +fn build_air + 'static>( + num_columns: usize, + interactions: Vec, + options: &ProofOptions, + constraint_set: CS, + name: &'static str, + root: Commitment, + num_prep: usize, +) -> LfmAir { + AirWithBuses::new( + num_columns, + AuxiliaryTraceBuildData { interactions }, + options, + 1, + constraint_set, + ) + .with_name(name) + .with_preprocessed(root, num_prep) +} + +impl LfmAirs { + /// Builds the chip set against the supplied (registry-resolved or + /// freshly built) instruction-column-group roots, in the frozen order, + /// with `KECCAK_RND` instantiated `keccak_rnd_chunks` times. + /// + /// A zero chunk count builds no `KECCAK_RND` at all; callers on the verify + /// path must reject that shape before getting here rather than relying on + /// the resulting AIR-count mismatch (`verify_against` does). + pub fn new( + roots: &[Commitment; NUM_LFM_CHIPS], + options: &ProofOptions, + keccak_rnd_chunks: usize, + ) -> Self { + Self::new_with_hasher(roots, options, keccak_rnd_chunks, HasherKind::default()) + } + + /// [`LfmAirs::new`] with the `LFM_HASH` permutation chosen explicitly. + /// + /// The hasher is a construction-time property of the AIR set because the + /// chip bakes its round constants into its constraints: the same `hasher` + /// must reach execution and trace generation, which is what + /// `proof::lfm_prove_with_hasher` guarantees. Nothing else in the set moves + /// — the preprocessed prefix is the instruction column group, which no + /// hasher changes, so the preprocessed roots and the program digest are + /// hasher-independent. + pub fn new_with_hasher( + roots: &[Commitment; NUM_LFM_CHIPS], + options: &ProofOptions, + keccak_rnd_chunks: usize, + hasher: HasherKind, + ) -> Self { + LfmAirs { + const_: build_air( + const_::cols::NUM_COLUMNS, + const_::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[0], + roots[0], + layout::const_::PREP_WIDTH, + ), + balu: build_air( + balu::cols::NUM_COLUMNS, + balu::bus_interactions(), + options, + balu::BaluConstraints, + LFM_CHIP_NAMES[1], + roots[1], + layout::balu::PREP_WIDTH, + ), + xalu: build_air( + xalu::cols::NUM_COLUMNS, + xalu::bus_interactions(), + options, + xalu::XaluConstraints, + LFM_CHIP_NAMES[2], + roots[2], + layout::xalu::PREP_WIDTH, + ), + select: build_air( + select::cols::NUM_COLUMNS, + select::bus_interactions(), + options, + select::SelectConstraints, + LFM_CHIP_NAMES[3], + roots[3], + layout::select::PREP_WIDTH, + ), + bitdec: build_air( + bitdec::cols::NUM_COLUMNS, + bitdec::bus_interactions(), + options, + bitdec::BitDecConstraints, + LFM_CHIP_NAMES[4], + roots[4], + layout::bitdec::PREP_WIDTH, + ), + hash: build_air( + hash::num_columns(hasher), + hash::bus_interactions(hasher), + options, + hash::HashConstraints { kind: hasher }, + LFM_CHIP_NAMES[5], + roots[5], + layout::hash::PREP_WIDTH, + ), + keccak: build_air( + keccak::cols::NUM_COLUMNS, + keccak::bus_interactions(), + options, + keccak::KeccakAdapterConstraints, + LFM_CHIP_NAMES[6], + roots[6], + layout::keccak::PREP_WIDTH, + ), + lanes: build_air( + lanes::cols::NUM_COLUMNS, + lanes::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[7], + roots[7], + layout::lanes::PREP_WIDTH, + ), + hint: build_air( + hint::cols::NUM_COLUMNS, + hint::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[8], + roots[8], + layout::hint::PREP_WIDTH, + ), + public: build_air( + public::cols::NUM_COLUMNS, + public::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[9], + roots[9], + layout::public::PREP_WIDTH, + ), + range: build_air( + range::cols::NUM_COLUMNS, + range::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[10], + roots[10], + layout::range::PREP_WIDTH, + ), + // KECCAK_RND has no preprocessed columns: `roots[KECCAK_RND_SLOT]` + // is the all-zero sentinel and is never consulted. Its correctness + // is entirely its own constraints plus bus balance, both + // program-independent, so there is nothing for a root to pin — + // and nothing that differs between chunks either, which is why + // every instance is built from the same arguments. + keccak_rnd: (0..keccak_rnd_chunks) + .map(|_| { + build_air_no_prep( + keccak_rnd::cols::NUM_COLUMNS, + keccak_rnd::bus_interactions(), + options, + keccak_rnd::KeccakRndConstraints, + LFM_CHIP_NAMES[11], + ) + }) + .collect(), + keccak_rc: build_air( + keccak_rc::cols::NUM_COLUMNS, + keccak_rc::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[12], + roots[12], + keccak_rc::NUM_PRECOMPUTED_COLS, + ), + bitwise: build_air( + bitwise::cols::NUM_COLUMNS, + bitwise::bus_interactions(), + options, + EmptyConstraints, + LFM_CHIP_NAMES[13], + roots[13], + bitwise::NUM_PRECOMPUTED_COLS, + ), + } + } + + /// Number of `KECCAK_RND` instances this set was built with. + pub fn keccak_rnd_chunks(&self) -> usize { + self.keccak_rnd.len() + } + + /// Verify-side projection, frozen order (must match `air_trace_pairs`). + pub fn air_refs(&self) -> Vec> { + let mut refs: Vec> = vec![ + &self.const_, + &self.balu, + &self.xalu, + &self.select, + &self.bitdec, + &self.hash, + &self.keccak, + &self.lanes, + &self.hint, + &self.public, + &self.range, + ]; + refs.extend(self.keccak_rnd.iter().map(|a| a as DynLfmAir<'_>)); + refs.push(&self.keccak_rc); + refs.push(&self.bitwise); + refs + } + + /// Prove-side projection, frozen order (must match `air_refs`). + /// + /// `traces.keccak_rnd` must have exactly one trace per chunk; a mismatch + /// would silently shorten the pair list under `zip`, so it is asserted. + #[allow(clippy::type_complexity)] + pub fn air_trace_pairs<'a>( + &'a self, + traces: &'a mut LfmTraces, + ) -> Vec<(DynLfmAir<'a>, &'a mut TraceTable, &'a ())> { + debug_assert_eq!( + self.keccak_rnd.len(), + traces.keccak_rnd.len(), + "KECCAK_RND chunk count differs between the AIR set and the traces \ + — artifacts and traces were built from different chunking policies" + ); + let mut pairs: Vec<(DynLfmAir<'a>, &'a mut TraceTable, &'a ())> = vec![ + (&self.const_, &mut traces.const_, &()), + (&self.balu, &mut traces.balu, &()), + (&self.xalu, &mut traces.xalu, &()), + (&self.select, &mut traces.select, &()), + (&self.bitdec, &mut traces.bitdec, &()), + (&self.hash, &mut traces.hash, &()), + (&self.keccak, &mut traces.keccak, &()), + (&self.lanes, &mut traces.lanes, &()), + (&self.hint, &mut traces.hint, &()), + (&self.public, &mut traces.public, &()), + (&self.range, &mut traces.range, &()), + ]; + pairs.extend( + self.keccak_rnd + .iter() + .zip(traces.keccak_rnd.iter_mut()) + .map(|(air, trace)| (air as DynLfmAir<'a>, trace, &())), + ); + pairs.push((&self.keccak_rc, &mut traces.keccak_rc, &())); + pairs.push((&self.bitwise, &mut traces.bitwise, &())); + pairs + } +} diff --git a/prover/src/lfm/blake3.rs b/prover/src/lfm/blake3.rs new file mode 100644 index 000000000..dbf1bab75 --- /dev/null +++ b/prover/src/lfm/blake3.rs @@ -0,0 +1,366 @@ +//! The BLAKE3 compression function the LFM chips are built from. +//! +//! **This module is a re-export.** The implementation lives at +//! [`crypto::hash::blake3`], which is the only crate the three callers that must +//! not disagree can all reach: the Merkle commitment backends (in `crypto`), the +//! `LFM_BLAKE3` chip and the `LFM_HASH` socket (here), and the CUDA kernels' +//! parity reference (in `math-cuda`, which `prover` depends on). A chip and a +//! commitment that hash identically because they call one function is a +//! different claim from two that agree today. +//! +//! Everything the chips used before is still reachable under this path and means +//! the same thing: [`blake3_compress_rounds`], [`blake3_compress_6round`], +//! [`BLAKE3_IV`], [`BLAKE3_MSG_PERMUTATION`], [`BLAKE3_ROUNDS`], +//! [`CANONICAL_VECTORS`] and [`canonical_expected_out`]. +//! +//! The round count travels with it: this crate's `blake3-6round` feature now +//! forwards to `crypto`'s, so [`BLAKE3_ROUNDS`] — and therefore +//! `blake3_socket::SOCKET_ROUNDS`, which is an alias of it — is one symbol for +//! the whole tree. Enabling `crypto/blake3-6round` alone would leave the chip at +//! 6 rounds and is caught by the `SOCKET_ROUNDS == BLAKE3_ROUNDS` assertion, +//! which is why that assertion stays. +//! +//! # Why the tests stayed here +//! +//! The falsification suite below — the negative controls that break one +//! convention at a time, and the `blake3` crate anchor — tests the primitive +//! *through this path*, which is the path the chips use. Keeping it here means +//! the re-export itself is covered: a shim that resolved to the wrong thing +//! would fail these, and moving them down would have made the chips' view of the +//! primitive untested. `crypto` has its own tests for the construction layer +//! ([`crypto::hash::blake3::chain`]) that this module does not use. + +pub use crypto::hash::blake3::*; + +#[cfg(test)] +mod tests { + use super::*; + + /// The conventions a wrong port could get wrong, as data. + /// + /// [`CANONICAL_VECTORS`] is supposed to pin every one of these. Naming them + /// in a struct is what lets the negative control break exactly one at a time. + #[derive(Clone, Copy)] + struct Conventions { + /// The four rotation amounts of `G`, in application order. + rot: [u32; 4], + /// The message-schedule permutation applied between rounds. + perm: [usize; 16], + rounds: usize, + } + + /// The conventions [`CANONICAL_VECTORS`] were generated under. `rounds` is + /// [`BLAKE3_SIX_ROUNDS`], not [`BLAKE3_ROUNDS`]: that table pins the 6-round + /// variant whatever the build is compiled for, and reading the knob here + /// would make the "7 rounds" control below silently stop discriminating at + /// the default. + const CANONICAL: Conventions = Conventions { + rot: [16, 12, 8, 7], + perm: BLAKE3_MSG_PERMUTATION, + rounds: BLAKE3_SIX_ROUNDS, + }; + + /// A deliberately *parameterised* compression, used only to build negative + /// controls: the same dataflow with [`Conventions`] as an input. + /// + /// It is NOT what [`blake3_compress_6round`] calls. Keeping the two apart + /// costs a duplicated loop and buys the thing rule 7 is about: the control + /// tests below compare this function's output against [`CANONICAL_VECTORS`] + /// — a constant that came from outside this file — so they stay meaningful + /// no matter how the real function is later refactored. + fn compress_variant(v: &Vector, c: Conventions) -> [u32; 16] { + let g = |s: &mut [u32; 16], a: usize, b: usize, cc: usize, d: usize, mx: u32, my: u32| { + s[a] = s[a].wrapping_add(s[b]).wrapping_add(mx); + s[d] = (s[d] ^ s[a]).rotate_right(c.rot[0]); + s[cc] = s[cc].wrapping_add(s[d]); + s[b] = (s[b] ^ s[cc]).rotate_right(c.rot[1]); + s[a] = s[a].wrapping_add(s[b]).wrapping_add(my); + s[d] = (s[d] ^ s[a]).rotate_right(c.rot[2]); + s[cc] = s[cc].wrapping_add(s[d]); + s[b] = (s[b] ^ s[cc]).rotate_right(c.rot[3]); + }; + let h = v.h; + let mut s: [u32; 16] = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + v.t as u32, + (v.t >> 32) as u32, + v.block_len, + v.flags, + ]; + let mut m = v.m; + for r in 0..c.rounds { + g(&mut s, 0, 4, 8, 12, m[0], m[1]); + g(&mut s, 1, 5, 9, 13, m[2], m[3]); + g(&mut s, 2, 6, 10, 14, m[4], m[5]); + g(&mut s, 3, 7, 11, 15, m[6], m[7]); + g(&mut s, 0, 5, 10, 15, m[8], m[9]); + g(&mut s, 1, 6, 11, 12, m[10], m[11]); + g(&mut s, 2, 7, 8, 13, m[12], m[13]); + g(&mut s, 3, 4, 9, 14, m[14], m[15]); + if r < c.rounds - 1 { + let prev = m; + for (i, &p) in c.perm.iter().enumerate() { + m[i] = prev[p]; + } + } + } + let mut out = [0u32; 16]; + for i in 0..8 { + out[i] = s[i] ^ s[i + 8]; + out[i + 8] = s[i + 8] ^ h[i]; + } + out + } + + /// The port reproduces all ten canonical vectors. + #[test] + fn the_compression_matches_the_canonical_six_round_vectors() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + assert_eq!( + blake3_compress_6round(&v.h, &v.m, v.t, v.block_len, v.flags), + v.out, + "canonical 6-round vector {i}" + ); + } + } + + /// The parameterised control, at canonical parameters, IS the port — so a + /// negative control below differs from the real thing in exactly the one + /// convention it names, and nothing else. + #[test] + fn the_variant_at_canonical_parameters_is_the_port() { + for v in CANONICAL_VECTORS.iter() { + assert_eq!( + compress_variant(v, CANONICAL), + v.out, + "the control must reproduce the vectors at canonical parameters" + ); + } + } + + /// NEGATIVE CONTROL (rule 9): each convention the vectors are supposed to + /// pin, broken one at a time, must stop reproducing them. + /// + /// Without this, "the vectors pass" would be evidence only that the vectors + /// are *reachable*, not that they discriminate. Each case names what would + /// silently be unpinned if it ever started passing. + #[test] + fn breaking_one_convention_at_a_time_breaks_the_vectors() { + // The message permutation transposed (its own inverse composition): + // same multiset of indices, same round count, different schedule. + let mut transposed = [0usize; 16]; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + transposed[p] = i; + } + let cases: [(&str, Conventions); 4] = [ + // rotr12 -> rotr13: the one rotation amount that is NOT a byte + // relabel in the chip, so a wrong value here is the wrong-rotation + // bug in its most consequential place. + ( + "rotr12 -> rotr13", + Conventions { + rot: [16, 13, 8, 7], + ..CANONICAL + }, + ), + // rotr16 and rotr8 swapped: both ARE free byte relabels in the + // chip, so transposing them costs no columns and no constraints — + // the cheapest possible way to be wrong. + ( + "rotr16 <-> rotr8", + Conventions { + rot: [8, 12, 16, 7], + ..CANONICAL + }, + ), + ( + "message schedule transposed", + Conventions { + perm: transposed, + ..CANONICAL + }, + ), + ( + "7 rounds (standard BLAKE3)", + Conventions { + rounds: 7, + ..CANONICAL + }, + ), + ]; + for (what, c) in cases { + let v = &CANONICAL_VECTORS[0]; + assert_ne!( + compress_variant(v, c), + v.out, + "{what} still reproduces the canonical vector — the vector does not pin it" + ); + } + } + + /// The port reproduces all ten canonical inputs at **7 rounds** too. + /// + /// [`CANONICAL_OUT_7ROUND`] came from two independently-written Python + /// references that agree on all ten, and whose 7-round paths are pinned by + /// the official BLAKE3 vectors. So this is the same shape of check as the + /// 6-round one above but with a stronger source, and together they are what + /// let `BLAKE3_ROUNDS` be flipped without the chip losing its vector pin. + #[test] + fn the_compression_matches_the_canonical_vectors_at_seven_rounds() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + assert_eq!( + blake3_compress_rounds( + &v.h, + &v.m, + v.t, + v.block_len, + v.flags, + BLAKE3_STANDARD_ROUNDS + ), + CANONICAL_OUT_7ROUND[i], + "7-round canonical vector {i}" + ); + } + } + + /// NEGATIVE CONTROL: the two tables really are different data. Without this, + /// a generation bug that emitted the 6-round outputs twice would leave the + /// test above passing and pinning nothing new. + #[test] + fn the_six_and_seven_round_vector_tables_differ_everywhere() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + assert_ne!(v.out, CANONICAL_OUT_7ROUND[i], "vector {i}"); + } + } + + /// `canonical_expected_out` selects the table matching the compiled knob. + /// This is the accessor `blake3_probe` asserts the chip's `OUT` columns + /// against, so a wrong branch here would silently unpin the chip. + #[test] + fn canonical_expected_out_follows_the_round_knob() { + for (i, v) in CANONICAL_VECTORS.iter().enumerate() { + let want = if BLAKE3_ROUNDS == BLAKE3_STANDARD_ROUNDS { + CANONICAL_OUT_7ROUND[i] + } else { + v.out + }; + assert_eq!(canonical_expected_out(i), want, "vector {i}"); + assert_eq!( + canonical_expected_out(i), + blake3_compress_rounds(&v.h, &v.m, v.t, v.block_len, v.flags, BLAKE3_ROUNDS), + "the accessor must agree with the primitive at the compiled round count" + ); + } + } + + /// ★ **The external anchor, direct.** At 7 rounds this module's compression + /// function IS standard BLAKE3, checked against the `blake3` crate with no + /// oracle, no JSON and no transcription in between. + /// + /// PLAN §2.2 step 4 asked for exactly this and Phase 1 deferred it for want + /// of a cargo dependency; this is that check, discharged. A message of at + /// most 64 bytes is one chunk and one block, so the whole tree hasher + /// collapses to a single `f` invocation: `h = IV`, the block zero-padded to + /// 64 bytes and read as 16 little-endian words, `t = 0`, `block_len` the + /// true length, `flags = CHUNK_START|CHUNK_END|ROOT`. The 32-byte digest is + /// `out[0..8]` in little-endian order. + /// + /// It runs over 65 lengths (0..=64) rather than one, because the length is + /// what `block_len` and the padding both key off, and a port that ignored + /// `block_len` would still pass at a single length. + #[test] + fn seven_rounds_is_the_blake3_crate() { + const CHUNK_START: u32 = 1; + const CHUNK_END: u32 = 2; + const ROOT: u32 = 8; + + for len in 0..=64usize { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect(); + let mut block = [0u8; 64]; + block[..len].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + + let out = blake3_compress_rounds( + &BLAKE3_IV, + &words, + 0, + len as u32, + CHUNK_START | CHUNK_END | ROOT, + BLAKE3_STANDARD_ROUNDS, + ); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + + assert_eq!( + ours, + *blake3::hash(&msg).as_bytes(), + "7-round compression must equal the blake3 crate at length {len}" + ); + } + } + + /// NEGATIVE CONTROL for the anchor above: at 6 rounds it must NOT match. + /// + /// Without this, `seven_rounds_is_the_blake3_crate` would pass just as well + /// if `rounds` were being ignored — which is the one bug that would make the + /// whole external-anchor argument vacuous, since the 6-round variant's only + /// defence is "the same code path with the loop bound changed". + #[test] + fn six_rounds_is_not_the_blake3_crate() { + let msg: [u8; 36] = core::array::from_fn(|i| i as u8); + let mut block = [0u8; 64]; + block[..36].copy_from_slice(&msg); + let words: [u32; 16] = core::array::from_fn(|i| { + u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap()) + }); + // BLAKE3_SIX_ROUNDS, not BLAKE3_ROUNDS: the knob defaults to 7, and + // reading it here would turn this control into a copy of the anchor. + let out = blake3_compress_rounds(&BLAKE3_IV, &words, 0, 36, 1 | 2 | 8, BLAKE3_SIX_ROUNDS); + let mut ours = [0u8; 32]; + for i in 0..8 { + ours[4 * i..4 * i + 4].copy_from_slice(&out[i].to_le_bytes()); + } + assert_ne!(ours, *blake3::hash(&msg).as_bytes()); + } + + /// The counter split is load-bearing and full-width: `t` reaches the state + /// as two 32-bit halves in low-then-high order, so swapping them must move + /// the output. Six of the ten canonical vectors have distinct halves. + #[test] + fn the_counter_halves_are_not_interchangeable() { + let mut checked = 0; + for v in CANONICAL_VECTORS.iter() { + let swapped = v.t.rotate_left(32); + if swapped == v.t { + continue; + } + checked += 1; + assert_ne!( + blake3_compress_6round(&v.h, &v.m, swapped, v.block_len, v.flags), + v.out, + "swapping the counter halves must change the output" + ); + } + assert!( + checked >= 8, + "expected most vectors to have distinct halves, got {checked}" + ); + } +} diff --git a/prover/src/lfm/blake3_chip.rs b/prover/src/lfm/blake3_chip.rs new file mode 100644 index 000000000..82a3ccd52 --- /dev/null +++ b/prover/src/lfm/blake3_chip.rs @@ -0,0 +1,1164 @@ +//! `LFM_BLAKE3` — the BLAKE3 compression chip, hosted on the LFM bus. +//! +//! Ported from PR #903's `prover/src/tables/blake3.rs` (`yetanotherco/lambda_vm`, +//! head `89aeeb8c2b0389e9d21a861c9e3a10a7b1b5704e`), which is the syscall +//! variant: it takes its inputs and returns its outputs through the VM's memory, +//! so its I/O side is an `Ecall` receiver, an x10 register read and 22 `Memw` +//! dword ops over a 176-byte state region. **The mixing core is unchanged.** +//! What this module replaces is the I/O side, with `LfmMem` word tokens in the +//! discipline `chips::keccak` (`LFM_KECCAK`) established: addresses and +//! multiplicities are preprocessed program data, and a machine word carries +//! four `u32` lanes. +//! +//! # What the swap costs and buys, send for send +//! +//! | | #903 (syscall) | here (LFM) | +//! |---|---|---| +//! | `Ecall` receiver | 1 | — | +//! | `Memw` x10 register read | 1 | — | +//! | `Memw` per state dword | 22 | — | +//! | `LfmMem` word tokens | — | 7 reads + 4 writes = 11 | +//! | `ByteAlu[XOR]` mixing + feed-forward | 832 | 832 | +//! | `AreBytes` shift halfwords | 384 | 384 | +//! | `AreBytes` message bytes | 32 | 32 | +//! | `AreBytes` OLD_OUT bytes | 32 | — | +//! | `AreBytes` addr bytes + alignment `AND` | 5 | — | +//! | `IsHalfword` pointer halfwords | 88 | — | +//! | **total interactions** | **1,397** | **1,259** | +//! | value columns | 3,219 | 3,056 | +//! +//! The dropped columns are `TIMESTAMP` (2), `ADDR` (8), `PTR` (88) and +//! `OLD_OUT` (64) — 162 — and `MU` moves into the preprocessed prefix, which +//! the census excludes, for 163 in total. +//! +//! # Why dropping those range checks is sound, not just cheaper +//! +//! Each dropped lookup guarded something that no longer exists: +//! +//! - **`OLD_OUT`'s 32 `AreBytes`.** #903 needs them because the previous memory +//! content of the out region appears only in the `Memw` write ops' `old` +//! field — never XOR-consumed, so its packed linear combinations could alias. +//! An `LfmMem` write carries no `old` field; there are no such columns here. +//! - **The address bytes, the alignment `AND` and the 88 pointer `IsHalfword`s.** +//! #903's state address is prover witness read out of x10 and must be +//! range-checked and shown 8-aligned before 22 pointers are derived from it. +//! Here every address is a *preprocessed* column supplied by the program and +//! vouched by the admission validator, exactly as for every other LFM chip — +//! a prover cannot choose it at all. +//! +//! What is NOT dropped is the byte-range coverage of the data columns, and it +//! carries over intact: +//! +//! - all 64 `m` bytes keep their explicit `AreBytes` (they are never XORed); +//! - `h`'s 32 bytes are XOR operands of the feed-forward (`out[i+8] = v[i+8] ^ h[i]`); +//! - `t_lo`, `t_hi`, `block_len`, `flags` are `v[12..16]`, each the `vd` operand +//! of a round-0 `G`, hence an operand of that `G`'s first XOR; +//! - all 64 `OUT` bytes are *results* of feed-forward XOR lookups. +//! +//! So every byte column reaching an `LfmMem` token is range-checked before the +//! token recomposes it, and a `u32` lane — four values below 2^8 with +//! coefficients 1, 2^8, 2^16, 2^24 — cannot reach 2^32. This is the same +//! transitive argument `chips::keccak` records for its 400 state bytes. +//! +//! # The single-dataflow rule, inherited +//! +//! The compression dataflow is written ONCE, in [`run_flow`], and interpreted +//! twice: [`WireFlow`] (columns — drives constraints and senders) and +//! [`ValueFlow`] (u32 witness — drives the trace and the BITWISE multiplicities). +//! The two cannot diverge on wiring, only on interpretation, which the probe's +//! bus-balance gate checks. That property is #903's and is worth preserving on +//! sight: it is why the sender list and the witness cannot drift apart. +//! +//! # Status +//! +//! This chip is **not registered** in the LFM fixed AIR set (`airs.rs` still +//! names 14 chips). It exists to be proved standalone by `blake3_probe` and +//! measured, which is what the hash matrix's blake column needs. Registration +//! would move every program digest and is a separate decision. +//! +//! ⚠ Round count follows [`super::blake3::BLAKE3_ROUNDS`]: 7 (standard BLAKE3) +//! by default, 6 under the `blake3-6round` feature. The 6-round instantiation +//! rests on the unratified security assumption **A6R**; the 7-round one carries +//! no assumption. Every column count in the table above is the 6-round one and +//! is quoted for continuity with #903 — `blake3_probe` pins both. + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use crate::constraints::templates::{INV_SHIFT_32, emit_is_bit}; +use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; +use crate::tables::types::{ + BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op, zeroed_fe_vec, +}; + +use super::blake3::{BLAKE3_IV, BLAKE3_MSG_PERMUTATION, BLAKE3_ROUNDS, blake3_compress_rounds}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// G-instances per compression: 8 per round, at the compiled round count. +pub const NUM_G: usize = BLAKE3_ROUNDS * 8; + +/// `u32` words the chip reads: `h[8] | m[16] | t_lo | t_hi | block_len | flags`. +pub const IN_U32: usize = 28; +/// `u32` words the chip writes: the full 16-word compression output. +pub const OUT_U32: usize = 16; +/// Machine words read (four `u32` lanes each). 28 / 4 divides exactly. +pub const IN_WORDS: usize = IN_U32 / 4; // 7 +/// Machine words written. 16 / 4 divides exactly. +pub const OUT_WORDS: usize = OUT_U32 / 4; // 4 + +/// The (a, b, c, d) state indices of the 8 G-calls of one round: +/// 4 column mixes then 4 diagonal mixes (BLAKE3 spec §2.1). +const G_INDICES: [(usize, usize, usize, usize); 8] = [ + (0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14), +]; + +/// Shift amounts of the two non-free rotations, as `rotl` inner shifts: +/// rotr12 = rotl20 = rotl16∘rotl4 (r=4); rotr7 = rotl25 = rotl16∘rotl9 (r=9). +pub(crate) const ROT_SHIFT_R: [u32; 2] = [4, 9]; + +// ========================================================================= +// Column layout +// ========================================================================= + +/// The chip's columns: a preprocessed instruction group, then value columns. +/// +/// The prefix mirrors `layout::keccak`'s discipline (addresses, per-output-word +/// read multiplicities, an is-real flag) and lives here rather than in +/// `layout.rs` because the chip is not registered in the machine — nothing else +/// shares these constants yet. +pub mod cols { + use super::{IN_WORDS, NUM_G, OUT_U32, OUT_WORDS}; + + // --- preprocessed (instruction column group) --- + /// Addresses of the 7 input machine words. + pub const IN_ADDR0: usize = 0; + /// Addresses of the 4 output machine words. + pub const OUT_ADDR0: usize = IN_ADDR0 + IN_WORDS; // 7 + /// Read count of each output word (its LogUp send multiplicity). + pub const MULT0: usize = OUT_ADDR0 + OUT_WORDS; // 11 + /// Is-real flag: gates every constraint and every read. + pub const MU: usize = MULT0 + OUT_WORDS; // 15 + pub const PREP_WIDTH: usize = MU + 1; // 16 + + // --- value columns --- + /// Input bytes: `h[32] | m[64] | t_lo[4] | t_hi[4] | block_len[4] | flags[4]`. + pub const IN: usize = PREP_WIDTH; // 16 + /// `NUM_G` G-blocks × 60 cells (56 bytes + 4 carry bits). + pub const G: usize = IN + 4 * super::IN_U32; // 128 + pub const G_SIZE: usize = 60; + /// Feed-forward output bytes `out[0..16]` (64 bytes). + pub const OUT: usize = G + NUM_G * G_SIZE; // 3008 + + pub const NUM_COLUMNS: usize = OUT + 4 * OUT_U32; // 3072 + + #[inline] + pub const fn in_addr(word: usize) -> usize { + IN_ADDR0 + word + } + #[inline] + pub const fn out_addr(word: usize) -> usize { + OUT_ADDR0 + word + } + #[inline] + pub const fn mult(word: usize) -> usize { + MULT0 + word + } + + /// Input word `i` (0..28: `h[0..8]`, `m[8..24]`, `t_lo=24`, `t_hi=25`, + /// `block_len=26`, `flags=27`), byte `b`. + #[inline] + pub const fn in_word(i: usize, b: usize) -> usize { + IN + i * 4 + b + } + + /// Feed-forward output word `i` (0..16), byte `b`. + #[inline] + pub const fn out_word(i: usize, b: usize) -> usize { + OUT + i * 4 + b + } + + /// Base column of G-block `g`. + #[inline] + pub const fn g_base(g: usize) -> usize { + G + g * G_SIZE + } + + // Offsets inside one G block (56 byte cells + 4 carry bits = 60): + /// add3 #1 output word (4 bytes). + pub const G_A1: usize = 0; + /// add3 #1 carry bits c1, c2. + pub const G_A1_C: usize = 4; + /// X1 = vd ^ A1 (4 bytes). + pub const G_X1: usize = 6; + /// add2 #1 output word (4 bytes). + pub const G_C1: usize = 10; + /// X2 = vb ^ C1 (4 bytes). + pub const G_X2: usize = 14; + /// rotr12 block: SLL_lo(2) SLLC_lo(2) SLL_hi(2) SLLC_hi(2) Y(4). + pub const G_R1: usize = 18; + /// add3 #2 output word (4 bytes). + pub const G_A2: usize = 30; + /// add3 #2 carry bits. + pub const G_A2_C: usize = 34; + /// X3 = vd ^ A2 (4 bytes). + pub const G_X3: usize = 36; + /// add2 #2 output word (4 bytes). + pub const G_C2: usize = 40; + /// X4 = B1 ^ C2 (4 bytes). + pub const G_X4: usize = 44; + /// rotr7 block: same layout as `G_R1`. + pub const G_R2: usize = 48; +} + +/// Value columns the census counts: everything past the preprocessed prefix. +pub const MAIN_COLUMNS: usize = cols::NUM_COLUMNS - cols::PREP_WIDTH; + +// ========================================================================= +// The single dataflow, interpreted twice (verbatim from #903) +// ========================================================================= + +/// The framing degrees of freedom [`run_flow`] itself decides, as opposed to +/// the ones an interpretation decides. +/// +/// These two live here, and not in each `Blake3Flow` impl, for one reason: they +/// change *which calls happen*, so an impl that got them wrong would silently +/// desynchronise the wire interpretation from the value interpretation and the +/// bus sends would stop matching the multiplicities. Deciding them in the single +/// dataflow is what keeps the single-dataflow rule true when there is more than +/// one framing (`blake3_chip`'s syscall shape and `blake3_socket`'s 2-to-1 +/// compress). +#[derive(Clone, Copy, Debug)] +pub(crate) struct FlowConfig { + /// Rounds of 8 G-calls. 6 for this chip; [`super::blake3_socket`] sweeps. + pub rounds: usize, + /// How many of the eight `out[i] = v[i] ^ v[i+8]` words to produce — the + /// truncation window. 8 here; 4 for the socket, whose digest is one cell. + pub out_window: usize, + /// Whether to produce `out[i+8] = v[i+8] ^ h[i]` as well. The socket does + /// not: those words are not part of a truncated 128-bit digest, and never + /// building them is where most of its saving over this chip comes from. + pub full_output: bool, +} + +impl FlowConfig { + /// The syscall-shaped chip's framing: the full 16-word output. + pub(crate) const fn full(rounds: usize) -> Self { + Self { + rounds, + out_window: 8, + full_output: true, + } + } +} + +/// The BLAKE3 compression dataflow, abstracted over its word representation. +pub(crate) trait Blake3Flow { + type Word: Copy; + + /// `h[i]` input word. + fn input_h(&mut self, i: usize) -> Self::Word; + /// `v[12..16]` init words: t_lo, t_hi, block_len, flags. + fn input_v12(&mut self, j: usize) -> Self::Word; + /// `IV[i]` constant (`v[8..12]`). + fn iv_const(&mut self, i: usize) -> Self::Word; + + /// 3-operand add `s = a + b + m[m_idx] mod 2^32` (half 0/1 = which add3 of G g). + fn add3( + &mut self, + g: usize, + half: usize, + a: Self::Word, + b: Self::Word, + m_idx: usize, + ) -> Self::Word; + /// 2-operand add `s = a + b mod 2^32`. + fn add2(&mut self, g: usize, half: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// XOR (slot 0..4 = X1..X4 of G g). Operand order is part of the wire format. + fn xor(&mut self, g: usize, slot: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// rotr16: free byte relabel `[b2,b3,b0,b1]`. + fn rotr16(&mut self, w: Self::Word) -> Self::Word; + /// rotr8: free byte relabel `[b1,b2,b3,b0]`. + fn rotr8(&mut self, w: Self::Word) -> Self::Word; + /// rotr12 (half=0) / rotr7 (half=1) via the inline shift identity. + fn rot_shift(&mut self, g: usize, half: usize, w: Self::Word) -> Self::Word; + /// Feed-forward, low half: `out[i] = v[i] ^ v[i+8]`. + fn feed_forward_low(&mut self, i: usize, vi: Self::Word, vi8: Self::Word); + /// Feed-forward, high half: `out[i+8] = v[i+8] ^ h[i]`. Called only under + /// [`FlowConfig::full_output`]. + fn feed_forward_high(&mut self, i: usize, vi8: Self::Word, hi: Self::Word); +} + +/// Drive the compression through `f`. The message schedule is tracked as +/// indices into the ORIGINAL m (permute^r composition), so both interpretations +/// reference original message words — never copies. +pub(crate) fn run_flow(f: &mut T, cfg: FlowConfig) { + let h: [T::Word; 8] = core::array::from_fn(|i| f.input_h(i)); + let mut v: [T::Word; 16] = core::array::from_fn(|i| { + if i < 8 { + h[i] + } else if i < 12 { + f.iv_const(i - 8) + } else { + f.input_v12(i - 12) + } + }); + + // sched[i] = index into the original m of the word consumed at position i + // this round. permute: m'[i] = m[P[i]] ⇒ sched'[i] = sched[P[i]]. + let mut sched: [usize; 16] = core::array::from_fn(|i| i); + + for r in 0..cfg.rounds { + for (j, &(ia, ib, ic, id)) in G_INDICES.iter().enumerate() { + let g = r * 8 + j; + let (va, vb, vc, vd) = (v[ia], v[ib], v[ic], v[id]); + let mx = sched[2 * j]; + let my = sched[2 * j + 1]; + + let a1 = f.add3(g, 0, va, vb, mx); + let x1 = f.xor(g, 0, vd, a1); + let vd1 = f.rotr16(x1); + let c1 = f.add2(g, 0, vc, vd1); + let x2 = f.xor(g, 1, vb, c1); + let b1 = f.rot_shift(g, 0, x2); // rotr12 + let a2 = f.add3(g, 1, a1, b1, my); + let x3 = f.xor(g, 2, vd1, a2); + let vd2 = f.rotr8(x3); + let c2 = f.add2(g, 1, c1, vd2); + let x4 = f.xor(g, 3, b1, c2); + let b2 = f.rot_shift(g, 1, x4); // rotr7 + + v[ia] = a2; + v[ib] = b2; + v[ic] = c2; + v[id] = vd2; + } + if r < cfg.rounds - 1 { + let prev = sched; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + sched[i] = prev[p]; + } + } + } + + for i in 0..cfg.out_window { + f.feed_forward_low(i, v[i], v[i + 8]); + if cfg.full_output { + f.feed_forward_high(i, v[i + 8], h[i]); + } + } +} + +// ========================================================================= +// Wire interpretation (columns) +// ========================================================================= + +/// A 32-bit word as wiring: four byte columns (LSB first), a constant, or a +/// constant selected by preprocessed mode columns. +/// Constants only ever appear as the IV `v[c]` operands of round-0 add2s. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WordRef { + Cols([usize; 4]), + Const(u32), + /// `Σ_k col_k · tag_k` — a constant selected by PREPROCESSED mode columns. + /// + /// The socket's domain tag, for when one tag is no longer enough. It is a + /// linear form over columns the prover cannot choose, so it is as unchosen + /// as the plain `Const` it replaces, and it costs the same: zero witness + /// columns, zero range checks, degree 1 where the constant was degree 0. + /// `add3`'s body is degree 1 in its operands either way, so the arm's max + /// degree does not move. + /// + /// It is a WHOLE-WORD form only — see [`WordRef::byte`]. + ModeSelected(&'static [(usize, u32)]), +} + +impl WordRef { + /// This word's byte `b`, for a byte-granular consumer (`ByteAlu[XOR]`). + /// + /// # Panics + /// + /// On [`WordRef::ModeSelected`]. A mode-selected word has no byte + /// decomposition without witnessing one, and the whole reason the tag lives + /// in a message word is that message words reach `add3` and nothing else. Panicking + /// says so out loud rather than letting a future byte consumer quietly + /// acquire four columns nobody committed. + pub(crate) fn byte(self, b: usize) -> ByteRef { + match self { + WordRef::Cols(c) => ByteRef::Col(c[b]), + WordRef::Const(w) => ByteRef::Const(((w >> (8 * b)) & 0xFF) as u8), + WordRef::ModeSelected(_) => unreachable!( + "a mode-selected word is a whole-word value: it reaches add3 and \ + nothing byte-granular, so it has no byte columns to name" + ), + } + } + + /// This word rotated right by `bytes` bytes — free wiring, no columns and + /// no constraint: `Cols` permutes its byte columns and `Const` rotates its + /// value. It is BLAKE3's `rotr16`/`rotr8`, whose shifts are byte-aligned. + /// + /// # Panics + /// + /// On [`WordRef::ModeSelected`], for the reason [`WordRef::byte`] gives: + /// the rotation is a byte permutation, and a mode-selected word has no + /// bytes to permute. + pub(crate) fn rotr_bytes(self, bytes: usize) -> WordRef { + match self { + WordRef::Cols(c) => WordRef::Cols(core::array::from_fn(|j| c[(j + bytes) % 4])), + WordRef::Const(v) => WordRef::Const(v.rotate_right(8 * bytes as u32)), + WordRef::ModeSelected(_) => unreachable!( + "a mode-selected word is a whole-word value: it has no byte \ + columns to rotate" + ), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ByteRef { + Col(usize), + Const(u8), +} + +/// One recorded 3-op add: operands (a, b, m), output columns, carries. +/// +/// `m` is a [`WordRef`] rather than four columns because the socket framing +/// makes every message word above the input lanes a compile-time constant — the +/// domain tag and the zero padding of a 52-byte message. Constant message words +/// cost no columns and no +/// range checks, which is the whole reason the tag is free there. +pub(crate) struct Add3Wire { + pub a: WordRef, + pub b: WordRef, + pub m: WordRef, + pub s: [usize; 4], + pub c1: usize, + pub c2: usize, +} + +/// One recorded 2-op add: operands, output columns (carry is an expression). +pub(crate) struct Add2Wire { + pub a: WordRef, + pub b: WordRef, + pub s: [usize; 4], +} + +/// One recorded XOR: per-byte operands and output columns. +pub(crate) struct XorWire { + pub a: WordRef, + pub b: WordRef, + pub out: [usize; 4], +} + +/// One recorded shift rotation: input word, the 8 shift-halfword byte columns +/// (SLL_lo, SLLC_lo, SLL_hi, SLLC_hi — 2 bytes each), output columns, r. +pub(crate) struct RotWire { + pub input: WordRef, + pub sll_lo: [usize; 2], + pub sllc_lo: [usize; 2], + pub sll_hi: [usize; 2], + pub sllc_hi: [usize; 2], + pub y: [usize; 4], + pub r: u32, +} + +/// The full wiring of one compression row, recorded in canonical order. +pub(crate) struct WireFlow { + pub add3s: Vec, + pub add2s: Vec, + pub xors: Vec, + pub rots: Vec, +} + +impl WireFlow { + pub(crate) fn build() -> Self { + let mut w = WireFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + 16), + rots: Vec::with_capacity(NUM_G * 2), + }; + run_flow(&mut w, FlowConfig::full(BLAKE3_ROUNDS)); + w + } +} + +#[inline] +pub(crate) fn word_cols(start: usize) -> [usize; 4] { + [start, start + 1, start + 2, start + 3] +} + +impl Blake3Flow for WireFlow { + type Word = WordRef; + + fn input_h(&mut self, i: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(i, 0))) + } + fn input_v12(&mut self, j: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(24 + j, 0))) + } + fn iv_const(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + fn add3(&mut self, g: usize, half: usize, a: WordRef, b: WordRef, m_idx: usize) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_A1 } else { cols::G_A2 }; + let cbase = cols::g_base(g) + + if half == 0 { + cols::G_A1_C + } else { + cols::G_A2_C + }; + let s = word_cols(base); + self.add3s.push(Add3Wire { + a, + b, + m: WordRef::Cols(word_cols(cols::in_word(8 + m_idx, 0))), + s, + c1: cbase, + c2: cbase + 1, + }); + WordRef::Cols(s) + } + + fn add2(&mut self, g: usize, half: usize, a: WordRef, b: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_C1 } else { cols::G_C2 }; + let s = word_cols(base); + self.add2s.push(Add2Wire { a, b, s }); + WordRef::Cols(s) + } + + fn xor(&mut self, g: usize, slot: usize, a: WordRef, b: WordRef) -> WordRef { + let off = match slot { + 0 => cols::G_X1, + 1 => cols::G_X2, + 2 => cols::G_X3, + _ => cols::G_X4, + }; + let out = word_cols(cols::g_base(g) + off); + self.xors.push(XorWire { a, b, out }); + WordRef::Cols(out) + } + + fn rotr16(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(2) + } + fn rotr8(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(1) + } + + fn rot_shift(&mut self, g: usize, half: usize, w: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_R1 } else { cols::G_R2 }; + let y = word_cols(base + 8); + self.rots.push(RotWire { + input: w, + sll_lo: [base, base + 1], + sllc_lo: [base + 2, base + 3], + sll_hi: [base + 4, base + 5], + sllc_hi: [base + 6, base + 7], + y, + r: ROT_SHIFT_R[half], + }); + WordRef::Cols(y) + } + + fn feed_forward_low(&mut self, i: usize, vi: WordRef, vi8: WordRef) { + self.xors.push(XorWire { + a: vi, + b: vi8, + out: word_cols(cols::out_word(i, 0)), + }); + } + + fn feed_forward_high(&mut self, i: usize, vi8: WordRef, hi: WordRef) { + self.xors.push(XorWire { + a: vi8, + b: hi, + out: word_cols(cols::out_word(i + 8, 0)), + }); + } +} + +// ========================================================================= +// Value interpretation (u32 witness) +// ========================================================================= + +/// Everything the trace filler and the BITWISE collector need for one +/// compression, recorded cell-exactly in the same canonical order as +/// [`WireFlow`]. +pub struct ValueFlow { + /// (s, c1, c2) per add3, canonical order. + pub add3s: Vec<(u32, u8, u8)>, + /// s per add2 (the carry is an expression, not a cell). + pub add2s: Vec, + /// (a, b, out) per XOR word, canonical order (Gs then feed-forward). + pub xors: Vec<(u32, u32, u32)>, + /// (sll_lo, sllc_lo, sll_hi, sllc_hi, y) per shift rotation. + pub rots: Vec<(u16, u16, u16, u16, u32)>, + /// The output words. Entries outside the framing's truncation window are + /// never computed and stay zero — reading one is a caller bug. + pub out: [u32; 16], + + h: [u32; 8], + m: [u32; 16], + v12: [u32; 4], +} + +impl ValueFlow { + /// The syscall-shaped chip's full 16-word compression. + pub fn compute(h: &[u32; 8], m: &[u32; 16], t: u64, block_len: u32, flags: u32) -> Self { + Self::compute_with(h, m, t, block_len, flags, FlowConfig::full(BLAKE3_ROUNDS)) + } + + /// [`ValueFlow::compute`] under an explicit framing. + pub(crate) fn compute_with( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, + cfg: FlowConfig, + ) -> Self { + let g = cfg.rounds * 8; + let mut f = ValueFlow { + add3s: Vec::with_capacity(g * 2), + add2s: Vec::with_capacity(g * 2), + xors: Vec::with_capacity(g * 4 + 16), + rots: Vec::with_capacity(g * 2), + out: [0; 16], + h: *h, + m: *m, + v12: [t as u32, (t >> 32) as u32, block_len, flags], + }; + run_flow(&mut f, cfg); + f + } +} + +impl Blake3Flow for ValueFlow { + type Word = u32; + + fn input_h(&mut self, i: usize) -> u32 { + self.h[i] + } + fn input_v12(&mut self, j: usize) -> u32 { + self.v12[j] + } + fn iv_const(&mut self, i: usize) -> u32 { + BLAKE3_IV[i] + } + + fn add3(&mut self, _g: usize, _half: usize, a: u32, b: u32, m_idx: usize) -> u32 { + let m = self.m[m_idx]; + let wide = a as u64 + b as u64 + m as u64; + let s = wide as u32; + let carry = (wide >> 32) as u8; // 0, 1 or 2 + // Two summed carry bits: c1 + c2 = carry. + let (c1, c2) = match carry { + 0 => (0, 0), + 1 => (1, 0), + _ => (1, 1), + }; + self.add3s.push((s, c1, c2)); + s + } + + fn add2(&mut self, _g: usize, _half: usize, a: u32, b: u32) -> u32 { + let s = a.wrapping_add(b); + self.add2s.push(s); + s + } + + fn xor(&mut self, _g: usize, _slot: usize, a: u32, b: u32) -> u32 { + let out = a ^ b; + self.xors.push((a, b, out)); + out + } + + fn rotr16(&mut self, w: u32) -> u32 { + w.rotate_right(16) + } + fn rotr8(&mut self, w: u32) -> u32 { + w.rotate_right(8) + } + + fn rot_shift(&mut self, _g: usize, half: usize, w: u32) -> u32 { + let r = ROT_SHIFT_R[half]; + let xlo = w & 0xFFFF; + let xhi = w >> 16; + // xlo·2^r = SLLC_lo·2^16 + SLL_lo (and same for hi): Euclidean split. + let sll_lo = ((xlo << r) & 0xFFFF) as u16; + let sllc_lo = ((xlo << r) >> 16) as u16; + let sll_hi = ((xhi << r) & 0xFFFF) as u16; + let sllc_hi = ((xhi << r) >> 16) as u16; + // Recombine + halfword swap: Ylo = SLL_hi + SLLC_lo, Yhi = SLL_lo + SLLC_hi. + let ylo = sll_hi as u32 + sllc_lo as u32; + let yhi = sll_lo as u32 + sllc_hi as u32; + let y = ylo | (yhi << 16); + debug_assert_eq!(y, w.rotate_right(if r == 4 { 12 } else { 7 })); + self.rots.push((sll_lo, sllc_lo, sll_hi, sllc_hi, y)); + y + } + + fn feed_forward_low(&mut self, i: usize, vi: u32, vi8: u32) { + let lo = vi ^ vi8; + self.xors.push((vi, vi8, lo)); + self.out[i] = lo; + } + + fn feed_forward_high(&mut self, i: usize, vi8: u32, hi: u32) { + let w = vi8 ^ hi; + self.xors.push((vi8, hi, w)); + self.out[i + 8] = w; + } +} + +// ========================================================================= +// Operation struct + trace generation +// ========================================================================= + +/// One compression, as the machine issues it. +/// +/// Addresses are program data, not witness: `in_addr`/`out_addr` land in the +/// preprocessed prefix. `read_counts` is the number of later reads of each +/// output word — the LogUp send multiplicity, which for a real machine comes +/// from the program's dataflow. +#[derive(Debug, Clone)] +pub struct Blake3Operation { + pub in_addr: [u64; IN_WORDS], + pub out_addr: [u64; OUT_WORDS], + pub read_counts: [u64; OUT_WORDS], + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, +} + +impl Blake3Operation { + /// The 28 input `u32` words in machine order: `h | m | t_lo | t_hi | len | flags`. + pub fn input_words(&self) -> [u32; IN_U32] { + let mut w = [0u32; IN_U32]; + w[0..8].copy_from_slice(&self.h); + w[8..24].copy_from_slice(&self.m); + w[24] = self.t as u32; + w[25] = (self.t >> 32) as u32; + w[26] = self.block_len; + w[27] = self.flags; + w + } + + /// The compression output. + pub fn output_words(&self) -> [u32; OUT_U32] { + blake3_compress_rounds( + &self.h, + &self.m, + self.t, + self.block_len, + self.flags, + BLAKE3_ROUNDS, + ) + } +} + +/// Write a 32-bit word as 4 byte cells at `col..col+4`. +#[inline] +fn set_word_bytes(table: &mut T, row: usize, col: usize, w: u32) { + for b in 0..4 { + table.set_u64(row, col + b, ((w >> (8 * b)) & 0xFF) as u64); + } +} + +/// One row per compression; padding rows are ALL ZERO. +/// +/// #903 needs a nonzero pad (`ptr[k] = 8k`) because its pointer columns carry an +/// ungated `addr + 8k` identity. Nothing here is ungated except `IS_BIT(MU)`, +/// which a zero row satisfies, so the pad is genuinely empty — and +/// `padding_rows_are_all_zero` in `blake3_probe` pins that rather than assuming +/// it. +pub fn generate_blake3_trace(ops: &[Blake3Operation]) -> TraceTable { + let num_rows = ops.len().next_power_of_two().max(4); + let mut trace = TraceTable::new_main( + zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + + for (row, op) in ops.iter().enumerate() { + for j in 0..IN_WORDS { + table.set_u64(row, cols::in_addr(j), op.in_addr[j]); + } + for j in 0..OUT_WORDS { + table.set_u64(row, cols::out_addr(j), op.out_addr[j]); + table.set_u64(row, cols::mult(j), op.read_counts[j]); + } + table.set_fe(row, cols::MU, FE::one()); + + for (i, &w) in op.input_words().iter().enumerate() { + set_word_bytes(table, row, cols::in_word(i, 0), w); + } + + // The mixing core, cell-exactly in canonical order. + let flow = ValueFlow::compute(&op.h, &op.m, op.t, op.block_len, op.flags); + let mut a3 = flow.add3s.iter(); + let mut a2 = flow.add2s.iter(); + let mut xo = flow.xors.iter(); + let mut ro = flow.rots.iter(); + for g in 0..NUM_G { + let base = cols::g_base(g); + for half in 0..2 { + let (s_off, c_off, x_off, c2_off, x2_off, r_off) = if half == 0 { + ( + cols::G_A1, + cols::G_A1_C, + cols::G_X1, + cols::G_C1, + cols::G_X2, + cols::G_R1, + ) + } else { + ( + cols::G_A2, + cols::G_A2_C, + cols::G_X3, + cols::G_C2, + cols::G_X4, + cols::G_R2, + ) + }; + let &(s, c1, c2) = a3.next().expect("add3 count"); + set_word_bytes(table, row, base + s_off, s); + table.set_u64(row, base + c_off, c1 as u64); + table.set_u64(row, base + c_off + 1, c2 as u64); + + let &(_, _, x) = xo.next().expect("xor count"); + set_word_bytes(table, row, base + x_off, x); + + let &c = a2.next().expect("add2 count"); + set_word_bytes(table, row, base + c2_off, c); + + let &(_, _, x2) = xo.next().expect("xor count"); + set_word_bytes(table, row, base + x2_off, x2); + + let &(sll_lo, sllc_lo, sll_hi, sllc_hi, y) = ro.next().expect("rot count"); + table.set_u64(row, base + r_off, (sll_lo & 0xFF) as u64); + table.set_u64(row, base + r_off + 1, (sll_lo >> 8) as u64); + table.set_u64(row, base + r_off + 2, (sllc_lo & 0xFF) as u64); + table.set_u64(row, base + r_off + 3, (sllc_lo >> 8) as u64); + table.set_u64(row, base + r_off + 4, (sll_hi & 0xFF) as u64); + table.set_u64(row, base + r_off + 5, (sll_hi >> 8) as u64); + table.set_u64(row, base + r_off + 6, (sllc_hi & 0xFF) as u64); + table.set_u64(row, base + r_off + 7, (sllc_hi >> 8) as u64); + set_word_bytes(table, row, base + r_off + 8, y); + } + } + for i in 0..OUT_U32 { + set_word_bytes(table, row, cols::out_word(i, 0), flow.out[i]); + } + } + + trace +} + +// ========================================================================= +// Bus interactions +// ========================================================================= + +fn direct(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: Packing::Direct, + } +} + +/// `u32` word `index` of the byte family at `bytes_start`, recomposed from its +/// four byte columns as `Σ byte_k · 256^k`. +/// +/// The same trick `chips::keccak::half_value` uses: the machine-side `u32` +/// never gets its own column, so there is nothing extra to keep consistent, and +/// the bytes are already range-checked by the lookups that consume them. +fn lane_value(bytes_start: usize, index: usize) -> BusValue { + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: bytes_start + index * 4 + k, + }) + .collect(), + ) +} + +/// An `LfmMem` token `(addr, v0..v3)` for machine word `word` of a byte family. +fn word_token(addr_col: usize, bytes_start: usize, word: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| lane_value(bytes_start, 4 * word + l))); + v +} + +/// Order groups: the `LfmMem` reads and writes, then the mixing core's ByteAlu +/// XORs (canonical `WireFlow` order), the shift `AreBytes`, and the message +/// `AreBytes`. +pub fn bus_interactions() -> Vec { + let wires = WireFlow::build(); + let mut interactions = Vec::with_capacity(1_259); + + let byte_bus_value = |b: ByteRef| -> BusValue { + match b { + ByteRef::Col(c) => direct(c), + ByteRef::Const(v) => BusValue::constant(v as u64), + } + }; + + // 1. Reads: the 7 input machine words. + for j in 0..IN_WORDS { + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MU), + word_token(cols::in_addr(j), cols::IN, j), + )); + } + // 2. Writes: the 4 output machine words, each with its own read count. + for j in 0..OUT_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::mult(j)), + word_token(cols::out_addr(j), cols::OUT, j), + )); + } + + // 3. Mixing core + feed-forward: ByteAlu[XOR] per byte, canonical order. + for xw in &wires.xors { + for b in 0..4 { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MU), + vec![ + BusValue::constant(alu_op::XOR as u64), + byte_bus_value(xw.a.byte(b)), + byte_bus_value(xw.b.byte(b)), + direct(xw.out[b]), + ], + )); + } + } + + // 4. Shift-halfword AreBytes: 4 pairs per rotation. + for rw in &wires.rots { + for pair in [rw.sll_lo, rw.sllc_lo, rw.sll_hi, rw.sllc_hi] { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![direct(pair[0]), direct(pair[1])], + )); + } + } + + // 5. Message AreBytes: m is never XORed, so its 64 bytes get no transitive + // range check (#903 DESIGN §4.7/§7.5). 32 pairs. + for i in 0..16 { + for p in 0..2 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + direct(cols::in_word(8 + i, 2 * p)), + direct(cols::in_word(8 + i, 2 * p + 1)), + ], + )); + } + } + + interactions +} + +/// The BITWISE lookups `bus_interactions` sends, mirrored send for send. +/// +/// Forked from #903's `collect_bitwise_from_blake3` with the address-shaped +/// lookups (the alignment `AND`, 4 addr `AreBytes`, 88 pointer `IsHalf`) and +/// the 32 `OLD_OUT` `AreBytes` dropped — the columns they guarded do not exist +/// here. Enumeration order is the senders' own, via the shared `ValueFlow`. +pub fn bitwise_ops_for(ops: &[Blake3Operation]) -> Vec { + let mut out = Vec::with_capacity(ops.len() * 1_248); + + for op in ops { + let flow = ValueFlow::compute(&op.h, &op.m, op.t, op.block_len, op.flags); + for &(a, b, _out) in &flow.xors { + for byte in 0..4 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((a >> (8 * byte)) & 0xFF) as u8, + ((b >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + for &word in &op.m { + for p in 0..2 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((word >> (16 * p)) & 0xFF) as u8, + ((word >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + } + + out +} + +// ========================================================================= +// Constraints +// ========================================================================= + +/// Word expression from a [`WordRef`]: `b0 + 256·b1 + 2^16·b2 + 2^24·b3`. +pub(crate) fn word_expr>(b: &B, w: &WordRef) -> B::Expr { + match w { + WordRef::Cols(c) => { + b.main(0, c[0]) + + b.main(0, c[1]) * b.const_base(256) + + b.main(0, c[2]) * b.const_base(65536) + + b.main(0, c[3]) * b.const_base(16777216) + } + WordRef::Const(v) => b.const_base(*v as u64), + WordRef::ModeSelected(terms) => { + let mut iter = terms.iter(); + let term = |b: &B, (col, tag): (usize, u32)| b.main(0, col) * b.const_base(tag as u64); + let &first = iter.next().expect("a mode-selected word selects something"); + iter.fold(term(b, first), |acc, &t| acc + term(b, t)) + } + } +} + +/// Halfword expression from 2 byte columns: `b0 + 256·b1`. +pub(crate) fn half_expr>(b: &B, c: &[usize; 2]) -> B::Expr { + b.main(0, c[0]) + b.main(0, c[1]) * b.const_base(256) +} + +/// The hosted chip's 769 transition constraints: +/// - idx 0..288: 96 add3 groups (sum identity + 2 carry booleanities); +/// - idx 288..384: 96 add2 expression-carry booleanities; +/// - idx 384..768: 96 rotations (2 shift identities + 2 recombine each); +/// - idx 768: `IS_BIT(MU)`, ungated. +/// +/// #903's first 45 constraints — the 22 `ptr[k] = addr + 8k` carry pairs and +/// the top-dword no-overflow check — have no counterpart: addresses here are +/// preprocessed, so there is nothing to derive and nothing a prover chooses. +/// +/// All μ-gated, max degree 3 (the booleanities; identities are degree 2). +#[derive(Clone, Copy)] +pub struct Blake3LfmConstraints; + +impl ConstraintSet for Blake3LfmConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let wires = WireFlow::build(); + let mu = |b: &B| b.main(0, cols::MU); + let mut idx = 0usize; + + let two_32 = b.const_base(1u64 << 32); + let inv_2_32 = b.const_base(INV_SHIFT_32); + + // add3: μ·(a + b + m − s − 2^32·(c1+c2)) = 0; μ·ci·(1−ci) = 0. + for aw in &wires.add3s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let m_w = word_expr(b, &aw.m); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let c1 = b.main(0, aw.c1); + let c2 = b.main(0, aw.c2); + let sum_id = a + bb + m_w - s - (c1.clone() + c2.clone()) * two_32.clone(); + let m = mu(b); + b.emit_base(idx, m * sum_id); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c1.clone() * (one - c1)); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c2.clone() * (one - c2)); + idx += 1; + } + + // add2: carry = (a + b − s)·2^−32; μ·carry·(1−carry) = 0. + for aw in &wires.add2s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let carry = (a + bb - s) * inv_2_32.clone(); + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * carry.clone() * (one - carry)); + idx += 1; + } + + // Rotations: 2 shift identities + 2 recombine identities each. + for rw in &wires.rots { + let (xlo, xhi) = match &rw.input { + WordRef::Cols(c) => (half_expr(b, &[c[0], c[1]]), half_expr(b, &[c[2], c[3]])), + WordRef::Const(_) | WordRef::ModeSelected(_) => { + unreachable!("shift inputs are always committed XOR outputs") + } + }; + let sll_lo = half_expr(b, &rw.sll_lo); + let sllc_lo = half_expr(b, &rw.sllc_lo); + let sll_hi = half_expr(b, &rw.sll_hi); + let sllc_hi = half_expr(b, &rw.sllc_hi); + let ylo = half_expr(b, &[rw.y[0], rw.y[1]]); + let yhi = half_expr(b, &[rw.y[2], rw.y[3]]); + let two_r = b.const_base(1u64 << rw.r); + let two_16 = b.const_base(65536); + + // μ·(xlo·2^r − SLLC_lo·2^16 − SLL_lo) = 0 (and hi). + let m = mu(b); + b.emit_base( + idx, + m * (xlo * two_r.clone() - sllc_lo.clone() * two_16.clone() - sll_lo.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base( + idx, + m * (xhi * two_r - sllc_hi.clone() * two_16 - sll_hi.clone()), + ); + idx += 1; + // μ·(Ylo − SLL_hi − SLLC_lo) = 0; μ·(Yhi − SLL_lo − SLLC_hi) = 0. + let m = mu(b); + b.emit_base(idx, m * (ylo - sll_hi - sllc_lo)); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (yhi - sll_lo - sllc_hi)); + idx += 1; + } + + // Ungated booleanity of the is-real flag. Preprocessed, so the + // registrar already vouches for it; kept because `chips::keccak` keeps + // its mode-sum booleanity for the same belt-over-suspenders reason. + emit_is_bit(b, idx, cols::MU, None); + } +} + +/// Constraints the chip emits — the number the degree/count tests pin. +pub const NUM_CONSTRAINTS: usize = 3 * (NUM_G * 2) + (NUM_G * 2) + 4 * (NUM_G * 2) + 1; diff --git a/prover/src/lfm/blake3_probe.rs b/prover/src/lfm/blake3_probe.rs new file mode 100644 index 000000000..d0af57f59 --- /dev/null +++ b/prover/src/lfm/blake3_probe.rs @@ -0,0 +1,1004 @@ +//! Prove + verify the LFM-hosted BLAKE3 compression chip standalone. +//! +//! The `keccak_probe` pattern, one hash later: [`super::blake3_chip`] carries +//! the chip's real bus interactions and its real constraints, and this module +//! closes both of its buses — `BusId::ByteAlu` / `BusId::AreBytes` against the +//! UNCHANGED production `BITWISE` table, and `BusId::LfmMem` against a mirror +//! AIR standing in for the machine's memory. The preprocessed prefix is +//! committed for real, so the addresses and multiplicities the chip reads are +//! program data here exactly as they would be in the machine. +//! +//! Standing-decisions rule 2 is why this exists: an execute-only test would +//! prove nothing about the chip, because [`super::blake3::blake3_compress_6round`] +//! and the chip's `ValueFlow` would simply agree with each other. Only a +//! prove+verify makes the chip's constraints and interactions load +//! bearing, which is what turns the measured width into a *column*. +//! +//! # What this probe cannot see +//! +//! - **Whether the machine can drive the chip.** The mirror AIR is a synthetic +//! memory: it sends whatever the ops say the inputs are. Nothing here checks +//! that an LFM program can produce those words at those addresses, that the +//! admission validator would accept the address assignment, or that the +//! multiplicities match real read counts. Those are registrar obligations and +//! they are exactly what registering the chip would exercise. +//! - **The epoch verifier's blake bill.** This measures cells per compression. +//! The permutation count comes from wave 8's rate-parameterised closed form +//! and is inherited, not re-established here. +//! - **Anything cryptographic about the 6-round variant** (assumption A6R). + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::config::Commitment; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, EmptyConstraints, RootKind, num_base_from_meta, +}; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::prover::{IsStarkProver, Prover}; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::bitwise; +use crate::tables::types::{BusId, FE, FEE, GoldilocksExtension, GoldilocksField, VmTable}; +use crate::test_utils::create_bitwise_air; + +use super::blake3::{BLAKE3_ROUNDS, CANONICAL_VECTORS, canonical_expected_out}; +use super::blake3_chip::{ + self, Blake3LfmConstraints, Blake3Operation, IN_WORDS, MAIN_COLUMNS, NUM_CONSTRAINTS, NUM_G, + OUT_WORDS, cols, +}; +use super::commit::commit_columns; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type DynAir<'a> = &'a dyn AIR; +type ChipAir = AirWithBuses; +type MirrorAir = AirWithBuses; + +const PROBE_TAG: &[u8] = b"LFM_BLAKE3_PROBE_V1"; +/// Compressions in the probe. Three real rows in a height-4 table leaves one +/// padding row, which `padding_row_turned_real_rejects` needs. +const NUM_OPS: usize = 3; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("probe options") +} + +fn transcript() -> DefaultTranscript { + let mut t = DefaultTranscript::::new(&[]); + t.append_bytes(PROBE_TAG); + t +} + +// ========================================================================= +// The ops +// ========================================================================= + +/// Three compressions taken from the canonical 6-round vectors, at disjoint +/// addresses. +/// +/// Using the pinned vectors rather than fresh randomness means the trace's own +/// OUT columns are checkable against a constant that came from outside this +/// repository's Rust (see [`super::blake3`]'s provenance note). +fn probe_ops() -> Vec { + (0..NUM_OPS) + .map(|i| { + let v = &CANONICAL_VECTORS[i]; + let base = 1_000 + (i as u64) * 100; + Blake3Operation { + in_addr: core::array::from_fn(|j| base + j as u64), + out_addr: core::array::from_fn(|j| base + 50 + j as u64), + // Distinct nonzero read counts: a uniform 1 would not notice a + // multiplicity mixed up between output words. + read_counts: core::array::from_fn(|j| 1 + j as u64), + h: v.h, + m: v.m, + t: v.t, + block_len: v.block_len, + flags: v.flags, + } + }) + .collect() +} + +// ========================================================================= +// The AIRs +// ========================================================================= + +/// The preprocessed prefix, column-major and padded — what the program would +/// supply and what the chip's addresses and multiplicities are read from. +fn prep_columns(ops: &[Blake3Operation], num_rows: usize) -> Vec> { + let mut columns = vec![vec![FE::zero(); num_rows]; cols::PREP_WIDTH]; + for (row, op) in ops.iter().enumerate() { + for j in 0..IN_WORDS { + columns[cols::in_addr(j)][row] = FE::from(op.in_addr[j]); + } + for j in 0..OUT_WORDS { + columns[cols::out_addr(j)][row] = FE::from(op.out_addr[j]); + columns[cols::mult(j)][row] = FE::from(op.read_counts[j]); + } + columns[cols::MU][row] = FE::one(); + } + columns +} + +fn chip_air(prep_root: Commitment, opts: &ProofOptions) -> ChipAir { + AirWithBuses::new( + cols::NUM_COLUMNS, + AuxiliaryTraceBuildData { + interactions: blake3_chip::bus_interactions(), + }, + opts, + 1, + Blake3LfmConstraints, + ) + .with_name("LFM_BLAKE3") + .with_preprocessed(prep_root, cols::PREP_WIDTH) +} + +/// A synthetic `LfmMem` counterparty: `[ADDR, V0..V3, SEND_MULT, RECV_MULT]`. +/// +/// One row per word the chip touches. Input words are SENT here (the chip +/// receives them); output words are RECEIVED here `read_counts` times (the chip +/// sends them once with that multiplicity). Nothing constrains the values — the +/// mirror is memory, and in the machine the `LfmMem` multiset IS the semantics. +mod mirror { + pub const ADDR: usize = 0; + pub const V0: usize = 1; // ..V3 + pub const SEND_MULT: usize = 5; + pub const RECV_MULT: usize = 6; + pub const NUM_COLUMNS: usize = 7; +} + +fn mirror_token() -> Vec { + let mut v = vec![BusValue::Packed { + start_column: mirror::ADDR, + packing: Packing::Direct, + }]; + v.extend((0..4).map(|l| BusValue::Packed { + start_column: mirror::V0 + l, + packing: Packing::Direct, + })); + v +} + +fn mirror_air(opts: &ProofOptions) -> MirrorAir { + let interactions = vec![ + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(mirror::SEND_MULT), + mirror_token(), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(mirror::RECV_MULT), + mirror_token(), + ), + ]; + AirWithBuses::new( + mirror::NUM_COLUMNS, + AuxiliaryTraceBuildData { interactions }, + opts, + 1, + EmptyConstraints, + ) + .with_name("LFM_MEM_MIRROR") +} + +/// Four `u32` lanes of machine word `word` out of a flat `u32` array. +fn lanes(words: &[u32], word: usize) -> [u64; 4] { + core::array::from_fn(|l| words[4 * word + l] as u64) +} + +fn mirror_trace(ops: &[Blake3Operation]) -> TraceTable { + let rows = (ops.len() * (IN_WORDS + OUT_WORDS)) + .next_power_of_two() + .max(4); + let mut trace = TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(rows * mirror::NUM_COLUMNS), + mirror::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + let mut row = 0usize; + for op in ops { + let inputs = op.input_words(); + for j in 0..IN_WORDS { + table.set_u64(row, mirror::ADDR, op.in_addr[j]); + for (l, v) in lanes(&inputs, j).into_iter().enumerate() { + table.set_u64(row, mirror::V0 + l, v); + } + table.set_fe(row, mirror::SEND_MULT, FE::one()); + row += 1; + } + let outputs = op.output_words(); + for j in 0..OUT_WORDS { + table.set_u64(row, mirror::ADDR, op.out_addr[j]); + for (l, v) in lanes(&outputs, j).into_iter().enumerate() { + table.set_u64(row, mirror::V0 + l, v); + } + table.set_u64(row, mirror::RECV_MULT, op.read_counts[j]); + row += 1; + } + } + trace +} + +fn bitwise_trace(ops: &[Blake3Operation]) -> TraceTable { + let mut hist = bitwise::BitwiseHistogram::new(); + hist.add_ops(&blake3_chip::bitwise_ops_for(ops)); + let mut bw = bitwise::generate_bitwise_trace(); + hist.fill_multiplicities(&mut bw); + bw +} + +/// The three traces, in AIR order: chip, mirror, BITWISE. +fn build_traces(ops: &[Blake3Operation]) -> [TraceTable; 3] { + [ + blake3_chip::generate_blake3_trace(ops), + mirror_trace(ops), + bitwise_trace(ops), + ] +} + +fn prove_traces( + opts: &ProofOptions, + chip: &ChipAir, + traces: &mut [TraceTable; 3], +) -> Result, stark::prover::ProvingError> { + let mirror = mirror_air(opts); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + let [t0, t1, t2] = traces; + let pairs: Vec<(DynAir, &mut TraceTable, &())> = + vec![(chip, t0, &()), (&mirror, t1, &()), (&bw_air, t2, &())]; + let mut t = transcript(); + Prover::multi_prove( + pairs, + &mut t, + #[cfg(feature = "disk-spill")] + Default::default(), + stark::residency_mode::ResidencyMode::Retain, + ) +} + +fn verify_proof( + opts: &ProofOptions, + chip: &ChipAir, + proof: &stark::proof::stark::MultiProof, +) -> bool { + let mirror = mirror_air(opts); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + let refs: Vec = vec![chip, &mirror, &bw_air]; + let mut vt = transcript(); + Verifier::multi_verify_views(&refs, MultiProofView::Owned(proof), &mut vt, &FEE::zero()) +} + +/// Prove + verify, optionally corrupting the chip trace in between. +/// +/// `Err` means the prover refused — which for this chip is the *expected* +/// outcome of most tampering, because unlike the keccak adapter it carries 769 +/// polynomial constraints that a wrong cell violates locally. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let ops = probe_ops(); + let num_rows = ops.len().next_power_of_two().max(4); + let root = commit_columns(&prep_columns(&ops, num_rows), &opts); + let chip = chip_air(root, &opts); + let mut traces = build_traces(&ops); + mutate(&mut traces[0]); + match prove_traces(&opts, &chip, &mut traces) { + Ok(proof) => Ok(verify_proof(&opts, &chip, &proof)), + Err(e) => Err(format!("{e:?}")), + } +} + +/// Assert a mutation does not end in an accepted proof. +/// +/// A refusal by the prover and a rejection by the verifier are both real +/// rejections and this chip produces both: a cell that violates one of its 769 +/// constraints is caught locally, while a cell that only breaks a bus reaches +/// the verifier. Each caller records which it observed in its own doc comment. +fn assert_not_accepted(what: &str, mutate: impl FnOnce(&mut TraceTable)) { + if let Ok(true) = round_trip(mutate) { + panic!("{what} must not produce an accepted proof, but the proof verified"); + } +} + +// ========================================================================= +// The measurement +// ========================================================================= + +/// The blake column's per-compression cell law, on our stack, at BOTH round +/// counts. +/// +/// `main + 3·aux` with `aux = ceil(interactions / 2)` is `airs.rs`'s census +/// formula — the same instrument that produced the keccak and Poseidon columns, +/// so the three are comparable by construction rather than by argument. +/// +/// The closed forms are written out as functions of the round count and the +/// literals for both are pinned, so the A6R price stays visible whichever way +/// the build is compiled; the built layout is then asserted to equal the +/// prediction at the compiled count. Two statements that can disagree. +#[test] +fn the_hosted_chip_cell_budget_at_both_round_counts() { + // 112 input bytes + `8·rounds` G-blocks of 60 + 64 feed-forward bytes. + const fn predicted_main(rounds: usize) -> usize { + 112 + 60 * (8 * rounds) + 64 + } + // 11 `LfmMem` tokens; `ByteAlu[XOR]` over `4·8·rounds` mixing words and 16 + // feed-forward words; `AreBytes` over `2·8·rounds` rotations; 32 message. + const fn predicted_interactions(rounds: usize) -> usize { + 11 + 4 * (4 * (8 * rounds) + 16) + 4 * (2 * (8 * rounds)) + 32 + } + const fn predicted_cells(rounds: usize) -> usize { + predicted_main(rounds) + 3 * predicted_interactions(rounds).div_ceil(2) + } + + // 6 rounds — the A6R variant. These four literals are #903's and were the + // measured figures before the round count became a knob. + assert_eq!(predicted_main(6), 3_056); + assert_eq!(predicted_interactions(6), 1_259); + assert_eq!(predicted_interactions(6).div_ceil(2), 630); + assert_eq!(predicted_cells(6), 4_946); + // Group by group at 6 rounds, so a layout change cannot move the total + // silently: 11 LfmMem + 832 ByteAlu + 384 shift AreBytes + 32 message. + assert_eq!(predicted_interactions(6), 11 + 832 + 384 + 32); + + // 7 rounds — standard BLAKE3, the default. PLAN §7 predicted exactly these + // on paper; this is the same arithmetic against the built layout. + assert_eq!(predicted_main(7), 3_536); + assert_eq!(predicted_interactions(7), 1_451); + assert_eq!(predicted_interactions(7).div_ceil(2), 726); + assert_eq!(predicted_cells(7), 5_714); + + // The built layout IS the prediction at the compiled round count. + let interactions = blake3_chip::bus_interactions().len(); + let aux = interactions.div_ceil(2); + assert_eq!(cols::PREP_WIDTH, 16, "preprocessed prefix"); + assert_eq!(cols::G - cols::IN, 112, "input bytes"); + assert_eq!(cols::OUT - cols::G, 60 * NUM_G, "G-blocks × 60 cells"); + assert_eq!( + cols::NUM_COLUMNS - cols::OUT, + 64, + "feed-forward output bytes" + ); + assert_eq!(MAIN_COLUMNS, predicted_main(BLAKE3_ROUNDS)); + assert_eq!(cols::NUM_COLUMNS, MAIN_COLUMNS + cols::PREP_WIDTH); + assert_eq!(IN_WORDS + OUT_WORDS, 11, "LfmMem tokens"); + assert_eq!(interactions, predicted_interactions(BLAKE3_ROUNDS)); + assert_eq!( + MAIN_COLUMNS + 3 * aux, + predicted_cells(BLAKE3_ROUNDS), + "base-field-equivalent cells" + ); + + // #903's syscall variant at 6 rounds, for the delta the hosting buys: 3,219 + // main and 1,397 interactions (699 aux) = 5,316. The difference is all I/O. + assert_eq!(3_219 + 3 * 1_397usize.div_ceil(2), 5_316); + + // ★ For the comparison this chip exists to support: the `LFM_HASH` BLAKE3 + // socket arm is cheaper at BOTH round counts — a constant initial state, a + // constant message tail, and twelve of the sixteen output words never built. + // + // Asserted against `blake3_socket_tests`' own census formula rather than + // against a transcription of its output. What stood here was "4,741 at 6 + // rounds and 5,509 at 7": those predated the leaf mode's canonicity block, + // nothing recomputed them, and they were wrong by 8 for as long as they + // stood — then wrong by 36 once the leaf RATE widened the socket. A cost + // figure no test derives is a comment, not a claim. + for rounds in [6, 7] { + let socket = super::blake3_socket_tests::predicted_cells(rounds); + assert!( + socket < predicted_cells(rounds), + "hosting must stay cheaper than the standalone chip at {rounds} \ + rounds: socket {socket}, standalone {}", + predicted_cells(rounds) + ); + } + assert_eq!(super::blake3_socket_tests::predicted_cells(6), 4_777); + assert_eq!(super::blake3_socket_tests::predicted_cells(7), 5_545); +} + +/// Every constraint index is emitted exactly once, and the count is the one the +/// module documents. #903 emits 814; the 45 address-derivation constraints have +/// no counterpart here. +#[test] +fn the_chip_emits_its_constraints_at_degree_3() { + // 16 per G-instance — two add3s (a sum identity and two carry booleanities + // each), two add2 carry booleanities, two rotations of four — plus the + // ungated `IS_BIT(MU)`. 769 at 6 rounds, 897 at 7; both written out. + assert_eq!(16 * (8 * 6) + 1, 769); + assert_eq!(16 * (8 * 7) + 1, 897); + assert_eq!(NUM_CONSTRAINTS, 16 * NUM_G + 1); + assert_eq!( + NUM_CONSTRAINTS, + 3 * (NUM_G * 2) + NUM_G * 2 + 4 * (NUM_G * 2) + 1 + ); + // #903's syscall variant emits 814 at 6 rounds; the 45 address-derivation + // constraints have no counterpart here. + if BLAKE3_ROUNDS == 6 { + assert_eq!(814 - 45, NUM_CONSTRAINTS, "vs #903's syscall variant"); + } + + let set = Blake3LfmConstraints; + let meta = ConstraintSet::::meta(&set); + assert_eq!(meta.len(), NUM_CONSTRAINTS, "constraints emitted"); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "meta must be dense and idx-ordered"); + assert_eq!(m.kind, RootKind::Base, "every blake constraint is base"); + } + + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), NUM_CONSTRAINTS, "one emit per constraint"); + + let declared = ConstraintSet::::max_degree(&set); + assert_eq!(declared, 3, "the wrap's blowup 2 depends on this staying 3"); + for &(idx, measured) in °rees { + assert!( + measured <= declared, + "constraint {idx}: measured degree {measured} EXCEEDS declared {declared}" + ); + } + // Not merely `<=`: the μ-gated carry booleanities really are cubic, so a + // set that quietly topped out at 2 would mean the carries had stopped being + // constrained. + assert_eq!( + degrees.iter().map(|&(_, d)| d).max(), + Some(3), + "some constraint must actually reach degree 3" + ); +} + +// ========================================================================= +// The round trip +// ========================================================================= + +#[test] +fn the_hosted_chip_proves_and_verifies() { + let ops = probe_ops(); + let traces = build_traces(&ops); + + // The chip's OUT columns are the real compression, byte for byte, against + // the canonical vectors. + for (row, op) in ops.iter().enumerate() { + let expected = canonical_expected_out(row); + assert_eq!( + expected, + blake3_chip::Blake3Operation::output_words(op), + "op {row}'s output must be the primitive's at the compiled round count" + ); + for (i, &word) in expected.iter().enumerate() { + for b in 0..4 { + assert_eq!( + traces[0].main_table.get_row(row)[cols::out_word(i, b)], + FE::from(u64::from((word >> (8 * b)) as u8)), + "OUT byte ({i}, {b}) of row {row}" + ); + } + } + } + + // The BITWISE feed is exactly the senders' count, with no address-shaped + // lookups: 1,248 per compression at 6 rounds and 1,440 at 7. Both literals + // are written out so the flip cannot quietly move the feed. + const fn predicted_bitwise(rounds: usize) -> usize { + 4 * (4 * (8 * rounds) + 16) + 4 * (2 * (8 * rounds)) + 32 + } + assert_eq!(predicted_bitwise(6), 1_248); + assert_eq!(predicted_bitwise(7), 1_440); + assert_eq!( + blake3_chip::bitwise_ops_for(&ops).len(), + ops.len() * predicted_bitwise(BLAKE3_ROUNDS), + "per-compression BITWISE lookup count" + ); + // And it is the interaction list less the 11 `LfmMem` tokens — the mirror + // property, which is what stops the feed and the senders from drifting. + assert_eq!( + predicted_bitwise(BLAKE3_ROUNDS) + 11, + blake3_chip::bus_interactions().len() + ); + + assert_eq!(round_trip(|_| {}), Ok(true), "honest proof must verify"); +} + +/// The padding rows carry nothing at all — no pointer pad, no witness. +#[test] +fn padding_rows_are_all_zero() { + let ops = probe_ops(); + let traces = build_traces(&ops); + let row = traces[0].main_table.get_row(NUM_OPS); + assert!( + row.iter().all(|c| *c == FE::zero()), + "the padding row must be entirely zero, so that μ = 0 is the only thing \ + standing between it and the constraint set" + ); +} + +// ========================================================================= +// Falsification (rule 1) +// ========================================================================= + +/// CONTROL. If this ever fails, every mutation below is reporting on a broken +/// harness rather than on the chip — check it first (rule 7's corollary). +#[test] +fn falsification_control_the_untampered_proof_verifies() { + assert_eq!(round_trip(|_| {}), Ok(true), "control must be green"); +} + +/// A flipped OUT byte: it is a feed-forward XOR *result*, so the XOR lookup +/// finds no BITWISE row, and the `LfmMem` word the mirror receives no longer +/// matches either. +#[test] +fn a_tampered_output_byte_rejects() { + assert_not_accepted("a flipped OUT byte", |t| { + let old = t.main_table.get_row(1)[cols::out_word(5, 2)]; + t.main_table + .set_fe(1, cols::out_word(5, 2), old + FE::one()); + }); +} + +/// A flipped message byte. `m` is never XORed, so the only things that see this +/// are its explicit `AreBytes` send, the add3 sum identity and the `LfmMem` +/// read. This is the test that would go green if the 32 message range checks +/// were ever dropped as "redundant" *and* the sum identity were loosened. +#[test] +fn a_tampered_message_byte_rejects() { + assert_not_accepted("a flipped message byte", |t| { + let old = t.main_table.get_row(0)[cols::in_word(8 + 3, 1)]; + t.main_table + .set_fe(0, cols::in_word(8 + 3, 1), old + FE::one()); + }); +} + +/// A padding row turned real. `MU` is preprocessed, so the prover recommits the +/// prefix and refuses before the constraint set is ever consulted — which IS +/// the point: an is-real flag a prover can choose is exactly what preprocessing +/// prevents, and the keccak adapter's `padding_row_multiplicity_rejects` +/// documents the weaker witness-side version. +#[test] +fn a_padding_row_turned_real_rejects() { + assert_not_accepted("an is-real padding row", |t| { + t.main_table.set_fe(NUM_OPS, cols::MU, FE::one()) + }); +} + +/// A bumped output-word read count, in the witness. Same shape: program data. +#[test] +fn a_tampered_read_multiplicity_rejects() { + assert_not_accepted("a bumped output-word read count", |t| { + let old = t.main_table.get_row(0)[cols::mult(2)]; + t.main_table.set_fe(0, cols::mult(2), old + FE::one()); + }); +} + +/// A carry bit flipped on an add3. The sum identity `a + b + m − s − 2^32·(c1+c2)` +/// is the only thing that sees it, and it is exactly the constraint #903's +/// "two summed committed carry bits" decision exists to keep at degree 3. +#[test] +fn a_tampered_add3_carry_bit_rejects() { + assert_not_accepted("a flipped add3 carry bit", |t| { + let col = cols::g_base(7) + cols::G_A1_C; + let old = t.main_table.get_row(2)[col]; + t.main_table.set_fe(2, col, old + FE::one()); + }); +} + +// ========================================================================= +// The column, at the production epoch shape +// ========================================================================= + +/// Two-term peak-RSS model (wave 7, after the one-parameter 33.7 B/cell fit was +/// falsified): `27 B` per base-field-equivalent cell plus `190 MB` per +/// sub-proof. +/// +/// ⚠ Both coefficients were calibrated on KECCAK-SHAPED runs — a machine whose +/// biggest tables are a 1,480-column round chip and a 2^20-row lookup table. +/// Nothing has checked that 27 B/cell survives a machine whose widest table is +/// a 3,056-column single-row chip, let alone one with no lookup table at all, +/// so every GiB below is a projection carrying that caveat and not a +/// measurement. +const BYTES_PER_CELL: f64 = 27.0; +const BYTES_PER_SUB_PROOF: f64 = 190_000_000.0; +const GIB: f64 = (1u64 << 30) as f64; + +fn projected_gib(cells: u64, sub_proofs: usize) -> f64 { + (cells as f64 * BYTES_PER_CELL + sub_proofs as f64 * BYTES_PER_SUB_PROOF) / GIB +} + +/// ★ The blake column, the residue split and the re-derived matrix, on the real +/// epoch verifier. +/// +/// `#[ignore]`d for the same reason `wrap_tests::the_wrap_census_at_blowup_8` +/// is: it proves a real inner epoch at blowup 8 and then emits ~2.25M +/// instructions. Run with +/// `cargo test -p lambda-vm-prover --lib the_blake_column -- --ignored --nocapture`. +/// +/// Every line labels its basis. Three MEASURED inputs feed it — the 4,946 +/// cells per compression proved above, this run's own census, and this run's +/// own permutation closed form — and everything else is arithmetic over them. +#[test] +#[ignore] +fn the_blake_column_and_the_residue_split() { + use super::airs::{lfm_cell_counts, lfm_chip_census}; + use super::epoch_verify::{query_permutations, query_permutations_at_rate}; + use super::instr::Instr; + use super::layout::padded_rows; + + let inner = crate::recursion::Preset::Blowup8.options(); + let e = super::epoch_tests::real_epoch_with(inner.clone()); + let profile = super::wrap_tests::epoch_profile(&e); + let program = super::epoch_tests::epoch_program(&e, true); + let spine = super::epoch_tests::epoch_program(&e, false); + + let census = lfm_chip_census(&program); + let (main, aux) = lfm_cell_counts(&program); + let total = main + 3 * aux; + let sub_proofs = census.len(); + + println!( + "\n★ EPOCH {profile}, inner blowup {}, {} queries, {sub_proofs} sub-proofs", + inner.blowup_factor, inner.fri_number_of_queries + ); + println!( + " {:>12} {:>12} {:>7} {:>6} {:>16} {:>8}", + "chip", "rows", "main", "aux", "base-equiv", "% total" + ); + // KECCAK_RND reports once per chunk; fold the chunks so the table reads as + // one line per chip class, which is what the matrix rows are about. + let mut folded: Vec<(&str, u64, usize, usize, u64)> = Vec::new(); + for c in &census { + let cells = c.main_cells() + 3 * c.aux_cells(); + match folded.iter_mut().find(|f| f.0 == c.name) { + Some(f) => { + f.1 += c.rows; + f.4 += cells; + } + None => folded.push((c.name, c.rows, c.main_cols, c.aux_cols, cells)), + } + } + for (name, rows, m, a, cells) in &folded { + println!( + " {name:>12} {rows:>12} {m:>7} {a:>6} {cells:>16} {:>7.2}%", + 100.0 * *cells as f64 / total as f64 + ); + } + println!(" {:>12} {:>50}", "TOTAL", total); + + let cells_of = |names: &[&str]| -> u64 { + folded + .iter() + .filter(|f| names.contains(&f.0)) + .map(|f| f.4) + .sum() + }; + // The keccak permutation itself, and the 2^20-row lookup table it shares + // with anything byte-oriented. Split because a field-native hash deletes + // BOTH while blake deletes only the first. + let keccak_perm = cells_of(&["LFM_KECCAK", "KECCAK_RND", "KECCAK_RC"]); + let bitwise = cells_of(&["BITWISE"]); + let residue = total - keccak_perm - bitwise; + + // ---- permutations, at both rates, from the closed form over the shapes. + let legs_17: usize = e.legs.iter().map(|l| query_permutations(&l.verify)).sum(); + let legs_8: usize = e + .legs + .iter() + .map(|l| query_permutations_at_rate(&l.verify, 8)) + .sum(); + let emitted = super::wrap_tests::permutations(&program); + let spine_perms = super::wrap_tests::permutations(&spine); + assert_eq!( + emitted - spine_perms, + legs_17, + "the rate-17 closed form must reproduce the emitted legs" + ); + // Rate 8 is BLAKE3's own: its socket absorbs two cells of message per + // compression, and option B1 did not change that. It is NOT the + // field-native chain's rate — that is `epoch_verify::LFM_HASH_RATE_FELTS`, + // which is 4 because the chain absorbs one cell per step. The two were the + // same number while the sponge was a three-cell duplex, and this line used + // to say "blake and field-native" on that basis; they have since diverged. + // + // The spine is absorption-bound, so at rate 8 it lies between 1.0x and + // 2.125x its rate-17 cost — wave 8's interval, restated on this run's own + // spine count rather than quoted. + let p_lo = legs_8 + spine_perms; + let p_hi = legs_8 + (spine_perms as f64 * 17.0 / 8.0).ceil() as usize; + println!( + "\n PERMUTATIONS emitted {emitted} = spine {spine_perms} + legs {legs_17} (MEASURED)\n \ + closed form legs @ rate 17 (keccak) {legs_17}, @ rate 8 (BLAKE3's socket) {legs_8}\n \ + P at rate 8 in [{p_lo}, {p_hi}] — legs exact, spine bounded" + ); + + // ---- the byteswap gadget: exactly the 64-bit decompositions. + // + // `sample_u64_pow2` asserts nbits <= 32 and every other production + // `bit_dec` site passes 32 or a Merkle depth, so a 64-bit decomposition in + // this program IS a `felt_be_halves` and nothing else. Counted rather than + // reasoned about, with the whole histogram printed so that a new 64-bit + // caller would show up instead of being silently folded in. + let mut hist = std::collections::BTreeMap::::new(); + for i in &program.instrs { + if let Instr::BitDec { bits, .. } = i { + *hist.entry(bits.len()).or_default() += 1; + } + } + println!("\n BitDec width histogram: {hist:?}"); + let swaps = hist.get(&64).copied().unwrap_or(0); + + let width = |name: &str| -> (u64, u64) { + let f = folded.iter().find(|f| f.0 == name).expect("chip in census"); + (f.2 as u64, f.3 as u64) + }; + let (bitdec_m, bitdec_a) = width("LFM_BITDEC"); + let (balu_m, balu_a) = width("LFM_BALU"); + let cell_law = |m: u64, a: u64, rows: u64| rows * (m + 3 * a); + + let bd_rows = program.groups.bitdec.real_rows; + let ba_rows = program.groups.balu.real_rows; + let before = cell_law(bitdec_m, bitdec_a, padded_rows(bd_rows) as u64) + + cell_law(balu_m, balu_a, padded_rows(ba_rows) as u64); + let after = cell_law(bitdec_m, bitdec_a, padded_rows(bd_rows - swaps) as u64) + + cell_law(balu_m, balu_a, padded_rows(ba_rows - 64 * swaps) as u64); + let unpadded = + cell_law(bitdec_m, bitdec_a, swaps as u64) + cell_law(balu_m, balu_a, 64 * swaps as u64); + let byteswap = before - after; + + println!( + "\n BYTESWAP GADGET {swaps} felts x (1 BitDec + 64 BALU)\n \ + LFM_BITDEC {bd_rows} real rows -> {} padded, {bitdec_m} main / {bitdec_a} aux\n \ + LFM_BALU {ba_rows} real rows -> {} padded, {balu_m} main / {balu_a} aux \ + ({:.2}% of all BALU rows)\n \ + unpadded closed form {unpadded}\n \ + padding-aware delta {byteswap} (the two chips together: before {before}, after {after})", + padded_rows(bd_rows), + padded_rows(ba_rows), + 100.0 * (64 * swaps) as f64 / ba_rows as f64, + ); + + // ---- the three residues the matrix needs. + let residue_field_native = residue - byteswap; + println!( + "\n RESIDUE (everything that is not the hash chip or its lookup table)\n \ + keccak permutation chips {keccak_perm:>16} {:>6.2}%\n \ + BITWISE (2^20 fixed) {bitwise:>16} {:>6.2}%\n \ + residue {residue:>16} {:>6.2}%\n \ + \x20 of which byteswap {byteswap:>16} {:>6.2}% OF THE RESIDUE\n \ + residue, byte-oriented {residue:>16} (blake keeps the gadget AND BITWISE)\n \ + residue, field-native {residue_field_native:>16} (gadget deleted, BITWISE deleted)", + 100.0 * keccak_perm as f64 / total as f64, + 100.0 * bitwise as f64 / total as f64, + 100.0 * residue as f64 / total as f64, + 100.0 * byteswap as f64 / residue as f64, + ); + println!( + " ⚠ the field-native line is DERIVED by subtraction from a keccak-shaped\n \ + emission, not measured on a re-emitted field-native verifier. It is an\n \ + UPPER bound on that residue: a field-native absorb also deletes the\n \ + Pack/Unpack traffic around the gadget, and LFM_LANES still costs {} here.", + cells_of(&["LFM_LANES"]) + ); + + // ---- the matrix, re-derived. + // + // Hash-chip cells at P permutations: `rows x (main + 3 x aux)`, with rows + // either padded to the next power of two (one AIR instance) or chunked the + // way KECCAK_RND is (several instances, ~1.9% waste). Both are printed + // because the choice is a policy, not a property of the hash. + let p = 192_000u64; + let chunked = |perms: u64| (perms as f64 * 1.01871).ceil() as u64; + let unchunked = |perms: u64| perms.next_power_of_two(); + let row = |name: &str, cells_per_perm: u64, resid: u64, table: u64| { + for (how, rows) in [("chunked", chunked(p)), ("padded", unchunked(p))] { + let hash_cells = rows * cells_per_perm; + let t = resid + table + hash_cells; + println!( + " {name:>28} {how:>8} hash {hash_cells:>13} total {t:>13} \ + {:>6.2}x under keccak ~{:.0} GiB", + total as f64 / t as f64, + projected_gib(t, sub_proofs), + ); + } + }; + println!( + "\n★ THE MATRIX, RE-DERIVED (P = {p}, {sub_proofs} sub-proofs, two-term RSS \ + {BYTES_PER_CELL} B/cell + {} MB/sub-proof)", + BYTES_PER_SUB_PROOF / 1e6 + ); + println!( + " {:>28} {:>8} keccak {:>11} total {:>13} {:>6.2}x ~{:.0} GiB", + "keccak (MEASURED, ours)", + "n/a", + keccak_perm + bitwise, + total, + 1.0, + projected_gib(total, sub_proofs), + ); + // Blake keeps the byte-oriented residue AND the BITWISE table it looks up in. + // The label and the figure follow the compiled round count, so a sweep + // cannot leave this row naming one variant and pricing another. + let blake_cells = (MAIN_COLUMNS + 3 * blake3_chip::bus_interactions().len().div_ceil(2)) as u64; + let blake_label = if BLAKE3_ROUNDS == 6 { + "BLAKE3-6r (MEASURED chip)" + } else { + "BLAKE3-7r (MEASURED chip)" + }; + row(blake_label, blake_cells, residue, bitwise); + // Field-native candidates delete both. Poseidon-original's 621 is wave 9's + // measured column; RPO's 152 and Monolith's ~850 stay INHERITED estimates. + row("Poseidon-orig (w9 MEASURED)", 621, residue_field_native, 0); + row("RPO (INHERITED estimate)", 152, residue_field_native, 0); + row("Monolith (INHERITED est.)", 850, residue_field_native, 0); +} + +// ========================================================================= +// The delegation topology, priced +// ========================================================================= + +/// ★ In-machine hosting vs an Airbender-style delegation circuit. +/// +/// The question (user request): instead of the epoch verifier carrying an +/// `LFM_BLAKE3` AIR, put the compressions in a SEPARATE specialized circuit and +/// verify that circuit's proof — Airbender's blake2s delegation circuit does +/// ~19 proofs' Merkle work in one 2^20 instance. +/// +/// The whole comparison is arithmetic over the same closed form the epoch's own +/// permutation count comes from ([`super::epoch_verify::blocks_at_rate`] and +/// the leaf/path/FRI decomposition), applied to the delegation proof's shape. +/// Every substituted input is named in the printout. +/// +/// ⚠ What this CANNOT see: prover wall time, proof size on the wire, and the +/// engineering cost of a second circuit and its glue. It prices cells only. +#[test] +#[ignore] +fn the_delegation_topology_priced_against_in_machine_hosting() { + use super::epoch_verify::{blocks_at_rate, group_leaf_felts}; + use super::sub_proof::GroupShape; + + let inner = crate::recursion::Preset::Blowup8.options(); + let e = super::epoch_tests::real_epoch_with(inner.clone()); + + // The epoch's widest leg supplies the two shape inputs this calculation + // does not derive: how many composition parts a sub-proof carries, and what + // one query's FRI leg costs. Both are INHERITED from a real proof rather + // than assumed. + let widest = e + .legs + .iter() + .max_by_key(|l| l.verify.sub.deep.log2_trace_length) + .expect("the epoch has legs"); + let parts = widest.verify.sub.deep.num_composition_parts; + let queries = widest.verify.num_queries; + let log2_blowup = inner.blowup_factor.trailing_zeros(); + + /// Compressions to verify ONE sub-proof of the given geometry, at BLAKE3's + /// rate 8 (two cells of message per compression — not the field-native + /// chain's 4, see `epoch_verify::LFM_HASH_RATE_FELTS`). + /// + /// `Σ_groups blocks_at_rate(leaf felts) + groups × merkle_depth + FRI`, the + /// same three terms `query_permutations_at_rate` sums, per query. + fn verify_cost( + groups: &[GroupShape], + log2_trace: u32, + log2_blowup: u32, + fri_per_query: usize, + queries: usize, + ) -> (usize, usize) { + let merkle_depth = (log2_trace + log2_blowup) as usize - 1; + let leaves: usize = groups + .iter() + .map(|g| blocks_at_rate(group_leaf_felts(g), 8)) + .sum(); + let per_query = leaves + groups.len() * merkle_depth + fri_per_query; + (per_query, per_query * queries) + } + + let fri_per_query = widest.verify.fri.permutations_per_query(); + + // --- the delegation circuit's two AIRs, at the epoch's own compression count. + let compressions = 192_000usize; + let log2_blake_trace = (compressions as u32).next_power_of_two().trailing_zeros(); // 18 + let blake_groups = vec![ + GroupShape { + num_columns: cols::PREP_WIDTH, + is_ext: false, + }, + GroupShape { + num_columns: MAIN_COLUMNS, + is_ext: false, + }, + GroupShape { + num_columns: blake3_chip::bus_interactions().len().div_ceil(2), + is_ext: true, + }, + GroupShape { + num_columns: parts, + is_ext: true, + }, + ]; + let bitwise_groups = vec![ + GroupShape { + num_columns: crate::tables::bitwise::NUM_PRECOMPUTED_COLS, + is_ext: false, + }, + GroupShape { + num_columns: 10, + is_ext: false, + }, + GroupShape { + num_columns: 5, + is_ext: true, + }, + GroupShape { + num_columns: parts, + is_ext: true, + }, + ]; + let (blake_pq, blake_total) = verify_cost( + &blake_groups, + log2_blake_trace, + log2_blowup, + fri_per_query, + queries, + ); + let (bw_pq, bw_total) = verify_cost(&bitwise_groups, 20, log2_blowup, fri_per_query, queries); + + let blake_aux = blake3_chip::bus_interactions().len().div_ceil(2); + let cells_per_compression = MAIN_COLUMNS as u64 + 3 * blake_aux as u64; + let delegation_trace = + (compressions as f64 * 1.01871).ceil() as u64 * cells_per_compression + 26_214_400; // its own BITWISE table + + println!( + "\n★ DELEGATION TOPOLOGY, at the epoch's {compressions} compressions\n\ + \x20 shared inputs (INHERITED from the epoch's 2^{} leg): {parts} composition parts, \ + {queries} queries, {fri_per_query} FRI compressions per query, blowup {}\n\n\ + \x20 IN-MACHINE the epoch verifier carries LFM_BLAKE3 as one more AIR of its\n\ + \x20 multi-proof, {compressions} rows x {cells_per_compression} cells = {} cells.\n\ + \x20 Nothing else changes: chip heights ARE program shape here, so the\n\ + \x20 hash competes with nothing for space.\n\n\ + \x20 DELEGATED (a) the delegation proof's own trace, LFM_BLAKE3 + BITWISE {delegation_trace:>12} cells\n\ + \x20 (b) verifying it inside the epoch verifier:\n\ + \x20 LFM_BLAKE3 AIR (2^{log2_blake_trace} rows, {MAIN_COLUMNS} main + {blake_aux} aux) \ + {blake_pq:>6}/query x {queries} = {blake_total:>8} compressions\n\ + \x20 BITWISE AIR (2^20 rows, 10 main + 5 aux) \ + {bw_pq:>6}/query x {queries} = {bw_total:>8} compressions\n\ + \x20 = {} extra compressions, i.e. {} extra cells in the\n\ + \x20 epoch verifier's OWN blake AIR, on top of (a).\n", + widest.verify.sub.deep.log2_trace_length, + inner.blowup_factor, + (compressions as f64 * 1.01871).ceil() as u64 * cells_per_compression, + blake_total + bw_total, + (blake_total + bw_total) as u64 * cells_per_compression, + ); + println!( + "\x20 VERDICT delegation costs (a) + (b) where in-machine costs (a) alone, so it is a\n\ + \x20 net LOSS of {:.0}M cells ({:.0}% on top) at these shapes. The reason is\n\ + \x20 structural, not a tuning accident: the thing Airbender's delegation\n\ + \x20 circuit buys is moving hash work out of a FIXED-SIZE main circuit (a\n\ + \x20 2^20-cycle RISC-V trace). The LFM has no fixed-size box — every chip's\n\ + \x20 height is program shape — so its multi-AIR proof already IS the\n\ + \x20 delegation pattern, and a second proof only adds a verification.\n\ + \x20 The leaf term is what makes (b) large: a {MAIN_COLUMNS}-column AIR has a\n\ + \x20 {}-felt main leaf, {} compressions to absorb, {} times per query.", + (blake_total + bw_total) as f64 * cells_per_compression as f64 / 1e6, + 100.0 * (blake_total + bw_total) as f64 * cells_per_compression as f64 + / delegation_trace as f64, + group_leaf_felts(&blake_groups[1]), + blocks_at_rate(group_leaf_felts(&blake_groups[1]), 8), + queries, + ); +} diff --git a/prover/src/lfm/blake3_socket.rs b/prover/src/lfm/blake3_socket.rs new file mode 100644 index 000000000..3dbbf3cef --- /dev/null +++ b/prover/src/lfm/blake3_socket.rs @@ -0,0 +1,1662 @@ +//! The BLAKE3 arm of `LFM_HASH` — the Option-A 2-to-1 compress socket. +//! +//! This is Route A of `thoughts/shared/lfm-real-hash/PLAN.md` §3: BLAKE3 hosted +//! *behind* the frozen `LFM_HASH` socket, exactly the way Poseidon is. The chip +//! count stays 14 and the 28-column shared value prefix keeps its offsets, so +//! the `LFM_HASH` tuple contract is untouched and everything BLAKE3 witnesses is +//! appended after the prefix. `PREP_WIDTH` is 13 — the transcript selector +//! (option B1) took it from 11 to 12 and the leaf selector (option C) to 13, +//! each moving every preprocessed root and every registered program's digest +//! once, in one re-bless. +//! +//! # What one row proves +//! +//! One row = one compression, in one of THREE domains, specified byte-level in +//! `thoughts/blake3/socket-kats/SOCKET.md` §2.1 and word-level in §2.2, at the +//! leaf RATE of `block-compression/commit-spec/COMMIT.md` §1.2: +//! +//! ```text +//! msg = LE32(lane0..lane11) ‖ tag (52 bytes) +//! digest = BLAKE3(msg)[0..16] (128 bits, 1 cell) +//! ``` +//! +//! | tag | row | the twelve lanes are | +//! |---|---|---| +//! | `"LFMC"` | Merkle parent / 2-to-1 compress | two digest cells, then four zeros | +//! | `"LFMT"` | a Fiat–Shamir transcript step | state ‖ operand ‖ four zeros | +//! | `"LFML"` | a **leaf** over four field elements | the chaining accumulator, then the felts' `lo`/`hi` halves | +//! +//! 52 bytes being one block, that is exactly one compression with `h = IV` (all +//! eight words), `m[0..12] = the lanes`, `m[12] = tag` as a little-endian `u32`, +//! `m[13..16] = 0`, `t = 0`, `block_len = 52`, +//! `flags = CHUNK_START|CHUNK_END|ROOT`, and the digest the LOW four output +//! words. **The three domains differ in `m[12]` and in nothing else**, so one +//! mixing core and one column layout serve all three. +//! +//! # The leaf RATE — why twelve lanes and not eight +//! +//! A leaf row absorbs **four felts and chains an accumulator in ONE +//! compression** (COMMIT.md §1.2): the accumulator cell rides in the message +//! rather than in `h`, so there is no separate fold. That is 4 felts per +//! compression against the 2 the accumulator-free row reached once its digest +//! had to be folded into a chain by an `"LFMC"` parent, and leaf absorption is +//! ~70% of a recursion tower node's bill. +//! +//! The lanes it costs are free of witness columns on the digest modes: lanes +//! 8–11 read `IN8..IN12`, the THIRD input cell, which +//! `chips::hash::emit_unread_input_pins` already pins to zero on every row that +//! does not read it. So a compress row's four new message words are forced to +//! zero by constraints that were already there — see the lane block in +//! [`eval`], and note that twelve is the last lane count for which this holds +//! (at thirteen, `IN0 + 12` is `S8` and the identity would start reading the +//! capacity state as an input felt). +//! +//! **At [`SOCKET_ROUNDS`] = 7 that is literally `blake3::hash(lanes ‖ tag)`,** +//! so the socket has a direct external anchor and needs no oracle in the chain — +//! and the transcript and leaf domains inherit that anchor unchanged, because +//! the tag is the only thing that moved. That is the whole reason the domain tag lives in the +//! *message* rather than in `flags`, `t` or `h`: a tag anywhere else would make +//! even the 7-round socket a nonstandard invocation of `f` that no library +//! computes, throwing the anchor away for nothing (SOCKET.md §2.3). +//! +//! The tag word is a linear form over the three PREPROCESSED mode columns rather than +//! a compile-time constant, which keeps it prover-unchosen and free — see +//! [`TAG_SELECTOR`]. +//! +//! # The LEAF mode, and the one thing not to conclude from it +//! +//! A leaf row reads TWO cells: a chaining accumulator, which is an ordinary +//! digest cell and fills lanes 0–3, and four arbitrary Goldilocks elements, each +//! split into a `lo`/`hi` `u32` pair so that eight halves fill lanes 4–11. +//! `p − 1 = 0xFFFFFFFF_00000000`, so for halves already known to be `u32`: +//! +//! ```text +//! v < p <==> NOT( hi = 2^32−1 AND lo >= 1 ) +//! ``` +//! +//! — "if `hi` is maximal then `lo` is zero", which is two witness columns and +//! four constraints per felt rather than a 64-bit decomposition. Without it one +//! field element would have TWO half-encodings and therefore two leaf digests, +//! which is a collision in the felt→digest map and exactly what a Merkle tree +//! must not have. +//! +//! ⚠ **The canonicity block ASSUMES the `u32` bound; it does not ESTABLISH it.** +//! `lo` and `hi` are ordinary input lanes, so the bound comes from the same O1 +//! machinery as every other lane: byte columns plus the `AreBytes` sends. That +//! is the whole reason this mode is cheap, and it is stated here because the +//! shape invites two opposite mistakes — adding a redundant range check on the +//! halves, or (far worse) **removing the lane identity or the `AreBytes` sends +//! on the theory that canonicity subsumes them. It does not.** With unbounded +//! halves, `hi = 2^32−1` stops being a reachable-and-detectable case and the +//! predicate above stops meaning `v < p` at all. +//! +//! # Why the socket is so much cheaper than the standalone chip +//! +//! [`super::blake3_chip`] is the syscall-shaped chip: 28 input `u32` words and +//! all 16 output words are committed columns. Here `h`, `t`, `block_len`, +//! `flags` and every message word above the lanes are **compile-time +//! constants**, and the truncation +//! window means only 4 of the 16 output words are ever built. What is left as +//! witness is 8 input lanes, the mixing core, and 4 output words. +//! +//! # The two soundness obligations this module discharges +//! +//! - **O1 — input lanes carry a committed byte decomposition.** A digest cell's +//! lane is a Goldilocks felt over `[0, p)` with `p ≈ 2^64`, and +//! `edsl::merkle_walk` feeds `compress` *arena-hinted* — that is, +//! prover-chosen — sibling cells. The contract has two halves and **they buy +//! different things**; conflating them is easy and is why this is spelled out. +//! +//! *The mu-gated linear identity* `IN_lane = Σ MB[k]·2^{8k}` ties the felt to +//! the bytes. Note what follows from it: the mixing core reads **the same +//! linear form** as the message word (`message_word_ref`), so `IN_lane` and +//! `m[lane]` are the same field element by construction. A lane therefore +//! cannot be hashed as anything but itself, and the textbook alias — `v` and +//! `v + 2^32` hashing alike — is **unconstructible here**, not merely +//! prevented. (It is real for a chip that derives the message bytes by +//! reduction mod 2^32 instead of by a checked decomposition, which is why the +//! identity is the right shape; it is not what the `AreBytes` sends buy.) +//! +//! *The `AreBytes` sends* are the message words' **only** range check — +//! `m` reaches `add3` and nothing else, never an XOR, so unlike almost every +//! other word in this design it gets no free byte bound from a lookup that +//! consumes it. And `add3`'s exactness needs `m < 2^32`: in round 0 the `a` +//! and `b` operands are compile-time constants and the output `s` is +//! byte-bounded by the XOR that consumes it, so an unbounded `m` lets a +//! prover solve `m ≡ s + 2^32·k − a − b (mod p)` for any chosen `s`, put the +//! whole value in `MB[0]` with the other three bytes zero — satisfying the +//! identity, since nothing bounds them — and hint the sibling cell to match. +//! The first `add3`'s output, and hence the entire compression, would be +//! prover-chosen. +//! +//! Stated as the one mechanism, since this is the spot the argument keeps +//! drifting: what the sends do is **transfer a bound onto the lane**. Without +//! them the identity is satisfiable for *every* felt `IN_lane` — put the whole +//! value in `MB[0]` — so it bounds nothing. With them the four bytes sum to +//! less than `2^32`, so it is satisfiable exactly when `IN_lane < 2^32`, and +//! then the decomposition is unique. +//! +//! So: neither half alone suffices, and `blake3_socket_tests:: +//! the_lane_range_check_is_load_bearing_on_its_own` pins the separation by +//! exhibiting a witness the eval set cannot see at all. +//! - **O3 — `compress_iv()` does not participate.** The IV enters through `h`, +//! all eight words, not through the state's capacity lanes, so this arm +//! overrides `compress` (and [`LfmHasher::compress_out`]) rather than +//! inheriting the trait's permute-and-truncate default. +//! +//! # ✓ O5 — RETIRED, and enforced by the tag rather than by review +//! +//! The obligation was: leaves and parents must be domain-separated, or a +//! variable-depth tree admits the classic Merkle second-preimage confusion — an +//! internal node replayed as a leaf. It is now discharged **mechanically**. A +//! leaf digest is `BLAKE3(…‖"LFML")` and a parent is `BLAKE3(…‖"LFMC")`, so an +//! internal node cannot be replayed as a leaf whatever the tree's shape, and the +//! tag is selected by a preprocessed column the prover does not choose. +//! +//! What this replaced is worth recording, because it was weaker than it looked. +//! Programs formed leaf digests by compressing raw data rows under the SAME +//! `"LFMC"` tag as parents, so leaves and parents were not separated at all; +//! that was sound only because every eDSL circuit is fixed-shape at build time, +//! so a node at one level could not be replayed at another. Fixed depth remains +//! true of every current program and remains worth having, but **it is no longer +//! load-bearing for second-preimage resistance.** +//! +//! The reviewer's job shrinks accordingly: from "is this a leaf path, and does +//! the tree have fixed depth?" to *"is this row's mode right?"* — which the +//! registrar's one-hot check and controls M9/M10 answer. +//! +//! (BLAKE3's own `PARENT` flag was rejected for the split: it cannot be reused +//! without leaving the standard-hash framing that makes the crate a direct KAT.) +//! +//! Equally on the record: the digest is 128 bits, so this socket offers +//! **64-bit collision resistance** by the birthday bound. That follows from +//! `HASH_DIGEST_FELTS = 4` and the machine's declared 128-bit target — it is +//! not introduced by BLAKE3 or by the truncation window. +//! +//! # ✗ There is no `permute` socket, and there never will be +//! +//! `LFM_HASH` has four modes and this arm implements **three**. The `permute` +//! socket — 12 felts in, 12 out — is unspecified: it has no mapping decision, +//! no KATs, and its security argument is not the same argument as `compress`'s +//! (SOCKET.md §7). Rather than invent one, the AIR forces `MODE_P = 0`, so a +//! program containing a `permute` is *unprovable* under BLAKE3, and +//! [`Blake3Permutation`] rejects one at execution with a message saying why. +//! +//! Option B1 (ratified 2026-08-11) made that permanent by removing the only +//! reason to want one: the Fiat–Shamir sponge is a **compress chain**, not a +//! permutation duplex, so `edsl::SpongeVar` runs on this socket like everything +//! else and `MODE_P` stays pinned forever. Option C then gave leaves their own +//! mode on the same socket rather than a second one. The tag `"LFMP"` that was +//! reserved for the permute socket is retired unused. + +use stark::constraints::builder::ConstraintBuilder; +use stark::lookup::{BusInteraction, BusValue, Multiplicity}; + +use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; +use crate::tables::types::{BusId, FE, GoldilocksExtension, GoldilocksField, alu_op}; + +use super::blake3::{BLAKE3_IV, BLAKE3_ROUNDS, blake3_compress_rounds}; +use super::blake3_chip::{ + Add2Wire, Add3Wire, Blake3Flow, ByteRef, FlowConfig, ROT_SHIFT_R, RotWire, ValueFlow, WireFlow, + WordRef, XorWire, half_expr, run_flow, word_cols, word_expr, +}; +use super::chips::hash::NUM_UNREAD_INPUT_PINS; +use super::hash::{HASH_DIGEST_FELTS, HASH_STATE_FELTS, LfmHasher}; +use super::instr::HashMode; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +// ========================================================================= +// The framing (SOCKET.md §2.2) — every one of these is a way to be wrong +// ========================================================================= + +/// Rounds the `LFM_HASH` BLAKE3 arm is compiled for. +/// +/// An alias for [`BLAKE3_ROUNDS`] — 7 (standard BLAKE3) by default, 6 under the +/// `blake3-6round` feature. The socket and the standalone `LFM_BLAKE3` probe +/// share ONE knob on purpose: two would let a sweep leave the machine's hash and +/// the chip it is priced against describing different functions, and the whole +/// value of the probe is that the two are comparable. +pub const SOCKET_ROUNDS: usize = BLAKE3_ROUNDS; + +/// Tripwire for the single-knob invariant. +/// +/// Trivially true while [`SOCKET_ROUNDS`] is an alias — which is the point. The +/// invariant is enforced by that one `=` and by nothing else: `NUM_G == 8 * +/// SOCKET_ROUNDS` and `cols::OUT - cols::G == 60 * NUM_G` are each internally +/// consistent and would pass happily with the socket and the standalone probe +/// compiled at different round counts. That is not hypothetical — it is the +/// shape this tree had before the A6R flip, and re-introducing a second `cfg` +/// pair here is a silent pricing lie: the probe would measure one hash and the +/// machine would use another. This assertion is what fails instead. +const _: () = assert!(SOCKET_ROUNDS == BLAKE3_ROUNDS); + +/// G-instances per compression: 8 per round. +pub const NUM_G: usize = SOCKET_ROUNDS * 8; + +/// The domain tag `"LFMC"`, read as one little-endian `u32` — the message word +/// straight after the lanes, `m[NUM_LANES]`. +/// +/// A tag is never reused for a second purpose, for the same reason +/// `HasherKind::as_tag` never reuses a discriminant. `"LFMT"` is the transcript +/// domain, `"LFML"` is the LEAF domain (live since option C), and `"LFMP"` is +/// RETIRED UNUSED — it was reserved for a permute socket that option B1 decided +/// never to build. Retired rather than deleted: freeing the value would let a +/// later allocation reuse it and create a domain nobody analysed. +pub const TAG_LFMC: u32 = u32::from_le_bytes(*b"LFMC"); + +/// The domain tag `"LFML"` — a Merkle LEAF over four field elements. +/// +/// The third live domain, and the one that retires obligation O5: a leaf digest +/// and a parent digest are different functions of the same bits, so an internal +/// node cannot be replayed as a leaf regardless of tree depth. That previously +/// rested on every eDSL circuit being fixed-shape — true, but enforced by +/// nothing. +/// +/// The message is the felts' checked `u32` halves, so its byte layout is +/// identical to a digest-mode compress and the crate anchor survives: at +/// [`SOCKET_ROUNDS`] = 7 a leaf is +/// `blake3::hash(LE32(lo0)‖LE32(hi0)‖…‖LE32(hi3)‖"LFML")` truncated. +pub const TAG_LFML: u32 = u32::from_le_bytes(*b"LFML"); + +/// The domain tag `"LFMT"` — one step of the Fiat–Shamir transcript chain. +/// +/// The transcript step is this socket in every respect except this word: same +/// `h = IV`, same `m[0..4] = state`, `m[4..8] = operand`, same `t`, +/// `block_len` and `flags`, same four-word truncation. So at +/// [`SOCKET_ROUNDS`] = 7 a transcript step is literally +/// `blake3::hash(state ‖ operand ‖ "LFMT")` truncated to 128 bits, and it +/// inherits the compress socket's external anchor unchanged — which is the +/// whole point of building the transcript out of this socket rather than out of +/// a second one. +pub const TAG_LFMT: u32 = u32::from_le_bytes(*b"LFMT"); + +/// `CHUNK_START | CHUNK_END | ROOT` — the flags a one-block, one-chunk, +/// root-position BLAKE3 hash uses. Matching the tree hasher exactly is what +/// keeps §2.1's byte-level form a plain library call. +pub const FLAGS_LFMC: u32 = 0x0B; + +/// The message length in bytes: one 4-byte word per lane, plus the tag. +/// +/// 52 at [`cols::NUM_LANES`] = 12. **Derived, never written as a literal**: it +/// is `v[14]`, hence the `vd` operand of round-0 G #2 and from there an XOR +/// operand, so it cannot be mode-dependent (`WordRef::byte` panics on a +/// `ModeSelected`) — all three domains move together and a hand-written 52 that +/// disagreed with the lane count would desynchronise the wire interpretation +/// from the host reference (COMMIT.md §1.4.4 **H9**). +/// +/// 52 < 64 keeps a row ONE BLAKE3 block, which is what keeps the crate-KAT +/// anchor: at [`SOCKET_ROUNDS`] = 7 a row is still a plain `blake3::hash` call. +pub const BLOCK_LEN_LFMC: u32 = 4 * (cols::NUM_LANES as u32 + 1); + +/// The counter. Zero: one block, one chunk, chunk index 0. +pub const COUNTER_LFMC: u64 = 0; + +/// The truncation window: the digest is the LOW four of the 16 output words. +pub const OUT_WINDOW: usize = HASH_DIGEST_FELTS; + +/// The dataflow framing [`run_flow`] itself decides — the round count and the +/// truncation window. One value, used by the wire interpretation, the value +/// interpretation and the trace filler alike, so they cannot desynchronise. +pub(crate) const FLOW: FlowConfig = FlowConfig { + rounds: SOCKET_ROUNDS, + out_window: OUT_WINDOW, + full_output: false, +}; + +/// The 16 message words of the socket's 52-byte block, under domain `tag`. +/// +/// The lanes first, the tag straight after them, zeros above — so the byte +/// string is `LE32(lanes) ‖ tag` whatever the mode, and the tag stays LAST as +/// COMMIT.md §1.2 specifies it. +pub fn socket_message(lanes: &[u32; cols::NUM_LANES], tag: u32) -> [u32; 16] { + let mut m = [0u32; 16]; + m[..cols::NUM_LANES].copy_from_slice(lanes); + m[cols::NUM_LANES] = tag; + m +} + +/// **The reference the chip is checked against**: the socket's 2-to-1 step, +/// word-level, at an explicit round count and in an explicit domain. +/// +/// `rounds` is an argument rather than [`SOCKET_ROUNDS`] so the KATs can pin +/// both variants in one test run; the chip itself is compiled for exactly one. +/// `tag` is an argument for the same reason it is a column on the chip: two +/// domains, one function. +pub fn socket_digest_rounds_tagged( + a: &[u32; 4], + b: &[u32; 4], + rounds: usize, + tag: u32, +) -> [u32; 4] { + socket_digest_lanes(&digest_row_lanes(a, b), rounds, tag) +} + +/// A digest row's twelve lanes: the two cells it reads, then the four zeros the +/// third input cell's pins force. +/// +/// Written once, so the host reference cannot disagree with the AIR about what a +/// compress row's new lanes hold. +pub fn digest_row_lanes(a: &[u32; 4], b: &[u32; 4]) -> [u32; cols::NUM_LANES] { + let mut lanes = [0u32; cols::NUM_LANES]; + lanes[0..4].copy_from_slice(a); + lanes[4..8].copy_from_slice(b); + lanes +} + +/// The socket over twelve explicit lanes — the one place the framing is applied. +pub fn socket_digest_lanes(lanes: &[u32; cols::NUM_LANES], rounds: usize, tag: u32) -> [u32; 4] { + let out = blake3_compress_rounds( + &BLAKE3_IV, + &socket_message(lanes, tag), + COUNTER_LFMC, + BLOCK_LEN_LFMC, + FLAGS_LFMC, + rounds, + ); + [out[0], out[1], out[2], out[3]] +} + +/// [`socket_digest_rounds_tagged`] in the MERKLE domain. +pub fn socket_digest_rounds(a: &[u32; 4], b: &[u32; 4], rounds: usize) -> [u32; 4] { + socket_digest_rounds_tagged(a, b, rounds, TAG_LFMC) +} + +/// [`socket_digest_rounds`] at the compiled-in round count — what a `Compress` +/// row proves, and what [`Blake3Permutation::compress`] computes. +pub fn socket_digest(a: &[u32; 4], b: &[u32; 4]) -> [u32; 4] { + socket_digest_rounds(a, b, SOCKET_ROUNDS) +} + +/// One transcript step at an explicit round count — the `"LFMT"` domain. +pub fn transcript_digest_rounds(state: &[u32; 4], operand: &[u32; 4], rounds: usize) -> [u32; 4] { + socket_digest_rounds_tagged(state, operand, rounds, TAG_LFMT) +} + +/// [`transcript_digest_rounds`] at the compiled-in round count — what a +/// `Transcript` row proves, and what [`Blake3Permutation::transcript`] +/// computes. +pub fn transcript_digest(state: &[u32; 4], operand: &[u32; 4]) -> [u32; 4] { + transcript_digest_rounds(state, operand, SOCKET_ROUNDS) +} + +/// The domain tag a row in `mode` hashes under, or `None` for a mode this arm +/// has no socket for. +/// +/// One function, so the executor, the trace filler, the multiplicity histogram +/// and the KATs cannot disagree about which tag a row carries. The AIR gets the +/// same mapping through [`TAG_SELECTOR`], written the one other way it has to +/// be written — as a linear form over the mode columns. +pub const fn tag_for_mode(mode: HashMode) -> Option { + match mode { + HashMode::Compress => Some(TAG_LFMC), + HashMode::Transcript => Some(TAG_LFMT), + HashMode::Leaf => Some(TAG_LFML), + HashMode::Permute => None, + } +} + +// ========================================================================= +// The felt boundary (the LEAF mode) — host side +// ========================================================================= + +/// Field elements one leaf row hashes — the leaf **RATE** (COMMIT.md §1.4.1). +/// +/// Four felts = eight halves, which with the four accumulator lanes fill the +/// socket's twelve message lanes. It is one whole machine cell, which is the +/// property the rate was chosen for: the leaf program reads its felt stream in +/// the natural 4-per-cell layout with no re-packing pass. +pub const FELTS_PER_LEAF: usize = 4; + +/// Goldilocks `p = 2^64 − 2^32 + 1`, as the halves see it: `p − 1` is +/// `hi = 2^32−1`, `lo = 0`. +const MAX_HALF: u32 = u32::MAX; + +/// The chip's canonicity predicate, stated exactly as its constraints do. +/// +/// For halves already known to be `u32`, `v = lo + 2^32·hi < p` **iff** NOT +/// (`hi` maximal AND `lo ≥ 1`) — because `p − 1 = 0xFFFFFFFF_00000000`. That one +/// line is the whole reason this mode is cheap: it costs two witness columns per +/// felt instead of a 64-bit decomposition. +/// +/// ⚠ It **assumes** the `u32` bound rather than establishing it — see the +/// module docs. +pub const fn is_canonical(lo: u32, hi: u32) -> bool { + !(hi == MAX_HALF && lo >= 1) +} + +/// `v → (lo, hi)`, or `None` when `v` is not a canonical Goldilocks element. +/// +/// **REJECTS, never reduces.** A non-canonical value has no satisfying witness, +/// so its row is unprovable; a host that wrapped instead would claim a digest no +/// proof can produce. Same shape as obligation O1's own reject-don't-reduce +/// rule, and the reason is the same. +pub fn felt_halves(v: u64) -> Option<(u32, u32)> { + let (lo, hi) = (v as u32, (v >> 32) as u32); + is_canonical(lo, hi).then_some((lo, hi)) +} + +/// Four felts → the eight message lanes ABOVE the accumulator, +/// `[lo0, hi0, …, lo3, hi3]` — the row's lanes 4–11. +/// +/// A felt's halves are ADJACENT, which is load-bearing: it lets the canonicity +/// gate read one pair of neighbouring lanes instead of reaching across the row. +pub fn leaf_lanes(felts: &LfmWord) -> Option<[u32; 2 * FELTS_PER_LEAF]> { + use math::field::traits::IsPrimeField; + let mut lanes = [0u32; 2 * FELTS_PER_LEAF]; + for (i, f) in felts.iter().enumerate() { + let (lo, hi) = felt_halves(GoldilocksField::canonical(f.value()))?; + lanes[2 * i] = lo; + lanes[2 * i + 1] = hi; + } + Some(lanes) +} + +/// A leaf row's twelve lanes: the accumulator cell, then the felts' halves. +/// +/// `None` if the accumulator is not four `u32` lanes (it is a previous digest, +/// so it is by construction) or if a felt is not canonical. +/// +/// The split is what makes the row a HYBRID and it is the reason this exists as +/// one function: the accumulator is read as digest lanes and the felts as +/// halves, on the same row, and the trace filler, the BITWISE histogram and the +/// host reference must all split it identically (COMMIT.md §1.4.4 **H5**). +pub fn leaf_row_lanes(acc: &LfmWord, felts: &LfmWord) -> Option<[u32; cols::NUM_LANES]> { + let mut lanes = [0u32; cols::NUM_LANES]; + lanes[..cols::NUM_ACC_LANES].copy_from_slice(&lanes_of(acc)?); + lanes[cols::NUM_ACC_LANES..].copy_from_slice(&leaf_lanes(felts)?); + Some(lanes) +} + +/// One leaf row at an explicit round count — the `"LFML"` domain over the +/// accumulator and the felts' halves. +/// +/// **The accumulator rides in the message, so there is no separate fold**: this +/// one compression both absorbs `felts` and chains `acc` (COMMIT.md §1.2). +pub fn leaf_digest_rounds(acc: &LfmWord, felts: &LfmWord, rounds: usize) -> Option<[u32; 4]> { + Some(socket_digest_lanes( + &leaf_row_lanes(acc, felts)?, + rounds, + TAG_LFML, + )) +} + +/// [`leaf_digest_rounds`] at the compiled-in round count — what a `Leaf` row +/// proves, and what [`Blake3Permutation::leaf`] computes. +pub fn leaf_digest(acc: &LfmWord, felts: &LfmWord) -> Option<[u32; 4]> { + leaf_digest_rounds(acc, felts, SOCKET_ROUNDS) +} + +// ========================================================================= +// The lane boundary (obligation O1), host side +// ========================================================================= + +/// A digest cell's four lanes as `u32`s, or `None` if any lane is out of range. +/// +/// **`None` must never be turned into a reduction.** The host and the chip have +/// to agree about what was hashed; a felt outside `[0, 2^32)` has no byte +/// decomposition the chip can commit, so reducing it here would make a +/// host-side assertion pass while the chip proved something else. Rejecting is +/// also what keeps the two consistent in the other direction: the chip refuses +/// such a lane (no byte string satisfies both the identity and `AreBytes`), so +/// a host that reduced would claim a digest no proof can produce. +pub fn lanes_of(word: &LfmWord) -> Option<[u32; 4]> { + use math::field::traits::IsPrimeField; + let mut out = [0u32; 4]; + for (o, felt) in out.iter_mut().zip(word.iter()) { + *o = u32::try_from(GoldilocksField::canonical(felt.value())).ok()?; + } + Some(out) +} + +/// A digest cell built from four `u32` lanes — the inverse of [`lanes_of`], and +/// the `keccak_host` convention (one felt = one `u32` = four little-endian +/// bytes), NOT `word::pack_digest`'s eight-bytes-per-lane serialisation. +pub fn word_of(lanes: &[u32; 4]) -> LfmWord { + core::array::from_fn(|i| FE::from(u64::from(lanes[i]))) +} + +// ========================================================================= +// The host-side hasher +// ========================================================================= + +/// BLAKE3 behind `LFM_HASH`, `compress` only. +/// +/// The trait's `permute` is **partial** for lane-restricted hashers, and this is +/// one: it has no permute socket at all. [`LfmHasher::admits`] is what makes the +/// partiality a rejection rather than a wrong answer. +pub struct Blake3Permutation; + +impl LfmHasher for Blake3Permutation { + /// ✗ Unreachable by construction: [`LfmHasher::admits`] rejects a `Permute` + /// row before the executor gets here, and the AIR forces `MODE_P = 0`. + /// + /// It panics rather than returning something, because every value it could + /// return would be a hash the chip does not prove. + fn permute(&self, _state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + panic!( + "BLAKE3 has no LFM_HASH permute socket: 12-felt permute is unspecified \ + (thoughts/blake3/socket-kats/SOCKET.md §7). Use compress, or select \ + another hasher." + ) + } + + /// `BLAKE3_IV[0..4]`, so the capacity columns carry something meaningful if + /// read — but it is **not** part of the compress framing (obligation O3). + /// The IV enters through `h`, all eight words, and this arm overrides + /// `compress`/`compress_out` rather than inheriting the trait's + /// permute-a‖b‖IV default. + fn compress_iv(&self) -> LfmWord { + core::array::from_fn(|i| FE::from(u64::from(BLAKE3_IV[i]))) + } + + fn compress(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + self.step(a, b, TAG_LFMC) + } + + /// The digest in lanes 0–3 and zeros above, which is exactly what the chip's + /// `OUT` columns carry: `MULT1`/`MULT2` are zero on a Compress row, so the + /// upper eight are sent nowhere, and the AIR pins them to zero. + fn compress_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + Self::widen(self.compress(a, b)) + } + + /// The same socket under the TRANSCRIPT tag — this is where BLAKE3 stops + /// inheriting the trait's single-domain default, and it is the only thing + /// that makes a transcript step un-replayable as a Merkle parent. + fn transcript(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + self.step(a, b, TAG_LFMT) + } + + fn transcript_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + Self::widen(self.transcript(a, b)) + } + + /// The LEAF domain, and the one override that is an ENCODING rather than a + /// tag: the four felts become eight checked `u32` halves before they reach + /// the socket. This is what lets arbitrary Goldilocks data be hashed at all + /// — obligation O1 restricts the *lanes*, and a leaf row satisfies it by + /// construction rather than by luck. + fn leaf(&self, acc: &LfmWord, felts: &LfmWord) -> LfmWord { + word_of(&leaf_digest(acc, felts).expect( + "leaf accumulator lane is not a u32, or a leaf felt is not canonical — admits() \ + should have rejected it (reject, never reduce)", + )) + } + + fn leaf_out(&self, acc: &LfmWord, felts: &LfmWord) -> [FE; HASH_STATE_FELTS] { + Self::widen(self.leaf(acc, felts)) + } + + fn admits(&self, mode: HashMode, state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + match mode { + HashMode::Permute => Err( + "BLAKE3 has no LFM_HASH permute socket (SOCKET.md §7); its AIR forces MODE_P = 0", + ), + // ★ A leaf row is a HYBRID and both halves have to be checked, in + // the cells the AIR actually reads them from: the ACCUMULATOR in + // cell 0 is an ordinary digest and carries the full O1 `u32` + // restriction, while the FELTS in cell 1 have none — that is the + // entire point of the mode, since they are split into checked halves + // inside the socket. + // + // ⚠ Checking the wrong cell is a prover PANIC rather than a clean + // rejection: a non-canonical felt would pass here and blow up later + // in the witness filler (COMMIT.md §1.4.4 **H7**). The house rule is + // reject, never reduce — and never panic where a rejection is + // available. + HashMode::Leaf => { + let acc: LfmWord = core::array::from_fn(|i| state[i]); + let felts: LfmWord = core::array::from_fn(|i| state[4 + i]); + if lanes_of(&acc).is_none() { + return Err( + "BLAKE3 leaf accumulator lane is not a u32 (SOCKET.md obligation O1)", + ); + } + if leaf_lanes(&felts).is_none() { + return Err( + "BLAKE3 leaf felt is not a canonical Goldilocks element (LEAF.md §1.1)", + ); + } + Ok(()) + } + HashMode::Compress | HashMode::Transcript => { + let (a, b): (LfmWord, LfmWord) = ( + core::array::from_fn(|i| state[i]), + core::array::from_fn(|i| state[4 + i]), + ); + if lanes_of(&a).is_none() || lanes_of(&b).is_none() { + // Obligation O1, host side, and it binds both two-to-one + // modes: a transcript step is the same socket over the same + // lane columns, so it inherits the same restriction. + // Rejecting rather than reducing is the point: reduction is + // the collision. Data that cannot satisfy this belongs in a + // LEAF row, which is what that mode exists for. + return Err( + "BLAKE3 compress input lane is not a u32 (SOCKET.md obligation O1)", + ); + } + Ok(()) + } + } + } +} + +impl Blake3Permutation { + /// The socket, once, in the named domain — the one place the host computes + /// it, so `compress` and `transcript` cannot drift into different framings. + fn step(&self, a: &LfmWord, b: &LfmWord, tag: u32) -> LfmWord { + let (a, b) = ( + lanes_of(a).expect("socket lane is not a u32 — admits() should have rejected it"), + lanes_of(b).expect("socket lane is not a u32 — admits() should have rejected it"), + ); + word_of(&socket_digest_rounds_tagged(&a, &b, SOCKET_ROUNDS, tag)) + } + + fn widen(digest: LfmWord) -> [FE; HASH_STATE_FELTS] { + let mut out = [FE::zero(); HASH_STATE_FELTS]; + out[0..HASH_DIGEST_FELTS].clone_from_slice(&digest); + out + } +} + +// ========================================================================= +// Column layout +// ========================================================================= + +/// The BLAKE3 arm's columns. +/// +/// The frozen prefix (`IN0..12`, `S8..12`, `OUT0..12`) keeps the offsets +/// `chips::hash::cols` gives it, so the `LFM_HASH` tuple contract stays +/// literally frozen and every existing `edsl::merkle_walk` caller works +/// unchanged. Everything BLAKE3 additionally witnesses is appended from +/// [`LANES`] on. +/// +/// Width, in blocks: 28 shared + 32 lane bytes + `NUM_G · 60` mixing + 16 +/// output bytes. +pub mod cols { + pub use crate::lfm::chips::hash::cols::{ + IN_ADDR0, IN_ADDR1, IN_ADDR2, IN0, MODE_C, MODE_L, MODE_P, MODE_T, MULT0, MULT1, MULT2, + OUT_ADDR0, OUT_ADDR1, OUT_ADDR2, OUT0, PREP_WIDTH, S8, SHARED_VALUE_COLUMNS, + }; + + use super::{FELTS_PER_LEAF, HASH_DIGEST_FELTS, NUM_G, OUT_WINDOW}; + + /// The is-real flag every constraint is gated by and every send's + /// multiplicity: `MODE_C + MODE_T + MODE_L`, the three modes this arm has a + /// socket for. `MODE_P` is pinned to zero, so the sum is a bit on every row + /// and zero on padding. + /// + /// All three are *preprocessed* columns, so a prover chooses neither the + /// gate nor — through the same columns — the domain tag it selects. + pub const MU_COLUMNS: [usize; 3] = [MODE_C, MODE_T, MODE_L]; + + /// The modes whose message lanes above the accumulator ARE the `IN` lanes — + /// the digest modes. A leaf row's lanes 4–11 are its felts' halves instead, + /// so the lane identity is gated on this above [`NUM_ACC_LANES`] rather than + /// on the full mu. Lanes 0–3 are a digest cell in EVERY mode and take the + /// full mu; see [`super::eval`] and COMMIT.md §1.4.4 **H6**. + pub const DIGEST_MODE_COLUMNS: [usize; 2] = [MODE_C, MODE_T]; + + /// First appended witness column: the byte decomposition of the input lanes, + /// 4 bytes each, little-endian (`lane_byte`). + pub const LANES: usize = PREP_WIDTH + SHARED_VALUE_COLUMNS; + + /// Lanes carrying a cell that is a DIGEST under every mode: `a` on a digest + /// row, the chaining accumulator on a leaf row. One identity serves both + /// readings, which is why they can share a gate. + pub const NUM_ACC_LANES: usize = HASH_DIGEST_FELTS; + + /// Input lanes that carry message words. + /// + /// The accumulator cell plus one felt cell's halves — the leaf RATE decides + /// this number, and the digest modes inherit it: `a ‖ b` fills the first + /// eight and the last four are the third input cell, which the unread-`IN` + /// pins force to zero (COMMIT.md §1.4.1). + pub const NUM_LANES: usize = NUM_ACC_LANES + 2 * FELTS_PER_LEAF; + + /// The mixing core: one 60-cell block per G-instance, laid out exactly as + /// `blake3_chip::cols` lays one out (56 byte cells + 4 carry bits). + pub const G: usize = LANES + 4 * NUM_LANES; + pub const G_SIZE: usize = 60; + + /// Feed-forward output bytes — only the truncation window's four words. + pub const OUTW: usize = G + NUM_G * G_SIZE; + + /// The LEAF mode's canonicity witnesses: `Z_i` and `GINV_i` per felt. + /// + /// `LFM_BITDEC`'s own `Z`/`GINV` idiom, applied to two halves instead of 64 + /// bits — the machine's established canonicity shape, not a new invention. + /// Two columns and four constraints per felt, **zero extra sends**. + /// + /// They exist on EVERY row, leaf or not, because a chip has one width. That + /// is the mode's whole marginal cost: +8 value cells per compress row. + pub const CANON: usize = OUTW + 4 * OUT_WINDOW; + + pub const NUM_COLUMNS: usize = CANON + 2 * FELTS_PER_LEAF; + + // Offsets inside one G block, shared verbatim with `blake3_chip::cols` so + // the two chips' blocks are the same shape and the wire interpretation + // below is the same code with different bases. + pub use crate::lfm::blake3_chip::cols::{ + G_A1, G_A1_C, G_A2, G_A2_C, G_C1, G_C2, G_R1, G_R2, G_X1, G_X2, G_X3, G_X4, + }; + + /// Byte `b` of input lane `lane` (0..[`NUM_LANES`]). + #[inline] + pub const fn lane_byte(lane: usize, b: usize) -> usize { + LANES + 4 * lane + b + } + + /// Base column of G-block `g`. + #[inline] + pub const fn g_base(g: usize) -> usize { + G + g * G_SIZE + } + + /// Byte `b` of digest word `i` (0..4). + #[inline] + pub const fn out_byte(i: usize, b: usize) -> usize { + OUTW + 4 * i + b + } + + /// Felt `i`'s canonicity flag: 1 exactly when its high half is maximal. + #[inline] + pub const fn canon_z(i: usize) -> usize { + CANON + 2 * i + } + + /// Felt `i`'s inverse witness for `(2^32 − 1) − hi`, zero when that is zero. + #[inline] + pub const fn canon_ginv(i: usize) -> usize { + CANON + 2 * i + 1 + } + + /// The `IN` column of leaf felt `i` — the SECOND input cell. + /// + /// The felts sit above the accumulator, which is what the halves binding and + /// [`super::lanes_from_cells`] must both read (COMMIT.md §1.4.4 **H4**). + #[inline] + pub const fn leaf_felt(i: usize) -> usize { + IN0 + NUM_ACC_LANES + i + } + + /// Message lane carrying felt `i`'s LOW half. Halves are adjacent, so the + /// canonicity gate reads neighbours rather than reaching across the row; + /// they start above the accumulator lanes. + #[inline] + pub const fn leaf_lo_lane(i: usize) -> usize { + NUM_ACC_LANES + 2 * i + } + + /// Message lane carrying felt `i`'s HIGH half. + #[inline] + pub const fn leaf_hi_lane(i: usize) -> usize { + NUM_ACC_LANES + 2 * i + 1 + } +} + +/// Value columns the census counts: everything past the preprocessed prefix. +pub const MAIN_COLUMNS: usize = cols::NUM_COLUMNS - cols::PREP_WIDTH; + +// ========================================================================= +// Wire interpretation — the socket's framing over the shared dataflow +// ========================================================================= + +/// `m[8] = MODE_C·"LFMC" + MODE_T·"LFMT"` — the row's domain tag. +/// +/// **Why this is not prover-chosen.** `MODE_C` and `MODE_T` are preprocessed +/// columns: a row's mode is fixed by its position in the preprocessed trace, +/// that trace is fixed by its commitment, and the commitment is folded into +/// `lfm_program_id`. The prover chooses neither, which is the same argument +/// that already makes the mu gate trustworthy. Two constraints make it bite — +/// the mode-sum booleanity (idx 4) forces at most one tag to be selected, and +/// `MODE_T` being preprocessed is what stops the selector itself being chosen. +/// Controls M5 and M6 in `blake3_socket_tests` are what make each of those +/// dependencies a checked claim rather than an assertion. +const TAG_SELECTOR: &[(usize, u32)] = &[ + (cols::MODE_C, TAG_LFMC), + (cols::MODE_T, TAG_LFMT), + (cols::MODE_L, TAG_LFML), + // ✗ `MODE_P` is deliberately absent, not forgotten: there is no permute + // socket and idx 5 pins the column to zero, so a term for it would be + // identically zero and would suggest a domain that does not exist. +]; + +/// The message word at schedule index `i`, as wiring. +/// +/// `i < NUM_LANES` are the input lanes' byte columns; `m[NUM_LANES]` is the +/// domain tag and everything above it is zero. None of them is a witness column, +/// which is what makes the domain separation free (no cells, no range checks, +/// SOCKET.md §2.3) — the tag went from a constant to a linear form over +/// preprocessed columns and kept that property, because a preprocessed column is +/// not a witness. +/// +/// **The tag sits immediately after the lanes, not at a fixed `m[8]`.** That is +/// what makes the message the byte string `LE32(lanes) ‖ tag` at any lane count, +/// which is the form COMMIT.md §1.2 specifies and the form the KATs pin. A +/// `ModeSelected` word is legal at any index for the same reason a `Const` one +/// is: message words reach `add3` and nothing else. +fn message_word_ref(i: usize) -> WordRef { + match i { + i if i < cols::NUM_LANES => WordRef::Cols(word_cols(cols::lane_byte(i, 0))), + i if i == cols::NUM_LANES => WordRef::ModeSelected(TAG_SELECTOR), + _ => WordRef::Const(0), + } +} + +/// The socket's wire interpretation: same [`run_flow`], different framing and +/// different column bases. +struct SocketWire(WireFlow); + +impl Blake3Flow for SocketWire { + type Word = WordRef; + + /// `h = IV`, all eight words — so the entire initial state is constant and + /// the socket costs zero input-state columns. + fn input_h(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + /// `v[12..16] = t_lo, t_hi, block_len, flags` — all constants here. + fn input_v12(&mut self, j: usize) -> WordRef { + WordRef::Const( + [ + COUNTER_LFMC as u32, + (COUNTER_LFMC >> 32) as u32, + BLOCK_LEN_LFMC, + FLAGS_LFMC, + ][j], + ) + } + + fn iv_const(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + fn add3(&mut self, g: usize, half: usize, a: WordRef, b: WordRef, m_idx: usize) -> WordRef { + let base = cols::g_base(g); + let s = word_cols(base + if half == 0 { cols::G_A1 } else { cols::G_A2 }); + let cbase = base + + if half == 0 { + cols::G_A1_C + } else { + cols::G_A2_C + }; + self.0.add3s.push(Add3Wire { + a, + b, + m: message_word_ref(m_idx), + s, + c1: cbase, + c2: cbase + 1, + }); + WordRef::Cols(s) + } + + fn add2(&mut self, g: usize, half: usize, a: WordRef, b: WordRef) -> WordRef { + let s = word_cols(cols::g_base(g) + if half == 0 { cols::G_C1 } else { cols::G_C2 }); + self.0.add2s.push(Add2Wire { a, b, s }); + WordRef::Cols(s) + } + + fn xor(&mut self, g: usize, slot: usize, a: WordRef, b: WordRef) -> WordRef { + let off = match slot { + 0 => cols::G_X1, + 1 => cols::G_X2, + 2 => cols::G_X3, + _ => cols::G_X4, + }; + let out = word_cols(cols::g_base(g) + off); + self.0.xors.push(XorWire { a, b, out }); + WordRef::Cols(out) + } + + fn rotr16(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(2) + } + + fn rotr8(&mut self, w: WordRef) -> WordRef { + w.rotr_bytes(1) + } + + fn rot_shift(&mut self, g: usize, half: usize, w: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_R1 } else { cols::G_R2 }; + let y = word_cols(base + 8); + self.0.rots.push(RotWire { + input: w, + sll_lo: [base, base + 1], + sllc_lo: [base + 2, base + 3], + sll_hi: [base + 4, base + 5], + sllc_hi: [base + 6, base + 7], + y, + r: ROT_SHIFT_R[half], + }); + WordRef::Cols(y) + } + + fn feed_forward_low(&mut self, i: usize, vi: WordRef, vi8: WordRef) { + self.0.xors.push(XorWire { + a: vi, + b: vi8, + out: word_cols(cols::out_byte(i, 0)), + }); + } + + /// ✗ Never called: [`FLOW`] has `full_output = false`. `out[i+8]` is not + /// part of a truncated 128-bit digest, and never building those twelve + /// words is where most of the socket's saving comes from. + fn feed_forward_high(&mut self, _i: usize, _vi8: WordRef, _hi: WordRef) { + unreachable!("the socket's truncation window produces no high output words") + } +} + +/// The socket's full wiring, in canonical order. Built from the single +/// dataflow, so the senders below and the witness written by +/// [`fill_socket_witness`] cannot drift apart. +fn socket_wires() -> WireFlow { + let mut w = SocketWire(WireFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + OUT_WINDOW), + rots: Vec::with_capacity(NUM_G * 2), + }); + run_flow(&mut w, FLOW); + w.0 +} + +/// The value interpretation of the same dataflow, for one row's twelve message +/// lanes in one domain. +/// +/// Lanes rather than cells because a LEAF row's lanes are not cells throughout — +/// lanes 0–3 are its accumulator and lanes 4–11 are four felts' halves. The +/// mixing core does not care which; it sees twelve `u32`s either way, and that is +/// exactly why the leaf mode needs no new layout. +/// +/// The tag is an input because it is a message word: it enters the very first +/// round's `add3` and every value downstream of it, so a row's witness and its +/// BITWISE lookups both depend on which domain the row hashes in. +fn socket_values(lanes: &[u32; cols::NUM_LANES], tag: u32) -> ValueFlow { + ValueFlow::compute_with( + &BLAKE3_IV, + &socket_message(lanes, tag), + COUNTER_LFMC, + BLOCK_LEN_LFMC, + FLAGS_LFMC, + FLOW, + ) +} + +// ========================================================================= +// Bus interactions — the BITWISE half of `chips::hash::bus_interactions` +// ========================================================================= + +fn direct(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: stark::lookup::Packing::Direct, + } +} + +fn byte_bus_value(b: ByteRef) -> BusValue { + match b { + ByteRef::Col(c) => direct(c), + ByteRef::Const(v) => BusValue::constant(u64::from(v)), + } +} + +/// The BITWISE lookups the BLAKE3 arm adds to `LFM_HASH`'s six `LfmMem` tuples. +/// +/// Three groups, in canonical [`socket_wires`] order: +/// +/// 1. `ByteAlu[XOR]` per XOR byte — the mixing core and the feed-forward. The +/// lookup pins the output *and* byte-range-checks both operands, which is +/// why nearly every word in this design needs no explicit `AreBytes`. +/// 2. `AreBytes` on the four shift halfwords of each rotation. The `SLL` bound +/// is tight and load-bearing: with `2^16` invertible mod `p` it is what pins +/// `SLL = (x · 2^r) mod 2^16` uniquely. +/// 3. `AreBytes` on the input lanes' bytes — obligation O1. These are the only +/// bytes with no XOR consumer, exactly as `m`'s are in `blake3_chip`. +pub fn bitwise_interactions() -> Vec { + let wires = socket_wires(); + let mut interactions = + Vec::with_capacity(4 * wires.xors.len() + 4 * wires.rots.len() + 2 * cols::NUM_LANES); + let mu = || { + Multiplicity::Sum3( + cols::MU_COLUMNS[0], + cols::MU_COLUMNS[1], + cols::MU_COLUMNS[2], + ) + }; + + for xw in &wires.xors { + for b in 0..4 { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + mu(), + vec![ + BusValue::constant(alu_op::XOR as u64), + byte_bus_value(xw.a.byte(b)), + byte_bus_value(xw.b.byte(b)), + direct(xw.out[b]), + ], + )); + } + } + + for rw in &wires.rots { + for pair in [rw.sll_lo, rw.sllc_lo, rw.sll_hi, rw.sllc_hi] { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + mu(), + vec![direct(pair[0]), direct(pair[1])], + )); + } + } + + for lane in 0..cols::NUM_LANES { + for p in 0..2 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + mu(), + vec![ + direct(cols::lane_byte(lane, 2 * p)), + direct(cols::lane_byte(lane, 2 * p + 1)), + ], + )); + } + } + + interactions +} + +/// The BITWISE lookups [`bitwise_interactions`] sends, mirrored send for send, +/// for the multiplicity histogram. Enumeration order is the senders' own, via +/// the shared [`ValueFlow`]. +/// +/// Each row is `(lanes, tag)`: the domain reaches the histogram because it +/// reaches the tag word, and every XOR byte downstream of round 0 differs between the +/// domains. A histogram built with the wrong tag balances against nothing. +pub fn bitwise_ops_for(rows: &[([u32; cols::NUM_LANES], u32)]) -> Vec { + let mut out = Vec::with_capacity( + rows.len() * (4 * (NUM_G * 4 + OUT_WINDOW) + 4 * NUM_G * 2 + 2 * cols::NUM_LANES), + ); + + for (lanes, tag) in rows { + let flow = socket_values(lanes, *tag); + for &(x, y, _out) in &flow.xors { + for byte in 0..4 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((x >> (8 * byte)) & 0xFF) as u8, + ((y >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + for &lane in lanes.iter() { + for p in 0..2 { + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((lane >> (16 * p)) & 0xFF) as u8, + ((lane >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + } + + out +} + +// ========================================================================= +// Trace +// ========================================================================= + +#[inline] +fn set_word_bytes(row: &mut [FE], col: usize, w: u32) { + for b in 0..4 { + row[col + b] = FE::from(u64::from((w >> (8 * b)) as u8)); + } +} + +/// Writes the BLAKE3 witness into a hash row whose `IN`/`S`/`OUT` columns are +/// already filled. +/// +/// The two input cells are read back out of the row's own `IN0..8` columns — +/// the exact cells the lane-decomposition constraints read — rather than from +/// the executor record, so the witness cannot describe a different input than +/// the one the AIR constrains. That is the discipline `fill_poseidon_witness` +/// established and it matters more here, because the lane boundary is where the +/// only new soundness surface lives. +/// +/// The DOMAIN is read back out of the row's own mode columns for the same +/// reason, and it is the half that matters most: the tag word is a linear form over +/// exactly those columns, so a witness built from them cannot describe a +/// different domain than the one the AIR evaluates. Taking the tag as an +/// argument — as this did at first — left a filler that could be handed the +/// wrong domain for a row whose selectors said otherwise. +/// +/// # Panics +/// +/// If a lane is not a `u32`, or if the row selects no domain this arm has a +/// socket for. `LfmHasher::admits` rejects both at execution, so reaching here +/// means the executor and the trace filler disagree. +pub fn fill_socket_witness(row: &mut [FE]) { + let tag = tag_from_row(row); + fill_socket_witness_tagged(row, tag); +} + +/// The row's domain tag, read off its preprocessed mode columns — the machine +/// side of [`TAG_SELECTOR`], and the same value the tag word evaluates to. +/// +/// # Panics +/// +/// If the row selects neither two-to-one domain. A padding row never reaches +/// the filler (`chip_trace` fills only real rows) and a permute row is +/// unprovable here, so either is a caller bug rather than a case to handle. +fn tag_from_row(row: &[FE]) -> u32 { + let one = FE::one(); + let set: Vec = TAG_SELECTOR + .iter() + .filter(|(col, _)| row[*col] == one) + .map(|(_, tag)| *tag) + .collect(); + match set[..] { + [tag] => tag, + _ => panic!( + "a BLAKE3 hash row must select EXACTLY ONE of the domains this arm \ + has a socket for (MODE_C, MODE_T, MODE_L). None set means a permute \ + or padding row reached the socket witness filler; more than one is a \ + row the registrar's one-hot check should already have refused. \ + Either way its AIR cannot prove the row." + ), + } +} + +/// The twelve message lanes of a row that reads `cells`, split the way its mode +/// reads them. +/// +/// ★ **The row is a HYBRID and this is the only place the split lives.** Lanes +/// 0–3 are `cells[0]` read as digest lanes under EVERY mode — `a` on a digest +/// row, the chaining accumulator on a leaf row. Above that the readings differ: +/// +/// - **digest modes** — lanes 4–11 are `cells[1]` and `cells[2]`, two more cells +/// of four `u32` lanes. The third is unread by these modes and the AIR pins it +/// to zero, so those four lanes are zero on every honest digest row. +/// - **leaf mode** — lanes 4–11 are `cells[1]` read as four FELTS and split into +/// `lo`/`hi` halves. +/// +/// The trace filler and the BITWISE histogram both come through here, so a +/// witness and the multiplicities it must balance against cannot split a row +/// differently (COMMIT.md §1.4.4 **H5**). +pub fn lanes_from_cells(is_leaf: bool, cells: &[LfmWord; 3]) -> [u32; cols::NUM_LANES] { + let mut lanes = [0u32; cols::NUM_LANES]; + lanes[..cols::NUM_ACC_LANES] + .copy_from_slice(&lanes_of(&cells[0]).expect("socket lane is not a u32 (O1)")); + if is_leaf { + lanes[cols::NUM_ACC_LANES..].copy_from_slice( + &leaf_lanes(&cells[1]).expect("leaf felt is not canonical (LEAF.md §1.1)"), + ); + } else { + for (k, cell) in cells[1..].iter().enumerate() { + let base = cols::NUM_ACC_LANES + 4 * k; + lanes[base..base + 4] + .copy_from_slice(&lanes_of(cell).expect("compress lane is not a u32 (O1)")); + } + } + lanes +} + +/// [`lanes_from_cells`] for a trace row, reading the input cells and the mode +/// off the row itself. +/// +/// Keyed on `MODE_L` rather than on a tag, so it is total for any row a control +/// can build — including one whose mode columns are fractional. +fn lanes_from_row(row: &[FE]) -> [u32; cols::NUM_LANES] { + let cell = |base: usize| -> LfmWord { core::array::from_fn(|i| row[base + i]) }; + lanes_from_cells( + row[cols::MODE_L] == FE::one(), + &[cell(cols::IN0), cell(cols::IN0 + 4), cell(cols::IN0 + 8)], + ) +} + +/// The canonicity witnesses for one leaf row's four felts. +/// +/// `Z_i = 1` exactly when felt `i`'s high half is maximal; `GINV_i` inverts +/// `G_i = (2^32 − 1) − hi_i` when that is nonzero and is zero when it is not. +/// The same `Z`/`GINV` pair `LFM_BITDEC` uses for its own canonicity check. +fn fill_canonicity_witness(row: &mut [FE], lanes: &[u32; cols::NUM_LANES]) { + for i in 0..FELTS_PER_LEAF { + let hi = lanes[cols::leaf_hi_lane(i)]; + let g = u64::from(MAX_HALF - hi); + let (z, ginv) = if g == 0 { + (FE::one(), FE::zero()) + } else { + ( + FE::zero(), + FE::from(g).inv().expect("a nonzero field element inverts"), + ) + }; + row[cols::canon_z(i)] = z; + row[cols::canon_ginv(i)] = ginv; + } +} + +/// [`fill_socket_witness`] under an EXPLICIT domain. +/// +/// Exists for the negative controls (M1/M2), which have to build a row whose +/// witness and whose mode columns deliberately disagree — the forgery the +/// domain separation is supposed to reject. Production goes through +/// [`fill_socket_witness`], which cannot construct that. +pub(crate) fn fill_socket_witness_tagged(row: &mut [FE], tag: u32) { + // The lane READING is a property of the row's mode; the `tag` argument is + // only the hash DOMAIN. Keeping them separate is what lets the mode- + // confusion controls build a row that reads its input correctly and hashes + // it in the wrong domain — which is the forgery, and it would be + // unconstructible if one argument decided both. + let lanes = lanes_from_row(row); + + for (lane, &v) in lanes.iter().enumerate() { + set_word_bytes(row, cols::lane_byte(lane, 0), v); + } + // The canonicity witnesses are filled on EVERY row, not only leaf rows: the + // columns exist chip-wide, and a digest row's felts are its lanes, whose + // high halves are never maximal-and-nonzero-low in a way that matters + // because the constraints are `MODE_L`-gated. Filling them uniformly keeps + // the filler branch-free and leaves no uninitialised witness anywhere. + fill_canonicity_witness(row, &lanes); + + let flow = socket_values(&lanes, tag); + let mut a3 = flow.add3s.iter(); + let mut a2 = flow.add2s.iter(); + let mut xo = flow.xors.iter(); + let mut ro = flow.rots.iter(); + for g in 0..NUM_G { + let base = cols::g_base(g); + for half in 0..2 { + let (s_off, c_off, x_off, c2_off, x2_off, r_off) = if half == 0 { + ( + cols::G_A1, + cols::G_A1_C, + cols::G_X1, + cols::G_C1, + cols::G_X2, + cols::G_R1, + ) + } else { + ( + cols::G_A2, + cols::G_A2_C, + cols::G_X3, + cols::G_C2, + cols::G_X4, + cols::G_R2, + ) + }; + let &(s, c1, c2) = a3.next().expect("add3 count"); + set_word_bytes(row, base + s_off, s); + row[base + c_off] = FE::from(u64::from(c1)); + row[base + c_off + 1] = FE::from(u64::from(c2)); + + let &(_, _, x) = xo.next().expect("xor count"); + set_word_bytes(row, base + x_off, x); + + let &c = a2.next().expect("add2 count"); + set_word_bytes(row, base + c2_off, c); + + let &(_, _, x2) = xo.next().expect("xor count"); + set_word_bytes(row, base + x2_off, x2); + + let &(sll_lo, sllc_lo, sll_hi, sllc_hi, y) = ro.next().expect("rot count"); + let hw = |v: u16, k: usize| FE::from(u64::from((v >> (8 * k)) as u8)); + for (k, v) in [sll_lo, sllc_lo, sll_hi, sllc_hi].into_iter().enumerate() { + row[base + r_off + 2 * k] = hw(v, 0); + row[base + r_off + 2 * k + 1] = hw(v, 1); + } + set_word_bytes(row, base + r_off + 8, y); + } + } + + for i in 0..OUT_WINDOW { + set_word_bytes(row, cols::out_byte(i, 0), flow.out[i]); + debug_assert_eq!( + row[cols::OUT0 + i], + FE::from(u64::from(flow.out[i])), + "the digest lane the executor wrote is the one the mixing core produced" + ); + } +} + +// ========================================================================= +// Constraints +// ========================================================================= + +/// Constraints the BLAKE3 arm emits. +/// +/// The framing block — 4 capacity copies, the mode-sum booleanity, the +/// `MODE_P = 0` pin, [`cols::NUM_LANES`] lane decompositions, 8 unused-output +/// pins, 4 digest recompositions, [`NUM_UNREAD_INPUT_PINS`] unread-`IN` pins and +/// 16 leaf felt/canonicity constraints — plus 16 per G-instance: per G, two +/// add3s (a sum identity and two carry booleanities each), two add2 carry +/// booleanities, and two rotations (two shift identities and two recombines +/// each). +pub const NUM_CONSTRAINTS: usize = CORE_IDX + 16 * NUM_G; + +/// First mixing-core constraint index — everything below it is framing. +const CORE_IDX: usize = LEAF_IDX + LEAF_CONSTRAINTS_PER_FELT * FELTS_PER_LEAF; + +/// First lane-decomposition index: after the capacity copies, the mode-sum +/// booleanity and the `MODE_P` pin. +/// +/// Public so the gate suite can name the identity for a specific lane rather +/// than locate it by a literal — the point of the H6 controls is that the +/// violated constraint IS the lane's own identity. +pub const LANE_IDX: usize = 6; + +/// First unused-output pin index. +/// +/// ★ **DERIVED FROM [`cols::NUM_LANES`], never written as a literal.** The lane +/// block grew from 8 to 12 with the leaf RATE, and the constraint COUNT did not +/// move — the unread-`IN` pins lost exactly the four the lanes gained — so a +/// hardcoded `14` here would have silently overwritten the first four output +/// pins with lane identities, leaving lanes 8–11 with no identity at all. +/// `EmitTracker`'s duplicate assert is `#[cfg(debug_assertions)]` and the house +/// convention runs the suite in release, so nothing would have failed +/// (COMMIT.md §1.4.4 **H1**). `blake3_socket_tests:: +/// every_hash_candidate_emits_each_constraint_index_exactly_once` is the +/// release-visible guard that would catch it if these ever go back to literals. +const OUT_PIN_IDX: usize = LANE_IDX + cols::NUM_LANES; + +/// First digest-recomposition index. +const DIGEST_IDX: usize = OUT_PIN_IDX + 8; + +/// First unread-`IN` pin index. +/// +/// Public so the controls can name the pins rather than locate them by a +/// literal — the point of those tests is that the violated set IS the pins. +pub const UNREAD_IDX: usize = DIGEST_IDX + OUT_WINDOW; + +/// First LEAF constraint index: four per felt (the halves binding and the three +/// canonicity constraints), after the shared unread-`IN` pins. +/// +/// Public so the controls can locate a specific canonicity constraint by name +/// rather than by a literal that silently rots when the framing grows. +pub const LEAF_IDX: usize = UNREAD_IDX + NUM_UNREAD_INPUT_PINS; + +/// Constraints per leaf felt: the halves binding, then `canon-a/b/c`. +pub const LEAF_CONSTRAINTS_PER_FELT: usize = 4; + +/// The BLAKE3 arm of `HashConstraints::eval`. +/// +/// Every constraint is mu-gated on `MU = MODE_C + MODE_T + MODE_L` and every +/// bus send carries the same sum, so an all-zero padding row satisfies the set +/// vacuously and emits nothing. Max degree is 3, reached by the mu-gated carry +/// booleanities and by the leaf canonicity block — the wrap's blowup 2 depends +/// on that staying 3, which is why the 3-operand add uses two summed carry BITS +/// rather than one ternary carry (`k(k−1)(k−2) = 0` is already degree 3, and +/// mu-gating would push it to 4). +pub fn eval>(b: &mut B) { + let mu = |b: &B| { + let [c, t, l] = cols::MU_COLUMNS; + b.main(0, c) + b.main(0, t) + b.main(0, l) + }; + let digest_mu = |b: &B| { + let [c, t] = cols::DIGEST_MODE_COLUMNS; + b.main(0, c) + b.main(0, t) + }; + let mode_c = b.main(0, cols::MODE_C); + let mode_t = b.main(0, cols::MODE_T); + let mode_l = b.main(0, cols::MODE_L); + let mode_p = b.main(0, cols::MODE_P); + + // idx 0–3: capacity-state copy, in the same shape every other arm uses — + // `S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i`. A transcript row + // and a leaf row are both compresses in framing, so their capacity prefix is + // still the IV; only the selector widens. With MODE_P pinned to zero below + // this reduces to `S_i = MU·IV_i`; it is written in the general form so the + // shared prefix means the same thing under every hasher. + for (k, iv) in BLAKE3_IV.iter().take(4).enumerate() { + let s = b.main(0, cols::S8 + k); + let in_i = b.main(0, cols::IN0 + 8 + k); + let iv_i = b.const_base(u64::from(*iv)); + let m = mu(b); + b.emit_base(k, s - (mode_p.clone() * in_i + m * iv_i)); + } + + // idx 4: mode sum-boolean (exactly-one-of is the registrar's). This is what + // excludes two selectors both being 1 — which would sum BOTH domain tags + // into the tag word — since the sum would be 2 and 2·(1−2) ≠ 0. ⚠ It does NOT + // force each selector to a bit: a fractional split still satisfies it and + // blends the tags, which is what control M5/M6 demonstrates and what the + // registrar's one-hot check is the actual answer to. + let mode_sum = mode_c + mode_t + mode_l.clone() + mode_p.clone(); + let one = b.one(); + b.emit_base(4, mode_sum.clone() * (one - mode_sum)); + + // idx 5: ✗ no permute socket, PERMANENTLY. Pinning the preprocessed mode + // selector makes a program containing a `permute` unprovable under BLAKE3 + // rather than silently proved against a framing nobody specified. Option B1 + // decided no permute socket is ever built, so this pin is not a placeholder + // waiting to be deleted — it is the decision, written down as a constraint. + b.emit_base(5, mode_p); + + // THE LANE BOUNDARY (obligation O1). One linear identity per input lane; the + // matching `AreBytes` sends are in `bitwise_interactions`. NEITHER ALONE + // SUFFICES, and the two buy DIFFERENT things — see the module docs. This + // identity makes `IN_lane` and `m[lane]` the same field element, because the + // core reads the same linear form; the sends bound the bytes, and are the + // message words' ONLY range check, which is what `add3`'s exactness needs. + // With both, the sum of four bytes weighted by 2^{8k} is < 2^32 ≪ p, so it + // cannot wrap and the lane is forced below 2^32. + // + // ★ THE GATE IS PER LANE RANGE, and getting it wrong in the permissive + // direction is a soundness break (COMMIT.md §1.4.4 **H6**): + // + // - **lanes 0–3 → the full mu.** They are `a` on a digest row and the + // chaining ACCUMULATOR on a leaf row, and the same identity is correct for + // both readings, so one gate serves. This is also the only thing that + // range-checks the accumulator: identity + `AreBytes` is what makes "the + // accumulator is a previous digest, hence `u32`" a constraint rather than + // a hope. Gate these on `digest_mu` and a leaf row's accumulator carries + // no identity at all — the prover picks the chain's message words freely + // and the whole leaf chain unbinds. + // - **lanes 4–11 → the digest modes only.** On a LEAF row they are four + // felts' halves, so `IN_lane` and `m[lane]` are deliberately NOT the same + // field element — the leaf block below states the relation those rows do + // satisfy. Gating these on mu instead would make every leaf row + // unprovable. + // + // ★ On a digest row lanes 8–11 read the THIRD input cell, which the unread- + // `IN` pins force to zero, so the identity reads `0 = Σ bytes·2^{8k}` and + // with the `AreBytes` bound in hand forces all sixteen bytes to zero. That + // is what keeps the four message words the leaf RATE added out of the + // prover's hands on a Merkle parent — free, but only because these + // identities exist. + for lane in 0..cols::NUM_LANES { + let felt = b.main(0, cols::IN0 + lane); + let bytes = word_expr(b, &WordRef::Cols(word_cols(cols::lane_byte(lane, 0)))); + let m = if lane < cols::NUM_ACC_LANES { + mu(b) + } else { + digest_mu(b) + }; + b.emit_base(LANE_IDX + lane, m * (felt - bytes)); + } + + // The digest is ONE cell, so the upper eight `OUT` lanes carry nothing. + // `MULT1`/`MULT2` are zero on a Compress row so they reach no bus, but + // pinning them costs eight degree-1 constraints and removes the question + // entirely. Ungated: they are zero on padding rows too. + for j in 0..8 { + let out = b.main(0, cols::OUT0 + HASH_DIGEST_FELTS + j); + b.emit_base(OUT_PIN_IDX + j, out); + } + + // The digest lanes. No range check is needed on `OUTW`'s bytes — they are + // `ByteAlu[XOR]` outputs, hence already bytes — and the sum is < 2^32 ≪ p, + // so `OUT_i` is forced to the honest u32. That is why the socket's OUTPUT + // always satisfies O1 (obligation O2) and only leaf digests and + // prover-hinted siblings need the input check. + for i in 0..OUT_WINDOW { + let felt = b.main(0, cols::OUT0 + i); + let bytes = word_expr(b, &WordRef::Cols(word_cols(cols::out_byte(i, 0)))); + let m = mu(b); + b.emit_base(DIGEST_IDX + i, m * (felt - bytes)); + } + + // The input cells this row's mode does not read. + // + // ⚠ **LOAD-BEARING, and not only here.** On THIS arm the unread columns + // reach no constraint, so the pin is what keeps them from being an open + // question. On an arm whose constraints read `IN` — `Test` and `Poseidon` + // both do, `A_i = IN_i` for `i < 8` — the same pin is the difference between + // a leaf digest that is a function of its input and one carrying four free + // prover-chosen felts. It shipped missing there once. That is why this is + // `chips::hash`'s single derivation from `HashMode::num_input_cells` and not + // four lines written out per arm. + let next = crate::lfm::chips::hash::emit_unread_input_pins(b, UNREAD_IDX); + debug_assert_eq!(next, LEAF_IDX); + + // ★ THE LEAF MODE. Per felt: the halves binding, then the three canonicity + // constraints. The felts are the SECOND input cell — the first is the + // chaining accumulator, whose lanes the identity block above binds. + // + // `v = lo + 2^32·hi` with `lo, hi < 2^32` is a decomposition, not yet a + // canonical one: `p − 1 = 0xFFFFFFFF_00000000`, so the pairs with `hi` + // maximal and `lo ≥ 1` encode field elements that ALSO have an ordinary + // encoding. Without the canonicity block one felt would have two half-pairs, + // hence two leaf digests — a collision in the felt→digest map, which is + // exactly what a Merkle tree must not have. + // + // `Z`/`GINV` is `LFM_BITDEC`'s own idiom over two halves instead of 64 bits: + // `canon_a` gives `G ≠ 0 ⇒ Z = 0`, `canon_b` gives `G = 0 ⇒ Z = 1`, and + // `canon_c` then reads "hi maximal ⇒ lo zero". + let two_32_leaf = b.const_base(1u64 << 32); + let max_half = b.const_base(u64::from(MAX_HALF)); + for i in 0..FELTS_PER_LEAF { + let lo = word_expr( + b, + &WordRef::Cols(word_cols(cols::lane_byte(cols::leaf_lo_lane(i), 0))), + ); + let hi = word_expr( + b, + &WordRef::Cols(word_cols(cols::lane_byte(cols::leaf_hi_lane(i), 0))), + ); + let v = b.main(0, cols::leaf_felt(i)); + let z = b.main(0, cols::canon_z(i)); + let ginv = b.main(0, cols::canon_ginv(i)); + let g = max_half.clone() - hi.clone(); + let base = LEAF_IDX + LEAF_CONSTRAINTS_PER_FELT * i; + + // binding: the felt IS its two halves. + b.emit_base( + base, + mode_l.clone() * (v - lo.clone() - hi * two_32_leaf.clone()), + ); + // canon-a: G ≠ 0 ⇒ Z = 0. + b.emit_base(base + 1, mode_l.clone() * z.clone() * g.clone()); + // canon-b: G = 0 ⇒ Z = 1. + let one = b.one(); + b.emit_base(base + 2, mode_l.clone() * (one - z.clone() - g * ginv)); + // canon-c: hi maximal ⇒ lo zero. THE constraint; the two above exist to + // make `Z` mean what this one needs it to mean. + b.emit_base(base + 3, mode_l.clone() * z * lo); + } + + // The mixing core, from the single dataflow. + let wires = socket_wires(); + let mut idx = CORE_IDX; + + let two_32 = b.const_base(1u64 << 32); + let inv_2_32 = b.const_base(crate::constraints::templates::INV_SHIFT_32); + + // add3: μ·(a + b + m − s − 2^32·(c1+c2)) = 0; μ·ci·(1−ci) = 0. + for aw in &wires.add3s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let m_w = word_expr(b, &aw.m); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let c1 = b.main(0, aw.c1); + let c2 = b.main(0, aw.c2); + let sum_id = a + bb + m_w - s - (c1.clone() + c2.clone()) * two_32.clone(); + let m = mu(b); + b.emit_base(idx, m * sum_id); + idx += 1; + for c in [c1, c2] { + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c.clone() * (one - c)); + idx += 1; + } + } + + // add2: the carry is an EXPRESSION, `(a + b − s)·2^−32`, not a column — + // `μ·carry·(1−carry) = 0` says `a + b − s ∈ {0, 2^32}`, which is the sum + // identity and the carry's booleanity in one degree-3 constraint. One + // column and one constraint per add2 cheaper than witnessing the carry, and + // exactly as strong. + for aw in &wires.add2s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let carry = (a + bb - s) * inv_2_32.clone(); + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * carry.clone() * (one - carry)); + idx += 1; + } + + // Rotations: two shift identities and two recombines each. Soundness needs + // 2^16 invertible mod p, which is a FIELD fact a bitvector model cannot + // see — it is what makes the tight `AreBytes` bound on `SLL` load-bearing. + for rw in &wires.rots { + let (xlo, xhi) = match &rw.input { + WordRef::Cols(c) => (half_expr(b, &[c[0], c[1]]), half_expr(b, &[c[2], c[3]])), + WordRef::Const(_) | WordRef::ModeSelected(_) => { + unreachable!("shift inputs are always committed XOR outputs") + } + }; + let sll_lo = half_expr(b, &rw.sll_lo); + let sllc_lo = half_expr(b, &rw.sllc_lo); + let sll_hi = half_expr(b, &rw.sll_hi); + let sllc_hi = half_expr(b, &rw.sllc_hi); + let ylo = half_expr(b, &[rw.y[0], rw.y[1]]); + let yhi = half_expr(b, &[rw.y[2], rw.y[3]]); + let two_r = b.const_base(1u64 << rw.r); + let two_16 = b.const_base(65536); + + let m = mu(b); + b.emit_base( + idx, + m * (xlo * two_r.clone() - sllc_lo.clone() * two_16.clone() - sll_lo.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base( + idx, + m * (xhi * two_r - sllc_hi.clone() * two_16 - sll_hi.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (ylo - sll_hi - sllc_lo)); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (yhi - sll_lo - sllc_hi)); + idx += 1; + } + + debug_assert_eq!( + idx, NUM_CONSTRAINTS, + "every declared constraint index must be emitted exactly once" + ); +} diff --git a/prover/src/lfm/blake3_socket_kats.rs b/prover/src/lfm/blake3_socket_kats.rs new file mode 100644 index 000000000..a188300a3 --- /dev/null +++ b/prover/src/lfm/blake3_socket_kats.rs @@ -0,0 +1,144 @@ +//! Socket KATs for the LFM 2-to-1 BLAKE3 compress, at 6 and 7 rounds. +//! +//! GENERATED — do not hand-edit. The INPUT pairs are the union of two +//! independently produced vector tables: +//! `thoughts/blake3/socket-kats/socket_kats.json` (Phase 1, upstream BLAKE3's C +//! at word level + its whole tree hasher at byte level) and the gate-oracle's +//! `socket_kats.json` (a separately written Python oracle). The two share 5 of +//! the 15 input pairs and agreed on every one of them at both round counts, +//! which is what makes this table two sources rather than one transcribed +//! twice. +//! +//! ⚠ **The DIGESTS were re-pinned when the socket widened to twelve lanes**, by +//! `leaf-spec/rate4_kat_gen.py` out of that same gate-oracle BLAKE3. All 30 +//! moved and no input did: `block_len` is `v[14]` and cannot be made +//! mode-dependent, so the Merkle domain re-blesses alongside the leaf domain +//! that needed the width (COMMIT.md §1.4.4 H9). The upstream-C leg of the +//! provenance is therefore historical for the digests and current for the +//! inputs. What still anchors the new values outside this tree is that a +//! 52-byte message is one BLAKE3 block, so at 7 rounds each vector remains a +//! plain `blake3::hash` call. +//! +//! Framing (`SOCKET.md` §2.2 at the COMMIT.md §1.2 width): h = BLAKE3_IV, +//! m[0..4] = a, m[4..8] = b, m[8..12] = 0 — the third input cell, which the +//! unread-`IN` pins force to zero — m[12] = "LFMC", m[13..16] = 0, t = 0, +//! block_len = 52, flags = 0x0B, digest = out[0..4]. + +/// One socket vector: the two input cells and the digest at each round count. +pub struct SocketVector { + pub name: &'static str, + pub a: [u32; 4], + pub b: [u32; 4], + /// Digest at 6 rounds (the A6R variant; no library computes it). + pub digest_6: [u32; 4], + /// Digest at 7 rounds — `blake3::hash(a ‖ b ‖ "LFMC")[..16]`. + pub digest_7: [u32; 4], +} + +pub const SOCKET_VECTORS: [SocketVector; 15] = [ + SocketVector { + name: "a_one/unit_a", // socket-kats+gate-oracle + a: [0x00000001, 0x00000000, 0x00000000, 0x00000000], + b: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + digest_6: [0x34064AA0, 0xD7155685, 0x37B1522B, 0x17147454], + digest_7: [0xFD482C6D, 0xF3D43A7D, 0xECE55DC3, 0xA2594A53], + }, + SocketVector { + name: "all_ones/all_ones", // socket-kats+gate-oracle + a: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + b: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + digest_6: [0xCCEF3565, 0x52BAE2BB, 0x4C0E5777, 0x2896C5CB], + digest_7: [0x6C73358D, 0x7AC15CE6, 0xDB7BFA9C, 0x65CD3364], + }, + SocketVector { + name: "b_one/unit_b", // socket-kats+gate-oracle + a: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + b: [0x00000001, 0x00000000, 0x00000000, 0x00000000], + digest_6: [0x3B0F1AEB, 0x7FBAC531, 0xF7576FDB, 0xC4054075], + digest_7: [0x43B017F6, 0x35DA5810, 0x1C2F8BF8, 0xB16F9B88], + }, + SocketVector { + name: "boundary", // gate-oracle + a: [0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF], + b: [0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF], + digest_6: [0x03FA80ED, 0x79B929E7, 0x5FAF60C4, 0xB1F6E5C2], + digest_7: [0x518B26CE, 0xDD289FC7, 0x5E623AAC, 0xD189075A], + }, + SocketVector { + name: "formula_0", // socket-kats + a: [0x9E3779B9, 0x3C6EF372, 0xDAA66D2B, 0x78DDE6E4], + b: [0x8FF34781, 0x2E2AC13A, 0xCC623AF3, 0x6A99B4AC], + digest_6: [0x937642E5, 0x03E78AFE, 0x85C6D9CD, 0x42824862], + digest_7: [0xAB67E603, 0x5D251992, 0xCC7CE527, 0xD4EF91ED], + }, + SocketVector { + name: "formula_1", // socket-kats + a: [0x81AF1549, 0x1FE68F02, 0xBE1E08BB, 0x5C558274], + b: [0x736AE311, 0x11A25CCA, 0xAFD9D683, 0x4E11503C], + digest_6: [0xACE60EA5, 0x8135DE83, 0xEAE0B6DA, 0x9934A2F0], + digest_7: [0x653C2C04, 0x78EF5846, 0x6A736E8A, 0x914D4DB9], + }, + SocketVector { + name: "formula_1", // gate-oracle + a: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + b: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + digest_6: [0xF227539A, 0x2585193F, 0x16FB766E, 0x4B577722], + digest_7: [0xADED3052, 0x697F82FA, 0x7AEA4B01, 0x04BFCDBC], + }, + SocketVector { + name: "formula_2", // socket-kats + a: [0x6526B0D9, 0x035E2A92, 0xA195A44B, 0x3FCD1E04], + b: [0x56E27EA1, 0xF519F85A, 0x93517213, 0x3188EBCC], + digest_6: [0xC594FDD3, 0xBB338F05, 0x33C5A455, 0x2E9D6C14], + digest_7: [0xE87B6708, 0xCE220AD3, 0x5B64ED4F, 0xD8D574C2], + }, + SocketVector { + name: "formula_2", // gate-oracle + a: [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE], + b: [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD], + digest_6: [0x7636E806, 0x8AAB225F, 0x7F947CE5, 0xA73023AF], + digest_7: [0xDBD49162, 0x7A22E380, 0x81E2ECB6, 0xF29F4F76], + }, + SocketVector { + name: "formula_3", // socket-kats + a: [0x489E4C69, 0xE6D5C622, 0x850D3FDB, 0x2344B994], + b: [0x3A5A1A31, 0xD89193EA, 0x76C90DA3, 0x1500875C], + digest_6: [0x4C93E057, 0xA31827EB, 0xFDE7CB52, 0x027F1933], + digest_7: [0x5754C2B6, 0x35C665F2, 0xFFB72630, 0xEC470985], + }, + SocketVector { + name: "formula_3", // gate-oracle + a: [0x7F800001, 0x00000002, 0x80000000, 0x7FFFFFFF], + b: [0x00FF00FF, 0xFF00FF00, 0x0F0F0F0F, 0xF0F0F0F0], + digest_6: [0xFEB2EB59, 0xD8D6F7E1, 0xDC2774D3, 0x09A984FE], + digest_7: [0x5213902B, 0xC0A84BF1, 0xA070DC22, 0x531B944C], + }, + SocketVector { + name: "formula_4", // socket-kats + a: [0x2C15E7F9, 0xCA4D61B2, 0x6884DB6B, 0x06BC5524], + b: [0x1DD1B5C1, 0xBC092F7A, 0x5A40A933, 0xF87822EC], + digest_6: [0x312C20F4, 0x077F08FF, 0x0608FFAF, 0x70423FD2], + digest_7: [0xF910DA3B, 0x5CCD211F, 0xB6D1E097, 0xA7304D4D], + }, + SocketVector { + name: "max_min", // gate-oracle + a: [0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000], + b: [0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF], + digest_6: [0xDCF8F70F, 0x18645178, 0xE0842849, 0x97FEC771], + digest_7: [0x5B047362, 0xE1662BCF, 0x385D410A, 0xFD185A9E], + }, + SocketVector { + name: "nibble_ramp/nibble_ramp", // socket-kats+gate-oracle + a: [0x00000000, 0x11111111, 0x22222222, 0x33333333], + b: [0x44444444, 0x55555555, 0x66666666, 0x77777777], + digest_6: [0xA218F925, 0x7819A69C, 0xC14B82EC, 0x60E8A949], + digest_7: [0x2E24E9F5, 0xA0D24A4B, 0x53909030, 0xB64285F3], + }, + SocketVector { + name: "zeros/zeros", // socket-kats+gate-oracle + a: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + b: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + digest_6: [0x9E1DF680, 0x17A425B2, 0x890775EA, 0xE2C6E09F], + digest_7: [0x9484C177, 0xBA11AECD, 0x45BB9F21, 0x031F727D], + }, +]; diff --git a/prover/src/lfm/blake3_socket_tests.rs b/prover/src/lfm/blake3_socket_tests.rs new file mode 100644 index 000000000..aae104038 --- /dev/null +++ b/prover/src/lfm/blake3_socket_tests.rs @@ -0,0 +1,1637 @@ +//! The BLAKE3 arm of `LFM_HASH`: its framing, its layout, its degree bound, +//! what it accepts, what it rejects, and the prove+verify that turns a +//! predicted cell count into a measured one. +//! +//! ## What pins what +//! +//! Three layers, and they are deliberately not the same evidence: +//! +//! 1. **The primitive** is pinned elsewhere, to the `blake3` crate: +//! `blake3::tests::seven_rounds_is_the_blake3_crate`. Nothing here re-checks +//! the G function or the message schedule. +//! 2. **The framing** — the six choices between "a correct `f`" and "a correct +//! 2-to-1 compress" — is pinned here by [`SOCKET_VECTORS`], which came from +//! two independent generators, plus one negative control per choice. A right +//! constant inside a wrong framing is the normal way this goes wrong, and +//! every primitive test stays green while it happens. +//! 3. **The chip** is pinned here too: that `NUM_CONSTRAINTS` constraints over +//! `MAIN_COLUMNS` value columns say exactly what that framing says, and that +//! they say it inside a real proof produced by the production prover. +//! +//! ## What this suite cannot see +//! +//! It says nothing about the machine's DEFAULT hash, which is still +//! `TestPermutation`; every test constructs the BLAKE3 configuration +//! explicitly. It covers the two two-to-one modes — `compress` and the +//! `transcript` step, which are the same socket under different domain tags — +//! and no `permute`, because option B1 settled that no permute socket is ever +//! built. See `blake3_socket`'s module docs. +//! +//! The transcript's own vectors and its end-to-end behaviour live in +//! `transcript_tests`; what is here is the CHIP side of it — the mode-selected +//! tag, and the M1–M7 controls the transcript spec pre-committed. + +use math::field::element::FieldElement; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, ProverEvalFolder, RootKind, check_dense_index_set, + num_base_from_meta, +}; +use stark::frame::Frame; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::table::TableView; +use stark::trace::TraceTable; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField, VmTable}; + +use super::airs::lfm_chip_census_with_hasher; +use super::blake3::{BLAKE3_IV, BLAKE3_MSG_PERMUTATION}; +use super::blake3_socket::{ + self, BLOCK_LEN_LFMC, Blake3Permutation, COUNTER_LFMC, FLAGS_LFMC, MAIN_COLUMNS, + NUM_CONSTRAINTS, NUM_G, SOCKET_ROUNDS, TAG_LFMC, cols, lanes_of, socket_digest, + socket_digest_rounds, word_of, +}; +use super::blake3_socket_kats::SOCKET_VECTORS; +use super::builder::{Cell, LfmBuilder, LfmProgramSource}; +use super::chips::hash::{self, HashConstraints}; +use super::compiler::{LfmProgram, compile}; +use super::executor::{LfmExecError, execute}; +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::instr::HashMode; +use super::programs::{permute_coverage_program, trivial_program}; +use super::proof::{lfm_prove_with_hasher, prove_traces_with_hasher, verify_against}; +use super::registry::{build_artifacts, build_artifacts_with_hasher}; +use super::trace::build_traces_with_hasher; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +const KIND: HasherKind = HasherKind::Blake3; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +// ========================================================================= +// The closed-form budget, written out for BOTH round counts +// ========================================================================= + +/// Value columns per compression, as a function of the round count. +/// +/// Written as a formula over named blocks rather than taken from the layout, +/// because a closed form taken from the code under test would agree with any +/// layout, including a wrong one. 28 shared prefix + `4·12` lane bytes + +/// `8·rounds` G-blocks of 60 + 16 digest bytes + 8 leaf-canonicity witnesses +/// (`Z`/`GINV` per felt — present on EVERY row, since a chip has one width). +/// +/// ★ The lane count is spelled `4 + 2·4` — the accumulator cell plus one felt +/// cell's halves — rather than read from `cols::NUM_LANES`, for exactly the +/// reason the rest is spelled out: taking it from the layout would make this +/// agree with ANY widening instead of with the one the RATE specifies. Note +/// what does NOT move with it: the canonicity witnesses stay at 8, because the +/// accumulator is a previous digest and needs byte decomposition but no +/// canonicity gate. That is the claim that made the RATE cost +16 columns +/// rather than +24. +const fn predicted_main(rounds: usize) -> usize { + 28 + 4 * (4 + 2 * 4) + 60 * (8 * rounds) + 16 + 8 +} + +/// Bus interactions per compression: the frozen six `LfmMem` tuples, four +/// `ByteAlu[XOR]` per XOR word (`4·8·rounds` mixing words + 4 feed-forward), +/// four `AreBytes` per rotation (`2·8·rounds` of them), and two lane `AreBytes` +/// per lane. +const fn predicted_interactions(rounds: usize) -> usize { + 6 + 4 * (4 * (8 * rounds) + 4) + 4 * (2 * (8 * rounds)) + 2 * (4 + 2 * 4) +} + +/// `main + 3·aux` with `aux = ceil(interactions / 2)` — `airs.rs`'s census +/// formula, the same instrument that produced the keccak, Poseidon and +/// standalone-blake columns, so all four are comparable by construction. +/// +/// `pub(super)` so [`super::blake3_probe`] can state the socket-vs-standalone +/// comparison against THIS number instead of a transcription of it. The copy it +/// carried had drifted by 8 — it predated the leaf mode's canonicity block — and +/// a hand-copied cost figure is exactly the kind that rots in silence, because +/// nothing recomputes it. +pub(super) const fn predicted_cells(rounds: usize) -> usize { + predicted_main(rounds) + 3 * predicted_interactions(rounds).div_ceil(2) +} + +const fn predicted_constraints(rounds: usize) -> usize { + 50 + 16 * (8 * rounds) +} + +/// The whole budget, at both round counts, as literals. +/// +/// These are the numbers the report carries and the A6R decision is priced +/// against, so they are written out rather than left as an expression: the +/// arithmetic and the layout are two statements, and a test is only worth +/// having if they can disagree. +#[test] +fn the_socket_budget_is_the_predicted_one_at_both_round_counts() { + // 6 rounds — the A6R variant. + assert_eq!(predicted_main(6), 2_980); + assert_eq!(predicted_interactions(6), 1_198); + assert_eq!(predicted_interactions(6).div_ceil(2), 599); + assert_eq!(predicted_cells(6), 4_777); + assert_eq!(predicted_constraints(6), 818); + + // 7 rounds — standard BLAKE3, the default. + assert_eq!(predicted_main(7), 3_460); + assert_eq!(predicted_interactions(7), 1_390); + assert_eq!(predicted_interactions(7).div_ceil(2), 695); + assert_eq!(predicted_cells(7), 5_545); + assert_eq!(predicted_constraints(7), 946); + + // ★★ WHAT THE LEAF RATE COST, priced here rather than asserted anywhere + // else: +16 main columns (four lanes × four bytes), +8 bus interactions + // (two `AreBytes` per new lane) and +28 census cells at 7 rounds, against a + // 2.0× cut in leaf absorption — which is ~70% of a recursion tower node's + // bill (COMMIT.md §1.4.1). The pre-RATE figures were 3,444 / 1,382 / 5,517. + // The CONSTRAINT count did not move at all, and that is not luck: the lane + // identities gained four while the unread-`IN` pins lost four, which is + // precisely why a hand-numbered framing block could have overwritten the + // pins in silence (§1.4.4 H1). + assert_eq!(predicted_main(7) - (28 + 32 + 60 * 56 + 16 + 8), 16); + assert_eq!(predicted_cells(7) - 5_517, 28); + assert_eq!(predicted_constraints(7), 946, "unchanged by the widening"); + + // ★ The A6R price, on this socket: going 6 → 7 rounds costs +16.07% per + // compression. The plan's paper estimate for the syscall-shaped chip was + // +15.5%; the socket pays slightly more because its constant framing makes + // the round-INDEPENDENT part smaller, so the rounds are a larger share. + assert_eq!( + (predicted_cells(7) - predicted_cells(6)) * 10_000 / predicted_cells(6), + 1_607, + "hundredths of a percent" + ); + + // Both are BELOW the standalone chip's measured 4,946, which is the point of + // hosting: constant `h`/`t`/`block_len`/`flags`, a constant message tail, and a + // truncation window that never builds twelve of the sixteen output words. + assert!(predicted_cells(6) < 4_946); +} + +/// The compiled arm IS the prediction at the round count it was compiled for. +#[test] +fn the_built_layout_matches_the_prediction() { + // The single-knob invariant, asserted where it can actually fail: the + // socket and the standalone probe must be priced at the SAME round count. + // `NUM_G == 8 * SOCKET_ROUNDS` below is internally consistent either way, + // so it cannot see the two chips drifting apart; this can. + assert_eq!(SOCKET_ROUNDS, super::blake3::BLAKE3_ROUNDS); + assert_eq!( + NUM_G, + super::blake3_chip::NUM_G, + "the socket arm and the standalone LFM_BLAKE3 probe must be compiled \ + for the same round count, or the probe prices a hash the machine does \ + not use" + ); + assert_eq!(NUM_G, 8 * SOCKET_ROUNDS); + assert_eq!(MAIN_COLUMNS, predicted_main(SOCKET_ROUNDS)); + assert_eq!( + hash::num_columns(KIND) - cols::PREP_WIDTH, + predicted_main(SOCKET_ROUNDS) + ); + assert_eq!( + hash::bus_interactions(KIND).len(), + predicted_interactions(SOCKET_ROUNDS) + ); + assert_eq!(NUM_CONSTRAINTS, predicted_constraints(SOCKET_ROUNDS)); + // 12 since option B1 added `MODE_T` (was 11). The prefix is the hasher- + // independent instruction group, so this number is the same under every + // candidate — `poseidon_chip_tests` pins the identical value, and the two + // together are what would catch one arm's layout drifting from the other's. + assert_eq!( + cols::PREP_WIDTH, + 13, + "the preprocessed prefix does not move" + ); + assert_eq!(cols::LANES, 41, "the shared value prefix is not reflowed"); +} + +/// The layout is injective and gapless — no column written twice, none unread. +/// +/// The width alone cannot see an off-by-one inside `lane_byte`/`g_base`/ +/// `out_byte`: two blocks could overlap and the total still come out right. +#[test] +fn the_layout_assigns_every_column_exactly_once() { + let mut seen = vec![0usize; cols::NUM_COLUMNS]; + let mut claim = |c: usize| seen[c] += 1; + for i in 0..HASH_STATE_FELTS { + claim(cols::IN0 + i); + claim(cols::OUT0 + i); + } + for k in 0..4 { + claim(cols::S8 + k); + } + for lane in 0..cols::NUM_LANES { + for b in 0..4 { + claim(cols::lane_byte(lane, b)); + } + } + for g in 0..NUM_G { + for off in 0..cols::G_SIZE { + claim(cols::g_base(g) + off); + } + } + for i in 0..4 { + for b in 0..4 { + claim(cols::out_byte(i, b)); + } + } + for i in 0..blake3_socket::FELTS_PER_LEAF { + claim(cols::canon_z(i)); + claim(cols::canon_ginv(i)); + } + for (c, &n) in seen.iter().enumerate().skip(cols::PREP_WIDTH) { + assert_eq!( + n, 1, + "value column {c} is claimed {n} times, want exactly 1" + ); + } + for (c, &n) in seen.iter().enumerate().take(cols::PREP_WIDTH) { + assert_eq!(n, 0, "preprocessed column {c} must not be claimed"); + } +} + +/// The census reports the arm at its real width and its real interaction count, +/// so the hash-matrix instrument prices BLAKE3 rather than a stale Test column. +#[test] +fn the_census_prices_the_blake3_arm() { + let opts = options(); + let program = compress_program(); + let census = lfm_chip_census_with_hasher(&program, KIND); + let hash_chip = census + .iter() + .find(|c| c.name == "LFM_HASH") + .expect("LFM_HASH is in the census"); + assert_eq!(hash_chip.main_cols, predicted_main(SOCKET_ROUNDS)); + assert_eq!( + hash_chip.aux_cols, + predicted_interactions(SOCKET_ROUNDS).div_ceil(2) + ); + assert_eq!( + hash_chip.main_cols + 3 * hash_chip.aux_cols, + predicted_cells(SOCKET_ROUNDS), + "base-field-equivalent cells per compression row" + ); + let _ = opts; +} + +// ========================================================================= +// The framing — SOCKET.md §2, and the controls that make it discriminating +// ========================================================================= + +/// Every framing degree of freedom, in one object, so a negative control can +/// break exactly one at a time and nothing else. +#[derive(Clone, Copy)] +struct Framing { + rounds: usize, + cv: [u32; 8], + tag_word: u32, + counter: u64, + block_len: u32, + flags: u32, + a_slot: usize, + b_slot: usize, + tag_slot: usize, + out_window: usize, + lane_le: bool, + msg_permutation: [usize; 16], +} + +const HONEST: Framing = Framing { + rounds: SOCKET_ROUNDS, + cv: BLAKE3_IV, + tag_word: TAG_LFMC, + counter: COUNTER_LFMC, + block_len: BLOCK_LEN_LFMC, + flags: FLAGS_LFMC, + a_slot: 0, + b_slot: 4, + // Straight after the twelve lanes, not at a fixed 8: the tag is the last + // word of the message under every lane count, which is what keeps the byte + // string `LE32(lanes) ‖ tag` (COMMIT.md §1.2). + tag_slot: 12, + out_window: 0, + lane_le: true, + msg_permutation: BLAKE3_MSG_PERMUTATION, +}; + +/// The message words under a framing. A big-endian lane serialisation changes +/// the message WORDS, because a word is read little-endian from the bytes. +fn framed_message(a: &[u32; 4], b: &[u32; 4], fr: Framing) -> [u32; 16] { + let lane = |v: u32| if fr.lane_le { v } else { v.swap_bytes() }; + let mut m = [0u32; 16]; + for i in 0..4 { + m[fr.a_slot + i] = lane(a[i]); + m[fr.b_slot + i] = lane(b[i]); + } + m[fr.tag_slot] = fr.tag_word; + m +} + +/// A deliberately *parameterised* socket compress, used only to build negative +/// controls: the same dataflow with [`Framing`] as an input. +/// +/// It is NOT what `socket_digest_rounds` calls. Keeping the two apart costs a +/// duplicated loop and buys the thing the controls are for — they compare +/// against [`SOCKET_VECTORS`], constants that came from outside this file, so +/// they stay meaningful no matter how the real function is later refactored. +fn framed_digest(a: &[u32; 4], b: &[u32; 4], fr: Framing) -> [u32; 4] { + let g = |s: &mut [u32; 16], ia: usize, ib: usize, ic: usize, id: usize, mx: u32, my: u32| { + s[ia] = s[ia].wrapping_add(s[ib]).wrapping_add(mx); + s[id] = (s[id] ^ s[ia]).rotate_right(16); + s[ic] = s[ic].wrapping_add(s[id]); + s[ib] = (s[ib] ^ s[ic]).rotate_right(12); + s[ia] = s[ia].wrapping_add(s[ib]).wrapping_add(my); + s[id] = (s[id] ^ s[ia]).rotate_right(8); + s[ic] = s[ic].wrapping_add(s[id]); + s[ib] = (s[ib] ^ s[ic]).rotate_right(7); + }; + let mut v: [u32; 16] = [ + fr.cv[0], + fr.cv[1], + fr.cv[2], + fr.cv[3], + fr.cv[4], + fr.cv[5], + fr.cv[6], + fr.cv[7], + BLAKE3_IV[0], + BLAKE3_IV[1], + BLAKE3_IV[2], + BLAKE3_IV[3], + fr.counter as u32, + (fr.counter >> 32) as u32, + fr.block_len, + fr.flags, + ]; + let mut m = framed_message(a, b, fr); + for r in 0..fr.rounds { + g(&mut v, 0, 4, 8, 12, m[0], m[1]); + g(&mut v, 1, 5, 9, 13, m[2], m[3]); + g(&mut v, 2, 6, 10, 14, m[4], m[5]); + g(&mut v, 3, 7, 11, 15, m[6], m[7]); + g(&mut v, 0, 5, 10, 15, m[8], m[9]); + g(&mut v, 1, 6, 11, 12, m[10], m[11]); + g(&mut v, 2, 7, 8, 13, m[12], m[13]); + g(&mut v, 3, 4, 9, 14, m[14], m[15]); + if r < fr.rounds - 1 { + let prev = m; + for (i, &p) in fr.msg_permutation.iter().enumerate() { + m[i] = prev[p]; + } + } + } + let w = fr.out_window; + core::array::from_fn(|i| v[w + i] ^ v[w + i + 8]) +} + +/// Everything `f` actually sees under a framing: the initial state, the message +/// schedule at *every* round, and the output window. +/// +/// Two framings with equal traces compute equal digests *necessarily*, so a +/// control whose trace equals the honest one on some input is genuinely +/// INAPPLICABLE there rather than undetected — which is what lets the control +/// suite assert "changes the digest" unconditionally everywhere else. Deriving +/// applicability this way rather than hand-listing it is deliberate: a +/// hand-list goes stale as controls are added, and a stale entry is a control +/// that looks covered and is not. +/// +/// The schedules, not the permutation, are what belong here. `a_one` is the +/// case that proves it: its message has `m[2] = m[6] = 0`, so transposing the +/// first two entries of the permutation produces the identical schedule and the +/// control cannot possibly fire. +fn effective_trace( + a: &[u32; 4], + b: &[u32; 4], + fr: Framing, +) -> (usize, [u32; 8], u64, u32, u32, usize, Vec<[u32; 16]>) { + let mut sched = framed_message(a, b, fr); + let mut scheds = Vec::with_capacity(fr.rounds); + for r in 0..fr.rounds { + scheds.push(sched); + if r < fr.rounds - 1 { + let prev = sched; + for (i, &p) in fr.msg_permutation.iter().enumerate() { + sched[i] = prev[p]; + } + } + } + ( + fr.rounds, + fr.cv, + fr.counter, + fr.block_len, + fr.flags, + fr.out_window, + scheds, + ) +} + +/// ★ **The socket KATs.** The real function reproduces every vector at both +/// round counts. +#[test] +fn the_socket_matches_the_vectors_at_both_round_counts() { + for v in SOCKET_VECTORS.iter() { + assert_eq!( + socket_digest_rounds(&v.a, &v.b, 6), + v.digest_6, + "6-round socket vector {}", + v.name + ); + assert_eq!( + socket_digest_rounds(&v.a, &v.b, 7), + v.digest_7, + "7-round socket vector {}", + v.name + ); + } + // And the compiled-in round count is one of the two, reaching the vectors + // through the entry point the chip and the host actually call. + for v in SOCKET_VECTORS.iter() { + let expected = if SOCKET_ROUNDS == 7 { + v.digest_7 + } else { + v.digest_6 + }; + assert_eq!(socket_digest(&v.a, &v.b), expected, "vector {}", v.name); + } +} + +/// ★ **The external anchor, direct.** At 7 rounds the socket is literally +/// `blake3::hash(a ‖ b ‖ "LFMC")` truncated to 16 bytes — a library call, no +/// oracle, no JSON. +/// +/// This is what SOCKET.md §6 lists as ✗ DEFERRED ("the same equality against +/// the Rust `blake3` crate — needs cargo"). It also re-derives the 52-byte +/// message from the byte-level specification rather than from +/// `socket_message`, so the word-level and byte-level forms are two statements +/// that can disagree. +/// +/// ★ **The anchor is what the leaf RATE had to keep.** The message grew by the +/// third input cell's four zero lanes — 36 bytes to 52 — and 52 is still under +/// 64, so a row is still ONE block and still a plain library call. Carrying the +/// leaf's accumulator in the chaining value `h` instead would have made the row +/// a chunk CONTINUATION and thrown this test away for the same rate. +#[test] +fn seven_rounds_is_blake3_of_the_domain_separated_message() { + for v in SOCKET_VECTORS.iter() { + let mut msg = Vec::with_capacity(52); + for lane in v.a.iter().chain(v.b.iter()) { + msg.extend_from_slice(&lane.to_le_bytes()); + } + // The third input cell: unread by a digest mode, and pinned to zero by + // the unread-`IN` pins, so its four lanes are zero on every honest row. + msg.extend_from_slice(&[0u8; 16]); + msg.extend_from_slice(b"LFMC"); + assert_eq!(msg.len(), 52, "the socket message is one 52-byte block"); + assert!(msg.len() < 64, "and one block is what the anchor needs"); + + let full = blake3::hash(&msg); + let want: [u32; 4] = core::array::from_fn(|i| { + u32::from_le_bytes(full.as_bytes()[4 * i..4 * i + 4].try_into().unwrap()) + }); + assert_eq!( + socket_digest_rounds(&v.a, &v.b, 7), + want, + "7-round socket vector {} must be blake3::hash of its message", + v.name + ); + assert_eq!(want, v.digest_7, "the table itself agrees with the crate"); + } +} + +/// The parameterised control, at canonical parameters, IS the real function — +/// so every control below differs in exactly the one choice it names. +#[test] +fn the_framing_variant_at_canonical_parameters_is_the_socket() { + for v in SOCKET_VECTORS.iter() { + assert_eq!( + framed_digest(&v.a, &v.b, HONEST), + socket_digest(&v.a, &v.b), + "control harness must reproduce the socket at canonical parameters" + ); + } +} + +/// NEGATIVE CONTROL, one per framing degree of freedom. +/// +/// Without this, "the vectors pass" would be evidence only that the vectors are +/// *reachable*, not that they discriminate — and framing is precisely where a +/// correct `f` still gives a wrong hash. Each control must change the digest on +/// every vector where its effective trace differs from the honest one, and must +/// discriminate on at least one vector overall. +#[test] +fn breaking_one_framing_choice_at_a_time_breaks_the_digest() { + let mut transposed = [0usize; 16]; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + transposed[p] = i; + } + let controls: [(&str, Framing); 14] = [ + ( + "swap_a_b", + Framing { + a_slot: 4, + b_slot: 0, + ..HONEST + }, + ), + ( + "tag_changed", + Framing { + tag_word: u32::from_le_bytes(*b"LFMP"), + ..HONEST + }, + ), + ( + "tag_omitted", + Framing { + tag_word: 0, + ..HONEST + }, + ), + ( + // 8 — where the tag sat before the socket widened to twelve lanes, + // so this control is also the discrimination between the two + // framings: a chip that widened the lanes and left the tag behind + // would compute this, and it is a different hash. + "tag_slot_moved", + Framing { + tag_slot: 8, + ..HONEST + }, + ), + ( + "truncate_high_half", + Framing { + out_window: 4, + ..HONEST + }, + ), + ("flags_parent", Framing { flags: 4, ..HONEST }), + ("flags_no_root", Framing { flags: 3, ..HONEST }), + ( + "block_len_64", + Framing { + block_len: 64, + ..HONEST + }, + ), + ( + "block_len_32", + Framing { + block_len: 32, + ..HONEST + }, + ), + ( + "counter_one", + Framing { + counter: 1, + ..HONEST + }, + ), + ( + "cv_zero", + Framing { + cv: [0; 8], + ..HONEST + }, + ), + ( + "lanes_big_endian", + Framing { + lane_le: false, + ..HONEST + }, + ), + ( + "msg_perm_swapped", + Framing { + msg_permutation: { + let mut p = BLAKE3_MSG_PERMUTATION; + p.swap(0, 1); + p + }, + ..HONEST + }, + ), + ( + "other_round_count", + Framing { + rounds: if SOCKET_ROUNDS == 7 { 6 } else { 7 }, + ..HONEST + }, + ), + ]; + + for (what, fr) in controls { + let mut discriminated = 0; + for v in SOCKET_VECTORS.iter() { + let honest = socket_digest(&v.a, &v.b); + if effective_trace(&v.a, &v.b, fr) == effective_trace(&v.a, &v.b, HONEST) { + // Provably inapplicable on this input — `swap_a_b` when a == b, + // `lanes_big_endian` when every lane is a byte-palindrome, + // `msg_perm_swapped` when the two transposed slots hold equal + // words. Asserted as an equality, not skipped: an inapplicable + // control must produce the SAME digest, which is a check in its + // own right on the applicability derivation. + assert_eq!(framed_digest(&v.a, &v.b, fr), honest); + continue; + } + assert_ne!( + framed_digest(&v.a, &v.b, fr), + honest, + "{what} still reproduces the digest on vector {} — the vectors do not pin it", + v.name + ); + discriminated += 1; + } + assert!( + discriminated > 0, + "{what} is discriminated by no vector at all" + ); + } +} + +/// The transposed message permutation is a real permutation and a different +/// one — otherwise `msg_perm_swapped` above would be testing nothing. +#[test] +fn the_message_permutation_control_is_a_different_permutation() { + let mut p = BLAKE3_MSG_PERMUTATION; + p.swap(0, 1); + assert_ne!(p, BLAKE3_MSG_PERMUTATION); + let mut sorted = p; + sorted.sort_unstable(); + assert_eq!(sorted, core::array::from_fn::(|i| i)); +} + +// ========================================================================= +// The lane boundary — obligation O1, host side +// ========================================================================= + +/// ★ **O1, host side.** An out-of-range lane is REJECTED, never reduced. +/// +/// `edsl::merkle_walk` feeds `compress` arena-hinted — prover-chosen — sibling +/// cells, and a lane is a Goldilocks felt over `[0, p)`. The chip can only +/// commit a byte decomposition for a lane below `2^32`, so a host that reduced +/// instead of rejecting would claim a digest no proof can produce. +#[test] +fn an_out_of_range_lane_is_rejected_rather_than_reduced() { + let ok: LfmWord = word_of(&[1, 2, 3, 4]); + assert_eq!(lanes_of(&ok), Some([1, 2, 3, 4])); + + // The alias that would exist under silent reduction. + let aliased: LfmWord = [ + FE::from(1u64 + (1u64 << 32)), + FE::from(2u64), + FE::from(3u64), + FE::from(4u64), + ]; + assert_eq!(lanes_of(&aliased), None, "2^32 + 1 is not a u32 lane"); + + let mut state = [FE::zero(); HASH_STATE_FELTS]; + state[0..4].copy_from_slice(&aliased); + state[4..8].copy_from_slice(&ok); + assert!( + Blake3Permutation + .admits(HashMode::Compress, &state) + .is_err(), + "a non-u32 lane must be refused by admits" + ); + + // HONEST CONTROL: the in-range pair is still accepted. Without it, this + // test would pass equally if `admits` rejected everything. + let mut good = [FE::zero(); HASH_STATE_FELTS]; + good[0..4].copy_from_slice(&ok); + good[4..8].copy_from_slice(&ok); + assert!(Blake3Permutation.admits(HashMode::Compress, &good).is_ok()); +} + +/// The whole-machine version of the same thing: an arena word with a non-`u32` +/// lane makes the program fail to execute, with a reason. +#[test] +fn a_non_u32_arena_word_fails_execution_under_blake3() { + let program = compress_program(); + let mut bad = arenas(); + bad[0][0][0] = FE::from(1u64 << 32); + assert!( + matches!( + execute(&program, &bad, &KIND), + Err(LfmExecError::HasherRejected(_)) + ), + "a non-u32 hinted lane must be rejected at execution" + ); + // HONEST CONTROL. + assert!(execute(&program, &arenas(), &KIND).is_ok()); +} + +/// Obligation O2: the socket is closed on its own output, so a digest fed back +/// in as a sibling always satisfies O1. That is why only leaf digests and +/// prover-hinted siblings need the input check. +#[test] +fn the_socket_output_is_always_a_valid_input() { + for v in SOCKET_VECTORS.iter() { + let d = socket_digest(&v.a, &v.b); + assert_eq!( + lanes_of(&word_of(&d)), + Some(d), + "a socket digest must round-trip as a u32-lane cell" + ); + } +} + +/// Obligation O3: the IV enters through `h`, not through the capacity lanes, so +/// this arm overrides `compress` rather than inheriting permute-and-truncate. +/// +/// Asserted through `HasherKind`'s dispatch, because that is the path the +/// executor takes and a candidate whose override was not honoured there would +/// prove one thing and record another. +#[test] +fn compress_is_overridden_and_the_upper_out_lanes_are_empty() { + let a = word_of(&[0x0102_0304, 0, 0, 0]); + let b = word_of(&[0, 0, 0, 0x0506_0708]); + let via_kind = LfmHasher::compress(&KIND, &a, &b); + assert_eq!( + via_kind, + word_of(&socket_digest( + &[0x0102_0304, 0, 0, 0], + &[0, 0, 0, 0x0506_0708] + )) + ); + + let out = LfmHasher::compress_out(&KIND, &a, &b); + assert_eq!(&out[0..4], &via_kind[..]); + for (j, felt) in out.iter().enumerate().skip(4) { + assert_eq!( + *felt, + FE::zero(), + "OUT lane {j} carries nothing on a compress row" + ); + } + + // `compress_iv` is meaningful if read, and is NOT what the framing uses. + assert_eq!( + LfmHasher::compress_iv(&KIND), + word_of(&[BLAKE3_IV[0], BLAKE3_IV[1], BLAKE3_IV[2], BLAKE3_IV[3]]) + ); +} + +/// ✗ There is no permute socket, and the refusal is explicit rather than a +/// wrong answer. `permute_coverage_program` contains one, so it is unprovable +/// under BLAKE3 — which is the settled state of option B1, not a defect. +/// +/// The program under test used to be `trivial_program`, which no longer has a +/// permute in it: B1 gave the registry's entries up to the real hash, and this +/// test moved to the unregistered fixture that took over permute coverage. +#[test] +fn a_permute_row_is_refused_under_blake3() { + assert!( + Blake3Permutation + .admits(HashMode::Permute, &[FE::zero(); HASH_STATE_FELTS]) + .is_err() + ); + assert!( + matches!( + execute(&permute_coverage_program(), &permute_arenas(), &KIND), + Err(LfmExecError::HasherRejected(_)) + ), + "a program containing a permute must be refused under BLAKE3" + ); + // HONEST CONTROL: the same program executes fine under the hashers that do + // have a permute socket, so the refusal is BLAKE3's domain and not a break. + assert!( + execute( + &permute_coverage_program(), + &permute_arenas(), + &HasherKind::Test + ) + .is_ok() + ); +} + +/// ★ The F3.4-retirement milestone at the registry level: `TrivialV0` — a +/// REGISTERED program — now executes under BLAKE3, which it could not while it +/// held a raw permute. +/// +/// Its arena has to be `u32`-laned (obligation O1); that is the socket's domain, +/// not a property of this program. +#[test] +fn the_trivial_program_runs_under_blake3_now_that_it_has_no_permute() { + assert!( + !trivial_program().instrs.iter().any( + |i| matches!(i, super::instr::Instr::Hash { mode, .. } if *mode == HashMode::Permute) + ), + "a registered program must not contain a permute" + ); + let arenas = vec![ + (0..4u32) + .map(|i| word_of(&[0x1000_0000 * (i + 1), 0x0BAD_F00D ^ i, i, 0xFFFF_FFFF - i])) + .collect(), + ]; + execute(&trivial_program(), &arenas, &KIND).expect("TrivialV0 executes under BLAKE3"); + // HONEST CONTROL: still fine under the default hasher, on its own arenas. + execute(&trivial_program(), &trivial_arenas(), &HasherKind::Test).expect("and under Test"); +} + +// ========================================================================= +// The constraints +// ========================================================================= + +/// Every constraint index is emitted exactly once, the count is the one the +/// module documents, and the degree really reaches — and does not exceed — 3. +#[test] +fn the_arm_emits_its_constraints_at_degree_3() { + let set = HashConstraints::BLAKE3; + assert_eq!(HashConstraints::num_constraints(KIND), NUM_CONSTRAINTS); + + let meta = ConstraintSet::::meta(&set); + assert_eq!(meta.len(), NUM_CONSTRAINTS, "constraints emitted"); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "meta must be dense and idx-ordered"); + assert_eq!(m.kind, RootKind::Base, "every hash constraint is base"); + } + + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), NUM_CONSTRAINTS, "one emit per constraint"); + + let declared = ConstraintSet::::max_degree(&set); + assert_eq!(declared, 3, "the wrap's blowup 2 depends on this staying 3"); + for &(idx, measured) in °rees { + assert!( + measured <= declared, + "constraint {idx}: measured degree {measured} EXCEEDS declared {declared}" + ); + } + // Not merely `<=`: the mu-gated carry booleanities really are cubic, so a + // set that quietly topped out at 2 would mean the carries had stopped being + // constrained. + assert_eq!(degrees.iter().map(|&(_, d)| d).max(), Some(3)); +} + +/// The mode column a row in `mode` sets. +pub(super) fn mode_col(mode: HashMode) -> usize { + match mode { + HashMode::Compress => cols::MODE_C, + HashMode::Transcript => cols::MODE_T, + HashMode::Leaf => cols::MODE_L, + HashMode::Permute => cols::MODE_P, + } +} + +/// A hash row in `mode`, exactly as `trace::build_traces_with_hasher` fills +/// one. +fn hash_row_mode(mode: HashMode, a: [u32; 4], b: [u32; 4]) -> Vec { + let tag = blake3_socket::tag_for_mode(mode).expect("BLAKE3 has a socket for this mode"); + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[mode_col(mode)] = FE::one(); + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&word_of(&a)); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(&word_of(&b)); + for (k, iv) in BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = blake3_socket::socket_digest_rounds_tagged(&a, &b, SOCKET_ROUNDS, tag); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, tag); + row +} + +/// A `Compress` row — the shape most of these tests are about. +fn hash_row(a: [u32; 4], b: [u32; 4]) -> Vec { + hash_row_mode(HashMode::Compress, a, b) +} + +fn evaluate(row: &[FE]) -> Vec { + let set = HashConstraints::BLAKE3; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out +} + +pub(super) fn violations(row: &[FE]) -> Vec { + evaluate(row) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +/// An honest row satisfies every constraint, and the digest it carries is the +/// KAT's — so the constraint set and the vectors agree about the same row. +#[test] +fn an_honest_row_satisfies_every_constraint() { + for v in SOCKET_VECTORS.iter() { + let row = hash_row(v.a, v.b); + assert_eq!(violations(&row), Vec::::new(), "vector {}", v.name); + let want = if SOCKET_ROUNDS == 7 { + v.digest_7 + } else { + v.digest_6 + }; + for (i, lane) in want.iter().enumerate() { + assert_eq!(row[cols::OUT0 + i], FE::from(u64::from(*lane))); + for byte in 0..4 { + assert_eq!( + row[cols::out_byte(i, byte)], + FE::from(u64::from((lane >> (8 * byte)) as u8)), + "digest byte ({i}, {byte}) of vector {}", + v.name + ); + } + } + } +} + +/// An all-zero padding row satisfies the set, and a row claiming to be real +/// with no witness does not. +/// +/// The second half is what stops the first from being vacuous: a constraint set +/// that accepted anything would pass the padding check just as well. +#[test] +fn padding_is_satisfied_and_a_real_marked_empty_row_is_not() { + assert_eq!( + violations(&vec![FE::zero(); cols::NUM_COLUMNS]), + Vec::::new(), + "an all-zero padding row must satisfy every constraint" + ); + + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_C] = FE::one(); + assert!( + !violations(&row).is_empty(), + "a real-marked row with an all-zero witness must be rejected" + ); +} + +/// The `MODE_P = 0` pin: a permute-marked row is rejected by the AIR itself, +/// independently of the executor's refusal. +#[test] +fn a_permute_marked_row_violates_the_air() { + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_P] = FE::one(); + assert!( + !violations(&row).is_empty(), + "MODE_P = 1 must be unsatisfiable under the BLAKE3 arm" + ); +} + +/// ★ **The lane-decomposition constraint bites.** Retagging a lane's bytes to a +/// different value, or the lane felt to `v + 2^32`, must violate the AIR. +/// +/// The second case is the identity's own job: a lane moved without its bytes. +/// See `the_lane_range_check_is_load_bearing_on_its_own` for the other half — +/// the witness this identity cannot see, which is what the `AreBytes` sends +/// are for. +#[test] +fn the_lane_decomposition_binds_the_felt_to_its_bytes() { + let base = hash_row([0x1234_5678, 1, 2, 3], [4, 5, 6, 7]); + + let mut tampered = base.clone(); + tampered[cols::lane_byte(0, 0)] += FE::one(); + assert!( + !violations(&tampered).is_empty(), + "moving a lane byte must violate the decomposition" + ); + + let mut aliased = base.clone(); + aliased[cols::IN0] += FE::from(1u64 << 32); + assert!( + !violations(&aliased).is_empty(), + "lane + 2^32 must violate the decomposition — this IS obligation O1" + ); + + // HONEST CONTROL. + assert_eq!(violations(&base), Vec::::new()); +} + +/// ★ **O1's OTHER half — the one the rest of this suite never exercises.** +/// +/// The lane contract is an eval constraint AND two `AreBytes` sends, and the +/// module comment says "NEITHER ALONE SUFFICES". Every other control here +/// breaks the linear identity, which the identity alone catches — so until this +/// test existed, the `AreBytes` half was asserted in prose and exercised +/// nowhere. +/// +/// The witness that separates them moves `2^8` from one byte column into the +/// next (`MB[0] += 256`, `MB[1] -= 1`). The weighted sum is unchanged +/// **exactly**, over the field, with no borrow — `256·(b1 − 1) + (b0 + 256) = +/// 256·b1 + b0` — so the lane identity passes, the message word the mixing core +/// reads is the same linear form and therefore also unchanged, and the honest +/// digest still comes out. Nothing in the eval set is wrong with the row. The +/// only defect is that `MB[0]` is no longer a byte, and only the range check +/// can see that. +/// +/// Recorded because it is not what I expected and it sharpens the argument: +/// **a carry-absorbing witness cannot be silent, because the lane bytes ARE the +/// message bytes.** Trying to alias a lane to `v + 2^32` and letting `MB[3]` +/// absorb the carry does satisfy the lane identity — and then breaks the mixing +/// core instead, because the word the core hashes moved by `2^32` too. So the +/// alias is caught either way; what the range check uniquely buys is the case +/// where the *sum* is preserved. +/// +/// And that case is not a curiosity — it is the whole attack surface. The +/// message words reach `add3` and nothing else (never an XOR), so these sends +/// are their ONLY bound. A sum-preserving witness is exactly the door to an +/// unbounded `m`, and `add3`'s exactness in round 0 — constant `a` and `b`, a +/// byte-bounded `s` — is what an unbounded `m` breaks: the prover solves for +/// any `s` it likes and owns the compression. +#[test] +fn the_lane_range_check_is_load_bearing_on_its_own() { + let base = hash_row([0x1234_5678, 1, 2, 3], [4, 5, 6, 7]); + /// Constraint index of input lane 0's decomposition (idx 6–13 are the eight + /// lanes); `CORE_IDX` is 26, so anything below it is framing. + const LANE0: usize = 6; + + // (a) THE ONE ONLY `AreBytes` CATCHES. Identity preserved, core preserved, + // eval set entirely silent. If this half ever starts failing, the proof + // below has stopped testing the range check and has become a duplicate of + // `the_lane_decomposition_binds_the_felt_to_its_bytes`. + let mut shifted = base.clone(); + shifted[cols::lane_byte(0, 0)] += FE::from(256u64); + shifted[cols::lane_byte(0, 1)] = shifted[cols::lane_byte(0, 1)] - FE::one(); + assert_eq!( + violations(&shifted), + Vec::::new(), + "the linear identity alone cannot see a byte column carrying 2^8 — \ + which is exactly why the AreBytes sends are not optional" + ); + + // (b) The naive alias: claim `v + 2^32` and leave the bytes alone. The + // IDENTITY catches this one, at lane 0's own index. + let mut naive = base.clone(); + naive[cols::IN0] += FE::from(1u64 << 32); + assert!( + violations(&naive).contains(&LANE0), + "a lane moved without its bytes must violate its own decomposition" + ); + + // (c) The alias with the carry absorbed, which is the interesting one: the + // lane identity is satisfied — `LANE0` is NOT among the violations — and the + // MIXING CORE rejects instead, because `MB[3]` is a message byte and the + // word being hashed moved by 2^32 as well. + let mut absorbed = base.clone(); + absorbed[cols::IN0] += FE::from(1u64 << 32); + absorbed[cols::lane_byte(0, 3)] += FE::from(256u64); + let v = violations(&absorbed); + assert!( + !v.contains(&LANE0), + "absorbing the carry must satisfy the lane identity — otherwise this \ + case is not demonstrating what it claims" + ); + assert!( + v.iter().all(|&i| i >= 26) && !v.is_empty(), + "and the mixing core must reject it instead, got {v:?}" + ); + + // (d) In a real proof, the `AreBytes` send catches (a). Only the byte + // shuffle is used: it leaves `IN0` untouched, so the `LfmMem` receive token + // is unchanged and the rejection can only come from the range check, not + // from a memory-bus mismatch. + assert_not_accepted("a byte column carrying 2^8, identity preserved", |t| { + let b0 = t.main_table.get_row(0)[cols::lane_byte(0, 0)]; + let b1 = t.main_table.get_row(0)[cols::lane_byte(0, 1)]; + t.main_table + .set_fe(0, cols::lane_byte(0, 0), b0 + FE::from(256u64)); + t.main_table + .set_fe(0, cols::lane_byte(0, 1), b1 - FE::one()); + }); +} + +/// The digest recomposition binds `OUT` to the mixing core's output bytes. +#[test] +fn the_digest_recomposition_binds_out_to_the_core() { + let base = hash_row([9, 8, 7, 6], [5, 4, 3, 2]); + let mut tampered = base.clone(); + tampered[cols::OUT0] += FE::one(); + assert!(!violations(&tampered).is_empty()); + + let mut upper = base.clone(); + upper[cols::OUT0 + 4] = FE::one(); + assert!( + !violations(&upper).is_empty(), + "the unused upper OUT lanes are pinned to zero" + ); + assert_eq!(violations(&base), Vec::::new()); +} + +// ========================================================================= +// M1–M7 — the mode-selected tag, PRE-COMMITTED controls +// +// Named in the transcript spec §5.3 before this chip existed, so they are +// inherited obligations rather than tests written to fit what got built. Each +// one is paired with an honest-path assertion: "the bad row is rejected" passes +// just as well when every row is rejected. +// ========================================================================= + +/// **M1 — a transcript row that hashed under the MERKLE tag is rejected.** +/// +/// Spec form: "`m[8]` pinned to `TAG_LFMC` while `MODE_T = 1` — SAT", i.e. in a +/// model where the tag is free, a transcript row can compute the Merkle +/// function. On the real chip the tag is NOT free, so the same statement is a +/// rejection, and that is what is asserted here. +#[test] +fn m1_a_transcript_row_computing_the_merkle_tag_is_rejected() { + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + let mut row = hash_row_mode(HashMode::Transcript, a, b); + // Recompute the whole witness under the WRONG domain, leaving MODE_T set. + let digest = socket_digest(&a, &b); // "LFMC" + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, TAG_LFMC); + assert!( + !violations(&row).is_empty(), + "a MODE_T row carrying the Merkle computation must be rejected" + ); + + // HONEST CONTROL: the same row under its own domain satisfies everything. + assert_eq!( + violations(&hash_row_mode(HashMode::Transcript, a, b)), + Vec::::new() + ); +} + +/// **M2 — the mirror: a compress row that hashed under the TRANSCRIPT tag is +/// rejected.** Both directions, because a one-directional separation is not one. +#[test] +fn m2_a_compress_row_computing_the_transcript_tag_is_rejected() { + let (a, b) = ([1u32, 2, 3, 4], [5u32, 6, 7, 8]); + let mut row = hash_row_mode(HashMode::Compress, a, b); + let digest = blake3_socket::transcript_digest(&a, &b); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, blake3_socket::TAG_LFMT); + assert!( + !violations(&row).is_empty(), + "a MODE_C row carrying the transcript computation must be rejected" + ); + + assert_eq!( + violations(&hash_row_mode(HashMode::Compress, a, b)), + Vec::::new() + ); +} + +/// **M3 — both mode bits set on one row is unsatisfiable**, and it is the +/// mode-sum booleanity (idx 4) that says so. +/// +/// This is the constraint that stops `m[8]` being `TAG_LFMC + TAG_LFMT`, a tag +/// in neither domain. +#[test] +fn m3_both_two_to_one_modes_on_one_row_is_unsatisfiable() { + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + let mut row = hash_row_mode(HashMode::Compress, a, b); + row[cols::MODE_T] = FE::one(); + assert!( + violations(&row).contains(&4), + "idx 4 — the mode-sum booleanity — must be the constraint that fires" + ); + + // HONEST CONTROL: clearing it again restores an accepted row. + row[cols::MODE_T] = FE::zero(); + assert_eq!(violations(&row), Vec::::new()); +} + +/// **M4 — the mu gate IS the sum of the two two-to-one selectors**, so it +/// cannot be 1 while both are 0. +/// +/// Structural rather than algebraic: on this chip `MU` is not a column a row +/// could set independently, it is the expression `MODE_C + MODE_T`. The test +/// pins that, and pins the consequence — with both zero the row is padding, it +/// satisfies the set vacuously and its bus sends carry multiplicity zero. +#[test] +fn m4_the_mu_gate_is_exactly_the_two_to_one_selector_sum() { + assert_eq!(cols::MU_COLUMNS, [cols::MODE_C, cols::MODE_T, cols::MODE_L]); + + // A row with garbage in every witness column but no mode set is padding. + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + let mut row = hash_row_mode(HashMode::Compress, a, b); + row[cols::MODE_C] = FE::zero(); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&[0, 0, 0, 0])); + for k in 0..4 { + row[cols::S8 + k] = FE::zero(); + } + assert_eq!( + violations(&row), + Vec::::new(), + "with no mode set the row is padding and every mu-gated constraint is vacuous" + ); + + // HONEST CONTROL: it is vacuous because it is UNGATED-satisfiable, not + // because the set accepts anything — restoring the mode makes the same + // garbage row fail. + row[cols::MODE_C] = FE::one(); + assert!(!violations(&row).is_empty()); +} + +/// **M5/M6 — ⚠ the AIR alone does not pin the tag; the PREPROCESSED binding +/// does.** This is the control that turns §3.3 from an assertion into a checked +/// claim, and it fires. +/// +/// Constraint idx 4 pins the mode SUM to a bit, not each selector to a bit. So +/// a row with `MODE_C = x`, `MODE_T = 1 − x` satisfies it for every field +/// element `x`, and `m[8]` becomes `x·"LFMC" + (1−x)·"LFMT"` — which, solving +/// for `x`, is **any 32-bit value the prover likes**. This test picks the tag +/// `"XXXX"`, derives the `x` that produces it, and shows the constraint set +/// ACCEPTS the resulting row. +/// +/// Two mechanisms stop a real prover doing this, and neither is in this file's +/// constraint set: +/// +/// - the mode columns are **preprocessed**, fixed by the row's position in a +/// trace whose commitment is folded into `lfm_program_id`; and +/// - the admission validator's one-hot check rejects any program whose +/// `LFM_HASH` group carries a non-boolean selector. +/// +/// Both are asserted below, so this is a live demonstration of *why* they are +/// load-bearing rather than a latent hole. +#[test] +fn m5_m6_the_mode_columns_must_be_preprocessed_or_the_tag_is_prover_chosen() { + const FORGED_TAG: u32 = u32::from_le_bytes(*b"XXXX"); + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + + // x such that x·LFMC + (1−x)·LFMT = FORGED_TAG. + let lfmc = FE::from(u64::from(TAG_LFMC)); + let lfmt = FE::from(u64::from(blake3_socket::TAG_LFMT)); + let x = (FE::from(u64::from(FORGED_TAG)) - &lfmt) + * (&lfmc - &lfmt).inv().expect("the two tags differ"); + + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_C] = x; + row[cols::MODE_T] = FE::one() - x; + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&word_of(&a)); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(&word_of(&b)); + for (k, iv) in BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = blake3_socket::socket_digest_rounds_tagged(&a, &b, SOCKET_ROUNDS, FORGED_TAG); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, FORGED_TAG); + + assert_eq!( + violations(&row), + Vec::::new(), + "⚠ the constraint set alone accepts a prover-chosen domain tag — the \ + mode columns being preprocessed is what stops this" + ); + assert_ne!( + digest, + socket_digest(&a, &b), + "the forged domain really is a different function" + ); + + // MECHANISM 1: the mode columns are inside the preprocessed prefix, so a + // prover supplies neither. + const { assert!(cols::MODE_C < cols::PREP_WIDTH) }; + const { assert!(cols::MODE_T < cols::PREP_WIDTH) }; + const { assert!(cols::MODE_P < cols::PREP_WIDTH) }; + + // MECHANISM 2: the admission validator rejects a non-one-hot selector, so + // the program above cannot be registered even if a prover could write it. + let mut program = compress_program(); + let g = &mut program.groups.hash; + let row0 = 0; + g.data[row0 * g.width + super::layout::hash::MODE_C] = x; + g.data[row0 * g.width + super::layout::hash::MODE_T] = FE::one() - x; + assert!( + matches!( + super::validator::validate(&program), + Err(super::validator::LfmViolation::NonOneHotSelector { + chip: "LFM_HASH", + .. + }) + ), + "the registrar must reject a fractional mode selector" + ); + + // HONEST CONTROL: the untouched program is admissible, so the rejection + // above is about the tampering and not about the program. + assert!(super::validator::validate(&compress_program()).is_ok()); +} + +/// **M7 — the capacity constraints (idx 0–3) bite on a transcript row.** +/// +/// A transcript row is still a compress, so its capacity prefix is still the +/// IV — the selector widened to `MODE_C + MODE_T` and nothing else did. If the +/// widening had been forgotten, a transcript row's `S` would be pinned to zero +/// instead of to the IV, and this is the test that would have said so. +#[test] +fn m7_the_capacity_constraints_bite_on_a_transcript_row() { + let (a, b) = ([9u32, 8, 7, 6], [5u32, 4, 3, 2]); + for k in 0..4 { + let mut row = hash_row_mode(HashMode::Transcript, a, b); + row[cols::S8 + k] += FE::one(); + assert_eq!( + violations(&row), + vec![k], + "a wrong capacity lane must violate exactly constraint {k}" + ); + } + + // A transcript row's capacity is the IV, same as a compress row's — the two + // differ in `m[8]` and in nothing else. + let transcript = hash_row_mode(HashMode::Transcript, a, b); + let compress = hash_row_mode(HashMode::Compress, a, b); + for k in 0..4 { + assert_eq!(transcript[cols::S8 + k], compress[cols::S8 + k]); + assert_eq!(transcript[cols::S8 + k], FE::from(u64::from(BLAKE3_IV[k]))); + } + assert_eq!(violations(&transcript), Vec::::new()); +} + +/// The degree bound survives the mode-selected tag: `m[8]` went from degree 0 +/// to degree 1, and the wrap's blowup 2 depends on the maximum staying 3. +#[test] +fn the_mode_selected_tag_does_not_raise_the_degree() { + let set = HashConstraints::BLAKE3; + assert_eq!(ConstraintSet::::max_degree(&set), 3); + // Measured, not declared — `the_declared_degree_bound_is_respected` walks + // the captured IR; this asserts the declaration it checks against. +} + +// ========================================================================= +// Prove and verify — rule 2: this is what makes the numbers measurements +// ========================================================================= + +/// A compress-only program: two leaf merges and a parent merge. +/// +/// This is the shape the socket exists for — `edsl::merkle_walk`'s parent +/// compression — and it exercises obligation O2 as well, since `d0` and `d1` are +/// socket outputs fed straight back in as inputs. `trivial_program` cannot be +/// used: it contains a `permute`, which BLAKE3 has no socket for. +fn compress_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(4); + let h: Vec = (0..4).map(|i| b.hint_word(arena, i)).collect(); + let d0 = b.compress(h[0].as_digest(), h[1].as_digest()); + let d1 = b.compress(h[2].as_digest(), h[3].as_digest()); + let root = b.compress(d0, d1); + b.public(root.as_cell()); + b.finish() +} + +fn compress_program() -> LfmProgram { + compile(compress_program_source()) +} + +/// Four arena words whose lanes are `u32`s — the socket's domain (O1). +fn arenas() -> Vec> { + vec![ + (0..4u32) + .map(|i| word_of(&[0x1000_0000 * (i + 1), 0x0BAD_F00D ^ i, i, 0xFFFF_FFFF - i])) + .collect(), + ] +} + +/// `permute_coverage_program`'s arenas — three state cells. +fn permute_arenas() -> Vec> { + vec![ + (0..3u64) + .map(|i| core::array::from_fn(|j| FE::from(500 * (i + 1) + j as u64))) + .collect(), + ] +} + +/// `trivial_program`'s arenas — arbitrary felts, which is exactly why BLAKE3 +/// cannot take them. +fn trivial_arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +/// ★ The production prover builds this AIR, proves a program through it, and +/// the production verifier accepts. +#[test] +fn the_blake3_socket_proves_and_verifies() { + let opts = options(); + let program = compress_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, KIND) + .expect("proving under BLAKE3 must succeed"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest BLAKE3-configured proof must verify" + ); + + // The public output is the Merkle root the socket computed, recomputed here + // from the vectors' own reference function — so the proof's public words + // are checked against the specification, not against the executor. + let a = arenas(); + let lanes = |i: usize| lanes_of(&a[0][i]).expect("u32 lanes"); + let d0 = socket_digest(&lanes(0), &lanes(1)); + let d1 = socket_digest(&lanes(2), &lanes(3)); + let root = socket_digest(&d0, &d1); + assert_eq!(proved.public_words, vec![(0u32, word_of(&root))]); +} + +/// A proof is bound to the hasher it was produced under, in both directions. +#[test] +fn a_blake3_proof_does_not_verify_under_another_hasher() { + let opts = options(); + let program = compress_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, KIND).expect("prove"); + + for other in [HasherKind::Test, HasherKind::Poseidon] { + // The digest stays the proved-under one: this isolates the AIR-set + // mismatch rather than passing because the statement also moved. + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + other, + ), + "a BLAKE3 proof must not verify under {other:?}" + ); + } +} + +/// The hasher tag moves the program digest and no preprocessed root — the +/// Phase-3 binding, now with a third candidate in it. +/// +/// The third candidate is the point: with only two, a width coincidence was +/// enough to separate them by accident. The tag is what separates them on +/// purpose. +#[test] +fn the_blake3_choice_moves_the_program_digest_and_no_root() { + let opts = options(); + let program = compress_program(); + let test = build_artifacts_with_hasher(&program, &opts, HasherKind::Test); + let blake = build_artifacts_with_hasher(&program, &opts, KIND); + let pos = build_artifacts_with_hasher(&program, &opts, HasherKind::Poseidon); + + assert_eq!(build_artifacts(&program, &opts).program_id, test.program_id); + assert_eq!(test.roots, blake.roots, "no root may move with the hasher"); + assert_eq!(test.log_heights, blake.log_heights); + assert_eq!(test.keccak_rnd_chunks, blake.keccak_rnd_chunks); + assert_ne!(test.program_id, blake.program_id); + assert_ne!(pos.program_id, blake.program_id); + assert_eq!(KIND.as_tag(), 2, "the wire tag is written out, not derived"); +} + +/// Prove the program with `mutate` applied to the hash trace, and report +/// whether the proof was ACCEPTED. A prover refusal and a verifier rejection +/// are both real rejections and this chip produces both. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let program = compress_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let exec = execute(&program, &arenas(), &KIND).expect("execute"); + let mut traces = build_traces_with_hasher(&program, &exec.records, KIND); + mutate(&mut traces.hash); + match prove_traces_with_hasher( + &artifacts, + &mut traces, + &exec.public_words, + &opts, + KIND, + stark::residency_mode::ResidencyMode::Retain, + ) { + Ok(proof) => Ok(verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + KIND, + )), + Err(e) => Err(format!("{e:?}")), + } +} + +fn assert_not_accepted(what: &str, mutate: impl FnOnce(&mut TraceTable)) { + if let Ok(true) = round_trip(mutate) { + panic!("{what} must not produce an accepted proof, but the proof verified"); + } +} + +/// Tamper rejection, one cell at a time, across the three column families the +/// arm adds: a lane byte, a mixing-core carry, and a digest byte. +/// +/// The honest control is `the_blake3_socket_proves_and_verifies` above: without +/// it these would pass just as well if the AIR rejected everything. +#[test] +fn tampering_with_the_witness_is_not_accepted() { + assert_not_accepted("a moved input lane byte", |t| { + let v = t.main_table.get_row(0)[cols::lane_byte(0, 0)]; + t.main_table.set_fe(0, cols::lane_byte(0, 0), v + FE::one()); + }); + assert_not_accepted("a flipped add3 carry bit", |t| { + let c = cols::g_base(0) + cols::G_A1_C; + let v = t.main_table.get_row(0)[c]; + t.main_table.set_fe(0, c, v + FE::one()); + }); + assert_not_accepted("a moved digest byte", |t| { + let v = t.main_table.get_row(0)[cols::out_byte(0, 0)]; + t.main_table.set_fe(0, cols::out_byte(0, 0), v + FE::one()); + }); + assert_not_accepted("a real flag on a padding row", |t| { + t.main_table.set_fe(3, cols::MODE_C, FE::one()); + }); +} + +// ========================================================================= +// F3.4 — what B1 retired, and the ONE thing it did not +// ========================================================================= + +/// ★ `TrivialV0` — a REGISTERED program — proves and verifies under BLAKE3. +/// +/// It could not while it ended on a raw `permute`; option B1 replaced that with +/// a third `compress`, and this is the milestone that states. +#[test] +fn the_trivial_program_proves_and_verifies_under_blake3() { + let opts = options(); + let program = trivial_program(); + let arenas = vec![ + (0..4u32) + .map(|i| word_of(&[0x1000_0000 * (i + 1), 0x0BAD_F00D ^ i, i, 0xFFFF_FFFF - i])) + .collect(), + ]; + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &arenas, &opts, KIND) + .expect("TrivialV0 must prove under BLAKE3"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest BLAKE3 proof of TrivialV0 must verify" + ); +} + +// The O1 tripwire that used to live here is GONE, and that is the deliverable. +// +// It asserted that `FriToyV0` was refused under BLAKE3 for obligation O1, and +// its own doc said it must be replaced by a prove+verify when O1 closed. Option +// C closed it: `leaf_tests::fri_toy_proves_and_verifies_under_blake3` is the +// replacement, and it carries the negative leg the tripwire's criteria asked +// for. + +/// ★ H1 guard — every `LFM_HASH` candidate emits each constraint index exactly +/// once, checked in RELEASE. +/// +/// `EmitTracker`'s duplicate assert is `#[cfg(debug_assertions)]` and this +/// workspace declares no `[profile.release]` override, so under the house +/// convention `cargo test --release` it is a no-op: a second +/// `emit_base(idx, ..)` silently overwrites the first. Nothing else notices, +/// because a body that emits one index twice and another never still fills the +/// declared number of slots — `num_constraints`, `predicted_constraints` and +/// `assert_complete` all still pass while a constraint has been deleted. +/// +/// This runs the real body through `ConstraintSet::meta` (no `cfg`) and demands +/// the emitted index multiset be exactly `0..num_constraints`. The checker's own +/// ability to fail — on this exact shape, a widened lane block overrunning the +/// pins after it — is established in +/// `stark::tests::constraint_index_tests::the_widened_lane_block_collides_and_the_checker_says_so`. +/// +/// Required by COMMIT.md §1.4.4 H1. It guards the chip as it stands today, and +/// it is what would catch the `NUM_LANES` widening if that lands before the +/// framing indices stop being written as literals. +#[test] +fn every_hash_candidate_emits_each_constraint_index_exactly_once() { + for (set, kind) in [ + (HashConstraints::TEST, HasherKind::Test), + (HashConstraints::POSEIDON, HasherKind::Poseidon), + (HashConstraints::BLAKE3, HasherKind::Blake3), + ] { + let declared = HashConstraints::num_constraints(kind); + let meta = >::meta(&set); + check_dense_index_set(&meta, declared) + .unwrap_or_else(|e| panic!("{kind:?} LFM_HASH constraint body: {e}")); + } +} diff --git a/prover/src/lfm/builder.rs b/prover/src/lfm/builder.rs new file mode 100644 index 000000000..bd4b54de0 --- /dev/null +++ b/prover/src/lfm/builder.rs @@ -0,0 +1,519 @@ +//! The LFM eDSL: typed SSA handles over write-once cells. +//! +//! The builder is an ordinary Rust API that *emits instructions*; host-side +//! `for` loops unroll and nothing loop-shaped reaches the machine. Every +//! emitted destination gets the next dense address (SSA — uniqueness and +//! acyclicity by construction), every operand use bumps that cell's read +//! count, and the compiler later backfills the counts as the static +//! multiplicities the write-once memory argument needs. + +use std::collections::HashMap; + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::instr::{Addr, ArenaId, BaseOp, ExtOp, HashMode, Instr, KeccakMode}; +use super::layout; +use super::word::{LfmWord, base_word, ext_word}; + +/// A cell holding a base field value `(v, 0, 0, 0)`. +#[derive(Debug, Clone, Copy)] +pub struct Felt(pub(crate) Addr); +/// A cell holding an Fp3 value `(a0, a1, a2, 0)`. +#[derive(Debug, Clone, Copy)] +pub struct Ext(pub(crate) Addr); +/// A cell holding a digest (all four lanes). +#[derive(Debug, Clone, Copy)] +pub struct DigestVal(pub(crate) Addr); +/// An untyped word cell. +#[derive(Debug, Clone, Copy)] +pub struct Cell(pub(crate) Addr); +/// A cell holding a boolean `(b, 0, 0, 0)`, `b ∈ {0, 1}`. +#[derive(Debug, Clone, Copy)] +pub struct Bit(pub(crate) Addr); + +macro_rules! handle_addr { + ($($t:ty),*) => {$( + impl $t { + /// The underlying cell address. + pub fn addr(&self) -> Addr { self.0 } + /// Erase the type: any handle is a word cell. + pub fn as_cell(&self) -> Cell { Cell(self.0) } + } + )*}; +} +handle_addr!(Felt, Ext, DigestVal, Cell, Bit); + +impl Bit { + /// A bit is a valid base felt. + pub fn as_felt(&self) -> Felt { + Felt(self.0) + } +} + +impl Cell { + /// Reinterpret as a digest cell (e.g. hint words feeding `compress`). + pub fn as_digest(&self) -> DigestVal { + DigestVal(self.0) + } + + /// Reinterpret as an ext value. Sound by construction: every ext-typed + /// bus receive carries a constant zero in lane 3, so a word whose lane 3 + /// is nonzero makes the program unprovable (and the executor errors). + pub fn as_ext(&self) -> Ext { + Ext(self.0) + } +} + +impl Felt { + /// A base cell `(v, 0, 0, 0)` is a valid ext cell `(v, 0, 0)`. + pub fn as_ext(&self) -> Ext { + Ext(self.0) + } +} + +/// Declared arena lengths (in words), fixed at build time. The executor +/// checks supplied arenas against this schema; the admission validator checks +/// every `Hint` lands inside it. +#[derive(Debug, Clone, Default)] +pub struct ArenaSchema { + pub lens: Vec, +} + +/// Everything the compiler needs: the emitted instructions plus the builder's +/// bookkeeping. Fields are public so tests can hand-build malformed sources +/// to exercise the compiler's invariant panics. +#[derive(Debug)] +pub struct LfmProgramSource { + pub instrs: Vec, + pub num_addrs: u64, + /// Reads per address, indexed BY address. + /// + /// Dense rather than a map because [`LfmBuilder::alloc`] hands out + /// addresses sequentially from zero, so the key space is exactly + /// `0..num_addrs` with no holes — a `HashMap` was paying ~16 bytes plus + /// control per entry, and its bucket array is the emitter's second-largest + /// allocation at production query counts. + pub read_counts: Vec, + pub arena_schema: ArenaSchema, + pub public_len: u32, +} + +#[derive(Default)] +pub struct LfmBuilder { + instrs: Vec, + next_addr: u64, + const_pool: HashMap<[u64; 4], Addr>, + /// Parallel to the address space; see [`LfmProgramSource::read_counts`]. + read_counts: Vec, + arena_schema: ArenaSchema, + public_len: u32, +} + +impl LfmBuilder { + pub fn new() -> Self { + Self::default() + } + + fn alloc(&mut self) -> Addr { + let addr = Addr(self.next_addr); + self.next_addr += 1; + // Keeps `read_counts` exactly as long as the address space, which is + // what lets `read` index instead of hash. + self.read_counts.push(0); + addr + } + + fn read(&mut self, addr: Addr) { + self.read_counts[addr.0 as usize] += 1; + } + + // ---- constants (interned; one LFM_CONST row per distinct word) ---- + + fn word_const(&mut self, value: LfmWord) -> Addr { + let key: [u64; 4] = core::array::from_fn(|i| GoldilocksField::canonical(value[i].value())); + if let Some(&addr) = self.const_pool.get(&key) { + return addr; + } + let out = self.alloc(); + self.instrs.push(Instr::Const { + out, + value, + mult: 0, + }); + self.const_pool.insert(key, out); + out + } + + pub fn felt_const(&mut self, v: FE) -> Felt { + Felt(self.word_const(base_word(v))) + } + + pub fn ext_const(&mut self, v: &FEE) -> Ext { + Ext(self.word_const(ext_word(v))) + } + + pub fn digest_const(&mut self, v: LfmWord) -> DigestVal { + DigestVal(self.word_const(v)) + } + + pub fn bit_const(&mut self, b: bool) -> Bit { + Bit(self.word_const(base_word(if b { FE::one() } else { FE::zero() }))) + } + + // ---- base ALU ---- + + fn balu(&mut self, op: BaseOp, a: Felt, b: Felt, c: Option) -> Felt { + self.read(a.0); + self.read(b.0); + if let Some(c) = c { + self.read(c.0); + } + let out = self.alloc(); + self.instrs.push(Instr::BaseAlu { + op, + out, + a: a.0, + b: b.0, + c: c.map_or(Addr(0), |c| c.0), + mult: 0, + }); + Felt(out) + } + + pub fn add(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Add, a, b, None) + } + pub fn sub(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Sub, a, b, None) + } + pub fn mul(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Mul, a, b, None) + } + /// `a / b` under the machine convention `0/0 = 1`, `x/0 = error`. + pub fn div(&mut self, a: Felt, b: Felt) -> Felt { + self.balu(BaseOp::Div, a, b, None) + } + /// `a·b + c` — the Horner step, first-class. + pub fn mul_add(&mut self, a: Felt, b: Felt, c: Felt) -> Felt { + self.balu(BaseOp::MulAdd, a, b, Some(c)) + } + + // ---- Fp3 ALU (lanes 0–2, w³ = 2) ---- + + fn xalu(&mut self, op: ExtOp, a: Addr, b: Addr, c: Option) -> Ext { + self.read(a); + self.read(b); + if let Some(c) = c { + self.read(c); + } + let out = self.alloc(); + self.instrs.push(Instr::ExtAlu { + op, + out, + a, + b, + c: c.unwrap_or(Addr(0)), + mult: 0, + }); + Ext(out) + } + + pub fn eadd(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Add, a.0, b.0, None) + } + pub fn esub(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Sub, a.0, b.0, None) + } + pub fn emul(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Mul, a.0, b.0, None) + } + /// `a / b` under `0/0 = (1, 0, 0)`, `x/0 = error`. + pub fn ediv(&mut self, a: Ext, b: Ext) -> Ext { + self.xalu(ExtOp::Div, a.0, b.0, None) + } + pub fn emul_add(&mut self, a: Ext, b: Ext, c: Ext) -> Ext { + self.xalu(ExtOp::MulAdd, a.0, b.0, Some(c.0)) + } + /// Extension × base — 3 base multiplies instead of 9. + pub fn emul_base(&mut self, a: Ext, b: Felt) -> Ext { + self.xalu(ExtOp::MulBase, a.0, b.0, None) + } + + // ---- assertions (lowered, no chip) ---- + + /// `assert_eq` lowers to `diff = a − b; _ = diff / ZERO`: provable (and + /// executable) iff `diff = 0` under the `0/0 = 1` convention. + pub fn assert_eq(&mut self, a: Felt, b: Felt) { + let diff = self.sub(a, b); + let zero = self.felt_const(FE::zero()); + let _ = self.div(diff, zero); + } + + pub fn assert_eq_ext(&mut self, a: Ext, b: Ext) { + let diff = self.esub(a, b); + let zero = self.ext_const(&FEE::zero()); + let _ = self.ediv(diff, zero); + } + + // ---- select / bitdec ---- + + /// Conditional swap: `bit = 0 ⇒ (l, r)`; `bit = 1 ⇒ (r, l)`. + pub fn select(&mut self, bit: Bit, l: Cell, r: Cell) -> (Cell, Cell) { + self.read(bit.0); + self.read(l.0); + self.read(r.0); + let out_l = self.alloc(); + let out_r = self.alloc(); + self.instrs.push(Instr::Select { + bit: bit.0, + out_l, + out_r, + in_l: l.0, + in_r: r.0, + mult_l: 0, + mult_r: 0, + }); + (Cell(out_l), Cell(out_r)) + } + + /// Canonical 64-bit decomposition; returns the low `nbits` bits as cells + /// (low-to-high). Only these become memory cells; all 64 bits exist as + /// constrained witness columns either way. + pub fn bit_dec(&mut self, x: Felt, nbits: usize) -> Vec { + assert!(nbits <= 64, "bit_dec: at most 64 bits"); + self.read(x.0); + let bits: Vec<(Addr, u64)> = (0..nbits).map(|_| (self.alloc(), 0)).collect(); + let handles = bits.iter().map(|(a, _)| Bit(*a)).collect(); + self.instrs.push(Instr::BitDec { input: x.0, bits }); + handles + } + + // ---- hash ---- + + /// Two digest cells → one digest cell. + pub fn compress(&mut self, a: DigestVal, b: DigestVal) -> DigestVal { + self.two_to_one(HashMode::Compress, a, b) + } + + /// One step of a Merkle LEAF chain: absorb one cell, read as four FIELD + /// ELEMENTS, into the running accumulator `acc`. + /// + /// The only mode whose second input is not a digest: each felt is split into + /// a checked `lo`/`hi` `u32` pair inside the chip, so arbitrary Goldilocks + /// data can be hashed by a socket whose lanes must be `u32`. The `"LFML"` + /// domain keeps a leaf un-replayable as a parent whatever the tree's shape. + /// + /// **One call absorbs four felts AND chains**, because the accumulator rides + /// in the message rather than being folded in afterwards: a wide leaf over + /// `k` cells costs `k` hashes against the `2k − 1` a felts-only leaf plus a + /// fold of the results costs (COMMIT.md §1.2). The chain binds cell ORDER + /// for free; what it does not bind is the leaf's SHAPE, which is the header + /// cell's job in the commitment layer above this. + pub fn leaf(&mut self, acc: DigestVal, felts: Cell) -> DigestVal { + self.read(acc.0); + self.read(felts.0); + let out = self.alloc(); + self.instrs.push(Instr::Hash { + mode: HashMode::Leaf, + ins: [acc.0, felts.0, Addr(0)], + outs: [out, Addr(0), Addr(0)], + mults: [0, 0, 0], + }); + DigestVal(out) + } + + /// One step of the Fiat–Shamir transcript chain: two cells → one cell, in + /// the TRANSCRIPT hash domain. + /// + /// The same socket and the same columns as [`LfmBuilder::compress`]; the + /// row's preprocessed mode selects the domain tag, so a transcript step and + /// a Merkle parent over the same two cells are different digests. Callers + /// go through [`super::edsl::SpongeVar`] rather than here — the chain's + /// operand sequence is what its security argument rests on, and a raw step + /// is an easy way to break it. + pub fn transcript_step(&mut self, a: DigestVal, b: DigestVal) -> DigestVal { + self.two_to_one(HashMode::Transcript, a, b) + } + + fn two_to_one(&mut self, mode: HashMode, a: DigestVal, b: DigestVal) -> DigestVal { + debug_assert!(mode.is_two_to_one()); + self.read(a.0); + self.read(b.0); + let out = self.alloc(); + self.instrs.push(Instr::Hash { + mode, + ins: [a.0, b.0, Addr(0)], + outs: [out, Addr(0), Addr(0)], + mults: [0, 0, 0], + }); + DigestVal(out) + } + + /// Full three-cell state permutation. + pub fn permute(&mut self, state: [Cell; 3]) -> [Cell; 3] { + for c in &state { + self.read(c.0); + } + let outs = [self.alloc(), self.alloc(), self.alloc()]; + self.instrs.push(Instr::Hash { + mode: HashMode::Permute, + ins: [state[0].0, state[1].0, state[2].0], + outs, + mults: [0, 0, 0], + }); + outs.map(Cell) + } + + // ---- lane conversion (LFM_LANES) ---- + + /// Split a word into its four lanes as base cells — the only route from + /// a hash-state/digest cell into the ALU. + pub fn unpack(&mut self, c: Cell) -> [Felt; 4] { + self.read(c.0); + let outs = [self.alloc(), self.alloc(), self.alloc(), self.alloc()]; + self.instrs.push(Instr::Unpack { + input: c.0, + outs, + mults: [0; 4], + }); + outs.map(Felt) + } + + /// Assemble a word from four base cells. + pub fn pack_word(&mut self, lanes: [Felt; 4]) -> Cell { + for l in &lanes { + self.read(l.0); + } + let out = self.alloc(); + self.instrs.push(Instr::Pack { + lanes: lanes.map(|f| f.0), + out, + mult: 0, + }); + Cell(out) + } + + /// Assemble an ext cell `(a0, a1, a2, 0)` from three base cells (lane 3 + /// is the shared zero constant). + pub fn pack_ext(&mut self, a0: Felt, a1: Felt, a2: Felt) -> Ext { + let zero = self.felt_const(FE::zero()); + Ext(self.pack_word([a0, a1, a2, zero]).0) + } + + // ---- keccak-f[1600] (LFM_KECCAK) ---- + + /// One `keccak-f[1600]` permutation over 13 state words. + /// + /// The 25 `u64` lanes travel as 50 `u32` halves packed four to a word: + /// word `j` carries halves `4j..4j+3`, half `h` is the low (`h` even) or + /// high (`h` odd) 32 bits of lane `h / 2`. The last word's top two lanes + /// are unused and must be zero — the bus pins them as tuple constants, and + /// the executor errors on a nonzero one. Every lane of every input word + /// must be a canonical value below `2^32`. + pub fn keccak_f(&mut self, state: [Cell; layout::keccak::NUM_WORDS]) -> [Cell; 13] { + self.emit_keccak(KeccakMode::Permute, state, [Cell(Addr(0)); 9], false) + .0 + } + + /// One sponge absorb step: XOR a 136-byte rate block (9 words of `u32` + /// halves, the top two half slots unused and zero) into the state's rate + /// region, then permute. + pub fn keccak_absorb( + &mut self, + state: [Cell; layout::keccak::NUM_WORDS], + block: [Cell; layout::keccak::BLOCK_WORDS], + ) -> [Cell; 13] { + self.emit_keccak(KeccakMode::Absorb, state, block, false).0 + } + + /// Absorb, and additionally materialize the byte-REVERSED digest of the + /// resulting state as two words — the production transcript's `sample()`, + /// which both returns those bytes and re-absorbs them as the next segment's + /// prefix. Free on the bus (see `layout::keccak::REV_ADDR0`). + pub fn keccak_absorb_rev( + &mut self, + state: [Cell; layout::keccak::NUM_WORDS], + block: [Cell; layout::keccak::BLOCK_WORDS], + ) -> ([Cell; 13], [Cell; 2]) { + let (outs, rev) = self.emit_keccak(KeccakMode::Absorb, state, block, true); + (outs, rev.expect("requested")) + } + + fn emit_keccak( + &mut self, + mode: KeccakMode, + state: [Cell; layout::keccak::NUM_WORDS], + block: [Cell; layout::keccak::BLOCK_WORDS], + want_rev: bool, + ) -> ([Cell; 13], Option<[Cell; 2]>) { + for c in &state { + self.read(c.0); + } + if mode == KeccakMode::Absorb { + for c in &block { + self.read(c.0); + } + } + let outs: [Addr; 13] = core::array::from_fn(|_| self.alloc()); + let rev_outs: Option<[Addr; 2]> = want_rev.then(|| core::array::from_fn(|_| self.alloc())); + self.instrs + .push(Instr::KeccakF(Box::new(super::instr::KeccakOperands { + mode, + ins: state.map(|c| c.0), + block: block.map(|c| c.0), + outs, + mults: [0; 13], + rev: rev_outs.map(|outs| super::instr::KeccakReversedDigest { + outs, + mults: [0; 2], + }), + }))); + (outs.map(Cell), rev_outs.map(|r| r.map(Cell))) + } + + // ---- hints / public ---- + + pub fn declare_arena(&mut self, len: u32) -> ArenaId { + self.arena_schema.lens.push(len); + (self.arena_schema.lens.len() - 1) as ArenaId + } + + /// One arena word → one memory cell. Arena values are unconstrained by + /// the reading chip; the arena rule (transitively hash-authenticate + /// everything hinted; never derive challenges from arenas) is what makes + /// this sound. + pub fn hint_word(&mut self, arena: ArenaId, index: u32) -> Cell { + let out = self.alloc(); + self.instrs.push(Instr::Hint { + arena, + index, + out, + mult: 0, + }); + Cell(out) + } + + pub fn hint_felt(&mut self, arena: ArenaId, index: u32) -> Felt { + Felt(self.hint_word(arena, index).0) + } + + /// Expose a cell on the public-output bus (auto-incrementing index). + pub fn public(&mut self, c: Cell) { + self.read(c.0); + let index = self.public_len; + self.public_len += 1; + self.instrs.push(Instr::Public { addr: c.0, index }); + } + + pub fn finish(self) -> LfmProgramSource { + LfmProgramSource { + instrs: self.instrs, + num_addrs: self.next_addr, + read_counts: self.read_counts, + arena_schema: self.arena_schema, + public_len: self.public_len, + } + } +} diff --git a/prover/src/lfm/chips.rs b/prover/src/lfm/chips.rs new file mode 100644 index 000000000..a4ce1b143 --- /dev/null +++ b/prover/src/lfm/chips.rs @@ -0,0 +1,1406 @@ +//! The LFM chips: bus interactions and constraint sets. +//! +//! Eleven chips live here. The other three slots of the fixed AIR set are the +//! production keccak family (`KECCAK_RND` / `KECCAK_RC` / `BITWISE`), hosted +//! unchanged from `tables/` and driven by `LFM_KECCAK` below. +//! +//! Shared conventions (see `SOUNDNESS.md` and the design doc): +//! - each chip's trace = its instruction column group (preprocessed, leading +//! columns, layout in [`super::layout`]) followed by the value columns +//! defined here; +//! - one sign convention machine-wide: writes are senders with +//! `Multiplicity::Column(mult)` (a preprocessed column), reads are +//! receivers gated by selectors / `is_real` (also preprocessed); no +//! `Negated` forms anywhere; +//! - the `LfmMem` token is `(addr, v0, v1, v2, v3)`; base values carry +//! constant-zero high lanes *in the tuple*, so a base cell cannot smuggle +//! extension lanes; +//! - in-AIR checks are per-op algebra plus belt-over-suspenders booleanity; +//! uniqueness/acyclicity/mult-equality/one-hot-ness are the registrar's +//! (admission validator), per the soundness split. + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; + +use crate::tables::types::{BusId, GoldilocksExtension, GoldilocksField}; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +fn direct(col: usize) -> BusValue { + BusValue::Packed { + start_column: col, + packing: Packing::Direct, + } +} + +fn zero() -> BusValue { + BusValue::constant(0) +} + +/// A word value spread over four adjacent columns. +fn word(cols_start: usize) -> [BusValue; 4] { + [ + direct(cols_start), + direct(cols_start + 1), + direct(cols_start + 2), + direct(cols_start + 3), + ] +} + +/// `Σ` of a run of selector columns, as a LogUp multiplicity (= is_real). +fn selector_sum(first: usize, count: usize) -> Multiplicity { + Multiplicity::Linear( + (0..count) + .map(|i| LinearTerm::ColumnUnsigned { + coefficient: 1, + column: first + i, + }) + .collect(), + ) +} + +/// A base-value memory token: `(addr, v, 0, 0, 0)`. +fn base_token(addr_col: usize, val_col: usize) -> Vec { + vec![direct(addr_col), direct(val_col), zero(), zero(), zero()] +} + +/// An ext-value memory token: `(addr, v0, v1, v2, 0)` — lane 3 is a tuple +/// constant, which is exactly what pins ext cells to lane-3-zero. +fn ext_token(addr_col: usize, lanes_start: usize) -> Vec { + vec![ + direct(addr_col), + direct(lanes_start), + direct(lanes_start + 1), + direct(lanes_start + 2), + zero(), + ] +} + +/// A full-word memory token: `(addr, v0..v3)`. +fn word_token(addr_col: usize, lanes_start: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend(word(lanes_start)); + v +} + +// ========================================================================= +// LFM_CONST — pooled constants (all instruction data preprocessed) +// ========================================================================= + +pub mod const_ { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::const_::*; + /// All-zero main column: the commit path expects a non-empty + /// non-preprocessed subset (KECCAK_RC precedent). + pub const PAD: usize = PREP_WIDTH; + pub const NUM_COLUMNS: usize = PREP_WIDTH + 1; + } + + pub fn bus_interactions() -> Vec { + vec![BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + word_token(cols::ADDR, cols::V0), + )] + } +} + +// ========================================================================= +// LFM_BALU — Goldilocks ALU +// ========================================================================= + +pub mod balu { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::balu::*; + pub const A: usize = PREP_WIDTH; + pub const B: usize = PREP_WIDTH + 1; + pub const C: usize = PREP_WIDTH + 2; + pub const OUT: usize = PREP_WIDTH + 3; + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + vec![ + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + base_token(cols::A_ADDR, cols::A), + ), + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + base_token(cols::B_ADDR, cols::B), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::SEL_MULADD), + base_token(cols::C_ADDR, cols::C), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + base_token(cols::OUT_ADDR, cols::OUT), + ), + ] + } + + pub struct BaluConstraints; + + impl ConstraintSet for BaluConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let a = b.main(0, cols::A); + let bb = b.main(0, cols::B); + let c = b.main(0, cols::C); + let out = b.main(0, cols::OUT); + let sel = |b: &B, i: usize| b.main(0, cols::SEL_ADD + i); + + // idx 0: add — sel·(a + b − out) + b.emit_base(0, sel(b, 0) * (a.clone() + bb.clone() - out.clone())); + // idx 1: sub — sel·(a − b − out) + b.emit_base(1, sel(b, 1) * (a.clone() - bb.clone() - out.clone())); + // idx 2: mul — sel·(a·b − out) + b.emit_base(2, sel(b, 2) * (a.clone() * bb.clone() - out.clone())); + // idx 3: div as reversed mul — sel·(b·out − a). With b = 0 this + // forces a = 0 and leaves out free (the executor pins 0/0 = 1): + // the assert-via-division mechanism. + b.emit_base(3, sel(b, 3) * (bb.clone() * out.clone() - a.clone())); + // idx 4: mul-add — sel·(a·b + c − out) (the Horner step) + b.emit_base(4, sel(b, 4) * (a * bb + c - out)); + // idx 5: selector sum-boolean (belt; one-hot is the registrar's) + let sum = (1..cols::NUM_SELECTORS).fold(sel(b, 0), |acc, i| acc + sel(b, i)); + let one = b.one(); + b.emit_base(5, sum.clone() * (one - sum)); + } + } +} + +// ========================================================================= +// LFM_XALU — Fp3 ALU on word lanes 0–2 (w³ = 2) +// ========================================================================= + +pub mod xalu { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::xalu::*; + pub const A0: usize = PREP_WIDTH; // ..A2 + pub const B0: usize = PREP_WIDTH + 3; // ..B2 + pub const C0: usize = PREP_WIDTH + 6; // ..C2 + pub const OUT0: usize = PREP_WIDTH + 9; // ..OUT2 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 12; + } + + pub fn bus_interactions() -> Vec { + vec![ + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + ext_token(cols::A_ADDR, cols::A0), + ), + BusInteraction::receiver( + BusId::LfmMem, + selector_sum(cols::SEL_ADD, cols::NUM_SELECTORS), + ext_token(cols::B_ADDR, cols::B0), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::SEL_MULADD), + ext_token(cols::C_ADDR, cols::C0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + ext_token(cols::OUT_ADDR, cols::OUT0), + ), + ] + } + + pub struct XaluConstraints; + + impl XaluConstraints { + /// The three product lanes of `X·Y` in `Fp[w]/(w³ − 2)`: + /// `p0 = x0y0 + 2(x1y2 + x2y1)`, `p1 = x0y1 + x1y0 + 2·x2y2`, + /// `p2 = x0y2 + x1y1 + x2y0` (matches `Degree3GoldilocksExtensionField::mul`). + fn product>(b: &B, x0: usize, y0: usize) -> [B::Expr; 3] { + let x = |i: usize| b.main(0, x0 + i); + let y = |i: usize| b.main(0, y0 + i); + let two = b.const_base(2); + [ + x(0) * y(0) + two.clone() * (x(1) * y(2) + x(2) * y(1)), + x(0) * y(1) + x(1) * y(0) + two * x(2) * y(2), + x(0) * y(2) + x(1) * y(1) + x(2) * y(0), + ] + } + } + + impl ConstraintSet for XaluConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + let sel = |b: &B, i: usize| b.main(0, cols::SEL_ADD + i); + let lane = |b: &B, base: usize, j: usize| b.main(0, base + j); + + // idx 0–2 add / 3–5 sub: componentwise. + for j in 0..3 { + let a = lane(b, cols::A0, j); + let bb = lane(b, cols::B0, j); + let out = lane(b, cols::OUT0, j); + b.emit_base(j, sel(b, 0) * (a.clone() + bb.clone() - out.clone())); + b.emit_base(3 + j, sel(b, 1) * (a - bb - out)); + } + // idx 6–8 mul: P(A, B) = OUT. + let p_ab = Self::product(b, cols::A0, cols::B0); + for (j, p) in p_ab.into_iter().enumerate() { + b.emit_base(6 + j, sel(b, 2) * (p - lane(b, cols::OUT0, j))); + } + // idx 9–11 div as reversed mul: P(B, OUT) = A (0/0 = (1,0,0) in + // the executor; x/0 unprovable — the ext assert mechanism). + let p_bout = Self::product(b, cols::B0, cols::OUT0); + for (j, p) in p_bout.into_iter().enumerate() { + b.emit_base(9 + j, sel(b, 3) * (p - lane(b, cols::A0, j))); + } + // idx 12–14 mul-add: P(A, B) + C = OUT. + let p_ab = Self::product(b, cols::A0, cols::B0); + for (j, p) in p_ab.into_iter().enumerate() { + b.emit_base( + 12 + j, + sel(b, 4) * (p + lane(b, cols::C0, j) - lane(b, cols::OUT0, j)), + ); + } + // idx 15–17 mul-base: OUT_j = A_j · B0. + for j in 0..3 { + b.emit_base( + 15 + j, + sel(b, 5) + * (lane(b, cols::A0, j) * lane(b, cols::B0, 0) - lane(b, cols::OUT0, j)), + ); + } + // idx 18–19: MulBase's operand is a base word — the shared B + // lanes 1–2 must vanish on its rows or the received token would + // not match any base writer's. + b.emit_base(18, sel(b, 5) * lane(b, cols::B0, 1)); + b.emit_base(19, sel(b, 5) * lane(b, cols::B0, 2)); + // idx 20: selector sum-boolean. + let sum = (1..cols::NUM_SELECTORS).fold(sel(b, 0), |acc, i| acc + sel(b, i)); + let one = b.one(); + b.emit_base(20, sum.clone() * (one - sum)); + } + } +} + +// ========================================================================= +// LFM_SELECT — conditional cell swap +// ========================================================================= + +pub mod select { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::select::*; + pub const BIT: usize = PREP_WIDTH; + pub const INL0: usize = PREP_WIDTH + 1; // ..+4 + pub const INR0: usize = PREP_WIDTH + 5; // ..+8 + pub const OUTL0: usize = PREP_WIDTH + 9; // ..+12 + pub const OUTR0: usize = PREP_WIDTH + 13; // ..+16 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 17; + } + + pub fn bus_interactions() -> Vec { + vec![ + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + base_token(cols::BIT_ADDR, cols::BIT), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + word_token(cols::INL_ADDR, cols::INL0), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + word_token(cols::INR_ADDR, cols::INR0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT_L), + word_token(cols::OUTL_ADDR, cols::OUTL0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT_R), + word_token(cols::OUTR_ADDR, cols::OUTR0), + ), + ] + } + + pub struct SelectConstraints; + + impl ConstraintSet for SelectConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + // idx 0: bit booleanity (belt over suspenders — a witness bit + // exists, so it is constrained here and not only vouched). + let bit = b.main(0, cols::BIT); + let one = b.one(); + b.emit_base(0, bit.clone() * (one - bit)); + // idx 1–4 / 5–8: out_l = in_l + bit·(in_r − in_l); out_r mirrored. + // Trivially satisfied on zero-filled padding rows. + for j in 0..4 { + let bit = b.main(0, cols::BIT); + let inl = b.main(0, cols::INL0 + j); + let inr = b.main(0, cols::INR0 + j); + let outl = b.main(0, cols::OUTL0 + j); + b.emit_base( + 1 + j, + outl - (inl.clone() + bit.clone() * (inr.clone() - inl.clone())), + ); + let outr = b.main(0, cols::OUTR0 + j); + b.emit_base(5 + j, outr - (inr.clone() + bit * (inl - inr))); + } + } + } +} + +// ========================================================================= +// LFM_BITDEC — canonical 64-bit decomposition over p = 2^64 − 2^32 + 1 +// ========================================================================= + +pub mod bitdec { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::bitdec::*; + pub const BITS0: usize = PREP_WIDTH; // 64 bit columns, low-to-high + pub const Z: usize = PREP_WIDTH + NUM_BITS; + pub const GINV: usize = PREP_WIDTH + NUM_BITS + 1; + pub const NUM_COLUMNS: usize = PREP_WIDTH + NUM_BITS + 2; + } + + pub fn bus_interactions() -> Vec { + // The received value is the recomposition Σ 2^i·B_i, expressed as a + // linear bus value over the bit columns — no input column needed. + let recomposition = BusValue::Linear( + (0..cols::NUM_BITS) + .map(|i| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << i, + column: cols::BITS0 + i, + }) + .collect(), + ); + let mut interactions = vec![BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + vec![direct(cols::IN_ADDR), recomposition, zero(), zero(), zero()], + )]; + for i in 0..cols::NUM_BITS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::bit_mult(i)), + base_token(cols::bit_addr(i), cols::BITS0 + i), + )); + } + interactions + } + + pub struct BitDecConstraints; + + impl ConstraintSet for BitDecConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + // idx 0–63: booleanity. + for i in 0..cols::NUM_BITS { + let bit = b.main(0, cols::BITS0 + i); + let one = b.one(); + b.emit_base(i, bit.clone() * (one - bit)); + } + // Canonicity: p − 1 = (2^32 − 1)·2^32, i.e. 32 ones ‖ 32 zeros, + // so value < p ⟺ (top 32 bits all ones ⇒ bottom 32 bits zero). + // G = (2^32 − 1) − Σ_{i=32..63} 2^{i−32}·B_i; witnesses Z ("top + // all ones"), GINV (= G⁻¹ when G ≠ 0). + let top = (0..32).fold(None::, |acc, k| { + let term = b.const_base(1u64 << k) * b.main(0, cols::BITS0 + 32 + k); + Some(match acc { + None => term, + Some(a) => a + term, + }) + }); + let g = b.const_base(0xFFFF_FFFF) - top.expect("nonempty"); + let z = b.main(0, cols::Z); + let ginv = b.main(0, cols::GINV); + // idx 64: Z·G = 0 — G ≠ 0 forces Z = 0. + b.emit_base(64, z.clone() * g.clone()); + // idx 65: IS_REAL·(1 − Z − G·GINV) = 0 — G = 0 forces Z = 1. + // Gated by IS_REAL so zero-filled padding rows satisfy it. + let is_real = b.main(0, cols::IS_REAL); + let one = b.one(); + b.emit_base(65, is_real * (one - z.clone() - g * ginv)); + // idx 66: Z·(Σ_{i<32} 2^i·B_i) = 0 — top all ones ⇒ bottom zero. + let low = (0..32).fold(None::, |acc, k| { + let term = b.const_base(1u64 << k) * b.main(0, cols::BITS0 + k); + Some(match acc { + None => term, + Some(a) => a + term, + }) + }); + b.emit_base(66, z * low.expect("nonempty")); + } + } +} + +// ========================================================================= +// LFM_HASH — the chiplet (frozen tuple contract; TestPermutation behind it) +// ========================================================================= + +pub mod hash { + use super::*; + use crate::lfm::hash::{HASH_STATE_FELTS, HasherKind, TestPermutation}; + use crate::lfm::instr::HashMode; + use crate::tables::types::FE; + use math::field::traits::IsPrimeField; + + pub mod cols { + pub use crate::lfm::layout::hash::*; + pub const IN0: usize = PREP_WIDTH; // ..IN11 + /// Materialized capacity-state columns for lanes 8–11: + /// `S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i` (degree-2 + /// copy), so the permutation constraint stays at degree 3. Transcript + /// and leaf rows are compresses in every structural respect, so they + /// take the IV too. + pub const S8: usize = PREP_WIDTH + 12; // ..S11 + pub const OUT0: usize = PREP_WIDTH + 16; // ..OUT11 + /// Value columns every hasher's layout shares: `IN`, `S`, `OUT`. The + /// bus tuples read only these (`bus_interactions`), which is why they + /// keep their offsets in EVERY layout — a candidate appends its + /// witness columns after them rather than reflowing the prefix. + pub const SHARED_VALUE_COLUMNS: usize = 28; + /// Width of the [`HasherKind::Test`] layout. Use [`super::num_columns`] + /// unless you specifically mean `TestPermutation`. + pub const TEST_NUM_COLUMNS: usize = PREP_WIDTH + SHARED_VALUE_COLUMNS; + } + + /// Column layout for the [`HasherKind::Poseidon`] configuration. + /// + /// The frozen prefix (`IN0..12`, `S8..12`, `OUT0..12`) keeps the offsets + /// `cols` gives it, so [`bus_interactions`] is hasher-INDEPENDENT and the + /// `LFM_HASH` tuple contract stays literally frozen. Everything Poseidon + /// additionally witnesses is appended from [`ROUNDS`] on: per round, the + /// `x²` and `x³` intermediates of its S-boxed lanes plus its post-MDS + /// output — except the LAST round, whose output IS `OUT0..12`. + /// + /// Width: `28 + 7·36 + 24 + 22·14 = 612` value columns, one row per + /// permutation. + /// + /// ⚠ This layout is a deliberate UPPER BOUND, roughly 2× a known-achievable + /// one (Miden's measured Poseidon2 at the same width is 256 main cells via + /// 16 columns × 16 rows, reusing state columns across rounds instead of + /// allocating fresh ones). It is not optimised because the epoch verifier's + /// already-measured non-hash residue dominates the total: halving the hash + /// term moves the epoch bill by ~3%. Measure here, optimise elsewhere. + pub mod poseidon_cols { + use crate::lfm::hash::HASH_STATE_FELTS; + use crate::lfm::poseidon::{NUM_ROUNDS, sboxed_lanes}; + + pub use super::cols::{ + IN0, MODE_C, MODE_L, MODE_P, MODE_T, OUT0, PREP_WIDTH, S8, SHARED_VALUE_COLUMNS, + }; + + /// First appended witness column. + pub const ROUNDS: usize = PREP_WIDTH + SHARED_VALUE_COLUMNS; + + /// Width of round `r`'s appended block: `x²` and `x³` for each S-boxed + /// lane, plus 12 output columns — none for the last round, which writes + /// its output into `OUT`. + pub const fn block_width(r: usize) -> usize { + let out = if r + 1 == NUM_ROUNDS { + 0 + } else { + HASH_STATE_FELTS + }; + 2 * sboxed_lanes(r) + out + } + + /// First column of round `r`'s appended block. + pub const fn block(r: usize) -> usize { + let mut off = ROUNDS; + let mut i = 0; + while i < r { + off += block_width(i); + i += 1; + } + off + } + + /// `a_lane²` for round `r`. Only lanes `< sboxed_lanes(r)` exist. + pub const fn x2(r: usize, lane: usize) -> usize { + block(r) + lane + } + + /// `a_lane³` for round `r`. Only lanes `< sboxed_lanes(r)` exist. + pub const fn x3(r: usize, lane: usize) -> usize { + block(r) + sboxed_lanes(r) + lane + } + + /// Round `r`'s post-MDS output lane `j` — `OUT` for the final round. + pub const fn out(r: usize, j: usize) -> usize { + if r + 1 == NUM_ROUNDS { + OUT0 + j + } else { + block(r) + 2 * sboxed_lanes(r) + j + } + } + + pub const NUM_COLUMNS: usize = block(NUM_ROUNDS); + + /// 4 capacity copies + 1 mode-boolean + per round (`2·sboxed` S-box + /// steps and 12 MDS outputs). + pub const NUM_CONSTRAINTS: usize = { + let mut n = 5 + super::NUM_UNREAD_INPUT_PINS; + let mut r = 0; + while r < NUM_ROUNDS { + n += 2 * sboxed_lanes(r) + HASH_STATE_FELTS; + r += 1; + } + n + }; + } + + /// The chip's total width under `kind` — the number the AIR is built with, + /// the census reads, and the trace filler allocates. + pub const fn num_columns(kind: HasherKind) -> usize { + match kind { + HasherKind::Test => cols::TEST_NUM_COLUMNS, + HasherKind::Poseidon => poseidon_cols::NUM_COLUMNS, + HasherKind::Blake3 => crate::lfm::blake3_socket::cols::NUM_COLUMNS, + } + } + + /// The chip's bus interactions under `kind`. + /// + /// **Hasher-DEPENDENT, and BLAKE3 is why.** The six `LfmMem` tuples below + /// are the frozen `LFM_HASH` contract and are the same under every + /// candidate; they read and write only the shared value prefix, whose + /// offsets no layout moves. But a candidate built out of byte operations + /// needs a lookup table, and BLAKE3 needs one per XOR byte and one per + /// range-checked byte pair — over a thousand of them, none of which + /// `TestPermutation` or Poseidon has, both being pure field arithmetic. + /// + /// Callers must thread the same `kind` they build the AIR's width and + /// constraints with; `LfmAirs::new_with_hasher` is the one place that does. + pub fn bus_interactions(kind: HasherKind) -> Vec { + let mut interactions = lfm_mem_interactions(); + if kind == HasherKind::Blake3 { + interactions.extend(crate::lfm::blake3_socket::bitwise_interactions()); + } + interactions + } + + /// The frozen `LFM_HASH` tuple contract: 2 (or 3) cells in, 1 (or 3) out. + /// + /// The FIRST TWO input cells are read in every mode, so their multiplicity + /// is the row's is-real flag: the sum of all four mode selectors, which the + /// AIR pins to a bit. The third is read only by a permutation. + /// + /// ⚠ The second cell's multiplicity used to EXCLUDE `MODE_L`, because a leaf + /// row read one cell of four felts and receiving a second would have claimed + /// a memory read it never made. Under the leaf RATE a leaf row reads two — + /// a chaining accumulator and a felt cell — so that exclusion became the + /// opposite bug: the felts would never be read from memory at all (COMMIT.md + /// §1.4.4 **H3**). The bus ARITY does not move; this multiplicity is the one + /// part of the frozen contract that the RATE does. + fn lfm_mem_interactions() -> Vec { + let is_real = || selector_sum(cols::MODE_C, cols::NUM_SELECTORS); + vec![ + BusInteraction::receiver( + BusId::LfmMem, + is_real(), + word_token(cols::IN_ADDR0, cols::IN0), + ), + BusInteraction::receiver( + BusId::LfmMem, + is_real(), + word_token(cols::IN_ADDR1, cols::IN0 + 4), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_P), + word_token(cols::IN_ADDR2, cols::IN0 + 8), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT0), + word_token(cols::OUT_ADDR0, cols::OUT0), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT1), + word_token(cols::OUT_ADDR1, cols::OUT0 + 4), + ), + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT2), + word_token(cols::OUT_ADDR2, cols::OUT0 + 8), + ), + ] + } + + fn canonical_u64(fe: &FE) -> u64 { + GoldilocksField::canonical(fe.value()) + } + + /// Every mode selector paired with the mode it selects. + /// + /// One table, so the input pins below and anything else that reasons per + /// mode read the same mapping rather than each carrying its own copy. + pub(crate) const MODE_SELECTORS: [(usize, HashMode); 4] = [ + (cols::MODE_C, HashMode::Compress), + (cols::MODE_T, HashMode::Transcript), + (cols::MODE_L, HashMode::Leaf), + (cols::MODE_P, HashMode::Permute), + ]; + + /// Input cell slots that SOME mode does not read, and which therefore need + /// pinning. Cell 0 is read by every mode and is never a candidate. + /// + /// Derived rather than written down: the leaf RATE took `Leaf` from one + /// input cell to two, which emptied slot 1's set. Left as a literal, the + /// emitter's `.expect("some mode reads fewer than three input cells")` would + /// have fired and AIR construction would have panicked (COMMIT.md §1.4.4 + /// **H2**). + const fn unread_input_slots() -> usize { + let mut slots = 0; + let mut slot = 1; + while slot < 3 { + let mut i = 0; + while i < MODE_SELECTORS.len() { + if MODE_SELECTORS[i].1.num_input_cells() <= slot { + slots += 1; + break; + } + i += 1; + } + slot += 1; + } + slots + } + + /// Constraints [`emit_unread_input_pins`] emits: four per input cell that + /// some mode does not read. + pub(crate) const NUM_UNREAD_INPUT_PINS: usize = 4 * unread_input_slots(); + + /// The first constraint index each arm places the unread-`IN` pins at. + /// + /// Each arm chooses where in its own numbering they land, so the one place + /// that knows all three is here, next to the emitter. The controls read it + /// to assert that a forged row's violated set IS the pins. + #[cfg(test)] + pub(crate) const fn unread_input_pin_base(kind: HasherKind) -> usize { + match kind { + HasherKind::Test => 17, + HasherKind::Poseidon => poseidon_cols::NUM_CONSTRAINTS - NUM_UNREAD_INPUT_PINS, + HasherKind::Blake3 => crate::lfm::blake3_socket::UNREAD_IDX, + } + } + + /// ★ **Pins the `IN` columns of every input cell a mode does not read.** + /// + /// **This is load-bearing on any arm whose constraints READ `IN`, and that + /// is not something to decide per arm.** A mode that reads fewer cells than + /// the layout provides leaves the rest receiving nothing from `LfmMem` — + /// their multiplicity excludes it — so if anything then reads those columns + /// they are four free felts of prover choice and the row's output stops + /// being a function of its input. + /// + /// That is not hypothetical: it shipped. `MODE_L` reads one cell, the bus + /// and the validator were both taught so, the BLAKE3 arm pinned the unread + /// columns — and the `Test` and `Poseidon` arms, whose round 0 reads + /// `A_i = IN_i` for `i < 8`, were not. Under those two a leaf row carried + /// four unconstrained felts that the permutation consumed, which is a + /// Fiat–Shamir break for any program that absorbs data. Deriving the pins + /// from [`HashMode::num_input_cells`] here, once, is what stops the next + /// mode repeating it: an arm cannot forget a pin it does not write. + /// + /// Degree 2 (a selector sum times a column), so no arm's bound moves. + /// + /// Returns the next free constraint index. + pub(crate) fn emit_unread_input_pins>( + b: &mut B, + first_idx: usize, + ) -> usize { + let mut idx = first_idx; + // Cell 0 is read by every mode, so it is never pinned. A slot EVERY mode + // reads is skipped rather than pinned to nothing — which is the shape + // slot 1 took when the leaf RATE gave `Leaf` a second input cell. + for slot in 1..3usize { + let Some(sel) = MODE_SELECTORS + .iter() + .filter(|(_, mode)| mode.num_input_cells() <= slot) + .fold(None::, |acc, (col, _)| { + let term = b.main(0, *col); + Some(match acc { + None => term, + Some(a) => a + term, + }) + }) + else { + continue; + }; + for j in 0..4 { + let in_col = b.main(0, cols::IN0 + 4 * slot + j); + b.emit_base(idx, sel.clone() * in_col); + idx += 1; + } + } + debug_assert_eq!(idx - first_idx, NUM_UNREAD_INPUT_PINS); + idx + } + + /// The permutation the chip proves, chosen at construction. + /// + /// One struct with a runtime discriminant rather than one type per hasher: + /// `LfmAirs` holds `LfmAir` as a single field, so a + /// per-hasher type would force a trait object or an enum there instead. + pub struct HashConstraints { + pub kind: HasherKind, + } + + impl HashConstraints { + /// The `TestPermutation` configuration — the machine's pre-decision + /// default. `HashConstraints::default()` is the same thing. + pub const TEST: Self = Self { + kind: HasherKind::Test, + }; + + /// The Poseidon-original configuration. + pub const POSEIDON: Self = Self { + kind: HasherKind::Poseidon, + }; + + /// The BLAKE3 2-to-1 compress configuration. + pub const BLAKE3: Self = Self { + kind: HasherKind::Blake3, + }; + + /// Constraints emitted under `kind` — the count the framework's + /// dense-index invariant requires `eval` to fill exactly. + pub const fn num_constraints(kind: HasherKind) -> usize { + match kind { + HasherKind::Test => 17 + NUM_UNREAD_INPUT_PINS, + HasherKind::Poseidon => poseidon_cols::NUM_CONSTRAINTS, + HasherKind::Blake3 => crate::lfm::blake3_socket::NUM_CONSTRAINTS, + } + } + } + + impl Default for HashConstraints { + fn default() -> Self { + Self::TEST + } + } + + impl ConstraintSet for HashConstraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + match self.kind { + HasherKind::Test => Self::eval_test(b), + HasherKind::Poseidon => Self::eval_poseidon(b), + // The BLAKE3 arm lives in its own module: it shares the mixing + // dataflow with `blake3_chip` rather than with anything here, + // and putting it beside its column layout, its senders and its + // trace filler is what keeps the four in step. + HasherKind::Blake3 => crate::lfm::blake3_socket::eval(b), + } + } + } + + impl HashConstraints { + fn eval_test>(b: &mut B) { + let mode_c = b.main(0, cols::MODE_C); + let mode_t = b.main(0, cols::MODE_T); + let mode_l = b.main(0, cols::MODE_L); + let mode_p = b.main(0, cols::MODE_P); + + // idx 0–3: capacity-state copy — + // S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i. Transcript + // and leaf rows are one-cell-out steps like a compress row, so they + // take the same capacity; `TestPermutation` has one hash domain and + // is field-native, so all three compute the same function (see + // `LfmHasher::transcript_out` / `leaf_out` and their recorded + // weakening). + for (k, iv_raw) in TestPermutation::compress_iv_raw().into_iter().enumerate() { + let s = b.main(0, cols::S8 + k); + let in_i = b.main(0, cols::IN0 + 8 + k); + let iv_i = b.const_base(iv_raw); + let m = mode_c.clone() + mode_t.clone() + mode_l.clone(); + b.emit_base(k, s - (mode_p.clone() * in_i + m * iv_i)); + } + + // idx 4–15: the TestPermutation round — t_i = (A_i + rc_i·m)³ + // with A_i = IN_i (i < 8) or S_i (i ≥ 8) and m the mode sum; + // OUT_j = t_j + Σ_i t_i (mixing matrix M = I + J). The round + // constant is scaled by the mode sum so zero-filled padding rows + // satisfy the constraint (0 = 0) without a degree-4 gate: on real + // rows m = 1 and the permutation is unchanged. + // NON-CRYPTOGRAPHIC — this block behind the bus contract above is + // the hash-swap surface. + let t: Vec = (0..12) + .map(|i| { + let a = if i < 8 { + b.main(0, cols::IN0 + i) + } else { + b.main(0, cols::S8 + (i - 8)) + }; + let rc = b.const_base(canonical_u64(&TestPermutation::round_constant(i))); + let m = b.main(0, cols::MODE_C) + + b.main(0, cols::MODE_T) + + b.main(0, cols::MODE_L) + + b.main(0, cols::MODE_P); + let x = a + rc * m; + x.clone() * x.clone() * x + }) + .collect(); + let sum = t[1..].iter().fold(t[0].clone(), |acc, ti| acc + ti.clone()); + for (j, tj) in t.into_iter().enumerate() { + let out = b.main(0, cols::OUT0 + j); + b.emit_base(4 + j, out - (tj + sum.clone())); + } + + // idx 16: mode sum-boolean (exactly-one-of is the registrar's). + let mode_sum = mode_c + mode_t + mode_l + mode_p; + let one = b.one(); + b.emit_base(16, mode_sum.clone() * (one - mode_sum)); + + // idx 17–24: the unread input cells. ★ REQUIRED HERE, because the + // round above reads `IN_i` for every `i < 8` — including the four a + // leaf row does not read. See `emit_unread_input_pins`. + emit_unread_input_pins(b, 17); + } + + /// Poseidon-original at width 12: 30 rounds of `x ↦ x⁷` (all lanes on + /// the 8 full rounds, lane 0 only on the 22 partial ones) followed by + /// the circulant MDS. + /// + /// **Degree is exactly 3, by construction.** `x⁷` is lowered as + /// `(x³)²·x` over the witnessed `x²`/`x³` columns, so the MDS output + /// constraint — the highest-degree one — is `column² · (degree-1 + /// expression)`. That keeps `max_degree() = 3` and leaves the wrap's + /// blowup 2 untouched, which is the whole reason the S-box is + /// decomposed instead of written `a⁷`. + /// + /// **The round constant is scaled by the mode sum, and that is + /// load-bearing.** With `m = MODE_C + MODE_P = 0` a zero-filled padding + /// row gives `a = 0`, hence `x² = x³ = 0` and `out = MDS·0 = 0`, + /// inductively through all 30 rounds — so padding satisfies every + /// constraint without a degree-4 `IS_REAL` gate anywhere. On a real row + /// `m = 1` and the permutation is unchanged. + fn eval_poseidon>(b: &mut B) { + use crate::lfm::poseidon::{MDS_CIRC_ROW, ROUND_CONSTANTS, sboxed_lanes}; + use poseidon_cols as pc; + + let mode_c = b.main(0, pc::MODE_C); + let mode_t = b.main(0, pc::MODE_T); + let mode_l = b.main(0, pc::MODE_L); + let mode_p = b.main(0, pc::MODE_P); + let m = mode_c + mode_t + mode_l + mode_p.clone(); + + // idx 0–3: capacity-state copy — S_i = MODE_P·IN_i. + // + // Poseidon's `compress_iv` is ZERO (plain sponge compression, no + // domain separation invented here), so the `MODE_C·IV_i` term the + // TestPermutation version carries vanishes: on a compress row + // MODE_P = 0 forces S_i = 0, which IS the IV. Transcript and leaf + // rows are the same shape and take the same zero capacity — + // Poseidon has one domain here, so it separates none of them. + for k in 0..4 { + let s = b.main(0, pc::S8 + k); + let in_i = b.main(0, pc::IN0 + 8 + k); + b.emit_base(k, s - mode_p.clone() * in_i); + } + + // idx 4: mode sum-boolean (exactly-one-of is the registrar's). + let one = b.one(); + b.emit_base(4, m.clone() * (one - m.clone())); + + let mut idx = 5; + for (r, rc_row) in ROUND_CONSTANTS.iter().enumerate() { + let sboxed = sboxed_lanes(r); + + // a_i = state_i + rc[r][i]·m, degree 1. Round 0 reads IN/S; + // later rounds read the previous round's MDS output. + let a: Vec = rc_row + .iter() + .enumerate() + .map(|(i, rc_i)| { + let state = if r == 0 { + if i < 8 { + b.main(0, pc::IN0 + i) + } else { + b.main(0, pc::S8 + (i - 8)) + } + } else { + b.main(0, pc::out(r - 1, i)) + }; + let rc = b.const_base(*rc_i); + state + rc * m.clone() + }) + .collect(); + + // The two S-box steps per S-boxed lane, both degree 2. + for (lane, a_lane) in a.iter().enumerate().take(sboxed) { + let x2 = b.main(0, pc::x2(r, lane)); + let x3 = b.main(0, pc::x3(r, lane)); + b.emit_base(idx, x2.clone() - a_lane.clone() * a_lane.clone()); + b.emit_base(idx + 1, x3 - x2 * a_lane.clone()); + idx += 2; + } + + // What enters the MDS: a^7 = (x³)²·a on S-boxed lanes + // (degree 3), the bare post-constant lane otherwise. + let f: Vec = (0..HASH_STATE_FELTS) + .map(|i| { + if i < sboxed { + let x3 = b.main(0, pc::x3(r, i)); + x3.clone() * x3 * a[i].clone() + } else { + a[i].clone() + } + }) + .collect(); + + // out_o = Σ_i MDS_CIRC_ROW[(i − o) mod 12] · f_i — the same + // orientation `poseidon::PoseidonGoldilocks::mds` uses, and one + // of the three conventions the external KAT pins. + for o in 0..HASH_STATE_FELTS { + let acc = f + .iter() + .enumerate() + .fold(None::, |acc, (i, fi)| { + let c = b.const_base( + MDS_CIRC_ROW[(i + HASH_STATE_FELTS - o) % HASH_STATE_FELTS], + ); + let term = c * fi.clone(); + Some(match acc { + None => term, + Some(x) => x + term, + }) + }) + .expect("twelve lanes"); + let out = b.main(0, pc::out(r, o)); + b.emit_base(idx, out - acc); + idx += 1; + } + } + idx = emit_unread_input_pins(b, idx); + debug_assert_eq!( + idx, + poseidon_cols::NUM_CONSTRAINTS, + "every declared constraint index must be emitted exactly once" + ); + } + } +} + +// ========================================================================= +// LFM_KECCAK — the keccak-f[1600] adapter +// ========================================================================= +// +// Replaces the production `KECCAK` core chip, which is VM-coupled (it moves +// the state through timestamped `MEMW` tokens) and therefore unusable here. +// This chip owns exactly the core's two `Keccak` bus tokens and binds them to +// `LfmMem` words instead of memory. The permutation itself is proved by the +// UNCHANGED production `KECCAK_RND` / `KECCAK_RC` / `BITWISE` AIRs — see +// `keccak_adapter` and `keccak_probe`, which pin that contract standalone. +// +// CONSTRAINTS: none, and none are needed for the 400 state byte columns. +// Byte-ness is transitive: every IN byte is an operand of a `BYTE_ALU[XOR]` +// lookup in the round chip's θ column-parity chain (which covers all 25 lanes) +// and again in θ-final, and every OUT byte is the *result* of a `BYTE_ALU[XOR]` +// lookup (χ, or ι for lane 0). BYTE_ALU tokens carry the result as a tuple +// element, so a non-byte value finds no row in the 2^20 BITWISE table and the +// bus cannot balance. That in turn makes each `u32` half — a fixed linear +// combination of four such bytes — free of any separate range check: four +// values below 2^8 with coefficients 1, 2^8, 2^16, 2^24 cannot reach 2^32. + +pub mod keccak { + use super::*; + use crate::lfm::layout::keccak::{ + BLOCK_HALVES, BLOCK_WORDS, NUM_HALVES, NUM_WORDS, RATE_BYTES, RATE_LANES, + }; + use crate::tables::types::alu_op; + + pub mod cols { + pub use crate::lfm::layout::keccak::*; + /// The state as received from memory, 200 byte columns, lane-major: + /// `STATE + lane * 8 + b`. + pub const STATE: usize = PREP_WIDTH; // 56 + /// The rate block as received, 136 byte columns. Block byte `k` is byte + /// `k % 8` of lane `k / 8` — rate bytes are lane-major and + /// little-endian within a lane, exactly like the state columns, so + /// block byte `k` pairs with state byte `k`. (The column-major traversal + /// that bites elsewhere is a property of the *token element order*, not + /// of this column layout — see `keccak_token`.) + pub const BLOCK: usize = STATE + 200; // 256 + /// What enters the permutation: `STATE ⊕ BLOCK` over the rate region on + /// absorb rows, `STATE` everywhere else. + pub const PERM_IN: usize = BLOCK + RATE_BYTES; // 392 + /// The permuted state, 200 byte columns. + pub const OUT: usize = PERM_IN + 200; // 592 + pub const NUM_COLUMNS: usize = OUT + 200; // 792 + + pub const fn state_byte(lane: usize, b: usize) -> usize { + STATE + lane * 8 + b + } + pub const fn perm_in_byte(lane: usize, b: usize) -> usize { + PERM_IN + lane * 8 + b + } + pub const fn out_byte(lane: usize, b: usize) -> usize { + OUT + lane * 8 + b + } + } + + /// The row's is-real flag: exactly one mode on a real row, neither on + /// padding. + fn is_real() -> Multiplicity { + Multiplicity::Sum(cols::MODE_PERM, cols::MODE_ABSORB) + } + + /// Half `h` of the byte family at `bytes_start`, recomposed from its four + /// byte columns as `Σ byte_k · 256^k`. + /// + /// This is the trick the dropped core chip used to rebuild addresses from + /// byte columns (`tables/keccak.rs`): the machine-side value never gets its + /// own column, so there is nothing extra to keep consistent. Half slots at + /// or above `num_halves` are the family's unused top lanes and become tuple + /// constants — a nonzero value there cannot balance. + /// + /// Note `(h / 2) * 8 + 4 * (h % 2) == 4 * h`; the long form is kept because + /// it names why: half `h` is the low or high 4 bytes of lane `h / 2`. + fn half_value(bytes_start: usize, h: usize, num_halves: usize) -> BusValue { + if h >= num_halves { + return zero(); + } + let byte0 = bytes_start + (h / 2) * 8 + 4 * (h % 2); + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: byte0 + k, + }) + .collect(), + ) + } + + /// An `LfmMem` token for word `word` of a byte family: `(addr, h0..h3)`. + fn word_token_from_bytes( + addr_col: usize, + bytes_start: usize, + word: usize, + num_halves: usize, + ) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| half_value(bytes_start, 4 * word + l, num_halves))); + v + } + + /// Half `h` of the byte-reversed digest: reversed byte `j` is digest byte + /// `31 − j`, so this half's bytes are `OUT[31 − 4h − k]` for `k = 0..3` with + /// the usual little-endian coefficients. Both the byte order WITHIN a half + /// and the order OF the halves come out reversed, which is exactly what + /// reversing all 32 bytes means. + fn reversed_half_value(h: usize) -> BusValue { + BusValue::Linear( + (0..4) + .map(|k| LinearTerm::ColumnUnsigned { + coefficient: 1u64 << (8 * k), + column: cols::OUT + 31 - 4 * h - k, + }) + .collect(), + ) + } + + /// An `LfmMem` token for word `w` of the reversed digest. + fn reversed_digest_token(addr_col: usize, w: usize) -> Vec { + let mut v = vec![direct(addr_col)]; + v.extend((0..4).map(|l| reversed_half_value(4 * w + l))); + v + } + + /// A `Keccak` bus token: `(tag_lo, tag_hi, round, state[200])`. + /// + /// The 200 state elements are traversed **column-major over lanes** — + /// element `3 + 8·(5x + y) + b` is byte `b` of lane `x + 5y`, so lanes come + /// in the order 0, 5, 10, 15, 20, 1, 6, … That asymmetry is inherited from + /// the production sender's `for x { for y { … } }` loop over a + /// `(x + 5y)·8 + b` column formula; emitting them in natural order instead + /// leaves the bus unbalanced (falsification-verified in R1a). + #[allow(clippy::needless_range_loop)] + fn keccak_token(round: u64, bytes_start: usize) -> Vec { + let mut values = vec![ + direct(cols::TAG_LO), + direct(cols::TAG_HI), + BusValue::constant(round), + ]; + for x in 0..5 { + for y in 0..5 { + for b in 0..8 { + values.push(direct(bytes_start + (x + 5 * y) * 8 + b)); + } + } + } + values + } + + pub fn bus_interactions() -> Vec { + let mut interactions = Vec::with_capacity(2 * NUM_WORDS + BLOCK_WORDS + RATE_BYTES + 2); + // Reads: the 13 state words. + for j in 0..NUM_WORDS { + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + is_real(), + word_token_from_bytes(cols::in_addr(j), cols::STATE, j, NUM_HALVES), + )); + } + // Reads: the 9 rate-block words — absorb rows only, so on a permute row + // the BLOCK columns are read by nothing (no token, no lookup) and are + // simply dead witness. + for j in 0..BLOCK_WORDS { + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_ABSORB), + word_token_from_bytes(cols::block_addr(j), cols::BLOCK, j, BLOCK_HALVES), + )); + } + // Writes: the 13 output words, each with its own read count. + for j in 0..NUM_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::mult(j)), + word_token_from_bytes(cols::out_addr(j), cols::OUT, j, NUM_HALVES), + )); + } + // The absorb XOR, one BITWISE lookup per rate byte: + // `PERM_IN[k] = STATE[k] ⊕ BLOCK[k]`. + for k in 0..RATE_BYTES { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MODE_ABSORB), + vec![ + BusValue::constant(alu_op::XOR as u64), + direct(cols::STATE + k), + direct(cols::BLOCK + k), + direct(cols::PERM_IN + k), + ], + )); + } + // The reversed digest: the first 32 output bytes read back-to-front, + // as two words. This is the production transcript's `sample()` — it + // finalizes, reverses the digest in place, absorbs the reversed bytes + // and returns them, so one value serves as both the challenge and the + // next segment's prefix. + // + // Reversal is FREE at the recomposition boundary: the bus already + // rebuilds each `u32` half as a linear combination of four byte + // columns, so flipping the coefficient order (and the half order) is a + // different Linear over the SAME columns — no new value columns, no + // BitDec, no extra permutation. Rows that need no reversed digest leave + // `REV_MULT` at zero and these two sends are inert. + for w in 0..cols::DIGEST_WORDS { + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::rev_mult(w)), + reversed_digest_token(cols::rev_addr(w), w), + )); + } + // The request/reply pair that drives the production keccak family. + interactions.push(BusInteraction::sender( + BusId::Keccak, + is_real(), + keccak_token(0, cols::PERM_IN), + )); + interactions.push(BusInteraction::receiver( + BusId::Keccak, + is_real(), + keccak_token(24, cols::OUT), + )); + interactions + } + + pub struct KeccakAdapterConstraints; + + impl ConstraintSet for KeccakAdapterConstraints { + fn max_degree(&self) -> usize { + 2 + } + + fn eval>(&self, b: &mut B) { + // idx 0..63: the capacity region never absorbs, in either mode — + // `PERM_IN = STATE` for lanes 17..24, ungated (and trivially true on + // zero-filled padding rows). + for i in 0..(25 - RATE_LANES) * 8 { + let k = RATE_BYTES + i; + let s = b.main(0, cols::STATE + k); + let p = b.main(0, cols::PERM_IN + k); + b.emit_base(i, p - s); + } + // idx 64..199: on a permute row nothing is absorbed, so the rate + // region passes through too. On an absorb row this is gated off and + // the BYTE_ALU[XOR] lookups above pin PERM_IN instead. Without this, + // a permute row could feed the family a state unrelated to the one + // it read from memory. + let base = (25 - RATE_LANES) * 8; + for k in 0..RATE_BYTES { + let mode_perm = b.main(0, cols::MODE_PERM); + let s = b.main(0, cols::STATE + k); + let p = b.main(0, cols::PERM_IN + k); + b.emit_base(base + k, mode_perm * (p - s)); + } + // idx 200: mode sum-boolean (exactly-one-of is the registrar's). + let sum = b.main(0, cols::MODE_PERM) + b.main(0, cols::MODE_ABSORB); + let one = b.one(); + b.emit_base(base + RATE_BYTES, sum.clone() * (one - sum)); + } + } +} + +// ========================================================================= +// LFM_LANES — word ↔ lane conversion (Pack / Unpack) +// ========================================================================= +// +// Discovered as a real ISA gap in Milestone C: challenges are squeezed from +// the sponge as *cells*, but the ALU consumes base/ext operands, and no +// composition of the original eight ops can cross that boundary. The chip +// has NO constraints — the shared value columns appearing in both the word +// token and the four lane tokens IS the semantics. + +pub mod lanes { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::lanes::*; + pub const V0: usize = PREP_WIDTH; // ..V3 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + let mut interactions = vec![ + // Pack rows write the assembled word; Unpack rows read one. + BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::WORD_MULT), + word_token(cols::WORD_ADDR, cols::V0), + ), + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_UNPACK), + word_token(cols::WORD_ADDR, cols::V0), + ), + ]; + for i in 0..4 { + // Unpack rows write the four lanes; Pack rows read them. + interactions.push(BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::LANE_MULT0 + i), + base_token(cols::LANE_ADDR0 + i, cols::V0 + i), + )); + interactions.push(BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::MODE_PACK), + base_token(cols::LANE_ADDR0 + i, cols::V0 + i), + )); + } + interactions + } +} + +// ========================================================================= +// LFM_HINT — arena ingestion (values unconstrained BY DESIGN; arena rule) +// ========================================================================= + +pub mod hint { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::hint::*; + pub const V0: usize = PREP_WIDTH; // ..V3 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + vec![BusInteraction::sender( + BusId::LfmMem, + Multiplicity::Column(cols::MULT), + word_token(cols::OUT_ADDR, cols::V0), + )] + } +} + +// ========================================================================= +// LFM_PUBLIC — attestation output (COMMIT-bus closure pattern) +// ========================================================================= + +pub mod public { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::public::*; + pub const V0: usize = PREP_WIDTH; // ..V3 + pub const NUM_COLUMNS: usize = PREP_WIDTH + 4; + } + + pub fn bus_interactions() -> Vec { + let mut send = vec![direct(cols::INDEX)]; + send.extend(word(cols::V0)); + vec![ + BusInteraction::receiver( + BusId::LfmMem, + Multiplicity::Column(cols::IS_REAL), + word_token(cols::IN_ADDR, cols::V0), + ), + BusInteraction::sender(BusId::LfmPublic, Multiplicity::Column(cols::IS_REAL), send), + ] + } +} + +// ========================================================================= +// LFM_RANGE — fixed 2^16 lookup table (idle in v0; the future hash chip's +// byte/limb tables land here) +// ========================================================================= + +pub mod range { + use super::*; + + pub mod cols { + pub use crate::lfm::layout::range::*; + pub const MU: usize = PREP_WIDTH; + pub const NUM_COLUMNS: usize = PREP_WIDTH + 1; + } + + pub fn bus_interactions() -> Vec { + vec![BusInteraction::receiver( + BusId::LfmRange, + Multiplicity::Column(cols::MU), + vec![direct(cols::VALUE)], + )] + } +} diff --git a/prover/src/lfm/chunking.rs b/prover/src/lfm/chunking.rs new file mode 100644 index 000000000..d9214805c --- /dev/null +++ b/prover/src/lfm/chunking.rs @@ -0,0 +1,168 @@ +//! `KECCAK_RND` chunking — how the hosted keccak family scales past one table. +//! +//! `KECCAK_RND` costs 24 rows per permutation at 1480 columns, so a single +//! instance saturates a 2^19-row table at ~21.8k permutations while a real +//! proof wrap needs ~460k. The RV64 VM solves the same problem for its own +//! tables by splitting them into chunk-AIRs; LFM does the same, with one +//! simplification: **the chunk count is static program shape**, fixed by +//! [`KeccakChunking`] at compile time, pinned in the registry and bound into +//! the program digest — never derived at prove time and never read off the +//! proof. +//! +//! # Why splitting the rows is free +//! +//! `KECCAK_RND` has no row-to-row transition constraints at all (its +//! [`ConstraintSet`](crate::tables::keccak_rnd::KeccakRndConstraints) is 20 +//! per-row `IS_BIT` checks). The 24-round chain is carried entirely by the +//! `Keccak` bus: row *r* receives `(tag, r, state)` and sends `(tag, r+1, +//! out)`, so consecutive rounds are linked by token *matching*, not by row +//! adjacency. LogUp balances the multiset over every AIR in the proof, so it +//! cannot tell which instance a row lived in. That is what makes chunking need +//! zero pairing logic — the same property the VM's chunked tables rely on. +//! +//! # What is *not* chunked +//! +//! `KECCAK_RC` and `BITWISE` stay single shared instances. Both are receivers +//! whose multiplicity columns count lookups from the whole proof: +//! `keccak_rc::update_multiplicities` writes the total permutation count into +//! every round row, and `bitwise::BitwiseHistogram` accumulates every operation +//! before the trace is filled. Per-chunk copies would each have to carry the +//! full histogram and would then over-receive. Their sizes are fixed anyway +//! (32 and 2^20 rows), so they never needed splitting. + +/// Trace rows one permutation occupies in `KECCAK_RND` — one per round. +pub const KECCAK_RND_ROWS_PER_PERMUTATION: usize = 24; + +/// Rows per `KECCAK_RND` chunk in the default policy. +/// +/// Retuning knob: this trades sub-proof count against per-chunk prover memory, +/// exactly like `max_rows` does for the VM's split tables. 2^19 rows is 21,845 +/// permutations per chunk. +pub const KECCAK_RND_MAX_CHUNK_ROWS: usize = 1 << 19; + +/// How a program's permutations are distributed over `KECCAK_RND` instances. +/// +/// Carried on [`LfmProgram`](super::compiler::LfmProgram), so trace generation +/// and artifact building read the same policy and cannot disagree. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeccakChunking { + permutations_per_chunk: usize, +} + +impl KeccakChunking { + /// The policy that fills chunks to at most `max_rows` trace rows. + /// + /// Panics at compile time (it is `const`) if `max_rows` cannot hold a + /// single permutation. + pub const fn from_max_rows(max_rows: usize) -> Self { + let permutations_per_chunk = max_rows / KECCAK_RND_ROWS_PER_PERMUTATION; + assert!( + permutations_per_chunk > 0, + "a KECCAK_RND chunk must hold at least one permutation (24 rows)" + ); + Self { + permutations_per_chunk, + } + } + + /// The policy that puts at most `permutations_per_chunk` permutations in + /// each chunk. The small-limit constructor tests use to force several + /// chunks out of a tiny program. + pub const fn from_permutations(permutations_per_chunk: usize) -> Self { + assert!( + permutations_per_chunk > 0, + "a KECCAK_RND chunk must hold at least one permutation" + ); + Self { + permutations_per_chunk, + } + } + + pub const fn permutations_per_chunk(self) -> usize { + self.permutations_per_chunk + } + + /// Number of `KECCAK_RND` instances a program with `num_permutations` + /// permutations gets — never zero, so the chip is present (and its + /// constraints verified) even for a program containing no keccak at all. + pub fn chunk_count(self, num_permutations: usize) -> usize { + num_permutations + .div_ceil(self.permutations_per_chunk) + .max(1) + } + + /// Splits per-permutation records into exactly [`Self::chunk_count`] + /// slices. The single rule both trace generation and the artifact/AIR + /// shape derive from; `split_agrees_with_chunk_count` pins the agreement. + pub fn split(self, permutations: &[T]) -> Vec<&[T]> { + if permutations.is_empty() { + vec![&permutations[..0]] + } else { + permutations.chunks(self.permutations_per_chunk).collect() + } + } +} + +impl Default for KeccakChunking { + fn default() -> Self { + Self::from_max_rows(KECCAK_RND_MAX_CHUNK_ROWS) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_policy_is_the_documented_geometry() { + let c = KeccakChunking::default(); + assert_eq!(c.permutations_per_chunk(), 21845); + assert!( + c.permutations_per_chunk() * KECCAK_RND_ROWS_PER_PERMUTATION + <= KECCAK_RND_MAX_CHUNK_ROWS + ); + // One chunk up to the limit, two past it. + assert_eq!(c.chunk_count(21845), 1); + assert_eq!(c.chunk_count(21846), 2); + // The ~460k-permutation wrap the design targets. + assert_eq!(c.chunk_count(460_000), 22); + } + + #[test] + fn empty_programs_still_get_one_chunk() { + for per in [1usize, 2, 7, 21845] { + let c = KeccakChunking::from_permutations(per); + assert_eq!(c.chunk_count(0), 1); + assert_eq!(c.split::(&[]).len(), 1); + assert!(c.split::(&[])[0].is_empty()); + } + } + + /// `split` and `chunk_count` are the same rule seen twice; if they ever + /// disagree the prover builds a different number of traces than the + /// verifier builds AIRs. + #[test] + fn split_agrees_with_chunk_count() { + for per in [1usize, 2, 3, 5, 24] { + let c = KeccakChunking::from_permutations(per); + for n in 0..40usize { + let ops: Vec = (0..n).collect(); + let split = c.split(&ops); + assert_eq!( + split.len(), + c.chunk_count(n), + "per={per} n={n}: split and chunk_count disagree" + ); + assert_eq!( + split.iter().map(|s| s.len()).sum::(), + n, + "per={per} n={n}: split lost or duplicated permutations" + ); + assert!( + split.iter().all(|s| s.len() <= per), + "per={per} n={n}: a chunk exceeded the limit" + ); + } + } + } +} diff --git a/prover/src/lfm/commit.rs b/prover/src/lfm/commit.rs new file mode 100644 index 000000000..4aff87b27 --- /dev/null +++ b/prover/src/lfm/commit.rs @@ -0,0 +1,58 @@ +//! Instruction-column-group commitment: interpolate → LDE → row-pair Merkle. +//! +//! The same pipeline the static preprocessed tables use (see +//! `tables/bitwise.rs::compute_preprocessed_commitment`), generalized over an +//! arbitrary column matrix so every LFM chip's group — and the registry +//! builder — shares one implementation. Host-side only; runs at program-build +//! and registry-regeneration time (seconds, not a ceremony — there is no +//! keygen in this framework). + +use math::polynomial::Polynomial; +use stark::commitment::{ROWS_PER_LEAF, commit_bit_reversed}; +use stark::config::Commitment; +use stark::proof::options::ProofOptions; +use stark::prover::evaluate_polynomial_on_lde_domain; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::compiler::ColumnGroup; + +/// Commits a column matrix (each inner `Vec` one column, power-of-two height). +pub fn commit_columns(columns: &[Vec], options: &ProofOptions) -> Commitment { + let num_rows = columns.first().map_or(0, Vec::len); + let polys: Vec> = columns + .iter() + .map(|col| { + Polynomial::interpolate_fft::(col) + .expect("FFT interpolation failed for LFM column group") + }) + .collect(); + let coset_offset = FE::from(options.coset_offset); + let lde_columns: Vec> = polys + .iter() + .map(|poly| { + evaluate_polynomial_on_lde_domain( + poly, + options.blowup_factor as usize, + num_rows, + &coset_offset, + ) + .expect("LDE evaluation failed for LFM column group") + }) + .collect(); + let (_, root) = commit_bit_reversed(&lde_columns, ROWS_PER_LEAF) + .expect("Merkle build failed for LFM column group"); + root +} + +/// A [`ColumnGroup`]'s data, column-major (the commit pipeline's input shape). +pub fn group_columns(group: &ColumnGroup) -> Vec> { + (0..group.width) + .map(|c| (0..group.padded_rows).map(|r| *group.at(r, c)).collect()) + .collect() +} + +/// Commits one instruction column group. +pub fn commit_group(group: &ColumnGroup, options: &ProofOptions) -> Commitment { + commit_columns(&group_columns(group), options) +} diff --git a/prover/src/lfm/compiler.rs b/prover/src/lfm/compiler.rs new file mode 100644 index 000000000..25e831642 --- /dev/null +++ b/prover/src/lfm/compiler.rs @@ -0,0 +1,501 @@ +//! The LFM straight-line compiler. +//! +//! Pass 1 backfills static multiplicities from the builder's read counters, +//! guarded by the two invariant panics (tripwires — the release-mode +//! admission validator is the gate, the registry is the record): +//! - **panic #1**: an address assigned twice (write-once violated in the +//! builder itself); +//! - **panic #2**: the read-count map is not drained after backfill (a +//! read of an address no instruction writes). +//! +//! Pass 2 emits the per-chip **instruction column groups** — the preprocessed +//! matrices whose Merkle roots become the program's identity. Layouts live in +//! [`super::layout`]; group commitment (interpolate → LDE → Merkle) is wired +//! at registry-build time (Milestone B) through the same pipeline the static +//! tables use. + +use crate::tables::types::FE; + +use super::builder::{ArenaSchema, LfmProgramSource}; +use super::chunking::KeccakChunking; +use super::instr::{Addr, BaseOp, ExtOp, HashMode, Instr, KeccakMode}; +use super::layout::{self, padded_rows}; + +/// One chip's instruction column group: a row-major matrix, zero-padded to a +/// power-of-two height (min 4). +#[derive(Debug, Clone)] +pub struct ColumnGroup { + pub width: usize, + pub real_rows: usize, + pub padded_rows: usize, + /// Row-major, `padded_rows × width`. + pub data: Vec, +} + +/// Accumulates one chip's rows directly into the flat row-major buffer the +/// finished [`ColumnGroup`] holds. +/// +/// The emitter used to collect `Vec>` and copy it row by row. That kept +/// two full materializations of every group alive at once and paid a heap +/// allocation per instruction — and the per-row `Vec`s over-allocate badly, +/// because they are grown by `extend`/`push` rather than sized: a 10-wide BALU +/// row lands at capacity 18. Appending into one buffer removes the second +/// materialization, the headers, the rounding waste and the ~271M malloc/free +/// pairs. +/// +/// Rows come out bit-identical: the same values are written at the same +/// row-major offsets, and the tail is zero-padded to the same height. +struct ColumnGroupBuilder { + width: usize, + real_rows: usize, + data: Vec, +} + +impl ColumnGroupBuilder { + fn new(width: usize) -> Self { + ColumnGroupBuilder { + width, + real_rows: 0, + data: Vec::new(), + } + } + + /// The ordinal the next row will take. `LFM_KECCAK` binds it into the row + /// as a structural tag, so it has to be read before [`Self::open_row`]. + fn next_row(&self) -> usize { + self.real_rows + } + + /// Append a zero-filled row, returning its base offset for [`Self::set`]. + fn open_row(&mut self) -> usize { + let base = self.data.len(); + self.data.resize(base + self.width, FE::zero()); + self.real_rows += 1; + base + } + + fn set(&mut self, base: usize, col: usize, v: FE) { + debug_assert!( + col < self.width, + "column {col} outside width {}", + self.width + ); + self.data[base + col] = v; + } + + fn finish(mut self) -> ColumnGroup { + let padded = padded_rows(self.real_rows); + self.data.resize(padded * self.width, FE::zero()); + ColumnGroup { + width: self.width, + real_rows: self.real_rows, + padded_rows: padded, + data: self.data, + } + } +} + +impl ColumnGroup { + pub fn at(&self, row: usize, col: usize) -> &FE { + &self.data[row * self.width + col] + } + + pub fn set(&mut self, row: usize, col: usize, v: FE) { + self.data[row * self.width + col] = v; + } +} + +/// The eight program-dependent instruction column groups, in the frozen chip +/// order. (`LFM_RANGE`'s group is program-independent and materialized at +/// commitment time.) +#[derive(Debug, Clone)] +pub struct LfmColumnGroups { + pub const_: ColumnGroup, + pub balu: ColumnGroup, + pub xalu: ColumnGroup, + pub select: ColumnGroup, + pub bitdec: ColumnGroup, + pub hash: ColumnGroup, + pub keccak: ColumnGroup, + pub lanes: ColumnGroup, + pub hint: ColumnGroup, + pub public: ColumnGroup, +} + +/// A compiled LFM program: multiplicity-backfilled instructions plus the +/// emitted instruction column groups. +#[derive(Debug)] +pub struct LfmProgram { + pub instrs: Vec, + pub num_addrs: u64, + pub arena_schema: ArenaSchema, + pub public_len: u32, + pub groups: LfmColumnGroups, + /// How this program's permutations are spread over `KECCAK_RND` + /// instances. Program shape, not a runtime knob: it is fixed here, bound + /// into the program digest and pinned in the registry. + pub chunking: KeccakChunking, +} + +impl LfmProgram { + /// Replaces the `KECCAK_RND` chunking policy. + /// + /// Chunking affects only how the round-chip rows are distributed over AIR + /// instances — never what is compiled — so it is safe to set after + /// compilation. Tests use it to force several chunks out of a program with + /// a handful of permutations; retuning uses it to size chunks per preset. + pub fn with_keccak_chunking(mut self, chunking: KeccakChunking) -> Self { + self.chunking = chunking; + self + } +} + +/// Emission backends. Backend 1 (column groups) is the machine; backend 2 is +/// the future circuit specialization, a stub by design so the option stays an +/// edit instead of a rewrite. +pub trait LfmBackend { + type Artifacts; + fn emit(&self, program: &LfmProgram) -> Self::Artifacts; +} + +pub struct ColumnGroupBackend; +impl LfmBackend for ColumnGroupBackend { + type Artifacts = LfmColumnGroups; + fn emit(&self, program: &LfmProgram) -> LfmColumnGroups { + program.groups.clone() + } +} + +/// The circuit backend does not exist yet; it panics so nothing can silently +/// depend on it. +pub struct CircuitBackend; +impl LfmBackend for CircuitBackend { + type Artifacts = (); + fn emit(&self, _program: &LfmProgram) -> () { + unimplemented!( + "LFM circuit backend is a v1+ specialization; only the column-group backend exists" + ) + } +} + +pub fn compile(source: LfmProgramSource) -> LfmProgram { + let LfmProgramSource { + mut instrs, + num_addrs, + mut read_counts, + arena_schema, + public_len, + } = source; + + // Pass 1: occupancy + multiplicity backfill. + let mut written = vec![false; num_addrs as usize]; + let take = |addr: Addr, written: &mut Vec, counts: &mut [u64]| -> u64 { + let slot = written + .get_mut(addr.0 as usize) + .unwrap_or_else(|| panic!("LFM compiler invariant: address {} out of range", addr.0)); + if *slot { + panic!("LFM compiler invariant: address {} written twice", addr.0); + } + *slot = true; + // Taking (not reading) is what drains the counter, so the emptiness + // check below still means "every read had a writer". + core::mem::take(&mut counts[addr.0 as usize]) + }; + for instr in &mut instrs { + match instr { + Instr::Const { out, mult, .. } + | Instr::BaseAlu { out, mult, .. } + | Instr::ExtAlu { out, mult, .. } + | Instr::Hint { out, mult, .. } + | Instr::Pack { out, mult, .. } => { + *mult = take(*out, &mut written, &mut read_counts); + } + Instr::Unpack { outs, mults, .. } => { + for i in 0..4 { + mults[i] = take(outs[i], &mut written, &mut read_counts); + } + } + Instr::KeccakF(k) => { + for i in 0..layout::keccak::NUM_WORDS { + k.mults[i] = take(k.outs[i], &mut written, &mut read_counts); + } + if let Some(rev) = &mut k.rev { + for i in 0..layout::keccak::DIGEST_WORDS { + rev.mults[i] = take(rev.outs[i], &mut written, &mut read_counts); + } + } + } + Instr::Select { + out_l, + out_r, + mult_l, + mult_r, + .. + } => { + *mult_l = take(*out_l, &mut written, &mut read_counts); + *mult_r = take(*out_r, &mut written, &mut read_counts); + } + Instr::BitDec { bits, .. } => { + for (addr, mult) in bits.iter_mut() { + *mult = take(*addr, &mut written, &mut read_counts); + } + } + Instr::Hash { + mode, outs, mults, .. + } => { + let num_outs = mode.num_output_cells(); + for i in 0..num_outs { + mults[i] = take(outs[i], &mut written, &mut read_counts); + } + } + Instr::Public { .. } => {} + } + } + assert!( + read_counts.iter().all(|&c| c == 0), + "LFM compiler invariant: read-count map not drained after backfill — reads of never-written addresses: {:?}", + read_counts + .iter() + .enumerate() + .filter(|(_, c)| **c != 0) + .map(|(a, _)| Addr(a as u64)) + .collect::>() + ); + + // Both are dead from here on and together outweigh the groups being built. + // Dropping them explicitly keeps the emitter's peak off the sum of the two + // materializations — the scope would otherwise hold them to the end. + drop(read_counts); + drop(written); + + let groups = emit_column_groups(&instrs, public_len); + + LfmProgram { + instrs, + num_addrs, + arena_schema, + public_len, + groups, + chunking: KeccakChunking::default(), + } +} + +fn fe(v: u64) -> FE { + FE::from(v) +} + +/// Pass 2: partition instructions per chip (program order preserved) and lay +/// out each chip's instruction fields per [`super::layout`]. +fn emit_column_groups(instrs: &[Instr], _public_len: u32) -> LfmColumnGroups { + let mut const_ = ColumnGroupBuilder::new(layout::const_::PREP_WIDTH); + let mut balu = ColumnGroupBuilder::new(layout::balu::PREP_WIDTH); + let mut xalu = ColumnGroupBuilder::new(layout::xalu::PREP_WIDTH); + let mut select = ColumnGroupBuilder::new(layout::select::PREP_WIDTH); + let mut bitdec = ColumnGroupBuilder::new(layout::bitdec::PREP_WIDTH); + let mut hash = ColumnGroupBuilder::new(layout::hash::PREP_WIDTH); + let mut keccak = ColumnGroupBuilder::new(layout::keccak::PREP_WIDTH); + let mut lanes = ColumnGroupBuilder::new(layout::lanes::PREP_WIDTH); + let mut hint = ColumnGroupBuilder::new(layout::hint::PREP_WIDTH); + let mut public = ColumnGroupBuilder::new(layout::public::PREP_WIDTH); + + for instr in instrs { + match instr { + Instr::Const { out, value, mult } => { + use layout::const_ as c; + let r = const_.open_row(); + const_.set(r, c::ADDR, fe(out.0)); + for (i, v) in value.iter().enumerate() { + const_.set(r, c::V0 + i, *v); + } + const_.set(r, c::MULT, fe(*mult)); + } + Instr::BaseAlu { + op, + out, + a, + b, + c, + mult, + } => { + use layout::balu as l; + let r = balu.open_row(); + balu.set(r, l::A_ADDR, fe(a.0)); + balu.set(r, l::B_ADDR, fe(b.0)); + balu.set(r, l::C_ADDR, fe(c.0)); + balu.set(r, l::OUT_ADDR, fe(out.0)); + let sel = match op { + BaseOp::Add => l::SEL_ADD, + BaseOp::Sub => l::SEL_SUB, + BaseOp::Mul => l::SEL_MUL, + BaseOp::Div => l::SEL_DIV, + BaseOp::MulAdd => l::SEL_MULADD, + }; + balu.set(r, sel, FE::one()); + balu.set(r, l::MULT, fe(*mult)); + } + Instr::ExtAlu { + op, + out, + a, + b, + c, + mult, + } => { + use layout::xalu as l; + let r = xalu.open_row(); + xalu.set(r, l::A_ADDR, fe(a.0)); + xalu.set(r, l::B_ADDR, fe(b.0)); + xalu.set(r, l::C_ADDR, fe(c.0)); + xalu.set(r, l::OUT_ADDR, fe(out.0)); + let sel = match op { + ExtOp::Add => l::SEL_ADD, + ExtOp::Sub => l::SEL_SUB, + ExtOp::Mul => l::SEL_MUL, + ExtOp::Div => l::SEL_DIV, + ExtOp::MulAdd => l::SEL_MULADD, + ExtOp::MulBase => l::SEL_MULBASE, + }; + xalu.set(r, sel, FE::one()); + xalu.set(r, l::MULT, fe(*mult)); + } + Instr::Select { + bit, + out_l, + out_r, + in_l, + in_r, + mult_l, + mult_r, + } => { + use layout::select as l; + let r = select.open_row(); + select.set(r, l::BIT_ADDR, fe(bit.0)); + select.set(r, l::INL_ADDR, fe(in_l.0)); + select.set(r, l::INR_ADDR, fe(in_r.0)); + select.set(r, l::OUTL_ADDR, fe(out_l.0)); + select.set(r, l::OUTR_ADDR, fe(out_r.0)); + select.set(r, l::MULT_L, fe(*mult_l)); + select.set(r, l::MULT_R, fe(*mult_r)); + select.set(r, l::IS_REAL, FE::one()); + } + Instr::BitDec { input, bits } => { + use layout::bitdec as l; + let r = bitdec.open_row(); + bitdec.set(r, l::IN_ADDR, fe(input.0)); + bitdec.set(r, l::IS_REAL, FE::one()); + for (i, (addr, mult)) in bits.iter().enumerate() { + bitdec.set(r, l::bit_addr(i), fe(addr.0)); + bitdec.set(r, l::bit_mult(i), fe(*mult)); + } + } + Instr::Hash { + mode, + ins, + outs, + mults, + } => { + // One-hot over the three modes. The AIR pins only the SUM to a + // bit; exactly-one-of is this emitter's job, re-checked by the + // admission validator. + use layout::hash as l; + let r = hash.open_row(); + hash.set(r, l::IN_ADDR0, fe(ins[0].0)); + hash.set(r, l::IN_ADDR1, fe(ins[1].0)); + hash.set(r, l::IN_ADDR2, fe(ins[2].0)); + hash.set(r, l::OUT_ADDR0, fe(outs[0].0)); + hash.set(r, l::OUT_ADDR1, fe(outs[1].0)); + hash.set(r, l::OUT_ADDR2, fe(outs[2].0)); + let mode_col = match mode { + HashMode::Compress => l::MODE_C, + HashMode::Transcript => l::MODE_T, + HashMode::Leaf => l::MODE_L, + HashMode::Permute => l::MODE_P, + }; + hash.set(r, mode_col, FE::one()); + hash.set(r, l::MULT0, fe(mults[0])); + hash.set(r, l::MULT1, fe(mults[1])); + hash.set(r, l::MULT2, fe(mults[2])); + } + Instr::KeccakF(op) => { + use layout::keccak as k; + // The tag is the row ordinal, so uniqueness is structural and + // the prover has no say (it is preprocessed data). See + // `layout::keccak::tag_for_row`. + let tag = k::tag_for_row(keccak.next_row()); + let r = keccak.open_row(); + keccak.set(r, k::TAG_LO, fe(tag & 0xFFFF_FFFF)); + keccak.set(r, k::TAG_HI, fe(tag >> 32)); + for j in 0..k::NUM_WORDS { + keccak.set(r, k::in_addr(j), fe(op.ins[j].0)); + keccak.set(r, k::out_addr(j), fe(op.outs[j].0)); + keccak.set(r, k::mult(j), fe(op.mults[j])); + } + if let Some(rev) = &op.rev { + for w in 0..k::DIGEST_WORDS { + keccak.set(r, k::rev_addr(w), fe(rev.outs[w].0)); + keccak.set(r, k::rev_mult(w), fe(rev.mults[w])); + } + } + match op.mode { + KeccakMode::Permute => keccak.set(r, k::MODE_PERM, FE::one()), + KeccakMode::Absorb => { + keccak.set(r, k::MODE_ABSORB, FE::one()); + for j in 0..k::BLOCK_WORDS { + keccak.set(r, k::block_addr(j), fe(op.block[j].0)); + } + } + } + } + Instr::Hint { out, mult, .. } => { + use layout::hint as l; + let r = hint.open_row(); + hint.set(r, l::OUT_ADDR, fe(out.0)); + hint.set(r, l::MULT, fe(*mult)); + } + Instr::Pack { + lanes: ls, + out, + mult, + } => { + use layout::lanes as l; + let r = lanes.open_row(); + lanes.set(r, l::WORD_ADDR, fe(out.0)); + for (i, lane) in ls.iter().enumerate() { + lanes.set(r, l::LANE_ADDR0 + i, fe(lane.0)); + } + lanes.set(r, l::MODE_PACK, FE::one()); + lanes.set(r, l::WORD_MULT, fe(*mult)); + } + Instr::Unpack { input, outs, mults } => { + use layout::lanes as l; + let r = lanes.open_row(); + lanes.set(r, l::WORD_ADDR, fe(input.0)); + for i in 0..4 { + lanes.set(r, l::LANE_ADDR0 + i, fe(outs[i].0)); + lanes.set(r, l::LANE_MULT0 + i, fe(mults[i])); + } + lanes.set(r, l::MODE_UNPACK, FE::one()); + } + Instr::Public { addr, index } => { + use layout::public as l; + let r = public.open_row(); + public.set(r, l::IN_ADDR, fe(addr.0)); + public.set(r, l::INDEX, fe(*index as u64)); + public.set(r, l::IS_REAL, FE::one()); + } + } + } + + LfmColumnGroups { + const_: const_.finish(), + balu: balu.finish(), + xalu: xalu.finish(), + select: select.finish(), + bitdec: bitdec.finish(), + hash: hash.finish(), + keccak: keccak.finish(), + lanes: lanes.finish(), + hint: hint.finish(), + public: public.finish(), + } +} diff --git a/prover/src/lfm/constraint_tests.rs b/prover/src/lfm/constraint_tests.rs new file mode 100644 index 000000000..ffdd95bbd --- /dev/null +++ b/prover/src/lfm/constraint_tests.rs @@ -0,0 +1,2033 @@ +//! The constraint-evaluation leg: lowering differential, cost census, and the +//! falsifications for each mechanism the lowering relies on. +//! +//! ## The oracle +//! +//! `eval_program_verifier` — the production CPU interpreter, on the OOD shape — +//! run over the DESERIALIZED artifact, never a local reimplementation of the +//! algebra. It is in turn pinned against the compiled folders by +//! `tests::constraint_artifact_tests`, so the chain from an AIR's Rust +//! constraints to the number this machine computes has no unpinned link. +//! +//! ## What this suite cannot see +//! +//! It executes; it does not prove. Per method rule 2, execution says nothing +//! about whether the CHIPS agree with the executor — the executor mirrors the +//! ALU it is checking. `constraint_leg_proves_and_verifies` is the test that +//! sees the chips, and it is deliberately on a small AIR: the differential's job +//! is coverage across all 28 tables, the proof's job is to close the +//! executor-vs-chip gap once. + +use stark::constraint_ir::{ConstraintArtifact, eval_program_verifier}; +use stark::frame::Frame; +use stark::proof::options::GoldilocksCubicProofOptions; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; +use crate::test_utils::{NUM_PRODUCTION_AIRS, production_airs}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::constraints::{ + Analysis, OodOperands, analyze, emit_analyzed, hint_ood_frame, ood_frame_words, +}; +use super::executor::execute; +use super::hash::TestPermutation; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +fn options() -> stark::proof::options::ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// Deterministic SplitMix64, matching the artifact suite's generator so the two +/// sweep the same kind of input. +struct SplitMix64(u64); + +impl SplitMix64 { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn fp3(&mut self) -> FEE { + FEE::new([ + FE::from(self.next_u64()), + FE::from(self.next_u64()), + FE::from(self.next_u64()), + ]) + } +} + +/// One AIR's OOD inputs: an all-extension frame with the verifier's next-row +/// PRUNING already applied, plus the per-proof uniforms. +/// +/// The pruning matters to the differential, not just to the cost: the verifier +/// reconstructs an undeclared next-row column as ZERO, so a host frame that put +/// a random value there would be comparing the machine against a frame no +/// verifier can produce. +struct OodFixture { + /// `steps[offset]` = `[main | aux]`, aux starting at `main_width`. + steps: Vec>, + main_width: usize, + aux_width: usize, + rap_challenges: Vec, + alpha_powers: Vec, + table_offset: FEE, +} + +impl OodFixture { + fn sample(artifact: &ConstraintArtifact, rng: &mut SplitMix64) -> Self { + let shape = &artifact.shape; + let main_width = shape.main_width as usize; + let aux_width = shape.aux_width as usize; + let width = main_width + aux_width; + let num_steps = shape.transition_offsets.len().max(1); + + let steps = (0..num_steps) + .map(|offset| { + (0..width) + .map(|col| { + let opened = offset == 0 || shape.next_row_columns.contains(&(col as u32)); + if opened { rng.fp3() } else { FEE::zero() } + }) + .collect() + }) + .collect(); + + Self { + steps, + main_width, + aux_width, + // [z, alpha] — the LogUp RAP challenges, in the verifier's order. + rap_challenges: vec![rng.fp3(), rng.fp3()], + alpha_powers: (0..shape.max_bus_elements as usize + 2) + .map(|_| rng.fp3()) + .collect(), + table_offset: rng.fp3(), + } + } + + /// The oracle's frame: the same values, in the verifier's own container. + fn frame(&self) -> Frame { + Frame::new( + self.steps + .iter() + .map(|s| { + TableView::new( + vec![s[..self.main_width].to_vec()], + vec![s[self.main_width..].to_vec()], + ) + }) + .collect(), + ) + } + + /// The arena the machine reads, in [`hint_ood_frame`]'s order: every opened + /// entry, step by step, column by column — pruned entries omitted because + /// the program supplies its own zero for them. + fn arena(&self, artifact: &ConstraintArtifact) -> Vec { + let shape = &artifact.shape; + let mut out = Vec::new(); + for (offset, step) in self.steps.iter().enumerate() { + for (col, v) in step.iter().enumerate() { + if offset == 0 || shape.next_row_columns.contains(&(col as u32)) { + out.push(ext_word(v)); + } + } + } + out + } + + fn uniform_arena(&self) -> Vec { + self.rap_challenges + .iter() + .chain(&self.alpha_powers) + .chain(std::iter::once(&self.table_offset)) + .map(ext_word) + .collect() + } +} + +/// Builds the differential program for one artifact: hint the frame and the +/// uniforms, lower the constraints, publish nothing. +/// +/// The uniforms are hinted HERE and only here. In the assembled verifier they +/// come from `TranscriptReplay` — an arena would let a prover choose its own +/// challenges — so this shortcut is a property of the isolated slice. The +/// guard asserting it (`challenges_are_not_an_arena_in_the_assembled_verifier`) +/// cannot exist until the assembled verifier does; it is owed as an OPEN entry +/// in `others/lfm-assembly-obligations.md`, not by this file. +fn differential_program( + artifact: &ConstraintArtifact, + an: &Analysis, +) -> (super::compiler::LfmProgram, Vec) { + let mut b = LfmBuilder::new(); + + let frame_arena = b.declare_arena(ood_frame_words(artifact)); + let (steps, words) = hint_ood_frame(&mut b, artifact, frame_arena, 0); + assert_eq!( + words, + ood_frame_words(artifact), + "ood_frame_words must predict what hint_ood_frame consumes" + ); + + let shape = &artifact.shape; + let num_uniforms = 2 + (shape.max_bus_elements + 2) + 1; + let uniform_arena = b.declare_arena(num_uniforms); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges = vec![take(&mut b), take(&mut b)]; + let alpha_powers: Vec<_> = (0..shape.max_bus_elements + 2) + .map(|_| take(&mut b)) + .collect(); + let table_offset = take(&mut b); + + let ood = OodOperands { + steps, + main_width: shape.main_width as usize, + rap_challenges, + alpha_powers, + table_offset, + }; + let evals = emit_analyzed(&mut b, an, &ood); + for e in &evals { + b.public(e.as_cell()); + } + (compile(b.finish()), evals) +} + +// ============================================================================= +// (a) + (b) — the lowering differential, every production AIR +// ============================================================================= + +/// ★ Every production AIR's lowered constraint program computes exactly what +/// the production interpreter computes, on random all-extension OOD frames. +/// +/// This is the acceptance criterion for the lowering pass. It runs over the +/// DESERIALIZED artifact, so the wire hop is inside the loop, and it compares +/// every constraint of every one of the 28 tables — including the three +/// continuation-only tables that no monolithic proof contains. +#[test] +fn lowered_constraints_match_the_verifier_interpreter() { + const TRIALS: usize = 4; + + let opts = options(); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let bytes = artifact.to_bytes().expect("serialize"); + let artifact = ConstraintArtifact::from_bytes(&bytes).expect("deserialize"); + let prog = artifact.program(); + let n = prog.roots.len(); + + let an = analyze(&artifact); + let (program, evals) = differential_program(&artifact, &an); + validate(&program).unwrap_or_else(|e| panic!("[{label}] lowered program invalid: {e:?}")); + + let mut rng = SplitMix64(0xC0FF_EE00 ^ label.len() as u64); + for trial in 0..TRIALS { + let fixture = OodFixture::sample(&artifact, &mut rng); + + let exec = execute( + &program, + &[fixture.arena(&artifact), fixture.uniform_arena()], + &TestPermutation, + ) + .unwrap_or_else(|e| panic!("[{label}] trial {trial}: execution failed: {e:?}")); + + // --- oracle: the production interpreter, verifier shape --- + let frame = fixture.frame(); + let ctx = TransitionEvaluationContext::::new_verifier( + &frame, + &fixture.rap_challenges, + &fixture.alpha_powers, + &fixture.table_offset, + ); + let mut expected = vec![FEE::zero(); n]; + eval_program_verifier(&prog, &ctx, &mut expected); + + for (c, want) in expected.iter().enumerate() { + let cell = exec.memory[evals[c].addr().0 as usize] + .unwrap_or_else(|| panic!("[{label}] constraint {c} cell unwritten")); + let got = word_as_ext(&cell).expect("an ext value has lane 3 zero"); + assert_eq!( + got, *want, + "[{label}] trial {trial}: constraint {c} disagrees with the interpreter" + ); + } + assert_eq!( + fixture.aux_width, artifact.shape.aux_width as usize, + "[{label}] fixture and artifact disagree on aux width" + ); + } + } +} + +/// The differential's own falsification: a lowering that drops the extension +/// arithmetic must be CAUGHT. Perturbing one constraint value by one and +/// re-checking proves the comparison above is load-bearing rather than +/// comparing two zeros. +#[test] +fn the_differential_rejects_a_perturbed_constraint_value() { + let opts = options(); + let airs = production_airs(&opts); + let (label, air) = airs + .iter() + .find(|(l, _)| *l == "L2G_GLOBAL") + .expect("L2G_GLOBAL is a production AIR"); + + let artifact = ConstraintArtifact::capture(&**air); + let prog = artifact.program(); + let an = analyze(&artifact); + let (program, evals) = differential_program(&artifact, &an); + + let mut rng = SplitMix64(1); + let fixture = OodFixture::sample(&artifact, &mut rng); + let exec = execute( + &program, + &[fixture.arena(&artifact), fixture.uniform_arena()], + &TestPermutation, + ) + .expect("execution"); + + let frame = fixture.frame(); + let ctx = TransitionEvaluationContext::::new_verifier( + &frame, + &fixture.rap_challenges, + &fixture.alpha_powers, + &fixture.table_offset, + ); + let mut expected = vec![FEE::zero(); prog.roots.len()]; + eval_program_verifier(&prog, &ctx, &mut expected); + + let got = word_as_ext(&exec.memory[evals[0].addr().0 as usize].expect("written")).expect("ext"); + assert_eq!(got, expected[0], "[{label}] baseline must agree"); + assert_ne!( + got, + &expected[0] + FEE::one(), + "[{label}] a one-off value must NOT compare equal — otherwise the \ + differential above proves nothing" + ); +} + +// ============================================================================= +// (b) — cost, against the design document's measured table +// ============================================================================= + +/// The per-AIR `instr` column of `others/lfm-constraint-lowering-design.md` §8.1, +/// as written there. Copied deliberately rather than recomputed: the point of +/// this test is to compare the emitter against the DESIGN's prediction and +/// report where they differ, which a self-consistent recomputation cannot do. +const DESIGN_INSTR: &[(&str, usize)] = &[ + ("CPU", 489), + ("BITWISE", 112), + ("LT", 116), + ("SHIFT", 321), + ("EQ", 88), + ("BYTEWISE", 138), + ("STORE", 149), + ("CPU32", 414), + ("MEMW", 448), + ("MEMW_A", 311), + ("MEMW_R", 153), + ("LOAD", 162), + ("DECODE", 18), + ("MUL", 320), + ("DVRM", 423), + ("BRANCH", 108), + ("HALT", 701), + ("COMMIT", 359), + ("PAGE", 41), + ("REGISTER", 29), + ("KECCAK", 3_146), + // 14_016 → 12_998: main replaced the θ/ρ HWSL lookups with inline μ-gated + // linear identities in `KeccakRndConstraints`, which nets −1018 constraint + // arithmetic rows (the same change whose receiver-side multiplicity drop is + // reconciled in `keccak_adapter::bitwise_ops_for`). + ("KECCAK_RND", 12_998), + ("KECCAK_RC", 26), + ("ECSM", 19_264), + ("ECDAS", 22_718), + ("HINT", 418), // main's new receiver AIR for the `hint` ecall. + ("L2G_GLOBAL", 27), + ("L2G_MEMORY", 65), + ("GLOBAL_MEMORY", 25), +]; + +/// ★ What the emitter actually costs per AIR, against the design's table. +/// +/// Two numbers per table. `unfused` is the design's own column — arithmetic +/// nodes that survive the verify-time-base fold, one row each — and must match +/// it exactly, because a mismatch means the design measured a different program +/// than the one being lowered. `emitted` is what the pass really writes, after +/// `MulAdd` fusion and dead-code elimination. +/// +/// ### What this instrument cannot see +/// +/// Nothing about how an EPOCH is assembled: it is per distinct AIR, and the +/// sub-proof count per epoch comes from `tests::constraint_artifact_tests`, not +/// from here. It also says nothing about padded CELL cost, which depends on how +/// these rows interleave with the rest of a program's. +#[test] +fn constraint_leg_instruction_census() { + let opts = options(); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + println!("\nconstraint-leg lowering cost, per AIR"); + println!( + "{:<14} {:>7} {:>7} {:>6} {:>6} {:>5} {:>6} {:>6} {:>8} {:>8} {:>8} {:>7}", + "table", + "nodes", + "leaves", + "fold", + "foldX", + "dead", + "unrK", + "fused", + "ext", + "mulbase", + "emitted", + "unfused" + ); + + let (mut t_unfused, mut t_emitted, mut t_fused, mut t_dead, mut t_foldx) = (0, 0, 0, 0, 0); + let (mut t_dead_const, mut t_cands, mut t_orphans) = (0, 0, 0); + let mut t_orphans_all = 0; + let mut mismatches: Vec = Vec::new(); + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let r = analyze(&artifact).report().clone(); + + println!( + "{:<14} {:>7} {:>7} {:>6} {:>6} {:>5} {:>6} {:>6} {:>8} {:>8} {:>8} {:>7}", + label, + r.nodes, + r.leaves, + r.fold_base, + r.fold_ext, + r.dead, + r.unreached_const, + r.fused, + r.ext_alu, + r.mul_base, + r.alu_rows(), + r.unfused_alu_rows() + ); + + t_unfused += r.unfused_alu_rows(); + t_emitted += r.alu_rows(); + t_fused += r.fused; + t_dead += r.dead; + t_dead_const += r.unreached_const; + t_cands += r.fuse_candidates; + t_orphans += r.orphans; + t_orphans_all += r.orphans_all_kinds; + t_foldx += r.fold_ext; + + let design = DESIGN_INSTR + .iter() + .find(|(l, _)| l == label) + .map(|(_, n)| *n) + .unwrap_or_else(|| panic!("no design entry for {label}")); + // The design's column counts every surviving arithmetic node once, with + // no DCE and no extension-valued folding, so add back what this pass + // removes on top of that. + let comparable = r.unfused_alu_rows() + r.dead + r.fold_ext + r.aliased; + if comparable != design { + mismatches.push(format!( + "{label}: design {design}, emitter {comparable} (delta {})", + comparable as i64 - design as i64 + )); + } + } + + println!( + "\nTOTALS unfused {t_unfused} emitted {t_emitted} (fusion saves {t_fused})\n\ + beyond the design's rule: {t_dead} dead ROWS eliminated, \ + {t_dead_const} root-unreachable constant nodes (free either way, \ + counted under the design's `fold`), {t_foldx} extension-valued constant \ + subtrees folded\n\ + fusion: {t_cands} candidate (Add, Mul) operand pairs, {t_fused} taken \ + — the gap is sums with TWO single-consumer products, which can absorb \ + only one\n\ + locally-orphaned nodes (the design's fanout-0 measure): {t_orphans} \ + arithmetic, {t_orphans_all} over all node kinds" + ); + + assert!( + mismatches.is_empty(), + "the emitter's per-AIR cost no longer matches the design's §8.1 table \ + (this is a real finding either way — the design is measured, not \ + guessed):\n {}", + mismatches.join("\n ") + ); +} + +// ============================================================================= +// Falsifications — one per mechanism the lowering relies on +// ============================================================================= + +/// Emitting a program for one artifact, purely to count what lands in it. +fn emitted_instrs(artifact: &ConstraintArtifact) -> Vec { + let an = analyze(artifact); + let (program, _) = differential_program(artifact, &an); + program.instrs +} + +fn count_ext(instrs: &[super::instr::Instr], want: super::instr::ExtOp) -> usize { + instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::ExtAlu { op, .. } if *op == want)) + .count() +} + +/// ★ `MulAdd` fusion happens, and the single-consumer guard is what stops it. +/// +/// Two constraint sets differing only in whether the shared product is read +/// twice. Hash-consing collapses the repeated `m0·m1` into ONE node, so the +/// second set is exactly the hazard `ConstraintArtifact`'s doc comment warns +/// about: fusing a shared `Mul` into each consumer would recompute it. +/// +/// The falsification is the second half. A test that only showed fusion +/// happening would pass just as well against an emitter that fused +/// unconditionally — which is the unsound one. +#[test] +fn muladd_fusion_requires_a_single_consumer() { + let single = fusion_air(false); + let shared = fusion_air(true); + + let a_single = ConstraintArtifact::capture(&single); + let a_shared = ConstraintArtifact::capture(&shared); + + let r_single = analyze(&a_single).report().clone(); + let r_shared = analyze(&a_shared).report().clone(); + + assert!( + r_single.fused >= 1, + "a single-consumer Mul under an Add must fuse; report {r_single:?}" + ); + assert_eq!( + r_shared.fused, 0, + "a Mul read by two Adds must NOT fuse — hash-consing makes that a \ + recomputation, not a saving; report {r_shared:?}" + ); + + let i_single = emitted_instrs(&a_single); + assert_eq!( + count_ext(&i_single, super::instr::ExtOp::MulAdd), + r_single.fused, + "every fusion the report claims must be a MulAdd row in the program" + ); + assert_eq!( + count_ext(&emitted_instrs(&a_shared), super::instr::ExtOp::MulAdd), + 0, + "the shared-Mul program must contain no MulAdd row" + ); +} + +/// ★ A constraint root is a consumer: fusing it away would delete the value the +/// quotient recombination reads. +#[test] +fn a_rooted_mul_is_never_fused_away() { + let air = rooted_mul_air(); + let artifact = ConstraintArtifact::capture(&air); + let an = analyze(&artifact); + + assert_eq!( + an.report().fused, + 0, + "the only Add's product operand is also a constraint root, so it must \ + survive as its own row" + ); + + // And the root still evaluates: emit, run, compare against the interpreter. + let prog = artifact.program(); + let (program, evals) = differential_program(&artifact, &an); + let mut rng = SplitMix64(7); + let fixture = OodFixture::sample(&artifact, &mut rng); + let exec = execute( + &program, + &[fixture.arena(&artifact), fixture.uniform_arena()], + &TestPermutation, + ) + .expect("execution"); + + let frame = fixture.frame(); + let ctx = TransitionEvaluationContext::::new_verifier( + &frame, + &fixture.rap_challenges, + &fixture.alpha_powers, + &fixture.table_offset, + ); + let mut expected = vec![FEE::zero(); prog.roots.len()]; + eval_program_verifier(&prog, &ctx, &mut expected); + for (c, want) in expected.iter().enumerate() { + let got = + word_as_ext(&exec.memory[evals[c].addr().0 as usize].expect("written")).expect("ext"); + assert_eq!(got, *want, "constraint {c}"); + } +} + +/// ★ `Op::Neg` lowers to a subtract from the pooled zero, and that is the only +/// thing it can lower to — the ISA has no unary negate. +#[test] +fn neg_lowers_to_a_subtract_from_zero() { + let air = neg_air(); + let artifact = ConstraintArtifact::capture(&air); + let prog = artifact.program(); + let negs = prog + .nodes + .iter() + .filter(|n| matches!(n, stark::constraint_ir::Op::Neg(_))) + .count(); + assert!( + negs >= 1, + "the fixture AIR must actually capture a Neg node" + ); + + let instrs = emitted_instrs(&artifact); + let subs = count_ext(&instrs, super::instr::ExtOp::Sub); + let ir_subs = prog + .nodes + .iter() + .filter(|n| matches!(n, stark::constraint_ir::Op::Sub(_, _))) + .count(); + assert_eq!( + subs, + ir_subs + negs, + "each Neg must add exactly one Sub row on top of the IR's own Subs" + ); + + // One pooled zero for all of them: the constant pool interns by value. + let zeros = instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Const { value, .. } if *value == ext_word(&FEE::zero()))) + .count(); + assert!( + zeros <= 1, + "the zero constant must be interned once, found {zeros}" + ); +} + +/// ★ Dead-code elimination drops an unreachable arithmetic node instead of +/// writing it with multiplicity zero. +/// +/// No production artifact exercises this: `constraint_leg_instruction_census` +/// measures ZERO root-unreachable arithmetic nodes across all 28 tables, and +/// only three orphaned nodes of any kind (which is the design §4.3 number, +/// reproduced — they are leaves, not arithmetic, so they never cost a row). +/// A defensive path nothing reaches is a path nothing has tested, so the +/// unreachable node is INJECTED here: one extra `Mul` appended past every root. +/// +/// The falsification is the first assertion. Without it the test would pass +/// against an emitter that lowered the injected node too, since an unread write +/// is legal — merely wasteful — and the differential would not notice. +#[test] +fn dead_nodes_are_eliminated() { + use stark::constraint_ir::ArtifactNode; + + let opts = options(); + let airs = production_airs(&opts); + let (_, air) = airs + .iter() + .find(|(l, _)| *l == "L2G_GLOBAL") + .expect("L2G_GLOBAL is a production AIR"); + + let clean = ConstraintArtifact::capture(&**air); + let baseline = analyze(&clean).report().clone(); + assert_eq!( + baseline.dead, 0, + "the unmodified artifact has no dead nodes" + ); + + // A product of the last two nodes, appended past every root. Operands are + // strictly earlier, so `validate_self` still accepts it. + let mut injected = clean.clone(); + let n = injected.nodes.len() as u32; + injected.nodes.push(ArtifactNode { + op: stark::constraint_ir::device::OP_MUL, + a: n - 2, + b: n - 1, + dim: stark::constraint_ir::artifact::DIM_EXT, + }); + injected + .validate_self() + .expect("an appended node keeps the artifact well-formed"); + + let report = analyze(&injected).report().clone(); + assert_eq!( + report.dead, 1, + "the injected node is reachable from no root and must be counted dead" + ); + assert_eq!( + report.alu_rows(), + baseline.alu_rows(), + "an unreachable node must cost no rows at all" + ); + + let an = analyze(&injected); + let (program, _) = differential_program(&injected, &an); + validate(&program).expect("a program with DCE applied is valid"); + for instr in &program.instrs { + if let super::instr::Instr::ExtAlu { op, mult, .. } = instr { + assert_ne!( + *mult, 0, + "an emitted {op:?} row is never read; DCE should have removed it" + ); + } + } +} + +/// ★ The next-row PRUNING is in the program text, not in the supplied arena. +/// +/// A column the AIR does not declare is reconstructed as ZERO by the verifier. +/// If the machine hinted a value there instead, a prover could supply a next-row +/// opening the real verifier never reads — so this is a soundness property, not +/// a size one. +#[test] +fn pruned_next_row_columns_are_program_zeros() { + let opts = options(); + let airs = production_airs(&opts); + let (label, air) = airs + .iter() + .find(|(l, _)| *l == "CPU") + .expect("CPU is a production AIR"); + let artifact = ConstraintArtifact::capture(&**air); + let shape = &artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let steps = shape.transition_offsets.len(); + assert!(steps >= 2, "[{label}] needs a next-row step to prune"); + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(ood_frame_words(&artifact)); + let (frame, words) = hint_ood_frame(&mut b, &artifact, arena, 0); + + assert_eq!( + words as usize, + width + (steps - 1) * shape.next_row_columns.len(), + "[{label}] only the opened entries may consume arena words" + ); + assert!( + (words as usize) < steps * width, + "[{label}] the pruning must actually save arena words" + ); + + let zero_addr = b.felt_const(FE::zero()).addr(); + for (col, cell) in frame[1].iter().enumerate().take(width) { + let declared = shape.next_row_columns.contains(&(col as u32)); + let is_zero = cell.addr() == zero_addr; + assert_eq!( + is_zero, !declared, + "[{label}] next-row column {col}: declared={declared} but \ + pruned={is_zero}" + ); + } +} + +// ============================================================================= +// Fixture AIRs — capture paths the production tables cannot reach +// ============================================================================= + +use math::field::traits::IsField; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; + +type FixtureAir = AirWithBuses; + +fn fixture_air>( + cols: usize, + set: C, + name: &'static str, +) -> FixtureAir { + AirWithBuses::new( + cols, + AuxiliaryTraceBuildData { + interactions: vec![], + }, + &options(), + 1, + set, + ) + .with_name(name) +} + +/// `shared = false`: one product under one sum — fusable. +/// `shared = true`: the SAME product under two sums — hash-consed to one node +/// with two consumers, so not fusable. +struct FusionConstraints { + shared: bool, +} + +impl ConstraintSet for FusionConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + let m2 = b.main(0, 2); + let m3 = b.main(0, 3); + b.emit_base(0, m0.clone() * m1.clone() + m2.clone()); + if self.shared { + // Structurally identical product: capture hash-conses it. + b.emit_base(1, m0 * m1 + m3); + } else { + b.emit_base(1, m2 * m3 + m0); + } + } +} + +fn fusion_air(shared: bool) -> FixtureAir { + fixture_air( + 4, + FusionConstraints { shared }, + if shared { "SHARED" } else { "SINGLE" }, + ) +} + +/// A product that is BOTH a constraint root and an operand of a sum. +struct RootedMulConstraints; + +impl ConstraintSet for RootedMulConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + let m2 = b.main(0, 2); + b.emit_base(0, m0.clone() * m1.clone()); + b.emit_base(1, m0 * m1 + m2); + } +} + +fn rooted_mul_air() -> FixtureAir { + fixture_air(3, RootedMulConstraints, "ROOTED_MUL") +} + +/// Negation, which no production table's captured IR happens to hold in +/// isolation. +struct NegConstraints; + +impl ConstraintSet for NegConstraints { + fn eval>(&self, b: &mut B) { + let m0 = b.main(0, 0); + let m1 = b.main(0, 1); + b.emit_base(0, -m0.clone() + m1.clone()); + b.emit_base(1, m0 - m1); + } +} + +fn neg_air() -> FixtureAir { + fixture_air(2, NegConstraints, "NEG") +} + +// ============================================================================= +// (c) + (d) — the quotient recombination, against a REAL proof +// ============================================================================= + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::domain::new_verifier_domain; +use stark::lookup::{BusPublicInputs, LOGUP_CHALLENGE_ALPHA, LOGUP_NUM_CHALLENGES}; +use stark::proof::stark::MultiProof; +use stark::proof::view::StarkProofView; +use stark::table::Table; +use stark::traits::AIR; +use stark::verifier::{Challenges, IsStarkVerifier, Verifier}; + +use super::constraints::{BoundaryTerm, QuotientShape, emit_quotient}; +use super::proof::{lfm_prove, verify_against}; +use super::registry::build_artifacts; + +/// A genuine STARK proof of a production AIR, opened up far enough that the +/// machine can be asked to redo the verifier's composition check on it. +/// +/// Everything here is READ OFF a real proof or replayed from a real transcript. +/// Nothing is synthesized: the OOD frame is the prover's, the composition parts +/// are the prover's, and the challenges come out of the production verifier's +/// own `replay_rounds_after_round_1` rather than a local Fiat-Shamir model. +pub(super) struct RealSubProof { + pub(super) artifact: ConstraintArtifact, + pub(super) ood_full: Table, + pub(super) main_width: usize, + pub(super) num_steps: usize, + pub(super) rap_challenges: Vec, + pub(super) alpha_powers: Vec, + pub(super) table_offset: FEE, + /// The table's total bus contribution `L`, undivided. The machine derives + /// `L/N` from THIS cell rather than reading a second arena word — see + /// `constraints::emit_table_offset` for why that is a soundness + /// requirement rather than a saving. + pub(super) contribution: FEE, + pub(super) zeta: FEE, + pub(super) beta: FEE, + pub(super) challenges: Challenges, + pub(super) claimed_parts: Vec, + pub(super) quotient: QuotientShape, +} + +/// Proves L2G_MEMORY — a real continuation table, and the only continuation AIR +/// with genuine constraints — over a real boundary-claim trace. +/// +/// Returns the AIR alongside the proof because the DEEP differential needs both: +/// its oracle is the production reconstruction, which takes the AIR's layout and +/// the proof's own openings. +pub(super) fn real_fixture() -> (BoxedAir, MultiProof) { + use crate::tables::local_to_global::{ + CellBoundary, FiniClaim, InitClaim, generate_local_to_global_trace, + }; + use crate::test_utils::{EPOCH_TEST_LABEL, multi_prove_ram}; + + let opts = options(); + let air = crate::continuation::l2g_memory_air(&opts, EPOCH_TEST_LABEL); + + let boundaries: Vec = (0..4u64) + .map(|i| CellBoundary { + address: 0x1000 + 8 * i, + init: InitClaim { + value: i + 1, + timestamp: 0, + originating_epoch: 0, + }, + fini: FiniClaim { + value: 2 * i + 3, + epoch: EPOCH_TEST_LABEL, + timestamp: 17 + i, + }, + }) + .collect(); + let mut trace = generate_local_to_global_trace(&boundaries); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("the L2G_MEMORY fixture must prove"); + + (Box::new(air), proof) +} + +pub(super) type BoxedAir = Box>; + +pub(super) fn real_sub_proof() -> RealSubProof { + let (air, proof) = real_fixture(); + open_sub_proof(&*air, &proof) +} + +/// Replays the production verifier's rounds over a real single-table proof and +/// packages everything the constraint leg needs. +pub(super) fn open_sub_proof( + air: &dyn AIR, + proof: &MultiProof, +) -> RealSubProof { + let view = StarkProofView::Owned(&proof.proofs[0]); + + // ---- Round 1, Phase A/B/C, transcribed from `multi_verify_views` for the + // single-table case (no per-table domain separator). + let mut transcript = DefaultTranscript::::new(&[]); + if air.is_preprocessed() { + transcript.append_bytes(&air.precomputed_commitment()); + } + transcript.append_bytes(view.lde_trace_main_merkle_root()); + let rap_challenges: Vec = if air.has_aux_trace() { + (0..LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect() + } else { + Vec::new() + }; + if let Some(root) = view.lde_trace_aux_merkle_root() { + transcript.append_bytes(root); + } + if let Some(contribution) = view.bus_table_contribution() { + transcript.append_field_element(&contribution); + } + + let trace_length = view.trace_length(); + let domain = new_verifier_domain(air, trace_length); + let layout = Verifier::ood_layout(air); + let challenges: Challenges = Verifier::replay_rounds_after_round_1( + air, + view, + &(), + &domain, + &mut transcript, + rap_challenges.clone(), + &layout, + ); + + // ---- β, recovered from the verifier's own coefficient run and CHECKED. + // + // `replay_rounds_after_round_1` expands one geometric run of β and splits it + // into the transition coefficients then the boundary ones. That split is + // exactly the term ordering `emit_quotient` folds, so asserting it here — + // against the verifier's values, not a model — is what pins the Horner. + let nt = challenges.transition_coeffs.len(); + assert_eq!( + challenges.transition_coeffs[0], + FEE::one(), + "the coefficient run starts at beta^0" + ); + let beta = challenges.transition_coeffs[1]; + for (c, coeff) in challenges.transition_coeffs.iter().enumerate() { + assert_eq!(*coeff, beta.pow(c as u64), "transition coefficient {c}"); + } + for (k, coeff) in challenges.boundary_coeffs.iter().enumerate() { + assert_eq!( + *coeff, + beta.pow((nt + k) as u64), + "boundary coefficient {k} must continue the same run past the \ + transition constraints" + ); + } + + // ---- the OOD grid, reconstructed by the verifier's own layout so the + // pruning is not modelled here. + let ood_current = view.trace_ood_evaluations(); + let ood_next = view.trace_ood_next_evaluations(); + let ood_full = layout.reconstruct_full( + ood_current.row_major_data(), + ood_current.width(), + ood_next.row_major_data(), + ); + + let (main_width, _) = air.trace_layout(); + let bus_public_inputs = view + .bus_table_contribution() + .map(BusPublicInputs::from_contribution); + let logup_alpha_powers: Vec = if rap_challenges.len() > LOGUP_CHALLENGE_ALPHA { + let alpha = rap_challenges[LOGUP_CHALLENGE_ALPHA]; + (0..air.max_bus_elements()) + .map(|i| alpha.pow(i as u64)) + .collect() + } else { + Vec::new() + }; + let contribution = view.bus_table_contribution().unwrap_or_else(FEE::zero); + let table_offset = FE::from(trace_length as u64) + .inv() + .expect("a nonzero trace length") + * contribution; + + let boundary_constraints = air.boundary_constraints( + &(), + &rap_challenges, + bus_public_inputs.as_ref(), + trace_length, + ); + // `VerifierDomain::trace_primitive_root` is crate-private, so the generator + // is rederived the same way `new_verifier_domain` does: the root of unity of + // order `trace_length`. + let generator = ::get_primitive_root_of_unity( + trace_length.trailing_zeros() as u64, + ) + .expect("a power-of-two trace length has a root of unity"); + let boundary: Vec = boundary_constraints + .constraints + .iter() + .map(|c| BoundaryTerm { + col: if c.is_aux { main_width + c.col } else { c.col }, + point: generator.pow(c.step as u64), + value: c.value, + }) + .collect(); + + let claimed_parts: Vec = view.composition_poly_parts_ood_evaluation().to_vec(); + let artifact = ConstraintArtifact::capture(air); + + RealSubProof { + num_steps: artifact.shape.transition_offsets.len(), + quotient: QuotientShape { + log2_trace_length: trace_length.trailing_zeros(), + num_composition_parts: claimed_parts.len(), + boundary, + }, + artifact, + ood_full, + main_width, + rap_challenges, + alpha_powers: logup_alpha_powers, + table_offset, + contribution, + zeta: challenges.z, + beta, + challenges, + claimed_parts, + } +} + +impl RealSubProof { + /// The OOD frame as the machine's arena sees it: opened entries only, in + /// [`hint_ood_frame`]'s order. + fn frame_arena(&self) -> Vec { + let shape = &self.artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let mut out = Vec::new(); + for offset in 0..self.num_steps { + let row = self.ood_full.get_row(offset); + for (col, v) in row.iter().enumerate().take(width) { + if offset == 0 || shape.next_row_columns.contains(&(col as u32)) { + out.push(ext_word(v)); + } + } + } + out + } + + fn uniform_arena(&self) -> Vec { + // `alpha_powers` are DERIVED in-machine from `rap_challenges[ALPHA]` + // (`constraints::emit_alpha_powers`), so they are deliberately absent + // here — a hinted power is a claim about alpha that nothing checks. + self.rap_challenges + .iter() + .chain([&self.contribution, &self.zeta, &self.beta]) + .map(ext_word) + .collect() + } + + fn parts_arena(&self) -> Vec { + self.claimed_parts.iter().map(ext_word).collect() + } + + pub(super) fn arenas(&self) -> Vec> { + vec![self.frame_arena(), self.uniform_arena(), self.parts_arena()] + } +} + +/// The full composition-check program for one sub-proof: lower the AIR's +/// transition constraints at ζ, recombine them against the shared zerofier and +/// the boundary quotient, and ASSERT the result equals the composition value the +/// proof claims. +/// +/// The assert is the point. A program that merely computed the composition would +/// be a calculator; asserting it against the claimed parts is what makes the +/// machine's acceptance mean something, and it is what the tamper vectors below +/// have to break. +fn composition_program_source(sp: &RealSubProof) -> super::builder::LfmProgramSource { + let mut b = LfmBuilder::new(); + + let frame_arena = b.declare_arena(ood_frame_words(&sp.artifact)); + let (steps, _) = hint_ood_frame(&mut b, &sp.artifact, frame_arena, 0); + + // The alpha POWERS are no longer hinted: they are derived from the one + // alpha challenge, so the uniform arena is that much shorter. + let num_uniforms = (sp.rap_challenges.len() + 3) as u32; + let uniform_arena = b.declare_arena(num_uniforms); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges: Vec<_> = (0..sp.rap_challenges.len()).map(|_| take(&mut b)).collect(); + let alpha_powers = super::constraints::emit_alpha_powers( + &mut b, + rap_challenges[stark::lookup::LOGUP_CHALLENGE_ALPHA], + sp.alpha_powers.len(), + ); + // `L`, undivided. The per-row offset is DERIVED from it so the constraint + // leg and the LogUp closure consume one cell rather than two independently + // hinted ones (`constraints::emit_table_offset`). + let contribution = take(&mut b); + let table_offset = + super::constraints::emit_table_offset(&mut b, contribution, sp.quotient.log2_trace_length); + let zeta = take(&mut b); + let beta = take(&mut b); + + let parts_arena = b.declare_arena(sp.claimed_parts.len() as u32); + let claimed_parts: Vec<_> = (0..sp.claimed_parts.len() as u32) + .map(|i| b.hint_word(parts_arena, i).as_ext()) + .collect(); + + let ood = OodOperands { + steps, + main_width: sp.main_width, + rap_challenges, + alpha_powers, + table_offset, + }; + let (evals, _) = super::constraints::emit_constraint_evals(&mut b, &sp.artifact, &ood); + let q = emit_quotient( + &mut b, + &sp.quotient, + &ood, + zeta, + beta, + &evals, + &claimed_parts, + ); + + b.assert_eq_ext(q.claimed, q.composition); + b.public(q.composition.as_cell()); + b.finish() +} + +/// ★ (c) The machine reproduces the verifier's composition check on a REAL +/// proof of a REAL production table. +/// +/// The oracle is the proof itself: an honestly generated proof satisfies +/// `Σ_j part_j·ζ^j = boundary_quotient + Σ_c β^c·C_c/Z`, so a machine that +/// computes either side differently cannot execute the in-machine assert. That +/// makes this a differential against the production prover and verifier +/// together, not against a transcription of one formula. +#[test] +fn composition_check_matches_a_real_proof() { + let sp = real_sub_proof(); + + // Make the coverage legible, and fail rather than silently degrade if the + // fixture ever stops exercising a term. + assert_eq!( + sp.quotient.boundary.len(), + 1, + "L2G_MEMORY has bus interactions, so it carries the framework's \ + acc[0] = 0 boundary constraint — without it the boundary half of the \ + recombination would be untested" + ); + assert!( + sp.quotient.log2_trace_length >= 2, + "the zerofier must cost more than a squaring or two" + ); + assert!( + !sp.alpha_powers.is_empty(), + "the LogUp uniforms must be live" + ); + + let program = compile(composition_program_source(&sp)); + validate(&program).expect("the composition program is admissible"); + + let leg = analyze(&sp.artifact).report().clone(); + println!( + "L2G_MEMORY composition check: {} instructions total, of which {} are \ + the constraint leg ({} constraints, {} parts, log2(N) = {})", + program.instrs.len(), + leg.alu_rows(), + sp.artifact.roots.len(), + sp.quotient.num_composition_parts, + sp.quotient.log2_trace_length, + ); + + let exec = execute(&program, &sp.arenas(), &TestPermutation) + .expect("an honest proof's composition check must execute"); + + // The published value is the recomputed composition; it must equal the + // Horner fold of the parts the proof carries. + let expected = sp + .claimed_parts + .iter() + .rev() + .fold(FEE::zero(), |acc, part| acc * sp.zeta + part); + let (_, word) = exec.public_words[0]; + assert_eq!( + word_as_ext(&word).expect("ext"), + expected, + "the machine's composition must equal the claimed composition" + ); + assert!( + expected != FEE::zero(), + "a zero composition would make the assert vacuous" + ); +} + +/// ★ (c) falsification: every input the check depends on, broken one at a time. +/// +/// Each vector leaves a genuine proof's data in place and changes exactly one +/// word. The in-machine `assert_eq_ext` lowers to `diff / ZERO`, which under the +/// machine's `x/0 = error` convention makes a mismatching run UNEXECUTABLE — the +/// earliest and loudest failure, and the one that shows the assert is carrying +/// the check rather than decorating it. +#[test] +fn a_tampered_composition_input_cannot_execute() { + let sp = real_sub_proof(); + let program = compile(composition_program_source(&sp)); + execute(&program, &sp.arenas(), &TestPermutation).expect("baseline honest run"); + + /// One tamper: a name and the single word it corrupts. + type Vector = (&'static str, Box>)>); + + let vectors: Vec = vec![ + ( + "a wrong OOD frame value", + Box::new(|a: &mut Vec>| a[0][0][0] += FE::one()), + ), + ( + "a wrong LogUp challenge", + Box::new(|a: &mut Vec>| a[1][0][0] += FE::one()), + ), + ( + "a wrong out-of-domain point zeta", + Box::new(|a: &mut Vec>| { + let i = a[1].len() - 2; + a[1][i][0] += FE::one(); + }), + ), + ( + "a wrong composition challenge beta", + Box::new(|a: &mut Vec>| { + let i = a[1].len() - 1; + a[1][i][0] += FE::one(); + }), + ), + ( + "a wrong claimed composition part", + Box::new(|a: &mut Vec>| a[2][0][0] += FE::one()), + ), + ]; + + // A zeta ON the trace domain, which makes the zerofier vanish. The + // out-of-domain sampler cannot produce it, but the constraint leg does not + // contain the sampler, so the reciprocal guard rather than an argument about + // a component elsewhere is what rules it out. `g` is chosen over `1` on + // purpose: at zeta = 1 the BOUNDARY denominator vanishes too, and the run + // would fail without saying which guard caught it. + { + let generator = ::get_primitive_root_of_unity( + sp.quotient.log2_trace_length as u64, + ) + .expect("root of unity"); + assert_eq!( + generator.pow(1u64 << sp.quotient.log2_trace_length), + FE::one(), + "g^N = 1, so the zerofier vanishes at zeta = g" + ); + assert_ne!( + generator, + FE::one(), + "but the boundary denominator does not" + ); + let mut arenas = sp.arenas(); + let i = arenas[1].len() - 2; + arenas[1][i] = ext_word(&generator.to_extension::()); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "a zeta on the trace domain must be rejected by the zerofier's \ + reciprocal guard, not silently return 0/0 = 1" + ); + } + + for (what, tamper) in vectors { + let mut arenas = sp.arenas(); + tamper(&mut arenas); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "{what} must make the composition check unexecutable" + ); + } +} + +/// ★ (d) The composition check PROVES and VERIFIES. +/// +/// Per method rule 2 this is the only test in this file that says anything about +/// the chips: everything above runs the executor, which mirrors the very ALU it +/// is checking. Here the emitted rows are proved by `LFM_XALU` and friends and +/// the proof is verified against the program's own committed artifacts. +/// +/// It uses `verify_against` rather than the registry. That is deliberate and is +/// the sanctioned path for a shape that is not registered: this program is one +/// AIR's leg, not the epoch verifier, and pinning its digest would pin a shape +/// that has to move once the DEEP and opening legs land. +#[test] +fn constraint_leg_proves_and_verifies() { + let opts = options(); + let sp = real_sub_proof(); + let program = compile(composition_program_source(&sp)); + let artifacts = build_artifacts(&program, &opts); + + let proved = lfm_prove(&program, &artifacts, &sp.arenas(), &opts) + .expect("the honest composition check must execute and prove"); + + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the proved composition check must verify" + ); + + // A verifier that claims a different composition value must reject, even + // though the proof itself is untouched: the claimed public words are what + // bind the machine's output to the statement. + let mut wrong = proved.public_words.clone(); + wrong[0].1[0] += FE::one(); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &wrong, + &opts, + artifacts.hasher, + ), + "a mismatched claimed composition must be rejected" + ); +} + +/// The emitted program is deterministic — same builder calls, same instructions, +/// same digest. That is the property registration would pin, asserted here for a +/// shape that is deliberately not in `LFM_REGISTRY`. +#[test] +fn composition_program_is_deterministic() { + let sp = real_sub_proof(); + let a = compile(composition_program_source(&sp)); + let b = compile(composition_program_source(&sp)); + assert_eq!(a.instrs.len(), b.instrs.len()); + assert_eq!(a.num_addrs, b.num_addrs); + let opts = options(); + assert_eq!( + build_artifacts(&a, &opts).program_id, + build_artifacts(&b, &opts).program_id, + "the same source must produce the same program identity" + ); +} + +// ============================================================================= +// (b) — the per-epoch budget +// ============================================================================= + +/// Rows the recombination costs for one sub-proof, MEASURED by emitting it into +/// a throwaway builder rather than counted off the source by eye. +/// +/// The operand plumbing (hints for the frame, the challenges, the constraint +/// values and the claimed parts) is built twice into two independent builders — +/// once alone and once followed by the quotient — and the difference is the +/// quotient's own rows. Emission is deterministic, so the two plumbings are +/// identical by construction. +fn quotient_rows(artifact: &ConstraintArtifact, log2_trace_length: u32) -> usize { + let shape = &artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let num_steps = shape.transition_offsets.len().max(1); + let num_parts = shape.composition_degree_multiplier as usize; + let num_constraints = artifact.roots.len(); + + let quotient = QuotientShape { + log2_trace_length, + num_composition_parts: num_parts, + // Every table with bus interactions carries the framework's single + // acc[0] = 0 constraint on the last aux column, and no production table + // declares any other boundary constraint. + boundary: if shape.has_trace_interaction { + vec![BoundaryTerm { + col: width - 1, + point: FE::one(), + value: FEE::zero(), + }] + } else { + Vec::new() + }, + }; + + let plumbing = |b: &mut LfmBuilder| { + let total = num_steps * width + 2 + num_constraints + num_parts + 1; + let arena = b.declare_arena(total as u32); + let mut idx = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, idx).as_ext(); + idx += 1; + c + }; + let steps: Vec> = (0..num_steps) + .map(|_| (0..width).map(|_| take(b)).collect()) + .collect(); + let ood = OodOperands { + steps, + main_width: shape.main_width as usize, + rap_challenges: Vec::new(), + alpha_powers: Vec::new(), + table_offset: take(b), + }; + let zeta = take(b); + let beta = take(b); + let evals: Vec<_> = (0..num_constraints).map(|_| take(b)).collect(); + let parts: Vec<_> = (0..num_parts).map(|_| take(b)).collect(); + (ood, zeta, beta, evals, parts) + }; + + let mut bare = LfmBuilder::new(); + let _ = plumbing(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut full = LfmBuilder::new(); + let (ood, zeta, beta, evals, parts) = plumbing(&mut full); + let q = emit_quotient(&mut full, "ient, &ood, zeta, beta, &evals, &parts); + full.assert_eq_ext(q.claimed, q.composition); + full.finish().instrs.len() - baseline +} + +/// ★ The constraint leg for a CONTINUATION EPOCH, against the design's budget. +/// +/// The composition is `others/lfm-constraint-lowering-design.md` §8.2.2's, which +/// `tests::constraint_artifact_tests::continuation_epoch_constraint_leg` derives +/// from the real epoch shape and pins against a measured 24/25 sub-proof count: +/// 14 split-table families at one chunk each, plus the nine fixed tables an +/// intermediate epoch carries (all ten on the final one), plus one L2G_MEMORY. +/// PAGE does not appear — epochs pass `page_configs = &[]`. +/// +/// ### What this instrument cannot see +/// +/// It assumes the MINIMUM epoch, one chunk per family. A larger epoch adds +/// chunks of the cheap tables, which the design measures at +642 instructions +/// per doubling past 2^19 cycles. It also fixes one trace length for the +/// zerofier across every sub-proof, so the recombination term is a +/// representative figure rather than a per-chunk one. +#[test] +fn continuation_epoch_constraint_leg_cost() { + /// Trace length assumed for the zerofier's squaring chain. + const LOG2_TRACE_LENGTH: u32 = 20; + + /// The 14 chunked split-table families. + const SPLIT_FAMILIES: &[&str] = &[ + "CPU", "LT", "SHIFT", "EQ", "BYTEWISE", "STORE", "CPU32", "MEMW", "MEMW_A", "MEMW_R", + "LOAD", "MUL", "DVRM", "BRANCH", + ]; + /// `FIXED_TABLE_COUNT`'s ten, which contribute exactly one sub-proof each + /// regardless of `TableCounts`. HALT is last: an intermediate epoch drops it. + const FIXED: &[&str] = &[ + "BITWISE", + "DECODE", + "COMMIT", + "KECCAK", + "KECCAK_RND", + "KECCAK_RC", + "REGISTER", + "ECSM", + "ECDAS", + "HALT", + ]; + + let opts = options(); + let airs = production_airs(&opts); + let cost: std::collections::BTreeMap<&str, (usize, usize, usize)> = airs + .iter() + .map(|(label, air)| { + let artifact = ConstraintArtifact::capture(&**air); + let r = analyze(&artifact).report().clone(); + ( + *label, + ( + r.alu_rows(), + r.unfused_alu_rows(), + quotient_rows(&artifact, LOG2_TRACE_LENGTH), + ), + ) + }) + .collect(); + + let sum = |labels: &[&str], pick: fn(&(usize, usize, usize)) -> usize| -> usize { + labels.iter().map(|l| pick(&cost[l])).sum() + }; + + let families = sum(SPLIT_FAMILIES, |c| c.0); + let fixed_no_halt = sum(&FIXED[..9], |c| c.0); + let halt = cost["HALT"].0; + let l2g = cost["L2G_MEMORY"].0; + + let families_unfused = sum(SPLIT_FAMILIES, |c| c.1); + let fixed_unfused = sum(&FIXED[..9], |c| c.1); + let l2g_unfused = cost["L2G_MEMORY"].1; + + let recombination = + sum(SPLIT_FAMILIES, |c| c.2) + sum(&FIXED[..9], |c| c.2) + cost["L2G_MEMORY"].2; + + let intermediate = families + fixed_no_halt + l2g; + let final_leg = intermediate + halt; + let final_total = final_leg + recombination + cost["HALT"].2; + let design_intermediate = families_unfused + fixed_unfused + l2g_unfused; + + println!( + "\ncontinuation epoch, constraint leg (minimum shape, 24 sub-proofs)\n\ + \x20 14 split families {families:>7} (unfused {families_unfused})\n\ + \x20 9 fixed, no HALT {fixed_no_halt:>7} (unfused {fixed_unfused})\n\ + \x20 1 L2G_MEMORY {l2g:>7} (unfused {l2g_unfused})\n\ + \x20 INTERMEDIATE leg {intermediate:>7} vs the design's {design_intermediate}\n\ + \x20 + recombination @ log2(N) = {LOG2_TRACE_LENGTH} {recombination:>7} \ + (zerofier, beta-fold, one division, claimed-parts Horner, assert)\n\ + \x20 INTERMEDIATE total {:>7} over 24 sub-proofs\n\ + \x20 FINAL epoch (+HALT) {final_leg:>7} leg, {final_total} total, \ + over 25 sub-proofs", + intermediate + recombination + ); + + // The design's §8.2.2 arithmetic, reproduced from the emitter's own unfused + // counts. A mismatch means the epoch composition changed, which is a finding + // about the epoch, not about this pass. + // + // 63_393 → 62_375 (−1018): attributed in full to KECCAK_RND, which is in + // `FIXED`. Main replaced its θ/ρ HWSL lookups with inline μ-gated linear + // identities in `KeccakRndConstraints`, netting −1018 constraint arithmetic + // rows — the exact same delta the per-AIR census records for KECCAK_RND + // (14_016 → 12_998). No other table moved; this is not a blind re-bless. + assert_eq!( + design_intermediate, 62_375, + "the design's intermediate-epoch budget no longer reproduces" + ); + assert!( + intermediate < design_intermediate, + "fusion must not make the leg more expensive" + ); +} + +// ============================================================================= +// The DEEP leg — differential against the production reconstruction +// ============================================================================= + +use super::deep::{DeepOpening, DeepShape, emit_deep_invariants, emit_deep_point}; + +/// The DEEP shape and the γ challenge, read off a real proof's replayed +/// challenges rather than modelled. +pub(super) fn deep_shape( + sp: &RealSubProof, + air: &dyn AIR, +) -> (DeepShape, FEE) { + let layout = Verifier::ood_layout(air); + let (main_width, aux_width) = air.trace_layout(); + let num_total_cols = main_width + aux_width; + + let shape = DeepShape { + step_size: layout.step_size(), + num_eval_points: sp.num_steps * layout.step_size(), + num_total_cols, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: sp.claimed_parts.len(), + log2_trace_length: sp.quotient.log2_trace_length, + }; + + // γ, recovered from the coefficient run and CHECKED against every entry the + // verifier built: coeff[c][r] is γ raised to a position-determined exponent, + // so if the emitter's exponent formula is wrong this assertion is what says + // so — not the differential, which would only say the answer differs. + let coeffs = &sp.challenges.trace_term_coeffs; + let gamma = coeffs[1][0]; + #[allow(clippy::needless_range_loop)] // `row` is a column-index, not a row-index, into `coeffs` + for row in 0..shape.num_eval_points { + let (cols, start, stride) = shape.block_for_test(row); + for (k, &c) in cols.iter().enumerate() { + assert_eq!( + coeffs[c][row], + gamma.pow((start + k * stride) as u64), + "trace_term_coeffs[{c}][{row}] disagrees with the emitter's \ + exponent formula" + ); + } + // Every column OUTSIDE the block must carry a zero coefficient — that is + // the pruning, and it is what makes folding the window alone exact. + if row >= shape.step_size { + for c in 0..num_total_cols { + if !cols.contains(&c) { + assert_eq!( + coeffs[c][row], + FEE::zero(), + "column {c} is pruned at row {row}" + ); + } + } + } + } + for (j, g) in sp.challenges.gammas.iter().enumerate() { + assert_eq!( + *g, + gamma.pow((shape.num_surviving() + j) as u64), + "composition gamma {j} must continue the same geometric run" + ); + } + + (shape, gamma) +} + +/// ★ The machine's DEEP reconstruction equals the production verifier's, on a +/// real proof's real query openings. +/// +/// The oracle is `reconstruct_deep_composition_poly_evaluation_pair` itself, +/// fed through `compute_query_invariant_deep_terms` — the exact pair of +/// functions `verify_rounds_2_to_4` calls, with the exact values a real proof +/// carries. Nothing about the algebra is transcribed into the test. +#[test] +fn deep_reconstruction_matches_the_production_verifier() { + let (air, proof) = real_fixture(); + let sp = open_sub_proof(&*air, &proof); + let (shape, gamma) = deep_shape(&sp, &*air); + + let view = StarkProofView::Owned(&proof.proofs[0]); + let layout = Verifier::ood_layout(&*air); + let invariants = Verifier::::compute_query_invariant_deep_terms( + &sp.challenges, + view, + &sp.ood_full, + layout.next_row_cols(), + layout.step_size(), + ) + .expect("a real proof's invariant terms"); + + let domain = new_verifier_domain(&*air, view.trace_length()); + let generator = ::get_primitive_root_of_unity( + sp.quotient.log2_trace_length as u64, + ) + .expect("root of unity"); + + let mut checked = 0usize; + for (q, iota) in sp.challenges.iotas.iter().enumerate() { + let opening = view.deep_poly_opening(q); + let precomputed: &[FE] = opening + .precomputed_trace_polys() + .map(|p| p.evaluations()) + .unwrap_or(&[]); + let main = opening.main_trace_polys().evaluations(); + let aux: &[FEE] = opening + .aux_trace_polys() + .map(|a| a.evaluations()) + .unwrap_or(&[]); + let precomputed_sym: &[FE] = opening + .precomputed_trace_polys() + .map(|p| p.evaluations_sym()) + .unwrap_or(&[]); + let main_sym = opening.main_trace_polys().evaluations_sym(); + let aux_sym: &[FEE] = opening + .aux_trace_polys() + .map(|a| a.evaluations_sym()) + .unwrap_or(&[]); + + type V = Verifier; + let point = V::query_challenge_to_evaluation_point(*iota, false, &domain); + let point_sym = V::query_challenge_to_evaluation_point(*iota, true, &domain); + + // --- oracle --- + let (want, want_sym) = V::reconstruct_deep_composition_poly_evaluation_pair( + &point, + &point_sym, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + precomputed, + main, + aux, + opening.composition_poly().evaluations(), + precomputed_sym, + main_sym, + aux_sym, + opening.composition_poly().evaluations_sym(), + ) + .expect("a real proof reconstructs"); + + // --- the machine --- + let trace: Vec = precomputed + .iter() + .chain(main.iter()) + .map(|v| v.to_extension::()) + .chain(aux.iter().copied()) + .collect(); + let trace_sym: Vec = precomputed_sym + .iter() + .chain(main_sym.iter()) + .map(|v| v.to_extension::()) + .chain(aux_sym.iter().copied()) + .collect(); + assert_eq!(trace.len(), shape.num_total_cols); + + let ood_words: Vec = (0..shape.num_eval_points) + .flat_map(|r| { + let row = sp.ood_full.get_row(r); + (0..shape.num_total_cols) + .map(|c| ext_word(&row[c])) + .collect::>() + }) + .collect(); + + let mut b = LfmBuilder::new(); + let words: Vec = std::iter::once(ext_word(&gamma)) + .chain(std::iter::once(ext_word(&sp.zeta))) + .chain(ood_words.iter().copied()) + .chain(sp.claimed_parts.iter().map(ext_word)) + .chain(std::iter::once(base_word(point))) + .chain(trace.iter().map(ext_word)) + .chain( + opening + .composition_poly() + .evaluations() + .iter() + .map(ext_word), + ) + .chain(std::iter::once(base_word(point_sym))) + .chain(trace_sym.iter().map(ext_word)) + .chain( + opening + .composition_poly() + .evaluations_sym() + .iter() + .map(ext_word), + ) + .collect(); + let arena = b.declare_arena(words.len() as u32); + let mut idx = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, idx).as_ext(); + idx += 1; + c + }; + let g_cell = take(&mut b); + let z_cell = take(&mut b); + let ood_steps: Vec> = (0..shape.num_eval_points) + .map(|_| (0..shape.num_total_cols).map(|_| take(&mut b)).collect()) + .collect(); + let parts: Vec<_> = (0..shape.num_composition_parts) + .map(|_| take(&mut b)) + .collect(); + let inv = emit_deep_invariants(&mut b, &shape, g_cell, z_cell, &ood_steps, &parts); + + let read_opening = |b: &mut LfmBuilder, idx: &mut u32| { + let p = super::builder::Felt(b.hint_word(arena, *idx).addr()); + *idx += 1; + let mut cells = Vec::with_capacity(shape.num_total_cols); + for _ in 0..shape.num_total_cols { + cells.push(b.hint_word(arena, *idx).as_ext()); + *idx += 1; + } + let mut ps = Vec::with_capacity(shape.num_composition_parts); + for _ in 0..shape.num_composition_parts { + ps.push(b.hint_word(arena, *idx).as_ext()); + *idx += 1; + } + DeepOpening { + point: p, + trace: cells, + parts: ps, + } + }; + let regular = read_opening(&mut b, &mut idx); + let symmetric = read_opening(&mut b, &mut idx); + let got = emit_deep_point(&mut b, &shape, g_cell, &inv, ®ular); + let got_sym = emit_deep_point(&mut b, &shape, g_cell, &inv, &symmetric); + b.public(got.as_cell()); + b.public(got_sym.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the DEEP program is admissible"); + let exec = execute(&program, &[words], &TestPermutation).expect("DEEP executes"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + want, + "query {q}: DEEP at the regular point" + ); + assert_eq!( + word_as_ext(&exec.public_words[1].1).expect("ext"), + want_sym, + "query {q}: DEEP at the symmetric point" + ); + assert_ne!(want, FEE::zero(), "query {q} must not be vacuously zero"); + + checked += 1; + if checked == 3 { + break; + } + } + assert!(checked > 0, "the fixture must carry at least one query"); + println!("DEEP differential: {checked} queries, both points each"); +} + +/// ★ The coefficient-exponent formula holds where no production AIR reaches: +/// `step_size = 2` with two next rows. +/// +/// The DEEP differential above runs on L2G_MEMORY, and every production AIR has +/// `step_size = 1` and a single next row — which collapses both strides to one. +/// A plain Horner in γ would therefore pass every test we have. This one builds +/// the verifier's own coefficient table at a wider step through +/// `build_pruned_trace_term_coeffs` and checks the emitter against it, then +/// shows the stride-1 reading DISAGREES. Without that second half the test would +/// pass against the wrong emitter. +#[test] +fn the_coefficient_exponent_formula_holds_at_a_wider_step() { + use stark::ood::build_pruned_trace_term_coeffs; + + const COLS: usize = 5; + const STEP: usize = 2; + const EVAL_POINTS: usize = 4; // two offsets x step 2 + let next_row_cols = vec![1usize, 3]; + + let shape = DeepShape { + step_size: STEP, + num_eval_points: EVAL_POINTS, + num_total_cols: COLS, + next_row_cols: next_row_cols.clone(), + num_composition_parts: 2, + log2_trace_length: 4, + }; + let surviving = shape.num_surviving(); + assert_eq!( + surviving, + COLS * STEP + next_row_cols.len() * (EVAL_POINTS - STEP) + ); + + let gamma = FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)]); + let powers: Vec = (0..surviving).map(|p| gamma.pow(p as u64)).collect(); + let coeffs = build_pruned_trace_term_coeffs(&powers, COLS, EVAL_POINTS, STEP, &next_row_cols); + + let mut stride_ever_exceeds_one = false; + let mut plain_horner_would_differ = false; + + #[allow(clippy::needless_range_loop)] // `row` is a column-index, not a row-index, into `coeffs` + for row in 0..EVAL_POINTS { + let (cols, start, stride) = shape.block_for_test(row); + if stride > 1 { + stride_ever_exceeds_one = true; + } + for (k, &c) in cols.iter().enumerate() { + assert_eq!( + coeffs[c][row], + gamma.pow((start + k * stride) as u64), + "coeffs[{c}][{row}] disagrees with the emitter's (start {start}, \ + stride {stride}) formula" + ); + // The falsification: what a stride-1 fold would have used. + if coeffs[c][row] != gamma.pow((start + k) as u64) { + plain_horner_would_differ = true; + } + } + // Pruned columns carry a zero coefficient on next rows. + if row >= STEP { + for c in 0..COLS { + if !cols.contains(&c) { + assert_eq!(coeffs[c][row], FEE::zero(), "column {c} at row {row}"); + } + } + } + } + + assert!( + stride_ever_exceeds_one, + "the fixture must actually produce a stride above one" + ); + assert!( + plain_horner_would_differ, + "a plain Horner in gamma must give a DIFFERENT coefficient here, or this \ + test does not show the stride is load-bearing" + ); +} + +/// ★ What a DEEP query costs, per sub-proof and per epoch. +/// +/// ### What this instrument cannot see +/// +/// The query COUNT. It is a proof-options property (219 at blowup 2, 73 at +/// blowup 8), not an AIR property, so the per-epoch line below is parameterised +/// on it rather than measured. It also excludes the Merkle authentication of the +/// openings this leg consumes, which is the R1f leg's cost, and the FRI folding +/// that consumes this leg's output. +#[test] +fn deep_leg_cost() { + /// Queries at blowup 2 — stated, not measured here. + const QUERIES: usize = 219; + + let opts = options(); + let airs = production_airs(&opts); + + println!("\nDEEP cost per query point, by AIR"); + println!( + "{:<14} {:>6} {:>7} {:>6} {:>9} {:>10}", + "table", "cols", "window", "parts", "rows/pt", "rows/query" + ); + + let mut total_per_query = 0usize; + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let layout = Verifier::::ood_layout(&**air); + let (main_width, aux_width) = air.trace_layout(); + let shape = DeepShape { + step_size: layout.step_size(), + num_eval_points: artifact.shape.transition_offsets.len() * layout.step_size(), + num_total_cols: main_width + aux_width, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: artifact.shape.composition_degree_multiplier as usize, + log2_trace_length: 20, + }; + + // Measure by emitting, twice, and differencing out the plumbing. + let plumb = |b: &mut LfmBuilder| { + let n = 2 + + shape.num_eval_points * shape.num_total_cols + + 2 * shape.num_composition_parts + + shape.num_total_cols + + 1; + let arena = b.declare_arena(n as u32); + let mut i = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, i).as_ext(); + i += 1; + c + }; + let g = take(b); + let z = take(b); + let steps: Vec> = (0..shape.num_eval_points) + .map(|_| (0..shape.num_total_cols).map(|_| take(b)).collect()) + .collect(); + let parts: Vec<_> = (0..shape.num_composition_parts).map(|_| take(b)).collect(); + let trace: Vec<_> = (0..shape.num_total_cols).map(|_| take(b)).collect(); + let qparts: Vec<_> = (0..shape.num_composition_parts).map(|_| take(b)).collect(); + let point = super::builder::Felt(take(b).addr()); + (g, z, steps, parts, trace, qparts, point) + }; + + let mut bare = LfmBuilder::new(); + let _ = plumb(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut inv_only = LfmBuilder::new(); + let (g, z, steps, parts, _, _, _) = plumb(&mut inv_only); + let _ = emit_deep_invariants(&mut inv_only, &shape, g, z, &steps, &parts); + let invariant_rows = inv_only.finish().instrs.len() - baseline; + + let mut full = LfmBuilder::new(); + let (g, z, steps, parts, trace, qparts, point) = plumb(&mut full); + let inv = emit_deep_invariants(&mut full, &shape, g, z, &steps, &parts); + emit_deep_point( + &mut full, + &shape, + g, + &inv, + &DeepOpening { + point, + trace, + parts: qparts, + }, + ); + let point_rows = full.finish().instrs.len() - baseline - invariant_rows; + + let per_query = 2 * point_rows; + total_per_query += per_query; + println!( + "{:<14} {:>6} {:>7} {:>6} {:>9} {:>10}", + label, + shape.num_total_cols, + shape.next_row_cols.len(), + shape.num_composition_parts, + point_rows, + per_query + ); + } + + println!( + "\nSum over all 28 AIRs, one query each (both points): {total_per_query} rows.\n\ + At {QUERIES} queries that is {} rows if every AIR appeared once — an\n\ + ORDER-OF-MAGNITUDE figure, not an epoch: an epoch's sub-proof set is not\n\ + the 28-AIR set, and this excludes the Merkle authentication of these same\n\ + openings and the FRI folding that consumes the result.", + total_per_query * QUERIES + ); +} diff --git a/prover/src/lfm/constraints.rs b/prover/src/lfm/constraints.rs new file mode 100644 index 000000000..f76eea7b8 --- /dev/null +++ b/prover/src/lfm/constraints.rs @@ -0,0 +1,888 @@ +//! Lowering a captured [`ConstraintArtifact`] into LFM instructions — the +//! constraint-evaluation leg of the epoch verifier. +//! +//! The pass runs on the HOST at registry-build time, so constant folding, +//! dead-code elimination, fanout analysis and peephole fusion are free: what +//! reaches the machine is fixed program text whose digest the registry pins. +//! +//! # The IR's `dim` tags describe the PROVER; the machine runs the VERIFIER +//! +//! [`Dim`] records what the prover computes over a base-field trace frame. The +//! machine evaluates at the out-of-domain point, where the frame holds only +//! extension elements — `eval_program_verifier` resolves every [`Op::Var`] to an +//! extension value regardless of `main`. Propagating that through the +//! interpreter's rule (base only when both operands are base *values* and the +//! declared dim is base), a node is base at verify time **only if its entire +//! subtree is constants**. Sizing this leg off the declared dims understates +//! extension traffic by roughly 14×. +//! +//! Those constant-only subtrees are exactly the nodes this pass folds, so they +//! cost no rows at all rather than costing base-ALU rows. +//! +//! # What costs a row and what does not +//! +//! | IR op | lowering | rows | +//! |---|---|---| +//! | [`Op::Var`] / [`Op::RapChallenge`] / [`Op::AlphaPow`] / [`Op::TableOffset`] | an address supplied by [`OodOperands`] | 0 | +//! | [`Op::ConstBase`] / [`Op::ConstExt`] | an interned `Instr::Const` word | 1 per distinct word, program-wide | +//! | [`Op::Embed`] | **nothing** — a base word already IS its extension embedding | 0 | +//! | [`Op::Neg`] | `ExtAlu{Sub}` against the pooled zero — the ISA has no unary negate | 1 | +//! | [`Op::Add`] | `ExtAlu{Add}`, or absorbed into a producer `Mul` as `MulAdd` | 1 or 0 | +//! | [`Op::Sub`] | `ExtAlu{Sub}` | 1 | +//! | [`Op::Mul`] | `ExtAlu{Mul}`, or `MulBase` when one operand is a base word | 1 | +//! +//! `MulAdd` costs the same single row as `Mul`, so fusing is not an optimization +//! — an unfused emitter simply pays two rows where one would do. It is sound +//! only under a single-consumer guard: the IR is HASH-CONSED, so a shared `Mul` +//! feeds several parents and fusing it into each would recompute it per parent. +//! The hazard is documented on [`ConstraintArtifact`] itself. +//! +//! # `MulBase` is cost-neutral here, not a saving +//! +//! `LFM_XALU` charges one row for `Mul` and one for `MulBase`, on the same chip +//! at the same width, and its `B` operand is received as an extension token +//! either way (`chips::xalu`). A base-valued word `(c, 0, 0, 0)` is therefore a +//! legal `Mul` operand and yields the same product. This pass routes the case +//! through `MulBase` because that states the intent and pins lanes 1–2 to zero +//! by constraint, but nothing breaks — and no row is added — if it does not. +//! See `others/lfm-constraint-lowering-design.md` §3, which overstates this as a +//! 4× obligation by comparing against a hand-lowering nobody would write. + +use std::collections::HashSet; + +use stark::constraint_ir::{ConstraintArtifact, ConstraintProgram, Dim, Op}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::{Ext, Felt, LfmBuilder}; + +type Prog = ConstraintProgram; + +/// Where a lowered constraint program reads its per-proof operands. +/// +/// The distinction between these four sources is a soundness boundary, not a +/// packaging convenience (`SOUNDNESS.md` §5): OOD frame values are arena-fed and +/// must be authenticated transitively by the DEEP/opening leg, whereas +/// challenges and alpha powers are computed in-machine by the transcript replay +/// and must NEVER come from an arena. This struct takes them as already-resolved +/// cells precisely so the lowering pass cannot invent either one. +pub struct OodOperands { + /// `steps[offset][col]` — the full-width `[main | aux]` OOD frame at each + /// transition offset, aux columns starting at `main_width`. This is the same + /// concatenated indexing the verifier's reconstructed grid uses. + /// + /// Next-row entries outside the AIR's declared `next_row_columns` are + /// reconstructed as ZERO by the verifier, so the caller supplies the pooled + /// zero cell there — see [`hint_ood_frame`]. + pub steps: Vec>, + /// Where the aux columns start inside each step. + pub main_width: usize, + /// The LogUp RAP challenges, transcript-derived. + pub rap_challenges: Vec, + /// Precomputed LogUp alpha powers. + pub alpha_powers: Vec, + /// The LogUp table offset `L/N`. + /// + /// ⚠ In any program that ALSO runs the LogUp closure, this must be the cell + /// [`emit_table_offset`] returns, not a separate hint. See that function for + /// why — the two legs consuming `L` independently makes the bus-balance + /// check vacuous. The synthetic IR-lowering differential is exempt: it has + /// no trace length and no closure, so there is no second consumer to agree + /// with. + pub table_offset: Ext, +} + +/// `L/N` — the LogUp per-row offset — DERIVED from the table's total bus +/// contribution `L` rather than hinted alongside it. +/// +/// # Why this is a soundness requirement and not a convenience +/// +/// `L` has two consumers that never meet. The circular accumulator constraint +/// (`lookup.rs`'s `emit_logup_accumulated`) enforces +/// `acc_next − acc_curr − Σterms + L/N = 0`, which together with the `acc[0] = 0` +/// boundary pins `L` to the aux trace: the accumulator wraps to zero after `N` +/// rows only if `L` really is that table's total. The bus-balance closure +/// separately checks `Σ_tables L = expected`. Production computes the offset +/// from the one `L` the proof carries (`verifier.rs`'s `logup_table_offset`), so +/// the two agree by construction. +/// +/// A machine that hinted `L/N` for the constraint leg and `L` for the closure +/// would hand the prover two independent arena words. Supply a truthful `L₁/N` +/// so every accumulator wraps, and an arbitrary `L₂` so the sum hits the target: +/// both legs pass in isolation and the bus balance is a statement about numbers +/// attached to nothing. Deriving one from the other is what denies that, and it +/// costs a single `MulBase` against a program constant, since `N` is shape. +/// +/// This is the same failure the DEEP/authentication join closes one leg over — +/// two consumers of one value, agreeing only because the host filling the arena +/// agreed with itself. +pub fn emit_table_offset(b: &mut LfmBuilder, contribution: Ext, log2_trace_length: u32) -> Ext { + let n = FE::from(1u64 << log2_trace_length); + let n_inv = n + .inv() + .expect("a power-of-two trace length is nonzero, so invertible"); + let c = b.felt_const(n_inv); + b.emul_base(contribution, c) +} + +/// `[α⁰, α¹, …, α^{n−1}]` — the LogUp alpha powers, DERIVED from the one α the +/// transcript produced rather than hinted one word each. +/// +/// # Why these cannot be arena words +/// +/// Same class as [`emit_table_offset`], and worse in degree. `Op::AlphaPow{idx}` +/// resolves to `alpha_powers[idx]`, and those powers are what build every LogUp +/// FINGERPRINT: a row's tuple `(v₀, v₁, …)` enters the constraint as +/// `z − Σ vⱼ·αʲ`. A prover who supplies the powers independently of α chooses +/// the fingerprints, and with the fingerprints goes the entire lookup argument — +/// any tuple can be made to match any other. The powers are not merely a second +/// consumer of α, they are a CLAIM about α that nothing else checks. +/// +/// Deriving them costs one `ExtAlu{Mul}` per power beyond the first two (α⁰ is +/// the interned one and α¹ is α itself), and `n = AIR::max_bus_elements()` is +/// shape, so the chain length is program text. +/// +/// The alpha this consumes must itself come from the transcript replay, never +/// from an arena — that part is an assembly obligation this function cannot +/// enforce, since it takes α as a cell and cannot see where the cell came from. +pub fn emit_alpha_powers(b: &mut LfmBuilder, alpha: Ext, n: usize) -> Vec { + let mut powers = Vec::with_capacity(n); + for i in 0..n { + powers.push(match i { + 0 => b.ext_const(&FEE::one()), + 1 => alpha, + _ => { + let prev = powers[i - 1]; + b.emul(prev, alpha) + } + }); + } + powers +} + +/// What one AIR's lowering cost, measured by the pass that emitted it. +/// +/// Every field is a count of what the pass DID, not a prediction: [`analyze`] +/// and [`emit_constraint_evals`] share one analysis, so a report can never drift +/// from the program it describes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoweringReport { + /// IR nodes in the artifact. + pub nodes: usize, + /// Leaf nodes (frame reads and uniforms) — addresses, never rows. + pub leaves: usize, + /// Arithmetic nodes unreachable from any root that would otherwise have + /// cost a ROW — the rows dead-code elimination actually saves. + pub dead: usize, + /// Constant nodes no root reaches — overwhelmingly the INTERIOR of a folded + /// subtree, whose value is absorbed into the fold rather than read. They + /// cost no row either way, and the design's census reports them under + /// `fold`, so a reader comparing the two totals must not add them again. + pub unreached_const: usize, + /// Constant-only subtrees folded to a BASE value at build time — the + /// "verify-time base" set, and the column the design doc calls `fold`. + pub fold_base: usize, + /// Constant-only subtrees whose declared dim widens them to the extension. + /// Folding these too is strictly cheaper and the design's census does not + /// count them; reported separately so the two remain comparable. + pub fold_ext: usize, + /// [`Op::Embed`] nodes lowered to a pure address alias. + pub aliased: usize, + /// `Mul`/`Add` pairs collapsed into one `MulAdd`. + pub fused: usize, + /// Fusable `(Add, Mul)` OPERAND pairs, before the one-multiply-per-row + /// limit is applied. An `Add` whose two operands are both single-consumer + /// `Mul`s offers two candidates and can absorb only one, so this exceeds + /// [`Self::fused`] by exactly the number of such sums — which is the whole + /// gap between a candidate count and a saving. + pub fuse_candidates: usize, + /// Arithmetic nodes no other node references and no root names — "fanout 0" + /// measured LOCALLY, which is the measure the design's §4.3 reports. + pub orphans: usize, + /// The same count over EVERY node kind, leaves and constants included. + pub orphans_all_kinds: usize, + /// Extension ALU rows: `Add`, `Sub`, `Mul`, `Neg` and the fused `MulAdd`s. + pub ext_alu: usize, + /// `MulBase`-routed multiplies (one base-valued operand). + pub mul_base: usize, + /// Distinct constant WORDS this AIR asks the builder for. The builder + /// interns program-wide, so a program covering several AIRs emits fewer + /// `Const` rows than the sum of these. + pub constants: usize, +} + +impl LoweringReport { + /// ALU rows the pass emits for this AIR: `ext_alu + mul_base`. Constants are + /// excluded because interning makes them a program-wide, not per-AIR, cost. + pub fn alu_rows(&self) -> usize { + self.ext_alu + self.mul_base + } + + /// ALU rows this AIR would cost with fusion switched off — the design doc's + /// per-AIR `instr` column, and an upper bound on [`Self::alu_rows`]. + pub fn unfused_alu_rows(&self) -> usize { + self.alu_rows() + self.fused + } +} + +/// A node's compile-time value, when it has one. +#[derive(Clone, Copy, Debug)] +enum Konst { + Base(FE), + Ext(FEE), +} + +impl Konst { + fn to_ext(self) -> FEE { + match self { + Konst::Base(x) => x.to_extension::(), + Konst::Ext(x) => x, + } + } + + fn is_base(self) -> bool { + matches!(self, Konst::Base(_)) + } +} + +/// Host-side analysis of one artifact: which nodes fold, which are dead, which +/// `Mul`s are absorbed by a consumer `Add`. +/// +/// Kept separate from emission so instruction counts can be measured without +/// building a program, and so the numbers reported are by construction the +/// numbers emitted. +pub struct Analysis { + prog: Prog, + konst: Vec>, + live: Vec, + /// On an `Add` that absorbs a producer `Mul`: that `Mul`'s node id. + fuse_src: Vec>, + /// Set on a `Mul` absorbed by its single consumer. + fused_away: Vec, + report: LoweringReport, +} + +impl Analysis { + /// The measured cost of this lowering. + pub fn report(&self) -> &LoweringReport { + &self.report + } + + /// The lifted program the analysis ran over. + pub fn program(&self) -> &Prog { + &self.prog + } +} + +/// Analyse an artifact without emitting anything. +/// +/// # Panics +/// +/// On a malformed artifact (out-of-range operand or constant index): the +/// artifact's own `validate_self` runs first and reports precisely what is +/// wrong. This is a build-time entry point, so failing loudly here is correct — +/// a silently mis-lowered constraint would be a wrong program with a valid +/// digest. +pub fn analyze(artifact: &ConstraintArtifact) -> Analysis { + artifact + .validate_self() + .expect("constraint artifact failed its own consistency check"); + let prog = artifact.program(); + let n = prog.nodes.len(); + + let mut report = LoweringReport { + nodes: n, + ..Default::default() + }; + + // ---- forward pass: compile-time values ---- + // + // Mirrors `interp::run` exactly, including the dim-driven widening in + // `binop`, so a folded value is bit-identical to what the interpreter would + // have computed for that node. + let mut konst: Vec> = vec![None; n]; + for i in 0..n { + let k = match prog.nodes[i] { + Op::ConstBase(idx) => Some(Konst::Base(prog.base_consts[idx as usize])), + Op::ConstExt(idx) => Some(Konst::Ext(prog.ext_consts[idx as usize])), + Op::Var { row, .. } => { + assert_eq!(row, 0, "node {i}: the capture path only reads row 0"); + report.leaves += 1; + None + } + Op::RapChallenge { .. } | Op::AlphaPow { .. } | Op::TableOffset => { + report.leaves += 1; + None + } + Op::Add(a, b) => fold_binop(&konst, a, b, prog.dims[i], |x, y| x + y, |x, y| x + y), + Op::Sub(a, b) => fold_binop(&konst, a, b, prog.dims[i], |x, y| x - y, |x, y| x - y), + Op::Mul(a, b) => fold_binop(&konst, a, b, prog.dims[i], |x, y| x * y, |x, y| x * y), + Op::Neg(a) => match (konst[a as usize], prog.dims[i]) { + (Some(Konst::Base(x)), Dim::Base) => Some(Konst::Base(-x)), + (Some(v), _) => Some(Konst::Ext(-v.to_ext())), + (None, _) => None, + }, + Op::Embed(a) => konst[a as usize].map(|v| Konst::Ext(v.to_ext())), + }; + konst[i] = k; + } + + // ---- backward pass: liveness from the roots ---- + // + // A folded node reads nothing at run time, so it does not keep its operands + // alive; that is what lets a whole constant subtree disappear rather than + // just its top node. + let mut live = vec![false; n]; + for &r in &prog.roots { + live[r as usize] = true; + } + for i in (0..n).rev() { + if !live[i] || konst[i].is_some() { + continue; + } + for a in operands(&prog.nodes[i]) { + live[a as usize] = true; + } + } + + // ---- fanout over EMITTED consumers, then fusion selection ---- + let mut fanout = vec![0u32; n]; + for i in 0..n { + if !live[i] || konst[i].is_some() { + continue; + } + for a in operands(&prog.nodes[i]) { + if konst[a as usize].is_none() { + fanout[a as usize] += 1; + } + } + } + // A root is a consumer: the quotient recombination reads it. + for &r in &prog.roots { + if konst[r as usize].is_none() { + fanout[r as usize] += 1; + } + } + + // Local fanout, the design's own measure: every reference from any node, + // folded or not, plus roots. Distinct from `fanout` above, which counts only + // the consumers that survive to read a cell. + { + let mut refs = vec![0u32; n]; + for i in 0..n { + for a in operands(&prog.nodes[i]) { + refs[a as usize] += 1; + } + } + for &r in &prog.roots { + refs[r as usize] += 1; + } + report.orphans = (0..n) + .filter(|&i| refs[i] == 0 && is_arith(&prog.nodes[i])) + .count(); + report.orphans_all_kinds = (0..n).filter(|&i| refs[i] == 0).count(); + } + + let mut fuse_src: Vec> = vec![None; n]; + let mut fused_away = vec![false; n]; + for i in 0..n { + if !live[i] || konst[i].is_some() { + continue; + } + let Op::Add(a, b) = prog.nodes[i] else { + continue; + }; + // `a` first, then `b` — `Add` is commutative and `MulAdd` computes + // `a·b + c`, so either side may supply the product. Only one can: the + // instruction carries a single multiply. + for cand in [a, b] { + if fusable(&prog, &konst, &fanout, cand) { + report.fuse_candidates += 1; + if fuse_src[i].is_none() { + fuse_src[i] = Some(cand); + fused_away[cand as usize] = true; + } + } + } + } + + // ---- cost accounting ---- + let mut constants: HashSet<[u64; 4]> = HashSet::new(); + let want_const = |k: Konst, set: &mut HashSet<[u64; 4]>| { + let w = match k { + Konst::Base(v) => super::word::base_word(v), + Konst::Ext(v) => super::word::ext_word(&v), + }; + set.insert(core::array::from_fn(|l| { + ::canonical(w[l].value()) + })); + }; + + for i in 0..n { + // Constants are classified BEFORE liveness, because a constant-only + // subtree costs no row whether or not a root reaches it — and because + // that is the split the design's census reports, so the two stay + // comparable. `dead` is then the DCE that actually saves rows. + if let Some(k) = konst[i] { + if is_arith(&prog.nodes[i]) { + if k.is_base() { + report.fold_base += 1; + } else { + report.fold_ext += 1; + } + if !live[i] { + report.unreached_const += 1; + } + } + continue; + } + if !live[i] { + if is_arith(&prog.nodes[i]) { + report.dead += 1; + } + continue; + } + if fused_away[i] { + report.fused += 1; + continue; + } + match prog.nodes[i] { + Op::Embed(_) => report.aliased += 1, + Op::Add(_, _) | Op::Sub(_, _) => report.ext_alu += 1, + Op::Neg(_) => report.ext_alu += 1, + Op::Mul(a, b) => { + if is_base_konst(&konst, a) != is_base_konst(&konst, b) { + report.mul_base += 1; + } else { + report.ext_alu += 1; + } + } + _ => {} + } + } + + // Constants actually referenced by an emitted node or a root, plus the + // pooled zero every `Neg` subtracts from. + let mut needs_zero = false; + for i in 0..n { + if !live[i] || konst[i].is_some() || fused_away[i] { + continue; + } + if matches!(prog.nodes[i], Op::Neg(_)) { + needs_zero = true; + } + for a in operands(&prog.nodes[i]) { + if let Some(k) = konst[a as usize] { + want_const(k, &mut constants); + } + } + } + for &r in &prog.roots { + if let Some(k) = konst[r as usize] { + want_const(k, &mut constants); + } + } + if needs_zero { + want_const(Konst::Base(FE::zero()), &mut constants); + } + report.constants = constants.len(); + + Analysis { + prog, + konst, + live, + fuse_src, + fused_away, + report, + } +} + +/// Lower an artifact's constraint program, returning one cell per constraint +/// root in `constraint_idx` order. +/// +/// The returned values are the AIR's transition-constraint evaluations at the +/// OOD point — the input to the zerofier/quotient recombination, not the +/// quotient itself. +pub fn emit_constraint_evals( + b: &mut LfmBuilder, + artifact: &ConstraintArtifact, + ood: &OodOperands, +) -> (Vec, LoweringReport) { + let analysis = analyze(artifact); + let evals = emit_analyzed(b, &analysis, ood); + (evals, analysis.report) +} + +/// [`emit_constraint_evals`] over an analysis the caller already has. +pub fn emit_analyzed(b: &mut LfmBuilder, an: &Analysis, ood: &OodOperands) -> Vec { + let prog = &an.prog; + let n = prog.nodes.len(); + let mut addr: Vec> = vec![None; n]; + + for i in 0..n { + if !an.live[i] || an.konst[i].is_some() || an.fused_away[i] { + continue; + } + let out = match prog.nodes[i] { + Op::Var { + main, offset, col, .. + } => { + let step = ood + .steps + .get(offset as usize) + .unwrap_or_else(|| panic!("node {i}: frame has no offset {offset}")); + let idx = if main { + col as usize + } else { + ood.main_width + col as usize + }; + *step + .get(idx) + .unwrap_or_else(|| panic!("node {i}: frame step {offset} has no column {idx}")) + } + Op::RapChallenge { idx } => ood.rap_challenges[idx as usize], + Op::AlphaPow { idx } => ood.alpha_powers[idx as usize], + Op::TableOffset => ood.table_offset, + // A base word IS its own extension embedding, so this is an address + // alias and not an instruction. + Op::Embed(a) => operand(b, an, &addr, a), + Op::Add(x, y) => match an.fuse_src[i] { + Some(m) => { + let (p, q) = match prog.nodes[m as usize] { + Op::Mul(p, q) => (p, q), + _ => unreachable!("fusion source is always a Mul"), + }; + let other = if m == x { y } else { x }; + let (p, q, c) = ( + operand(b, an, &addr, p), + operand(b, an, &addr, q), + operand(b, an, &addr, other), + ); + b.emul_add(p, q, c) + } + None => { + let (x, y) = (operand(b, an, &addr, x), operand(b, an, &addr, y)); + b.eadd(x, y) + } + }, + Op::Sub(x, y) => { + let (x, y) = (operand(b, an, &addr, x), operand(b, an, &addr, y)); + b.esub(x, y) + } + Op::Mul(x, y) => { + match (is_base_konst(&an.konst, x), is_base_konst(&an.konst, y)) { + (false, true) => { + let (a, s) = (operand(b, an, &addr, x), base_operand(b, an, y)); + b.emul_base(a, s) + } + (true, false) => { + let (a, s) = (operand(b, an, &addr, y), base_operand(b, an, x)); + b.emul_base(a, s) + } + // Both extension, or both base — a base word is a legal + // extension operand, so the plain product is correct. + _ => { + let (x, y) = (operand(b, an, &addr, x), operand(b, an, &addr, y)); + b.emul(x, y) + } + } + } + // The ISA has no unary negate: subtract from the pooled zero. + Op::Neg(x) => { + let zero = b.felt_const(FE::zero()).as_ext(); + let x = operand(b, an, &addr, x); + b.esub(zero, x) + } + Op::ConstBase(_) | Op::ConstExt(_) => unreachable!("constants fold"), + }; + addr[i] = Some(out); + } + + prog.roots + .iter() + .map(|&r| operand(b, an, &addr, r)) + .collect() +} + +// ============================================================================= +// helpers +// ============================================================================= + +fn operands(op: &Op) -> Vec { + match *op { + Op::Add(a, b) | Op::Sub(a, b) | Op::Mul(a, b) => vec![a, b], + Op::Neg(a) | Op::Embed(a) => vec![a], + _ => Vec::new(), + } +} + +fn is_arith(op: &Op) -> bool { + matches!( + op, + Op::Add(_, _) | Op::Sub(_, _) | Op::Mul(_, _) | Op::Neg(_) | Op::Embed(_) + ) +} + +fn is_base_konst(konst: &[Option], i: u32) -> bool { + konst[i as usize].is_some_and(Konst::is_base) +} + +fn fold_binop( + konst: &[Option], + a: u32, + b: u32, + dim: Dim, + base_op: impl Fn(FE, FE) -> FE, + ext_op: impl Fn(FEE, FEE) -> FEE, +) -> Option { + let (ka, kb) = (konst[a as usize]?, konst[b as usize]?); + Some(match (ka, kb, dim) { + (Konst::Base(x), Konst::Base(y), Dim::Base) => Konst::Base(base_op(x, y)), + _ => Konst::Ext(ext_op(ka.to_ext(), kb.to_ext())), + }) +} + +/// Whether `cand` may be absorbed into a consumer `Add` as a `MulAdd`. +/// +/// The single-consumer guard is what makes this sound: the IR is hash-consed, so +/// a shared `Mul` feeds several parents and fusing it into each would recompute +/// it per parent — a loss, not a saving. `fanout` counts roots as consumers, so +/// a constraint root is never fused away. +fn fusable(prog: &Prog, konst: &[Option], fanout: &[u32], cand: u32) -> bool { + let i = cand as usize; + konst[i].is_none() && matches!(prog.nodes[i], Op::Mul(_, _)) && fanout[i] == 1 +} + +fn operand(b: &mut LfmBuilder, an: &Analysis, addr: &[Option], i: u32) -> Ext { + match an.konst[i as usize] { + Some(Konst::Base(v)) => b.felt_const(v).as_ext(), + Some(Konst::Ext(v)) => b.ext_const(&v), + None => addr[i as usize].unwrap_or_else(|| { + panic!("node {i} is read before it is emitted; the IR claims topological order") + }), + } +} + +fn base_operand(b: &mut LfmBuilder, an: &Analysis, i: u32) -> Felt { + match an.konst[i as usize] { + Some(Konst::Base(v)) => b.felt_const(v), + _ => unreachable!("base_operand is only called on a base-valued constant"), + } +} + +// ============================================================================= +// frame supply +// ============================================================================= + +/// Hint one AIR's OOD frame into the machine, honouring the verifier's next-row +/// PRUNING: at frame offsets past the first, only the columns the AIR declares +/// in `next_row_columns` are opened, and every other column is reconstructed as +/// ZERO. +/// +/// Getting that wrong in the permissive direction is a soundness bug rather than +/// a cost one — a column the AIR omits from its declaration is read as zero by +/// the real verifier, so a machine that hinted a value there would accept frames +/// the verifier rejects. Emitting the pooled zero constant makes the pruning +/// part of the program text instead of a property of the supplied arena. +/// +/// Returns the frame and the number of arena words consumed. +pub fn hint_ood_frame( + b: &mut LfmBuilder, + artifact: &ConstraintArtifact, + arena: super::instr::ArenaId, + first_index: u32, +) -> (Vec>, u32) { + let shape = &artifact.shape; + let width = (shape.main_width + shape.aux_width) as usize; + let steps = shape.transition_offsets.len().max(1); + let next_row: HashSet = shape.next_row_columns.iter().copied().collect(); + + let zero = b.felt_const(FE::zero()).as_ext(); + let mut index = first_index; + let mut out = Vec::with_capacity(steps); + for offset in 0..steps { + let mut step = Vec::with_capacity(width); + for col in 0..width { + let opened = offset == 0 || next_row.contains(&(col as u32)); + if opened { + step.push(b.hint_word(arena, index).as_ext()); + index += 1; + } else { + step.push(zero); + } + } + out.push(step); + } + (out, index - first_index) +} + +/// Arena words [`hint_ood_frame`] consumes for this AIR — the frame's opened +/// entries, which is `width + (steps − 1) · |next_row_columns|`, not `steps · +/// width`. +pub fn ood_frame_words(artifact: &ConstraintArtifact) -> u32 { + let shape = &artifact.shape; + let width = shape.main_width + shape.aux_width; + let steps = shape.transition_offsets.len().max(1) as u32; + width + (steps - 1) * shape.next_row_columns.len() as u32 +} + +// ============================================================================= +// zerofier and quotient recombination +// ============================================================================= + +/// One boundary constraint, as program SHAPE. +/// +/// Boundary constraints are deliberately NOT part of a [`ConstraintArtifact`] — +/// `AIR::boundary_constraints` is a function of the public inputs, so it is not +/// a static property of the AIR and serializing it is a separate problem. Every +/// production VM table uses `NullBoundaryConstraintBuilder`, whose only output is +/// the framework's `acc[0] = 0` on the last aux column, so in practice this is a +/// zero- or one-element list whose `point` is `g^0 = 1` and whose `value` is +/// zero. The general form is carried anyway: an emitter that silently assumed +/// the degenerate case would be wrong the first time an AIR grew a real one. +#[derive(Clone, Debug)] +pub struct BoundaryTerm { + /// Full-width `[main | aux]` column index of the value opened at ζ. + pub col: usize, + /// The trace-domain point `g^step` the constraint is anchored at. + pub point: FE, + /// The value that column must take there. + pub value: FEE, +} + +/// Everything about the recombination that is compile-time for one sub-proof. +#[derive(Clone, Debug)] +pub struct QuotientShape { + /// `log2(N)`. The zerofier costs exactly this many squarings, so the trace + /// length is program SHAPE — a machine that read it from an arena would be + /// letting the prover pick the domain it is checked against. + pub log2_trace_length: u32, + /// `composition_poly_degree_bound(N) / N`, i.e. how many parts the claimed + /// composition evaluation is split into. Also shape: the verifier rejects a + /// proof whose part count disagrees with the AIR. + pub num_composition_parts: usize, + /// The AIR's boundary constraints. + pub boundary: Vec, +} + +/// What the recombination computed. +pub struct QuotientEval { + /// `ζ^N − 1`. + pub zerofier: Ext, + /// `Σ_c β^c·C_c / Z + Σ_k β^{n+k}·(t_k(ζ) − v_k)/(ζ − p_k)`. + pub composition: Ext, + /// `Σ_j part_j·ζ^j`, the claimed value the proof carries. + pub claimed: Ext, +} + +/// Emit the zerofier, the β-power fold and the claimed-composition Horner for +/// one sub-proof. +/// +/// # One division, not one per constraint +/// +/// Every production constraint applies to every row (`RowDomain::ALL`, measured +/// across all 28 tables), so `end_exemptions` is zero everywhere and all of an +/// AIR's constraints share the zerofier `Z = ζ^N − 1`. That lets the division +/// factor out of the β-power sum: +/// +/// ```text +/// Σ_c β^c·C_c/Z = (Σ_c β^c·C_c)/Z +/// ``` +/// +/// one division per AIR rather than one per constraint. The boundary terms do +/// NOT share `Z` — they have their own denominators — so they are pre-scaled by +/// `Z` before entering the same fold, which keeps the single division while +/// still giving each boundary term its own `β` power. Naively recomputing `ζ^N` +/// and a full extension inversion per constraint, as the verifier does today, +/// would cost about 24 rows per constraint instead of per AIR. +/// +/// # Why the reciprocal rather than a direct divide +/// +/// `Z` and each `ζ − p_k` are inverted against the interned one, and the +/// quotient is then a multiply. That costs one extra row apiece and closes a +/// hole: the machine's convention is `0/0 = 1`, so a direct `Div` would silently +/// return 1 for a zero denominator with a zero numerator, whereas `1/0` has no +/// satisfying assignment (`B·OUT = A` becomes `0 = 1`) and is therefore +/// unprovable. `z` is sampled outside the trace domain so neither denominator +/// can vanish in an honest proof — but "the sampler prevents it" is a property +/// of a component this leg does not contain, and the guard costs two rows. +/// +/// `constraint_evals` are in `constraint_idx` order, `claimed_parts` in the +/// order the proof carries them (part `j` multiplying `ζ^j`). +pub fn emit_quotient( + b: &mut LfmBuilder, + shape: &QuotientShape, + ood: &OodOperands, + zeta: Ext, + beta: Ext, + constraint_evals: &[Ext], + claimed_parts: &[Ext], +) -> QuotientEval { + assert!( + !constraint_evals.is_empty() || !shape.boundary.is_empty(), + "a sub-proof with neither transition nor boundary constraints has no \ + composition to check" + ); + assert!( + !claimed_parts.is_empty(), + "the composition is claimed in at least one part" + ); + assert_eq!( + claimed_parts.len(), + shape.num_composition_parts, + "the supplied parts must match the AIR's part count, which is shape and \ + never read off the proof" + ); + + let one = b.ext_const(&FEE::one()); + + // Z = ζ^N − 1, by repeated squaring. log2(N) rows, not N. + let mut power = zeta; + for _ in 0..shape.log2_trace_length { + power = b.emul(power, power); + } + let zerofier = b.esub(power, one); + let z_inv = b.ediv(one, zerofier); + + // Terms of Σ_k β^k·X_k, highest power first: the boundary terms occupy the + // indices past the transition constraints, exactly as `replay_rounds_after_ + // round_1` splits one geometric run of β into transition then boundary + // coefficients. + let mut terms: Vec = Vec::with_capacity(constraint_evals.len() + shape.boundary.len()); + for term in shape.boundary.iter().rev() { + let opened = *ood.steps[0] + .get(term.col) + .unwrap_or_else(|| panic!("boundary column {} is outside the frame", term.col)); + let numerator = if term.value == FEE::zero() { + opened + } else { + let v = b.ext_const(&term.value); + b.esub(opened, v) + }; + let point = b.felt_const(term.point).as_ext(); + let denominator = b.esub(zeta, point); + let den_inv = b.ediv(one, denominator); + let quotient = b.emul(numerator, den_inv); + terms.push(b.emul(quotient, zerofier)); + } + terms.extend(constraint_evals.iter().rev().copied()); + + let mut acc = terms[0]; + for t in &terms[1..] { + acc = b.emul_add(acc, beta, *t); + } + let composition = b.emul(acc, z_inv); + + // claimed = Σ_j part_j·ζ^j, the same Horner the verifier folds. + let mut iter = claimed_parts.iter().rev(); + let mut claimed = *iter.next().expect("checked non-empty"); + for p in iter { + claimed = b.emul_add(claimed, zeta, *p); + } + + QuotientEval { + zerofier, + composition, + claimed, + } +} diff --git a/prover/src/lfm/deep.rs b/prover/src/lfm/deep.rs new file mode 100644 index 000000000..cb219b0da --- /dev/null +++ b/prover/src/lfm/deep.rs @@ -0,0 +1,350 @@ +//! The DEEP leg: reconstructing the deep-composition polynomial at one query +//! point, in LFM instructions. +//! +//! This is where the values opened by a FRI query meet the out-of-domain frame +//! the [constraint leg](super::constraints) evaluates. For a query point `υ` the +//! verifier computes +//! +//! ```text +//! DEEP(υ) = Σ_r (Σ_c coeff[c][r]·opened[c] − oodRowSum[r]) / (υ − g^r·z) +//! + (Σ_j γ^{T+j}·H_j(υ) − hSumZpow) / (υ − z^P) +//! ``` +//! +//! and the same at the symmetric point `−υ`, which shares every query-invariant +//! term. `crypto/stark/src/verifier.rs`'s +//! `reconstruct_deep_composition_poly_evaluation_pair` is the definition and the +//! oracle; nothing here re-derives it. +//! +//! # The coefficients are powers, so nothing has to be stored +//! +//! `replay_rounds_after_round_1` samples ONE challenge γ and expands a single +//! geometric run of `num_surviving() + num_parts` powers, handing the leading +//! `num_surviving()` to `build_trace_term_coeffs` and the rest to `gammas`. So +//! `coeff[c][r]` is `γ^p` for a position-determined `p`, and every sum here is a +//! **Horner fold** — one `MulAdd` per opened value, with no coefficient table to +//! materialize, hint or authenticate. That is the single biggest structural +//! saving in this leg, and it comes from a property of the transcript rather +//! than from anything the machine does. +//! +//! The power index runs COLUMN-MAJOR within each block: for every column, each +//! row of the block. So along a fixed row the exponent advances by the block's +//! row count, and a row folds as a Horner in `γ^stride` scaled by `γ^start`. +//! Both strides are one for every production AIR, which is exactly why the +//! stride is carried explicitly rather than assumed — see [`DeepShape::block`]. +//! +//! The query POINT is an input here, not something this leg derives. Production +//! reaches it through a bit-reversal of the query index into the LDE coset +//! (`query_challenge_to_evaluation_point`), which belongs to the FRI/query leg. +//! +//! # Base openings need no conversion +//! +//! Precomputed and main-trace openings are BASE field elements; aux openings are +//! extension. Both enter the same `MulAdd`: a base word `(v, 0, 0, 0)` already +//! IS its extension embedding, so a base column costs exactly what an aux column +//! costs and no `MulBase` routing or repacking appears anywhere in this leg. +//! +//! # Reciprocals, not divisions +//! +//! Production batch-inverts the denominators and REJECTS the proof if any is +//! zero (`inplace_batch_inverse(...).ok()?`). Under the machine's `0/0 = 1` +//! convention a direct divide would instead return 1 whenever the numerator +//! vanished too — accepting exactly the malformed proof the production guard +//! exists to reject. So each denominator is inverted against the interned one, +//! which is unprovable at zero, and the quotient is a multiply. + +use math::field::traits::IsFFTField; + +use crate::tables::types::{FEE, GoldilocksField}; + +use super::builder::{Ext, Felt, LfmBuilder}; + +/// The compile-time shape of one sub-proof's DEEP reconstruction. +/// +/// Every field is program SHAPE: it fixes how many `MulAdd` rows a query costs +/// and which columns are folded. A machine that read any of it from an arena +/// would let the prover choose the sum it is checked against. +#[derive(Clone, Debug)] +pub struct DeepShape { + /// `AIR::step_size`. Rows below this open every column; rows at or above it + /// open only the transition window. + pub step_size: usize, + /// `num_transition_offsets · step_size` — rows in the full OOD grid. + pub num_eval_points: usize, + /// Full `[main | aux]` trace width, precomputed columns included. + pub num_total_cols: usize, + /// The transition-window columns, sorted — the only ones a next row opens. + pub next_row_cols: Vec, + /// Composition-polynomial parts. + pub num_composition_parts: usize, + /// `log2` of the trace length, for `g^r` and `z^P`. + pub log2_trace_length: u32, +} + +impl DeepShape { + /// Terms the trace-term coefficient run covers — `OodLayout::num_surviving`. + /// Also the exponent γ is raised to for the first composition gamma. + pub fn num_surviving(&self) -> usize { + let next_rows = self.num_eval_points - self.step_size; + self.num_total_cols * self.step_size + self.next_row_cols.len() * next_rows + } + + /// Row `r`'s coefficient run: which columns it opens, the γ exponent of its + /// first term, and the STRIDE between consecutive terms' exponents. + /// + /// The stride is not always one. `build_pruned_trace_term_coeffs` walks + /// column-major within each block — for each column, every row of the block + /// — so along a fixed row the exponent advances by the block's row count, + /// not by one. Every production AIR has `step_size = 1` and a single next + /// row, which collapses both strides to one; folding a row as a plain Horner + /// in γ would therefore pass every test we have and be wrong for the first + /// AIR that widened a step. Carrying the stride costs one extra power per + /// distinct value and removes the assumption. + pub(crate) fn block(&self, row: usize) -> (Vec, usize, usize) { + let next_rows = self.num_eval_points - self.step_size; + if row < self.step_size { + ((0..self.num_total_cols).collect(), row, self.step_size) + } else { + let start = self.num_total_cols * self.step_size + (row - self.step_size); + (self.next_row_cols.clone(), start, next_rows) + } + } + + /// [`Self::block`], exposed so the differential can check the emitter's + /// exponent formula against the verifier's own coefficient table. + #[cfg(test)] + pub fn block_for_test(&self, row: usize) -> (Vec, usize, usize) { + self.block(row) + } +} + +/// Terms shared by every query of one sub-proof, and by a query's two points. +/// +/// Hoisting these is not an optimization the machine invents: production hoists +/// exactly this set into `QueryInvariantDeepTerms`, for the same reason (they do +/// not depend on the query index). Emitting them once per sub-proof rather than +/// once per query is what keeps a 219-query proof affordable. +pub struct DeepInvariants { + /// `Σ_c coeff[c][r]·oodFull[r][c]`, one per OOD row. + pub ood_row_sum: Vec, + /// `Σ_j γ^{T+j}·H_j(z^P)` over the claimed composition parts. + pub h_sum_zpow: Ext, + /// `z^P`. + pub z_pow: Ext, + /// `g^r·z`, one per OOD row. + pub row_points: Vec, + /// `γ^T`, the exponent the composition gammas start at. + pub gamma_pow_surviving: Ext, + /// `γ^{start}` for each OOD row's block, so a block folds relative to its + /// own start and is scaled once. + pub gamma_pow_block: Vec, + /// `γ^{stride}` for each OOD row's block — the base its Horner runs in. + pub gamma_stride: Vec, +} + +/// `Σ_k terms[k]·γ^k`, terms low-to-high — one `MulAdd` per term after the +/// first, and no power table. +fn horner(b: &mut LfmBuilder, gamma: Ext, terms: &[Ext]) -> Ext { + let mut iter = terms.iter().rev(); + let mut acc = *iter.next().expect("a fold needs at least one term"); + for t in iter { + acc = b.emul_add(acc, gamma, *t); + } + acc +} + +/// `x^n` for a compile-time `n`, square-and-multiply. `n` is shape, so the row +/// count is program text rather than data-dependent. +fn pow_const(b: &mut LfmBuilder, x: Ext, n: usize) -> Ext { + assert!(n > 0, "pow_const is not defined at zero here"); + let mut result: Option = None; + let mut base = x; + let mut bits = n; + while bits > 0 { + if bits & 1 == 1 { + result = Some(match result { + None => base, + Some(acc) => b.emul(acc, base), + }); + } + bits >>= 1; + if bits > 0 { + base = b.emul(base, base); + } + } + result.expect("n > 0 sets at least one bit") +} + +/// Emit the per-sub-proof terms every query reuses. +/// +/// `ood_steps[r]` is the reconstructed OOD grid's row `r` — the same full-width +/// `[main | aux]` row the constraint leg reads, with pruned next-row entries +/// already the pooled zero. Those zeros are load-bearing here too: the verifier +/// pairs them with zero COEFFICIENTS, so folding the window alone is exact, and +/// a machine that hinted values into pruned slots would compute a different sum. +pub fn emit_deep_invariants( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + zeta: Ext, + ood_steps: &[Vec], + claimed_parts: &[Ext], +) -> DeepInvariants { + assert_eq!( + ood_steps.len(), + shape.num_eval_points, + "the OOD grid must have one row per evaluation point" + ); + assert_eq!( + claimed_parts.len(), + shape.num_composition_parts, + "the part count is shape and is never read off the proof" + ); + + let generator = ::get_primitive_root_of_unity( + shape.log2_trace_length as u64, + ) + .expect("a power-of-two trace length has a root of unity"); + + // Block scalars: γ^start for each row's coefficient run. + let mut gamma_pow_block = Vec::with_capacity(shape.num_eval_points); + let mut gamma_stride = Vec::with_capacity(shape.num_eval_points); + let mut ood_row_sum = Vec::with_capacity(shape.num_eval_points); + let mut row_points = Vec::with_capacity(shape.num_eval_points); + + #[allow(clippy::needless_range_loop)] // `row` indexes three parallel vectors, not one + for row in 0..shape.num_eval_points { + let (cols, start, stride) = shape.block(row); + let scale = if start == 0 { + None + } else { + Some(pow_const(b, gamma, start)) + }; + let base = if stride == 1 { + gamma + } else { + pow_const(b, gamma, stride) + }; + let terms: Vec = cols.iter().map(|&c| ood_steps[row][c]).collect(); + let folded = horner(b, base, &terms); + ood_row_sum.push(match scale { + None => folded, + Some(s) => b.emul(folded, s), + }); + gamma_pow_block.push(scale.unwrap_or_else(|| b.ext_const(&FEE::one()))); + gamma_stride.push(base); + + // g^r·z. `g^r` is a program constant, so this is one MulBase per row + // rather than a chain of multiplies whose length is data. + let g_r = generator.pow(row as u64); + row_points.push(if row == 0 { + zeta + } else { + let c = b.felt_const(g_r); + b.emul_base(zeta, c) + }); + } + + let gamma_pow_surviving = pow_const(b, gamma, shape.num_surviving()); + let folded_parts = horner(b, gamma, claimed_parts); + let h_sum_zpow = b.emul(folded_parts, gamma_pow_surviving); + let z_pow = pow_const(b, zeta, shape.num_composition_parts); + + DeepInvariants { + ood_row_sum, + h_sum_zpow, + z_pow, + row_points, + gamma_pow_surviving, + gamma_pow_block, + gamma_stride, + } +} + +/// One query point's opened values, in the order the proof carries them. +/// +/// `trace` is the concatenation `precomputed ‖ main ‖ aux` — the same order +/// `reconstruct_deep_composition_poly_evaluation_pair`'s `base_at` walks, with +/// base and aux openings alike presented as extension cells because a base word +/// is already one. +pub struct DeepOpening { + /// The query's domain point, a base-field element. + pub point: Felt, + /// `precomputed ‖ main ‖ aux`, one cell per full-width column. + pub trace: Vec, + /// The composition parts opened at this point. + pub parts: Vec, +} + +/// Emit `DEEP(υ)` for one query point. +/// +/// Returns the reconstructed value; the caller feeds it to the FRI leg, which +/// is where it is finally checked. Cost is `num_surviving() + num_parts` +/// `MulAdd` rows plus a small constant per row, so this is the leg that scales +/// with query count and trace width. +pub fn emit_deep_point( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + inv: &DeepInvariants, + opening: &DeepOpening, +) -> Ext { + assert_eq!( + opening.trace.len(), + shape.num_total_cols, + "an opening covers every trace column" + ); + assert_eq!(opening.parts.len(), shape.num_composition_parts); + + let one = b.ext_const(&FEE::one()); + let point = opening.point.as_ext(); + + let mut trace_term: Option = None; + for row in 0..shape.num_eval_points { + let (cols, start, _) = shape.block(row); + let terms: Vec = cols.iter().map(|&c| opening.trace[c]).collect(); + let folded = horner(b, inv.gamma_stride[row], &terms); + let scaled = if start == 0 { + folded + } else { + b.emul(folded, inv.gamma_pow_block[row]) + }; + let numerator = b.esub(scaled, inv.ood_row_sum[row]); + // υ − g^r·z, inverted against one so a vanishing denominator is + // unprovable rather than silently 0/0 = 1. + let denominator = b.esub(point, inv.row_points[row]); + let den_inv = b.ediv(one, denominator); + trace_term = Some(match trace_term { + None => b.emul(numerator, den_inv), + Some(acc) => b.emul_add(numerator, den_inv, acc), + }); + } + + let folded_parts = horner(b, gamma, &opening.parts); + let h_sum = b.emul(folded_parts, inv.gamma_pow_surviving); + let h_numerator = b.esub(h_sum, inv.h_sum_zpow); + let h_denominator = b.esub(point, inv.z_pow); + let h_den_inv = b.ediv(one, h_denominator); + + let trace_term = trace_term.expect("at least one evaluation point"); + b.emul_add(h_numerator, h_den_inv, trace_term) +} + +/// The two points of one query — the domain point and its symmetric partner — +/// sharing every invariant. +/// +/// `υ_sym = −υ`, which is why the pair costs no extra invariants: production +/// splits the term as `denom·(coeff·base − coeff·ood)` precisely so the OOD walk +/// and the coefficient run are done once for both. +pub fn emit_deep_query( + b: &mut LfmBuilder, + shape: &DeepShape, + gamma: Ext, + inv: &DeepInvariants, + regular: &DeepOpening, + symmetric: &DeepOpening, +) -> (Ext, Ext) { + ( + emit_deep_point(b, shape, gamma, inv, regular), + emit_deep_point(b, shape, gamma, inv, symmetric), + ) +} diff --git a/prover/src/lfm/edsl.rs b/prover/src/lfm/edsl.rs new file mode 100644 index 000000000..79cb759c8 --- /dev/null +++ b/prover/src/lfm/edsl.rs @@ -0,0 +1,547 @@ +//! eDSL libraries: transcript, Merkle and FRI expressed as ordinary Rust +//! that *emits instructions*. Host-side `for` loops unroll — nothing +//! loop-shaped reaches the machine; shapes (path depths, query counts, +//! domain parameters) are compile-time constants of the emitted program. +//! +//! The Fiat–Shamir transcript here is the machine side of the protocol loop and +//! is mirrored bit-exactly by `fixture::HostSponge`. Since option B1 it is a +//! **compress chain**, specified in +//! `thoughts/shared/lfm-real-hash/transcript-spec/`; see [`SpongeVar`]. + +use crate::tables::types::FE; + +use super::builder::{Bit, Cell, DigestVal, Ext, Felt, LfmBuilder}; + +/// The advance marker `"SQZ0"`, read as one little-endian `u32`. +/// +/// Lane 0 of the constant cell a squeeze advances with. It distinguishes an +/// advance operand from an absorbed digest as **defence in depth only** — the +/// load-bearing absorb/squeeze separation is that the operation sequence is a +/// compile-time constant of the program (see [`SpongeVar`]). +pub const SQUEEZE_MARK: u32 = u32::from_le_bytes(*b"SQZ0"); + +/// The Fiat–Shamir transcript over `LFM_HASH`: a **compress chain**, state = 1 +/// cell. +/// +/// ```text +/// absorb(c) state ← T(state, c) 1 step +/// absorb2(c0, c1) state ← T(T(state, c0), c1) 2 steps +/// squeeze() out = state ; state ← T(state, SQ(i)) 1 step +/// ``` +/// +/// where `T` is one `LFM_HASH` two-to-one step in the TRANSCRIPT domain +/// ([`LfmBuilder::transcript_step`]) and `SQ(i) = [SQUEEZE_MARK, i, 0, 0]`. +/// Squeeze outputs BEFORE advancing. +/// +/// # Why a chain and not a sponge +/// +/// This replaced an overwrite-rate duplex over a 3-cell permutation (option B1, +/// ratified 2026-08-11). A chain over a collision-resistant compression is the +/// textbook Fiat–Shamir transcript and needs no assumption beyond the one the +/// hash already carries — no T-sponge theorem, no capacity argument. It needs no +/// *permutation* either, which is what lets the machine's real hash have a +/// compress socket and no permute socket at all. Being public-coin is what makes +/// this legitimate: every absorbed value is a public commitment and every +/// squeezed value a public challenge, so there is no secret for a capacity to +/// protect. +/// +/// The state is one cell = 128 bits, so ~**64-bit collision resistance** by the +/// birthday bound. That is `HASH_DIGEST_FELTS = 4` speaking, not this +/// construction: the digest already had that bound. +/// +/// # Why the squeeze counter, and why it is free +/// +/// The eDSL fully unrolls, so `i` is a compile-time constant and `SQ(i)` is a +/// program constant pinned by `program_id` — a constant cell was going to be +/// emitted either way, and this one carries a counter. What it buys: without it +/// a run of consecutive squeezes iterates ONE fixed public non-injective map, +/// whose functional graph an adversary can precompute — the structure the +/// FSE-2014 T-sponge attacks on GLUON-64 exploit. With it every step is a +/// different map and no single functional graph exists. +/// +/// ⚠ **Squeeze runs still lose entropy, and the bound scales with the query +/// count.** A run of `k` consecutive squeezes shrinks the reachable state by +/// `−log₂ α_k` bits, `α_k ~ 2/k`: 1.7 bits at `k = 4`, 7 at `k = 256`, 15 at +/// `k = 2^16`. The counter does not change those numbers (composing distinct +/// random maps obeys the same recursion) — it removes the attack structure. The +/// FRI query loop squeezes once per query with no absorb between, so **its run +/// length IS the query count**. A program whose runs exceed `k = 2^16` must +/// revisit the analysis in the transcript spec §4.2; below that the 64-bit +/// collision bound above dominates and this changes nothing. +pub struct SpongeVar { + state: Cell, + /// The next squeeze's index — host-side bookkeeping, so it appears in the + /// program only as the constant it selects. + squeeze_index: u32, +} + +impl SpongeVar { + pub fn new(b: &mut LfmBuilder) -> Self { + SpongeVar { + state: b.felt_const(FE::zero()).as_cell(), + squeeze_index: 0, + } + } + + /// Absorb one cell: one transcript step against the current state. + pub fn absorb(&mut self, b: &mut LfmBuilder, c: Cell) { + self.state = b + .transcript_step(self.state.as_digest(), c.as_digest()) + .as_cell(); + } + + /// Absorb a cell of four arbitrary FIELD ELEMENTS. + /// + /// Data enters the transcript the same way it enters a Merkle tree: through + /// the LEAF encoding. The cell is hashed to a digest in the `"LFML"` domain + /// and that digest is absorbed, so the chain binds the data up to the leaf + /// hash's collision resistance. + /// + /// ⚠ **Use this for DATA and [`SpongeVar::absorb`] for DIGESTS.** Absorbing + /// raw field elements would hand the socket lanes that are not `u32`, which + /// under the machine's real hash is not a preference but an unprovable row + /// (obligation O1). A transcript that absorbs commitments needs `absorb`; one + /// that absorbs polynomial coefficients, evaluations or any other field data + /// needs this. + pub fn absorb_felts(&mut self, b: &mut LfmBuilder, c: Cell) { + let acc = leaf_chain_start(b); + let d = b.leaf(acc, c); + self.absorb(b, d.as_cell()); + } + + /// Absorb two cells, in order. Two steps, not one: the chain takes one + /// operand per step, and the ORDER is what the transcript binds. + pub fn absorb2(&mut self, b: &mut LfmBuilder, c0: Cell, c1: Cell) { + self.absorb(b, c0); + self.absorb(b, c1); + } + + /// Squeeze one cell: the current state, then advance past it with `SQ(i)`. + /// + /// Output-then-advance rather than advance-then-output, so no squeezed + /// value is ever the state a later step absorbs into. + pub fn squeeze_cell(&mut self, b: &mut LfmBuilder) -> Cell { + let out = self.state; + // `SQ(i)`, interned like every other program constant — one `LFM_CONST` + // row per distinct squeeze index, and nothing else. + let sq = b.digest_const([ + FE::from(u64::from(SQUEEZE_MARK)), + FE::from(u64::from(self.squeeze_index)), + FE::zero(), + FE::zero(), + ]); + self.state = b.transcript_step(self.state.as_digest(), sq).as_cell(); + self.squeeze_index += 1; + out + } + + /// Squeeze an ext challenge: lanes 0–2 of a squeezed cell. + pub fn squeeze_ext(&mut self, b: &mut LfmBuilder) -> Ext { + let c = self.squeeze_cell(b); + let [l0, l1, l2, _] = b.unpack(c); + b.pack_ext(l0, l1, l2) + } + + /// Squeeze `nbits` index bits: the canonical bit decomposition of lane 0 + /// of a squeezed cell (masking to a power-of-two bound, so no rejection + /// loop — the convention the RV64 verifier's query sampling already uses). + pub fn squeeze_bits(&mut self, b: &mut LfmBuilder, nbits: usize) -> Vec { + let c = self.squeeze_cell(b); + let [l0, _, _, _] = b.unpack(c); + b.bit_dec(l0, nbits) + } +} + +/// Where a leaf chain starts: the zero cell. +/// +/// ⚠ **This is a chain START, not a shape HEADER.** COMMIT.md §1.3 opens the +/// chain at `[LEAF_MARK, num_cols, kind, rows_per_leaf]` so that the leaf's +/// width and element kind are bound *inside* the hash — the whole point of that +/// construction. Nothing here has a width to bind: these leaves are fixed-shape +/// by the program that builds them, exactly as they were before the chain +/// existed. The commitment layer that hashes arbitrary-width openings supplies +/// the header instead of this, and it must, or its leaves bind no shape. +/// +/// Interned like every other program constant, so a program's whole leaf traffic +/// costs one `LFM_CONST` row for this. +pub fn leaf_chain_start(b: &mut LfmBuilder) -> DigestVal { + b.digest_const([FE::zero(); 4]) +} + +/// The Merkle LEAF digest of a pair of data cells — eight field elements. +/// +/// **Two compressions, and the shape is the point:** the cells are absorbed in +/// order into one `"LFML"` chain, four felts per hash, each step chaining the +/// last. So a leaf's *data* never enters a compress as a digest, and a parent +/// never enters as data — which is what makes an internal node un-replayable as +/// a leaf regardless of the tree's depth (obligation O5, discharged by the tag +/// rather than by fixed depth). +/// +/// It cost THREE while the accumulator was not in the message: each cell hashed +/// to its own leaf digest and an `"LFMC"` parent folded the two. Absorbing and +/// chaining in the same compression is what took leaf absorption from 2 felts +/// per hash to 4 (COMMIT.md §1.4.1), and leaf absorption is ~70% of a recursion +/// tower node's bill. The chain binds the cells' ORDER for free, where the fold +/// bound it through the parent's operand order. +/// +/// Before either, this was `compress(cell0, cell1)` — one compression that +/// treated arbitrary field elements as if they were `u32` digest lanes. Under a +/// hash whose lanes must BE `u32` that is not merely undesirable, it is +/// unprovable, which is why FRI data could not be hashed at all before the leaf +/// mode existed. +pub fn leaf_hash_pair(b: &mut LfmBuilder, c0: Cell, c1: Cell) -> DigestVal { + let acc = leaf_chain_start(b); + let d0 = b.leaf(acc, c0); + b.leaf(d0, c1) +} + +/// Walk one Merkle authentication path. `bits` are the leaf-index bits +/// low-to-high (level 0 first): bit = 0 ⇒ the current node is the LEFT +/// child. Sibling digests come as (arena-hinted) cells; every hinted value +/// ends up inside a `compress`, which is what authenticates it. +pub fn merkle_walk( + b: &mut LfmBuilder, + leaf: DigestVal, + bits: &[Bit], + siblings: &[Cell], +) -> DigestVal { + assert_eq!(bits.len(), siblings.len(), "one sibling per level"); + let mut current = leaf; + for (bit, sibling) in bits.iter().zip(siblings) { + let (left, right) = b.select(*bit, current.as_cell(), *sibling); + current = b.compress(left.as_digest(), right.as_digest()); + } + current +} + +// ===================== production keccak Merkle ===================== + +/// A 32-byte keccak digest as it lives in the machine: two words of four `u32` +/// halves each, half `h` carrying digest bytes `4h..4h+4`. +pub type KeccakDigest = [Cell; 2]; + +/// Halves in a 32-byte digest. +pub const DIGEST_HALVES: usize = 8; + +/// The eight halves of a keccak digest, ready to be streamed into another +/// `keccak256`. +pub fn keccak_digest_halves(b: &mut LfmBuilder, d: KeccakDigest) -> [Felt; DIGEST_HALVES] { + let lo = b.unpack(d[0]); + let hi = b.unpack(d[1]); + core::array::from_fn(|h| if h < 4 { lo[h] } else { hi[h - 4] }) +} + +/// The Merkle LEAF hash of a row pair, in the production commitment layout. +/// +/// `values` is `evaluations ‖ evaluations_sym` — the two bit-reversed rows the +/// leaf covers, each written column by column. Every element is a base field +/// element rendered as its canonical `u64` in BIG-endian bytes +/// (`FieldElement::stream_bytes`), so each costs one +/// [`super::transcript_replay::felt_be_halves`]: one `LFM_BITDEC` row and 64 +/// `LFM_BALU` rows. The hash itself is `keccak256` over `8 · values.len()` +/// bytes. +/// +/// ## The byteswapping is NOT what this costs — measured, against expectation +/// +/// A `c`-column table gives `2c` elements, so `2c` decompositions and `128c` +/// ALU rows against only `⌈(16c + 1) / 136⌉` permutations. On row counts the +/// byteswapping looks overwhelming, which is what the R1f handoff predicted. +/// That reading is wrong: rows of different chips are not comparable units. An +/// `LFM_BALU` row carries 4 non-preprocessed columns, while one permutation +/// expands into 24 `KECCAK_RND` rounds of 1480 columns — so in main-trace cells +/// a permutation costs 113 byteswaps, and the hash term dominates at every +/// table width. `machine_tests::keccak_merkle_opening_cost` measures it and +/// asserts the inequality holds. +/// +/// The swap is real work regardless, and it is not avoidable by pre-swapping in +/// the arena: the same opened values are consumed as FIELD ELEMENTS by the FRI +/// algebra and as BYTES by this hash, so something has to connect the two +/// representations, and only the machine can do it in a way the proof binds. +pub fn keccak_leaf_hash(b: &mut LfmBuilder, values: &[Felt]) -> KeccakDigest { + use super::keccak_host::BYTES_PER_HALF; + use super::transcript_replay::felt_be_halves; + + assert!(!values.is_empty(), "a leaf covers at least one column"); + let mut stream = Vec::with_capacity(2 * values.len()); + for v in values { + stream.extend(felt_be_halves(b, *v)); + } + let len_bytes = BYTES_PER_HALF * stream.len(); + keccak256(b, &stream, len_bytes) +} + +/// Walk one Merkle authentication path under the PRODUCTION hash. +/// +/// This is the keccak counterpart of [`merkle_walk`], and the two are not +/// interchangeable: `merkle_walk` compresses with `LFM_HASH`/`TestPermutation`, +/// the deliberately non-cryptographic Milestone-C placeholder, so it can only +/// ever authenticate the Milestone-C fixture tree. Production trees are keccak +/// throughout, and this is the walk that authenticates them. +/// +/// `bits` are the leaf index low-to-high, level 0 first; `bit = 0` means the +/// current node is the LEFT child, matching `verify_merkle_path_from_leaf_hash` +/// (`index % 2 == 0 ⇒ hash(current, sibling)`). +/// +/// ## The parent step +/// +/// `hash_new_parent(l, r) = keccak(l ‖ r)` — 64 bytes, no domain separation and +/// no ordering flag, so the ordering is carried entirely by the index bit. 64 +/// bytes sits inside one 136-byte rate block, so a level is exactly ONE +/// permutation. Per level the machine pays two `Select`s (a digest is two words +/// and both must swap on the same bit), four `Unpack`s and that permutation. +pub fn keccak_merkle_walk( + b: &mut LfmBuilder, + leaf: KeccakDigest, + bits: &[Bit], + siblings: &[KeccakDigest], +) -> KeccakDigest { + assert_eq!(bits.len(), siblings.len(), "one sibling per level"); + let mut current = leaf; + for (bit, sibling) in bits.iter().zip(siblings) { + // Both halves of the digest must swap on the SAME bit. + let (l0, r0) = b.select(*bit, current[0], sibling[0]); + let (l1, r1) = b.select(*bit, current[1], sibling[1]); + current = keccak_hash_pair(b, [l0, l1], [r0, r1]); + } + current +} + +/// The production Merkle PARENT hash: `keccak(left ‖ right)`. +/// +/// `hash_new_parent` streams the two 32-byte nodes into one digest with no +/// domain separation and no ordering flag, so 64 bytes sit inside a single +/// 136-byte rate block and a parent is exactly ONE permutation. +/// +/// This is the step [`keccak_merkle_walk`] performs once per level after its +/// `Select`, and the step a whole-tree build performs once per internal node +/// with no `Select` at all — a tree's child ORDER is known when the program is +/// emitted, so there is no bit to swap on. Keeping the two callers on one +/// primitive is what makes "the walk and the build hash the same way" a +/// property of the code rather than of a comment. +pub fn keccak_hash_pair( + b: &mut LfmBuilder, + left: KeccakDigest, + right: KeccakDigest, +) -> KeccakDigest { + let left_halves = keccak_digest_halves(b, left); + let right_halves = keccak_digest_halves(b, right); + let mut stream = Vec::with_capacity(2 * DIGEST_HALVES); + stream.extend(left_halves); + stream.extend(right_halves); + keccak256(b, &stream, 2 * COMMITMENT_BYTES) +} + +/// Build a whole Merkle TREE bottom-up and return its root. +/// +/// The counterpart of [`keccak_merkle_walk`]: the walk authenticates ONE leaf +/// against a root it is given, this CONSTRUCTS the root from every leaf. A +/// derivation needs the second — there is no root to authenticate against, +/// producing it is the point. +/// +/// Cost is `leaves − 1` permutations on top of the leaves' own, so a tree over +/// `L` leaves is `2L − 1` permutations in total. +/// +/// ## Power-of-two leaves +/// +/// `MerkleTree::build_from_hashed_leaves` runs `complete_until_power_of_two` +/// first, which pads by REPEATING the last leaf. This asserts a power of two +/// instead of emitting that padding: leaf counts here are shape (an LDE row +/// count over `ROWS_PER_LEAF`), so a non-power-of-two is a caller bug rather +/// than a case to handle, and emitting duplicate-leaf padding no production +/// commitment can reach would be dead program text. +pub fn keccak_merkle_tree_root(b: &mut LfmBuilder, leaves: &[KeccakDigest]) -> KeccakDigest { + assert!(!leaves.is_empty(), "a tree has at least one leaf"); + assert!( + leaves.len().is_power_of_two(), + "leaf counts are shape and must be a power of two; production would \ + pad by repeating the last leaf and no caller here needs that" + ); + let mut level = leaves.to_vec(); + while level.len() > 1 { + level = level + .chunks_exact(2) + .map(|pair| keccak_hash_pair(b, pair[0], pair[1])) + .collect(); + } + level[0] +} + +/// Bytes in a commitment / Merkle node. +pub const COMMITMENT_BYTES: usize = 32; + +/// Assert two words are equal, lane by lane (2 unpacks + 4 lowered asserts). +pub fn assert_word_eq(b: &mut LfmBuilder, x: Cell, y: Cell) { + let yl = b.unpack(y); + assert_word_eq_lanes(b, x, &yl); +} + +/// Assert a word equals four already-unpacked lanes (hoist the reference +/// word's unpack out of a loop — e.g. one root compared per query). +pub fn assert_word_eq_lanes(b: &mut LfmBuilder, x: Cell, y_lanes: &[Felt; 4]) { + let xl = b.unpack(x); + for i in 0..4 { + b.assert_eq(xl[i], y_lanes[i]); + } +} + +/// `scale · Π factors[i]^{bits[i]}` — one Select + one Mul per bit. Used to +/// derive domain points (and their inverses) from query-index bits; the +/// factors are program constants, so nothing here touches an arena. +pub fn pow_bits(b: &mut LfmBuilder, bits: &[Bit], factors: &[FE], scale: FE) -> Felt { + assert_eq!(bits.len(), factors.len()); + let mut acc = b.felt_const(scale); + for (bit, factor) in bits.iter().zip(factors) { + let one = b.felt_const(FE::one()); + let f = b.felt_const(*factor); + let (chosen, _) = b.select(*bit, one.as_cell(), f.as_cell()); + acc = b.mul(acc, Felt(chosen.0)); + } + acc +} + +/// `Σ_i coeffs[i]·α^i` over ext, coeffs given low-to-high (base cells are +/// valid ext operands). One `MulAdd` per coefficient — the Horner shape. +pub fn horner_ext(b: &mut LfmBuilder, alpha: Ext, coeffs_low_to_high: &[Ext]) -> Ext { + let mut iter = coeffs_low_to_high.iter().rev(); + let mut acc = *iter.next().expect("at least one coefficient"); + for c in iter { + acc = b.emul_add(acc, alpha, *c); + } + acc +} + +/// One unnormalized FRI fold — our production convention exactly: +/// `(lo + hi) + inv_x·ζ·(lo − hi)` (the missing ½ is absorbed into the +/// terminal polynomial). +pub fn fri_fold(b: &mut LfmBuilder, lo: Ext, hi: Ext, zeta: Ext, inv_x: Felt) -> Ext { + let sum = b.eadd(lo, hi); + let diff = b.esub(lo, hi); + let zd = b.emul(zeta, diff); + let scaled = b.emul_base(zd, inv_x); + b.eadd(sum, scaled) +} + +// ============================== keccak256 ============================== + +/// `keccak256` over a byte stream supplied as `u32`-half felts (four bytes +/// each, little-endian — see [`super::keccak_host::pack_stream`]). Returns the +/// 32-byte digest as two machine words of halves. +/// +/// Shapes are compile-time, as everywhere in this machine: `len_bytes` fixes +/// the block count and the padding positions, so `pad10*1` is emitted as +/// interned program CONSTANTS rather than computed. A different length is a +/// different program with a different digest — which is the straight-line +/// discipline working as intended, not a limitation to route around. +/// +/// The digest is the state's first 32 bytes = halves 0..7 = words 0 and 1 of +/// the state's word representation, which is exactly `PlatformKeccak256`'s +/// output byte order (byte `j` = byte `j % 4` of half `j / 4`). +pub fn keccak256(b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> [Cell; 2] { + let (state, _) = keccak256_absorb_all(b, stream, len_bytes, false); + [state[0], state[1]] +} + +/// `keccak256`, additionally returning the byte-REVERSED digest — the value the +/// production `DefaultTranscript::sample()` both returns as the challenge and +/// re-absorbs as the next segment's prefix. +/// +/// `sample()` is byte-for-byte identical before and after #841, so this is +/// independent of which transcript revision the caller targets. +pub fn keccak256_rev(b: &mut LfmBuilder, stream: &[Felt], len_bytes: usize) -> [Cell; 2] { + let (_, rev) = keccak256_absorb_all(b, stream, len_bytes, true); + rev.expect("requested") +} + +/// `keccak256` returning BOTH digests — plain and byte-reversed — off the one +/// keccak row that produces them. +/// +/// The transcript replay needs both at once and they are not interchangeable: +/// the reversed digest is what `sample()` returns and re-absorbs, while +/// candidates are read off the PLAIN digest (the reversal and the big-endian +/// candidate read cancel — see [`super::keccak_host::candidate_from_state`]). +pub fn keccak256_with_rev( + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, +) -> ([Cell; 2], [Cell; 2]) { + let (state, rev) = keccak256_absorb_all(b, stream, len_bytes, true); + ([state[0], state[1]], rev.expect("requested")) +} + +/// `Σ_i 2^i · bits[i]`, bits low-to-high — the value a bit decomposition stands +/// for. Horner from the top: one `MulAdd` per bit after the first. +pub fn bits_to_felt(b: &mut LfmBuilder, bits: &[Bit]) -> Felt { + let two = b.felt_const(FE::from(2u64)); + let mut iter = bits.iter().rev(); + let mut acc = iter.next().expect("at least one bit").as_felt(); + for bit in iter { + acc = b.mul_add(acc, two, bit.as_felt()); + } + acc +} + +fn keccak256_absorb_all( + b: &mut LfmBuilder, + stream: &[Felt], + len_bytes: usize, + want_rev: bool, +) -> ([Cell; 13], Option<[Cell; 2]>) { + use super::keccak_host::{num_blocks, num_stream_halves, pad_half}; + use super::layout::keccak::{BLOCK_HALVES, BLOCK_WORDS, NUM_WORDS}; + + assert_eq!( + stream.len(), + num_stream_halves(len_bytes), + "stream must hold exactly ceil(len_bytes / 4) halves" + ); + + let zero = b.felt_const(FE::zero()); + let mut state: [Cell; NUM_WORDS] = [zero.as_cell(); NUM_WORDS]; + let mut rev: Option<[Cell; 2]> = None; + + for block in 0..num_blocks(len_bytes) { + // Half `h` of this block is half `block * BLOCK_HALVES + h` of the + // padded message; both the rate (136 bytes) and a half (4 bytes) divide + // evenly, so the two indexings line up with no straddling across blocks. + let halves: Vec = (0..BLOCK_HALVES) + .map(|h| { + let g = block * BLOCK_HALVES + h; + let pad = pad_half(len_bytes, g); + match (g < num_stream_halves(len_bytes), pad) { + // Entirely inside the message. + (true, 0) => stream[g], + // Straddles the end: the stream half's high bytes are zero + // by the packing convention, so adding merges the padding in + // without carrying. + (true, p) => { + let c = b.felt_const(FE::from(p)); + b.add(stream[g], c) + } + // Entirely padding (possibly all-zero). + (false, p) => b.felt_const(FE::from(p)), + } + }) + .collect(); + + // 34 halves into 9 words; the last word's top two slots are the unused + // half slots the chip pins to zero. + let block_words: [Cell; BLOCK_WORDS] = core::array::from_fn(|w| { + let lane = |l: usize| { + let h = 4 * w + l; + if h < BLOCK_HALVES { halves[h] } else { zero } + }; + b.pack_word([lane(0), lane(1), lane(2), lane(3)]) + }); + + let last = block + 1 == num_blocks(len_bytes); + if last && want_rev { + let (next, rev_words) = b.keccak_absorb_rev(state, block_words); + state = next; + rev = Some(rev_words); + } else { + state = b.keccak_absorb(state, block_words); + } + } + + (state, rev) +} diff --git a/prover/src/lfm/epoch.rs b/prover/src/lfm/epoch.rs new file mode 100644 index 000000000..0e28b2d03 --- /dev/null +++ b/prover/src/lfm/epoch.rs @@ -0,0 +1,692 @@ +//! Assembly — the per-table challenge replay that turns the legs into a +//! verifier. +//! +//! Every leg so far took its challenges as ARENA WORDS: `emit_sub_proof` hints +//! `γ` and `ζ`, `declare_fri` hints the folding challenges, and the query index +//! arrives as a hinted felt that the walk decomposes. That is fine for a +//! differential against production, which supplies the true values, and it is +//! fatal in a verifier: a prover who chooses `γ` chooses the DEEP fold, one who +//! chooses `ζ_k` chooses the FRI fold, and one who chooses `ι` chooses which +//! rows are ever opened. This module is where those words stop being data. +//! +//! ## What it replays +//! +//! `verifier.rs`'s per-table body, in production's order and nothing else: +//! +//! - the FORK (`:1263-1266`) — clone the shared post-Phase-A transcript, then a +//! domain separator `idx.to_le_bytes()` when the epoch has more than one +//! table; +//! - the aux root (`:1269-1271`) and the bus contribution `L` (`:1274-1276`); +//! - Round 2 (`:1380-1404`): sample `β`, then absorb the composition root; +//! - Round 3 (`:1411-1434`): sample `z`, then absorb the two pruned OOD blocks +//! COLUMN-major, then the claimed composition parts; +//! - Round 4 (`:1445-1504`): sample `γ`; then per committed FRI layer sample +//! `ζ_k` and absorb root `k`; then `ζ_C` if and only if the codeword folds; +//! then every terminal coefficient; then grinding; then the query indices. +//! +//! The interleaving in Round 4 is the part no leg-side test could catch, and it +//! is load-bearing in both directions: a `ζ_k` sampled after its own layer root +//! is a challenge the prover can answer, and a layer root that is never absorbed +//! leaves the query indices independent of the codeword they index. +//! +//! ## Why the values are returned as CELLS +//! +//! The point of the module is that there is exactly ONE cell per value and both +//! consumers read it. `L` is absorbed here and summed by the LogUp closure; the +//! OOD block cells are absorbed here and folded by the constraint leg and by +//! DEEP; the composition parts likewise; the layer roots are absorbed here and +//! compared against the FRI walk. Nothing is hinted twice, which is the +//! assembly obligation (`others/lfm-assembly-obligations.md`, OPEN 3) stated as +//! a construction rather than as a rule to remember. +//! +//! ## Zero-rejection, one level up +//! +//! `sample_z_ood_with_domain_params` REJECTS a `z` that lands in the trace +//! domain or on the LDE coset and draws again. A straight-line program cannot, +//! so [`emit_z_ood`] draws once and CONSTRAINS both rejection predicates to be +//! false — the same disposition as the sampler's canonicity guard, and the same +//! completeness-only cost (`SOUNDNESS.md` §6.3). + +use crate::tables::types::{FE, FEE, GoldilocksExtension}; + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::fri::FriShape; +use super::layout::keccak::DIGEST_WORDS; +use super::transcript_replay::{ByteString, TranscriptReplay}; + +/// The grinding prefix, `crypto/stark/src/grinding.rs`'s `PREFIX`. +const GRINDING_PREFIX: [u8; 8] = 0x0123_4567_89ab_cded_u64.to_be_bytes(); + +/// A commitment root as the machine holds it: eight `u32` lanes, produced ONCE. +/// +/// Both consumers of a root — the transcript absorb and the Merkle comparison — +/// want a different view of the same 32 bytes, and a root that was hinted twice +/// (or unpacked twice) would let those views drift. So there is one unpack per +/// root and every consumer reads its lanes. +/// +/// The three constructors are the three SOURCES a root can have, which is +/// assembly ledger entry 7's whole content: program text ([`Self::constant`]), +/// an in-machine derivation ([`Self::from_digest`]) or the proof's arena +/// ([`Self::hint`]). Which one a given commitment may use is a property of what +/// the commitment is a function of, not a convenience. +#[derive(Clone)] +pub struct RootCells { + pub lanes: [[Felt; 4]; DIGEST_WORDS], +} + +impl RootCells { + /// Read a root out of an arena at `base` (two words) and hoist its unpack. + pub fn hint(b: &mut LfmBuilder, arena: super::instr::ArenaId, base: u32) -> Self { + let words = [b.hint_word(arena, base), b.hint_word(arena, base + 1)]; + RootCells { + lanes: [b.unpack(words[0]), b.unpack(words[1])], + } + } + + /// A root that is PROGRAM TEXT — its eight halves interned as constants. + /// + /// Admissible only for a commitment that is a function of the proof OPTIONS + /// and nothing else, because a program constant is part of program identity: + /// interning a root derived from per-proof data (an ELF, a register file) + /// would give the machine one program per proof instead of one per epoch + /// SHAPE. The two that qualify are BITWISE and KECCAK_RC + /// (`bitwise::preprocessed_commitment(options)`, + /// `tables::keccak_rc::preprocessed_commitment(options)`), plus PAGE's + /// zero-init root, which every zero-initialised page shares. + /// + /// Half `h` is bytes `4h..4h+4` little-endian — + /// `proof_arena::commitment_words`' layout, which is how a keccak digest + /// reaches the chip. + pub fn constant(b: &mut LfmBuilder, root: &[u8; 4 * 4 * DIGEST_WORDS]) -> Self { + let halves: Vec = root + .chunks(4) + .map(|c| { + let mut bytes = [0u8; 4]; + bytes.copy_from_slice(c); + b.felt_const(FE::from(u64::from(u32::from_le_bytes(bytes)))) + }) + .collect(); + let mut lanes = [[halves[0]; 4]; DIGEST_WORDS]; + for (w, word) in lanes.iter_mut().enumerate() { + for (j, lane) in word.iter_mut().enumerate() { + *lane = halves[4 * w + j]; + } + } + RootCells { lanes } + } + + /// A root the machine COMPUTED — the derivation's two digest words, unpacked + /// once so every consumer reads the same lanes. + /// + /// This is REGISTER's source: the commitment is a function of the previous + /// epoch's `reg_fini`, and computing it from those cells is what binds them + /// (`programs::emit_register_commitment`). A hinted REGISTER root would leave + /// the register boundary — the carried commit index among it — a free arena + /// word. + pub fn from_digest(b: &mut LfmBuilder, digest: super::edsl::KeccakDigest) -> Self { + RootCells { + lanes: [b.unpack(digest[0]), b.unpack(digest[1])], + } + } + + /// The 32 bytes as the eight `u32` halves the transcript absorbs, in order. + pub fn halves(&self) -> Vec { + let mut out = Vec::with_capacity(2 * 4); + for lanes in &self.lanes { + out.extend_from_slice(lanes); + } + out + } +} + +/// The shape of one sub-proof's challenge replay. Every field is a program +/// constant: shape, never proof data. +#[derive(Clone, Debug)] +pub struct TableChallengeShape { + /// Position in the epoch's table list — the fork's domain separator. + pub index: usize, + /// How many sub-proofs the epoch has. Production skips the separator + /// entirely at one table (`verifier.rs:1264`), so this changes the bytes. + pub num_tables: usize, + /// Whether the sub-proof carries an aux (LogUp) trace root. + pub has_aux_root: bool, + /// Whether the sub-proof carries a bus contribution `L`. + pub has_contribution: bool, + /// `log2` of the trace length. + pub log2_trace_length: u32, + /// `log2` of the blowup factor. + pub log2_blowup: u32, + /// `ProofOptions::coset_offset`. + pub coset_offset: FE, + /// `(width, height)` of the current-row OOD block, as the proof carries it. + pub ood_current_dims: (usize, usize), + /// `(width, height)` of the pruned next-row OOD block. + pub ood_next_dims: (usize, usize), + /// Composition-poly parts — `air.composition_poly_degree_bound / N`. + pub num_parts: usize, + /// The FRI shape, which fixes how many `ζ`s are drawn and in what order the + /// layer roots are absorbed. + pub fri: FriShape, + /// `ProofOptions::grinding_factor`. Zero means no nonce at all. + pub grinding_factor: u8, + /// `ProofOptions::fri_number_of_queries`. + pub num_queries: usize, +} + +impl TableChallengeShape { + /// `log2` of the LDE domain. + pub fn log2_lde_length(&self) -> u32 { + self.log2_trace_length + self.log2_blowup + } + + /// Bits one query index carries — `sample_u64(lde_length >> 1)` + /// (`verifier.rs:138-141`), so one bit narrower than the domain, which is + /// exactly the Merkle depth the walk consumes. + pub fn index_bits(&self) -> usize { + self.log2_lde_length() as usize - 1 + } + + fn check(&self) { + assert!( + self.index < self.num_tables, + "the table index must be in range" + ); + assert_eq!( + self.fri.log2_lde_length, + self.log2_lde_length(), + "the FRI shape and the trace shape must describe one domain" + ); + assert_eq!( + self.fri.num_queries, self.num_queries, + "the query count is one shape, declared once" + ); + assert!(self.num_parts > 0, "a composition polynomial has parts"); + } +} + +/// The proof-carried cells one table's replay absorbs. +/// +/// These are the caller's cells, hinted once and handed here — never re-hinted. +/// The struct is the assembly join surface: the same values go on to the +/// constraint leg, the DEEP fold, the FRI walk and the LogUp closure. +pub struct TableAbsorbs<'a> { + /// The aux trace root, present exactly when the AIR has an aux trace. + pub aux_root: Option<&'a RootCells>, + /// The bus contribution `L`. The LogUp closure sums THIS cell. + pub contribution: Option, + /// The composition polynomial's committed root. + pub composition_root: &'a RootCells, + /// The current-row OOD block, ROW-major as the proof carries it + /// (`width · height` cells). + pub ood_current: &'a [Ext], + /// The pruned next-row OOD block, row-major. + pub ood_next: &'a [Ext], + /// The claimed composition parts at `z^P`. + pub parts: &'a [Ext], + /// The committed FRI layer roots, in fold order. + pub fri_roots: &'a [RootCells], + /// The terminal polynomial's coefficients, low-to-high. + pub fri_coeffs: &'a [Ext], + /// The grinding nonce, present exactly when `grinding_factor > 0`. + /// + /// Carried as a FELT, so a nonce at or above `p` cannot be expressed. That + /// is a completeness restriction and not a soundness one — such a nonce + /// yields no LFM proof, never a wrong verdict — and it is unreachable in + /// practice: the prover searches nonces upward from zero, so reaching `p` + /// would mean grinding 64 bits. + pub nonce: Option, +} + +/// One table's challenges, as the cells the verification legs consume. +pub struct TableChallenges { + /// The constraint-coefficient base. Production expands `β⁰ .. β^{n−1}` and + /// splits the run into transition then boundary coefficients. + pub beta: Ext, + /// The OOD point. + pub z: Ext, + /// The DEEP batching challenge. + pub gamma: Ext, + /// `ζ₀ .. ζ_C`, or empty when the codeword never folds. + pub zetas: Vec, + /// Per query, the index bits low-to-high — `index_bits()` of them. + /// + /// Bits, never a felt: production draws `sample_u64(lde >> 1)`, whose + /// output is `nbits` bits by construction, and the walk consumes bits. A + /// felt would readmit the standalone driver's aliasing (ledger entry 5), + /// where `ι` and `ι + 2^(n−1)` are the same query. + pub iota_bits: Vec>, +} + +/// Fork the shared transcript for table `index` — `verifier.rs:1263-1266`. +/// +/// The clone emits nothing: the shared prefix's keccak rows were emitted once, +/// when Phase A ran, and every fork carries the same cells for them. The +/// separator is a program constant because the table index is shape. +pub fn fork_table(shared: &TranscriptReplay, index: usize, num_tables: usize) -> TranscriptReplay { + assert!(index < num_tables, "the table index must be in range"); + let mut fork = shared.clone(); + if num_tables > 1 { + fork.append_const_bytes(&(index as u64).to_le_bytes()); + } + fork +} + +/// `z ∉ trace domain ∪ LDE coset`, drawn once and constrained. +/// +/// Production loops until both predicates fail +/// (`is_transcript.rs:61-74`). This machine draws one `z` and proves the two +/// non-memberships, which is the same accepted set — a `z` production would +/// have rejected makes the program unprovable rather than making it accept. +/// +/// Both predicates are equalities over `z^N`, so one `N`-power chain +/// (`log2_trace_length` extension squarings) serves both, and each +/// non-equality is one extension division: `1/(a − b)` is provable exactly when +/// `a ≠ b`, the idiom `assert_canonical` uses on the candidate halves. +pub fn emit_z_ood( + b: &mut LfmBuilder, + t: &mut TranscriptReplay, + shape: &TableChallengeShape, +) -> Ext { + let z = t.sample_ext(b); + assert_z_outside_domains(b, z, shape); + z +} + +/// The two non-memberships alone, so they can be driven with a chosen `z`. +/// +/// A transcript-derived `z` is generic with overwhelming probability, so the +/// guard is unreachable from [`emit_z_ood`] — the reason it lives in its own +/// function is that `the_z_guard_rejects_a_point_in_either_domain` can then +/// feed it the points production would have rejected. +pub fn assert_z_outside_domains(b: &mut LfmBuilder, z: Ext, shape: &TableChallengeShape) { + // z^N by repeated squaring; N = 2^log2_trace_length. + let mut z_pow_trace = z; + for _ in 0..shape.log2_trace_length { + z_pow_trace = b.emul(z_pow_trace, z_pow_trace); + } + let one = b.ext_const(&FEE::one()); + assert_ne_ext(b, z_pow_trace, one); + + // (z^N)^blowup against coset_offset^lde — the offset power is a program + // constant because the domain is shape. + let mut z_pow_lde = z_pow_trace; + for _ in 0..shape.log2_blowup { + z_pow_lde = b.emul(z_pow_lde, z_pow_lde); + } + let offset_pow = shape + .coset_offset + .pow(1u64 << shape.log2_lde_length()) + .to_extension::(); + let offset_pow = b.ext_const(&offset_pow); + assert_ne_ext(b, z_pow_lde, offset_pow); +} + +/// Constrain `a ≠ b` by exhibiting `(a − b)⁻¹`. +/// +/// `Div` is constrained as `OUT · B = A`, which for `A = 1` has no witness at +/// `B = 0`: the program is unprovable exactly when the two are equal. +fn assert_ne_ext(b: &mut LfmBuilder, x: Ext, y: Ext) { + let d = b.esub(x, y); + let one = b.ext_const(&FEE::one()); + let _ = b.ediv(one, d); +} + +/// Verify the grinding nonce — `grinding::is_valid_nonce`. +/// +/// Two keccaks: the inner hash over `PREFIX ‖ state ‖ factor` (41 bytes) and +/// the outer over `inner ‖ nonce_be` (40 bytes). +/// +/// The predicate is `u64::from_be_bytes(digest[..8]) < 2^(64 − g)` — "the top +/// `g` bits of the digest's first eight bytes, read big-endian, are zero". Those +/// eight bytes are lanes 0 and 1 of the digest's first WORD, and a lane is four +/// bytes LITTLE-endian, so byte `i` of the big-endian run is bit-range +/// `[8·(i mod 4), 8·(i mod 4) + 8)` of lane `i / 4`. The check is therefore a +/// bit decomposition of at most two lanes plus a run of zero assertions — no +/// comparison and no 64-bit arithmetic, because the bound is a power of two. +/// +/// Skipping the check would not merely be untidy: the nonce is absorbed, so +/// the query indices depend on it, and an unchecked nonce is a free re-roll of +/// every query index at zero cost. +fn emit_grinding_check( + b: &mut LfmBuilder, + seed: [Cell; DIGEST_WORDS], + nonce_halves: [Felt; 2], + factor: u8, +) { + assert!( + (1..=64).contains(&factor), + "a grinding factor is in 1..=64 (grinding.rs:22-25), got {factor}" + ); + + let mut inner = ByteString::new(); + inner.push_const(&GRINDING_PREFIX); + let mut seed_halves = Vec::with_capacity(8); + for w in seed { + seed_halves.extend_from_slice(&b.unpack(w)); + } + inner.push_halves(&seed_halves); + inner.push_const(&[factor]); + let inner_hash = inner.keccak256(b); + + let mut outer = ByteString::new(); + let mut inner_halves = Vec::with_capacity(8); + for w in inner_hash { + inner_halves.extend_from_slice(&b.unpack(w)); + } + outer.push_halves(&inner_halves); + outer.push_halves(&nonce_halves); + let digest = outer.keccak256(b); + + // The zero bits, as `(byte, bit-within-byte)` pairs of the big-endian run: + // `factor / 8` whole leading bytes, then the top `factor % 8` bits of the + // next one. + let whole = factor as usize / 8; + let rest = factor as usize % 8; + let mut wanted: Vec<(usize, usize)> = Vec::with_capacity(factor as usize); + for byte in 0..whole { + wanted.extend((0..8).map(|bit| (byte, bit))); + } + wanted.extend((8 - rest..8).map(|bit| (whole, bit))); + + let lanes = b.unpack(digest[0]); + let zero = b.felt_const(FE::zero()); + let mut decomposed: [Option>; 2] = [None, None]; + for (byte, bit) in wanted { + let lane = byte / 4; + let bits = match &decomposed[lane] { + Some(bits) => bits, + None => { + decomposed[lane] = Some(b.bit_dec(lanes[lane], 32)); + decomposed[lane].as_ref().expect("just decomposed") + } + }; + let v = Felt(bits[8 * (byte % 4) + bit].addr()); + b.assert_eq(v, zero); + } +} + +/// `nonce.to_be_bytes()` as the two `u32` halves the transcript absorbs. +/// +/// The transcript reads halves as four LITTLE-endian bytes, so the big-endian +/// rendering is the felt's two halves in reversed ORDER, each byte-swapped — +/// which is exactly what `felt_be_halves` produces. +fn nonce_halves(b: &mut LfmBuilder, nonce: Felt) -> [Felt; 2] { + super::transcript_replay::felt_be_halves(b, nonce) +} + +/// Replay one table's rounds 2 to 4 against a FORKED transcript. +/// +/// `t` must be the fork ([`fork_table`]), not the shared transcript. Returns +/// the challenges as cells; every absorbed value came from the caller. +pub fn emit_table_challenges( + b: &mut LfmBuilder, + t: &mut TranscriptReplay, + shape: &TableChallengeShape, + absorbs: &TableAbsorbs<'_>, +) -> TableChallenges { + shape.check(); + assert_eq!( + absorbs.aux_root.is_some(), + shape.has_aux_root, + "the aux root's presence is shape" + ); + assert_eq!( + absorbs.contribution.is_some(), + shape.has_contribution, + "the contribution's presence is shape" + ); + assert_eq!( + absorbs.ood_current.len(), + shape.ood_current_dims.0 * shape.ood_current_dims.1, + "the current-row OOD block must match its declared dimensions" + ); + assert_eq!( + absorbs.ood_next.len(), + shape.ood_next_dims.0 * shape.ood_next_dims.1, + "the next-row OOD block must match its declared dimensions" + ); + assert_eq!(absorbs.parts.len(), shape.num_parts, "one cell per part"); + assert_eq!( + absorbs.fri_roots.len(), + shape.fri.num_committed(), + "one root per committed FRI layer" + ); + assert_eq!( + absorbs.fri_coeffs.len(), + shape.fri.num_terminal_coeffs(), + "the terminal polynomial's coefficient count is shape" + ); + assert_eq!( + absorbs.nonce.is_some(), + shape.grinding_factor > 0, + "a nonce exists exactly when grinding is on" + ); + + // ---- Phase C and the contribution bind, inside the fork. + if let Some(root) = absorbs.aux_root { + t.append_halves(&root.halves()); + } + if let Some(l) = absorbs.contribution { + append_ext_cell(b, t, l); + } + + // ---- Round 2: β, then the composition root. + let beta = t.sample_ext(b); + t.append_halves(&absorbs.composition_root.halves()); + + // ---- Round 3: z, then both OOD blocks COLUMN-major, then the parts. + let z = emit_z_ood(b, t, shape); + for (dims, block) in [ + (shape.ood_current_dims, absorbs.ood_current), + (shape.ood_next_dims, absorbs.ood_next), + ] { + let (width, height) = dims; + for col in 0..width { + for row in 0..height { + append_ext_cell(b, t, block[row * width + col]); + } + } + } + for part in absorbs.parts { + append_ext_cell(b, t, *part); + } + + // ---- Round 4: γ, the interleaved FRI commit phase, then the queries. + let gamma = t.sample_ext(b); + + let mut zetas = Vec::with_capacity(shape.fri.num_committed() + 1); + for root in absorbs.fri_roots { + // Sample FIRST, absorb SECOND — a ζ drawn after its own layer root is a + // challenge the prover answers rather than one that binds them. + zetas.push(t.sample_ext(b)); + t.append_halves(&root.halves()); + } + if shape.fri.total_folds() > 0 { + zetas.push(t.sample_ext(b)); + } + for c in absorbs.fri_coeffs { + append_ext_cell(b, t, *c); + } + + if let Some(nonce) = absorbs.nonce { + let seed = t.state(b); + let halves = nonce_halves(b, nonce); + emit_grinding_check(b, seed, halves, shape.grinding_factor); + t.append_halves(&halves); + } + + let iota_bits = (0..shape.num_queries) + .map(|_| t.sample_u64_pow2(b, shape.index_bits())) + .collect(); + + TableChallenges { + beta, + z, + gamma, + zetas, + iota_bits, + } +} + +/// Rebuild the full OOD grid from the two pruned blocks the proof carries. +/// +/// The in-machine analogue of `ood::reconstruct_ood_full` +/// (`crypto/stark/src/ood.rs`), and the seam between the spine and the two legs +/// that fold the grid: the same cells [`emit_table_challenges`] absorbed +/// column-major come back here as `num_eval_points × num_total_cols` rows, which +/// is the shape `constraints::emit_analyzed` reads `Op::Var{offset, col}` out of +/// and the shape `deep::emit_deep_invariants` sums. +/// +/// ## The zeros are program text, not arena data +/// +/// A pruned next-row entry is reconstructed as ZERO by the real verifier — no +/// transition constraint reads a pruned column at the next row, and DEEP pairs +/// those positions with zero coefficients. Emitting the pooled zero constant +/// makes the pruning part of the program rather than a property of the supplied +/// arena, which is the standing decision ("next-row pruning likewise, because the +/// verifier reconstructs an undeclared column as ZERO"). The permissive +/// direction is the dangerous one: a machine that hinted a value into a pruned +/// slot would fold a frame the real verifier cannot see. +/// +/// This emits no instruction beyond interning that zero — it is cell plumbing, +/// which is the point. The blocks are the transcript's own cells, so there is no +/// second copy of the grid for a prover to disagree with. +pub fn emit_reconstruct_ood( + b: &mut LfmBuilder, + deep: &super::deep::DeepShape, + current: &[Ext], + next: &[Ext], +) -> Vec> { + let width = deep.num_total_cols; + let mask_width = deep.next_row_cols.len(); + let next_rows = deep + .num_eval_points + .checked_sub(deep.step_size) + .expect("the OOD grid is at least the current-row block"); + assert_eq!( + current.len(), + deep.step_size * width, + "the current-row block is step_size × num_total_cols" + ); + assert_eq!( + next.len(), + next_rows * mask_width, + "the next-row block is (num_eval_points − step_size) × |next_row_cols|" + ); + + let zero = b.felt_const(FE::zero()).as_ext(); + let mut rows = Vec::with_capacity(deep.num_eval_points); + for r in 0..deep.step_size { + rows.push(current[r * width..(r + 1) * width].to_vec()); + } + for r in 0..next_rows { + let mut row = vec![zero; width]; + for (m, &col) in deep.next_row_cols.iter().enumerate() { + assert!( + col < width, + "a next-row column must index into the trace: {col} against {width}" + ); + row[col] = next[r * mask_width + m]; + } + rows.push(row); + } + rows +} + +/// Absorb an extension cell the way `append_field_element` streams it: three +/// coordinates, each eight big-endian bytes. +fn append_ext_cell(b: &mut LfmBuilder, t: &mut TranscriptReplay, v: Ext) { + let coords = b.unpack(v.as_cell()); + t.append_ext(b, [coords[0], coords[1], coords[2]]); +} + +/// Constrain `v < 2^32` — the felt-width guard assembly ledger entry 1 owes. +/// +/// An LFM arena is untyped felts; production's `reg_fini` and `register_init` are +/// `Vec`, and that TYPE is the whole enforcement on their side. So without +/// this the machine's accepted set is strictly wider than production's: a prover +/// could derive the REGISTER preprocessed commitment over a boundary value no +/// production epoch can hold. +/// +/// One `BitDec` plus one recomposition: `bit_dec` exposes the low 32 bits, and +/// asserting `v` equals their weighted sum has no witness above `2^32 − 1`. The +/// same idiom `emit_output_bytes` uses on the public-output halves, and for the +/// same reason. +/// +/// Deliberately at the ASSEMBLY call site rather than inside +/// `programs::emit_register_commitment`: the isolated derivation's width gap is +/// pinned by a guard test that asserts the hazard still exists +/// (`machine_tests::the_derivation_extends_a_non_u32_register_value_demonstrating_ +/// hazard`), which is correct — an isolated derivation binds nothing, and closing +/// the gap is what assembly is for. +pub fn assert_u32(b: &mut LfmBuilder, v: Felt) { + let bits = b.bit_dec(v, 32); + let two = b.felt_const(FE::from(2u64)); + let mut acc = Felt(bits[31].addr()); + for k in (0..31).rev() { + let bit = Felt(bits[k].addr()); + acc = b.mul_add(acc, two, bit); + } + b.assert_eq(v, acc); +} + +/// `β⁰ .. β^{n−1}` — production's `compute_alpha_powers(&beta, n)`, which the +/// quotient fold consumes as transition coefficients then boundary ones. +/// +/// Derived in-machine from the ONE `β` the transcript produced, for the reason +/// `constraints::emit_alpha_powers` exists: a hinted power run is a prover's +/// free choice of every constraint coefficient. +pub fn emit_beta_powers(b: &mut LfmBuilder, beta: Ext, n: usize) -> Vec { + super::constraints::emit_alpha_powers(b, beta, n) +} + +/// The public output as one cell per BYTE, derived from the `u32` halves the +/// statement absorbed. +/// +/// The output has two consumers with incompatible shapes: the statement absorb +/// wants four-byte halves, and the COMMIT-bus target folds one term per byte +/// (`logup::emit_commit_bus_target`). Two arenas would be two claims about the +/// same string — the assembly obligation's fifth instance. So the halves are +/// the arena, and the bytes are DERIVED here. +/// +/// Each half costs one `BitDec` and one `MulAdd`, and the recomposition assert +/// is what makes it a range check as well: a hinted half at or above `2^32` +/// cannot equal the sum of the four bytes read out of its low 32 bits, so the +/// program is unprovable rather than absorbing one string and folding another. +/// +/// A trailing partial half is masked the same way the transcript masks it: only +/// `len_bytes` bytes are returned, and the unused high bytes of the last half +/// are pinned to zero — without that a prover could absorb one string while the +/// length prefix claimed another. +pub fn emit_output_bytes(b: &mut LfmBuilder, halves: &[Felt], len_bytes: usize) -> Vec { + assert_eq!( + halves.len(), + len_bytes.div_ceil(4), + "one half per four output bytes, the last one partial" + ); + let mut out = Vec::with_capacity(len_bytes); + let zero = b.felt_const(FE::zero()); + for (h, half) in halves.iter().enumerate() { + let bits = b.bit_dec(*half, 32); + let bytes: Vec = (0..4) + .map(|k| super::edsl::bits_to_felt(b, &bits[8 * k..8 * k + 8])) + .collect(); + // half = Σ byteₖ·2^{8k}, which pins the half to its four bytes AND to + // the range `[0, 2^32)`. + let mut acc = bytes[3]; + for k in (0..3).rev() { + let shift = b.felt_const(FE::from(256u64)); + acc = b.mul_add(acc, shift, bytes[k]); + } + b.assert_eq(*half, acc); + + for (k, byte) in bytes.into_iter().enumerate() { + if 4 * h + k < len_bytes { + out.push(byte); + } else { + b.assert_eq(byte, zero); + } + } + } + out +} diff --git a/prover/src/lfm/epoch_tests.rs b/prover/src/lfm/epoch_tests.rs new file mode 100644 index 000000000..57505120e --- /dev/null +++ b/prover/src/lfm/epoch_tests.rs @@ -0,0 +1,2147 @@ +//! The assembled verifier's challenge replay, differentialled against +//! production's own. +//! +//! ## The oracle +//! +//! `Verifier::replay_rounds_after_round_1` — the function `multi_verify_views` +//! itself calls. Nothing here models Fiat-Shamir; the expected `β`, `z`, `γ`, +//! `ζ_k` and `ι_s` are the values the production verifier computed for a real +//! proof of a real AIR, and the machine is asked to reproduce them from the +//! proof's own bytes. +//! +//! ## What this suite can see that no leg-side suite could +//! +//! The FRI commit phase's INTERLEAVING (ledger entry 4). `fri_tests` supplies +//! `ζ_k` from the same replay it checks against, so absorbing the layer roots +//! in the wrong order — or not at all — moves nothing there. Here every +//! challenge is derived from the absorbed bytes, so a misordered absorb changes +//! `ζ`, and a `ζ` change moves the fold. The four fixtures span +//! `num_committed = 0, 1, 2, 3`, so the loop is exercised at zero, one and +//! several layers. +//! +//! ## What it cannot see +//! +//! It stops at the challenges. That the legs then CONSUME these cells is +//! [`the_legs_consume_the_replayed_challenges`]'s job, and the whole-epoch +//! composition (24 sub-proofs behind one statement) is not built here. + +use stark::config::Commitment; +use stark::proof::stark::MultiProof; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::edsl; +use super::epoch::{ + RootCells, TableAbsorbs, TableChallengeShape, emit_table_challenges, fork_table, +}; +use super::executor::execute; +use super::fri::FriShape; +use super::hash::TestPermutation; +use super::instr::ArenaId; +use super::transcript_replay::TranscriptReplay; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// Everything one real sub-proof supplies to the replay, plus the challenges +/// production derived from it. +#[derive(Clone)] +pub(super) struct HostTable { + pub(super) shape: TableChallengeShape, + /// The verifier's HARDCODED precomputed commitment, when the AIR is + /// preprocessed. A program constant, not arena data: the verifier does not + /// take this from the proof (`verifier.rs:1187`). + precomputed_root: Option, + main_root: Commitment, + aux_root: Option, + pub(super) contribution: Option, + composition_root: Commitment, + /// Row-major, as `row_major_data` carries it. + pub(super) ood_current: Vec, + pub(super) ood_next: Vec, + pub(super) parts: Vec, + fri_roots: Vec, + pub(super) fri_coeffs: Vec, + nonce: Option, + needs_lookup_challenges: bool, + + // ---- the oracle ---- + pub(super) beta: FEE, + pub(super) z: FEE, + pub(super) gamma: FEE, + pub(super) zetas: Vec, + pub(super) iotas: Vec, +} + +/// Read a real single-table proof into [`HostTable`], taking the challenges +/// from the production verifier rather than recomputing them. +fn host_table( + air: &dyn AIR, + proof: &MultiProof, +) -> HostTable { + let sp = super::constraint_tests::open_sub_proof(air, proof); + let view = StarkProofView::Owned(&proof.proofs[0]); + let opts = air.options(); + + let trace_length = view.trace_length(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + let fri = FriShape::from_options(opts, log2_trace_length + log2_blowup); + + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + + // `γ` comes back from the DEEP shape derivation, which recovers it from the + // verifier's own coefficient run — the same route `join_tests` uses. + let (_deep, gamma) = super::constraint_tests::deep_shape(&sp, air); + + let shape = TableChallengeShape { + index: 0, + num_tables: 1, + has_aux_root: view.lde_trace_aux_merkle_root().is_some(), + has_contribution: view.bus_table_contribution().is_some(), + log2_trace_length, + log2_blowup, + coset_offset: FE::from(opts.coset_offset), + ood_current_dims: (ood_c.width(), ood_c.height()), + ood_next_dims: (ood_n.width(), ood_n.height()), + num_parts: view.composition_poly_parts_ood_evaluation().len(), + fri, + grinding_factor: opts.grinding_factor, + num_queries: opts.fri_number_of_queries, + }; + + HostTable { + shape, + precomputed_root: air.is_preprocessed().then(|| air.precomputed_commitment()), + main_root: *view.lde_trace_main_merkle_root(), + aux_root: view.lde_trace_aux_merkle_root().copied(), + contribution: view.bus_table_contribution(), + composition_root: *view.composition_poly_root(), + ood_current: ood_c.row_major_data().to_vec(), + ood_next: ood_n.row_major_data().to_vec(), + parts: view.composition_poly_parts_ood_evaluation().to_vec(), + fri_roots: view.fri_layers_merkle_roots().to_vec(), + fri_coeffs: view.fri_final_poly_coeffs().to_vec(), + nonce: view.nonce(), + needs_lookup_challenges: air.has_aux_trace(), + beta: sp.beta, + z: sp.challenges.z, + gamma, + zetas: sp.challenges.zetas.clone(), + iotas: sp.challenges.iotas.clone(), + } +} + +/// Arena identifiers of the challenge program, in declaration order. +struct Arenas { + main_root: ArenaId, + aux_root: Option, + contribution: Option, + composition_root: ArenaId, + ood_current: ArenaId, + ood_next: ArenaId, + parts: ArenaId, + fri_roots: ArenaId, + fri_coeffs: ArenaId, + nonce: Option, + /// The verification legs' two arenas, present only in the ASSEMBLED + /// verifier — the trace openings and the FRI layer openings. `None` in the + /// spine-only program, which verifies nothing and so opens nothing. + legs: Option, +} + +/// A program that replays ONE table's challenges and publishes them. +/// +/// The transcript prefix is `multi_verify_views`' single-table Phase A: the +/// hardcoded precomputed commitment when the AIR is preprocessed, the main +/// root, then the shared LogUp challenges. The fork follows, then rounds 2-4. +fn challenge_program(h: &HostTable) -> LfmProgram { + let mut b = LfmBuilder::new(); + let shape = &h.shape; + + let a = Arenas { + main_root: b.declare_arena(2), + aux_root: shape.has_aux_root.then(|| b.declare_arena(2)), + contribution: shape.has_contribution.then(|| b.declare_arena(1)), + composition_root: b.declare_arena(2), + ood_current: b.declare_arena((shape.ood_current_dims.0 * shape.ood_current_dims.1) as u32), + ood_next: b.declare_arena((shape.ood_next_dims.0 * shape.ood_next_dims.1) as u32), + parts: b.declare_arena(shape.num_parts as u32), + fri_roots: b.declare_arena(2 * shape.fri.num_committed() as u32), + fri_coeffs: b.declare_arena(shape.fri.num_terminal_coeffs() as u32), + nonce: (shape.grinding_factor > 0).then(|| b.declare_arena(1)), + legs: None, + }; + + let mut t = TranscriptReplay::new(&[]); + if let Some(prep) = h.precomputed_root { + t.append_const_bytes(&prep); + } + let main = RootCells::hint(&mut b, a.main_root, 0); + t.append_halves(&main.halves()); + if h.needs_lookup_challenges { + for _ in 0..stark::lookup::LOGUP_NUM_CHALLENGES { + t.sample_ext(&mut b); + } + } + + let aux = a.aux_root.map(|id| RootCells::hint(&mut b, id, 0)); + let contribution = a.contribution.map(|id| b.hint_word(id, 0).as_ext()); + let composition = RootCells::hint(&mut b, a.composition_root, 0); + let ood_current: Vec<_> = (0..(shape.ood_current_dims.0 * shape.ood_current_dims.1) as u32) + .map(|i| b.hint_word(a.ood_current, i).as_ext()) + .collect(); + let ood_next: Vec<_> = (0..(shape.ood_next_dims.0 * shape.ood_next_dims.1) as u32) + .map(|i| b.hint_word(a.ood_next, i).as_ext()) + .collect(); + let parts: Vec<_> = (0..shape.num_parts as u32) + .map(|i| b.hint_word(a.parts, i).as_ext()) + .collect(); + let fri_roots: Vec<_> = (0..shape.fri.num_committed()) + .map(|i| RootCells::hint(&mut b, a.fri_roots, 2 * i as u32)) + .collect(); + let fri_coeffs: Vec<_> = (0..shape.fri.num_terminal_coeffs() as u32) + .map(|i| b.hint_word(a.fri_coeffs, i).as_ext()) + .collect(); + let nonce = a.nonce.map(|id| b.hint_felt(id, 0)); + + let mut fork = fork_table(&t, shape.index, shape.num_tables); + let ch = emit_table_challenges( + &mut b, + &mut fork, + shape, + &TableAbsorbs { + aux_root: aux.as_ref(), + contribution, + composition_root: &composition, + ood_current: &ood_current, + ood_next: &ood_next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }, + ); + + b.public(ch.beta.as_cell()); + b.public(ch.z.as_cell()); + b.public(ch.gamma.as_cell()); + for zeta in &ch.zetas { + b.public(zeta.as_cell()); + } + for bits in &ch.iota_bits { + let felt = edsl::bits_to_felt(&mut b, bits); + b.public(felt.as_cell()); + } + + let program = compile(b.finish()); + validate(&program).expect("the challenge program must be admissible"); + program +} + +/// The arenas [`challenge_program`] declares, in the same order. +fn challenge_arenas(h: &HostTable) -> Vec> { + let mut out = vec![super::proof_arena::commitments_to_arena(&[h.main_root])]; + if let Some(r) = h.aux_root { + out.push(super::proof_arena::commitments_to_arena(&[r])); + } + if let Some(c) = h.contribution { + out.push(vec![ext_word(&c)]); + } + out.push(super::proof_arena::commitments_to_arena(&[ + h.composition_root + ])); + out.push(h.ood_current.iter().map(ext_word).collect()); + out.push(h.ood_next.iter().map(ext_word).collect()); + out.push(h.parts.iter().map(ext_word).collect()); + out.push(super::proof_arena::commitments_to_arena(&h.fri_roots)); + out.push(h.fri_coeffs.iter().map(ext_word).collect()); + if let Some(n) = h.nonce { + out.push(vec![base_word(FE::from(n))]); + } + out +} + +/// Run the program and read the published challenges back. +fn run(h: &HostTable) -> (FEE, FEE, FEE, Vec, Vec) { + let program = challenge_program(h); + let arenas = challenge_arenas(h); + let exec = execute(&program, &arenas, &TestPermutation).expect("the replay must execute"); + + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + let beta = pub_ext(0); + let z = pub_ext(1); + let gamma = pub_ext(2); + let zetas: Vec = (0..h.zetas.len()).map(|k| pub_ext(3 + k)).collect(); + let base = 3 + h.zetas.len(); + let iotas: Vec = (0..h.shape.num_queries) + .map(|q| { + let w = exec.public_words[base + q].1; + let felt = super::word::word_as_base(&w).expect("an index is a base felt"); + felt.to_raw() + }) + .collect(); + (beta, z, gamma, zetas, iotas) +} + +/// ★ The whole point of the leg: every challenge the legs consume is the one +/// production derived, and it came out of the transcript rather than an arena. +/// +/// Swept over four real proofs whose committed FRI layer counts are 0, 1, 2 and +/// 3, because the Round-4 interleaving only has anything to get wrong once a +/// layer exists. +#[test] +fn the_challenge_replay_matches_production() { + for (boundaries, committed) in [(4usize, 0usize), (512, 1), (1024, 2), (2048, 3)] { + let (air, proof) = super::fri_tests::folding_fixture(boundaries, 2); + let h = host_table(&*air, &proof); + assert_eq!( + h.shape.fri.num_committed(), + committed, + "fixture of {boundaries} boundaries must commit {committed} layers" + ); + assert_eq!( + h.zetas.len(), + if h.shape.fri.total_folds() > 0 { + committed + 1 + } else { + 0 + }, + "folds exceed committed layers by one, and vanish when nothing folds" + ); + + let (beta, z, gamma, zetas, iotas) = run(&h); + assert_eq!(beta, h.beta, "beta at {boundaries} boundaries"); + assert_eq!(z, h.z, "z at {boundaries} boundaries"); + assert_eq!(gamma, h.gamma, "gamma at {boundaries} boundaries"); + assert_eq!(zetas, h.zetas, "the FRI zetas at {boundaries} boundaries"); + let want: Vec = h.iotas.iter().map(|i| *i as u64).collect(); + assert_eq!(iotas, want, "the query indices at {boundaries} boundaries"); + } +} + +/// ★ Two defects the differential above CANNOT see, pinned so they are not +/// mistaken for coverage. +/// +/// Both are degenerate parameters of the single-table L2G fixture, and the +/// falsification runs found them rather than reasoning predicting them: +/// injecting a ROW-major OOD absorb and deleting the fork's domain separator +/// both left `the_challenge_replay_matches_production` green. +#[test] +fn the_single_table_fixture_is_blind_to_two_defects() { + let (air, proof) = super::fri_tests::folding_fixture(2048, 2); + let h = host_table(&*air, &proof); + println!( + "ood_current {:?} ood_next {:?} num_tables {}", + h.shape.ood_current_dims, h.shape.ood_next_dims, h.shape.num_tables + ); + assert_eq!( + h.shape.ood_current_dims.1, 1, + "a one-ROW OOD block reads the same column-major as row-major, so this \ + fixture cannot witness the absorb order" + ); + assert_eq!( + h.shape.ood_next_dims.1, 1, + "likewise for the next-row block" + ); + assert_eq!( + h.shape.num_tables, 1, + "production emits no domain separator at one table (verifier.rs:1264), \ + so this fixture cannot witness the fork's separator" + ); +} + +/// ★ The `z` guard, driven with the points production rejects. +/// +/// `sample_z_ood_with_domain_params` loops until `z` is outside both the trace +/// domain and the LDE coset. The machine cannot loop, so it constrains the +/// first draw — and that constraint is unreachable from a real transcript, +/// which is why the guard is emitted against a HINTED `z` here. Both rejection +/// branches are exercised separately, with a generic `z` as the positive +/// control: without it, a guard that rejected everything would look identical. +#[test] +fn the_z_guard_rejects_a_point_in_either_domain() { + use math::field::traits::IsFFTField; + + let shape = TableChallengeShape { + index: 0, + num_tables: 1, + has_aux_root: false, + has_contribution: false, + log2_trace_length: 4, + log2_blowup: 1, + coset_offset: FE::from(3u64), + ood_current_dims: (1, 1), + ood_next_dims: (0, 0), + num_parts: 1, + fri: FriShape::from_options(&super::proof_fixture::fixture_options(), 5), + grinding_factor: 0, + num_queries: 1, + }; + + let program = { + let mut b = LfmBuilder::new(); + let a = b.declare_arena(1); + let z = b.hint_word(a, 0).as_ext(); + super::epoch::assert_z_outside_domains(&mut b, z, &shape); + let program = compile(b.finish()); + validate(&program).expect("the guard program must be admissible"); + program + }; + let runs = |z: FEE| execute(&program, &[vec![ext_word(&z)]], &TestPermutation).is_ok(); + + // Positive control: a generic point passes, so a guard that rejected + // everything would not be mistaken for a working one. + assert!( + runs(FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)])), + "a generic z must pass both non-memberships" + ); + + // In the trace domain: a 16th root of unity, so z^16 = 1. + let g = ::get_primitive_root_of_unity(4).expect("root of unity"); + for k in [0u64, 1, 5] { + let z = g.pow(k).to_extension::(); + assert!( + !runs(z), + "z = g^{k} is in the trace domain and production would have redrawn" + ); + } + + // On the LDE coset: z = offset · ω^k with ω the 32nd root of unity, so + // z^32 = offset^32. + let w = ::get_primitive_root_of_unity(5).expect("root of unity"); + for k in [0u64, 3, 17] { + let z = (FE::from(3u64) * w.pow(k)).to_extension::(); + assert!( + !runs(z), + "z = 3·ω^{k} is on the LDE coset and production would have redrawn" + ); + } +} + +/// ★ The grinding check, which the challenge differential is structurally +/// blind to. +/// +/// A wrong nonce changes the query indices, so the differential above would +/// simply compare different-but-consistent values; deleting +/// `emit_grinding_check` entirely left it green (falsification run +/// `grinding_check`). What makes a wrong nonce REJECT is the proof-of-work +/// predicate, and this drives it: the proof's own nonce runs, and eight +/// neighbouring nonces — which absorb just as happily — do not. +/// +/// This matters beyond tidiness. The nonce is absorbed before the query +/// indices are drawn, so an unchecked nonce is a free re-roll of every query +/// index: a prover with a bad codeword re-grinds until the indices miss it, +/// at the cost production charges 2^20 hashes for. +#[test] +fn a_nonce_that_did_not_grind_is_rejected() { + let (air, proof) = super::fri_tests::folding_fixture(4, 2); + let h = host_table(&*air, &proof); + assert!( + h.shape.grinding_factor > 0, + "the fixture must actually grind, or this test proves nothing" + ); + let real = h.nonce.expect("a grinding proof carries a nonce"); + + let program = challenge_program(&h); + let runs = |nonce: u64| { + let mut h2 = h.clone(); + h2.nonce = Some(nonce); + execute(&program, &challenge_arenas(&h2), &TestPermutation).is_ok() + }; + + assert!(runs(real), "the proof's own nonce must satisfy the check"); + let mut rejected = 0; + for delta in 1..=8u64 { + if !runs(real.wrapping_add(delta)) { + rejected += 1; + } + } + assert_eq!( + rejected, 8, + "at a grinding factor of {} a neighbouring nonce passes with \ + probability 2^-{}, so all eight must be rejected", + h.shape.grinding_factor, h.shape.grinding_factor + ); +} + +// ============================================================================= +// The whole epoch: one statement, Phase A over every sub-proof, then a fork per +// table. +// ============================================================================= + +/// A real continuation epoch, proved over the production epoch AIR set, with +/// production's own per-table challenges extracted for every sub-proof. +/// +/// Built the way `prove_continuation` builds epoch 0 — the same construction +/// `logup_tests::a_zero_row_fixed_table_carries_some_zero_not_none` proves and +/// production ACCEPTS. Nothing here is synthetic: the statement is the real +/// one, the forks carry the real domain separators, and the challenges come +/// from `replay_rounds_after_round_1` on each fork. +/// ★ Where a preprocessed commitment COMES FROM — assembly ledger entry 7, +/// as a type. +/// +/// Production absorbs every preprocessed root from the AIR and never from the +/// proof, so the machine owes each one a provenance of the same strength. The +/// three variants are the three that exist, and which one a commitment gets is +/// decided by what the commitment is a function of: +/// +/// - options only ⇒ [`Self::Constant`], interned as program text. Safe because +/// the proof options are already program SHAPE. +/// - the previous epoch's register boundary ⇒ [`Self::Register`], DERIVED +/// in-machine. Interning it would pin one LFM program per register file; +/// hinting it would leave the boundary — the carried commit index among it — a +/// free arena word (ledger entry 2). +/// - the inner ELF ⇒ [`Self::ElfDependent`], an arena cell bound one level up by +/// the attestation's `program_id` fold. Interning it would make LFM program +/// identity a function of the guest ELF, which is an always-stop item. +/// +/// [`prep_source`] decides the variant by MATCHING the AIR's own commitment +/// against production's candidate functions, so an epoch that grew a preprocessed +/// table with no known provenance panics instead of quietly hinting an unbound +/// root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum PrepSource { + /// A function of `ProofOptions` alone: BITWISE, KECCAK_RC, or PAGE's + /// shared zero-init root. + Constant(Commitment), + /// REGISTER: `compute_precomputed_commitment_with_fini(options, INIT, FINI)`. + Register(Commitment), + /// A function of the inner ELF: DECODE, or an ELF-data page's root. + ElfDependent(Commitment), +} + +impl PrepSource { + /// Whether this root occupies arena words — true for exactly the + /// ELF-dependent family. + fn is_arena(self) -> bool { + matches!(self, PrepSource::ElfDependent(_)) + } +} + +/// How many of an epoch's preprocessed roots come from each source — +/// `(options-only, derived, ELF-dependent)`. +pub(super) fn prep_source_census(e: &RealEpoch) -> (usize, usize, usize) { + let mut census = (0, 0, 0); + for source in e.phase_a.iter().filter_map(|(p, _)| *p) { + match source { + PrepSource::Constant(_) => census.0 += 1, + PrepSource::Register(_) => census.1 += 1, + PrepSource::ElfDependent(_) => census.2 += 1, + } + } + census +} + +/// Classify one preprocessed commitment by recomputing every candidate +/// production has and seeing which one it IS. +/// +/// A match is not a heuristic: these are keccak Merkle roots over different +/// tables, so two candidates agreeing would be a collision. What the function +/// really buys is the failure mode — a preprocessed AIR whose root matches +/// nothing known is a root the machine has no binding for, and this panics +/// rather than hinting it. +fn prep_source( + root: Commitment, + opts: &crate::ProofOptions, + elf: &executor::elf::Elf, + register_init: &[u32], + reg_fini: &[u32], +) -> PrepSource { + use crate::tables::{bitwise, decode, keccak_rc, page, register}; + + if root == bitwise::preprocessed_commitment(opts) + || root == keccak_rc::preprocessed_commitment(opts) + || root == page::zero_init_preprocessed_commitment(opts) + { + return PrepSource::Constant(root); + } + if root == register::compute_precomputed_commitment_with_fini(opts, register_init, reg_fini) { + return PrepSource::Register(root); + } + if root == decode::commitment_from_elf(elf, opts).expect("the DECODE commitment must compute") { + return PrepSource::ElfDependent(root); + } + panic!( + "a preprocessed sub-proof carries a root matching none of production's \ + candidate sources (BITWISE, KECCAK_RC, PAGE zero-init, REGISTER-with-FINI, \ + DECODE-from-ELF). The machine has no binding for it, so it must not be \ + hinted: extend the taxonomy (assembly ledger entry 7) rather than this list" + ); +} + +pub(super) struct RealEpoch { + pub(super) statement: super::statement_replay::EpochStatementShape, + elf_digest: [u8; 32], + pub(super) public_output: Vec, + epoch_label: u64, + /// Per table, in sub-proof order: the preprocessed commitment and WHERE IT + /// COMES FROM (when the AIR is preprocessed), and the proof's main trace root. + phase_a: Vec<(Option, Commitment)>, + /// The epoch's INIT register file — production's `register_init`. The whole + /// vector, not just the carried commit index: the REGISTER preprocessed + /// commitment is derived from it. + register_init: Vec, + /// The epoch's FINAL register file, the other half of that derivation. + reg_fini: Vec, + /// The inner ELF's entry point — `program_id`'s `pc_start`. + pc_start: u64, + /// The ELF-data page genesis roots the attestation folds. EMPTY for a + /// continuation epoch's own verification: continuation epochs carry no PAGE + /// sub-proof at all (`continuation.rs:695-702`), so these belong to the + /// GLOBAL proof and reach the fold from outside. + page_commitments: Vec<(u64, Commitment)>, + /// The inner proof's LDE domain, for the REGISTER derivation. Both fields are + /// proof OPTIONS, hence program shape. + reg_shape: super::programs::RegisterDerivationShape, + /// `recursion::program_id_from_digest` over this epoch's own inputs — the + /// oracle for the attestation fold. + pub(super) expected_program_id: [u8; 32], + /// Per table, everything the fork absorbs plus the oracle challenges. + pub(super) tables: Vec, + /// Per table, everything the VERIFICATION LEGS read — the shapes, the + /// constraint analysis and the per-query openings. Built in the same pass as + /// `tables` because it needs the AIRs and the proof view, which do not + /// outlive this function. + pub(super) legs: Vec, + /// The shared LogUp challenges Phase A ends on. + pub(super) z_alpha: (FEE, FEE), + /// The carried commit index — `reg_init[X254_INDEX]` of this epoch, which + /// is the PREVIOUS epoch's `reg_fini[64]`. + pub(super) start_index: u64, + /// The COMMIT-bus target production computed, and therefore the value the + /// closure must reach. + pub(super) expected_bus_balance: FEE, +} + +pub(super) fn real_epoch() -> RealEpoch { + real_epoch_with(super::proof_fixture::fixture_options()) +} + +/// What an epoch is built FROM: the inner guest, its private input, and the +/// epoch size. Separated from the proof options because the two axes are +/// independent — options change what verifying the epoch costs, these change +/// what the epoch IS. +/// +/// [`EpochInputs::fixture`] is the 16-cycle fibonacci fixture every existing +/// test builds; [`EpochInputs::from_env`] is that with the three overrides a +/// measurement run needs, and it is what [`real_epoch_with`] uses, so with +/// nothing set every caller keeps the exact path it had. +pub(super) struct EpochInputs { + pub(super) elf_bytes: Vec, + pub(super) private_input: Vec, + pub(super) epoch_log2: u32, + /// Names the guest in printed measurements, since a real-block run and the + /// fixture otherwise report identically shaped numbers. + pub(super) label: String, +} + +impl EpochInputs { + /// The fibonacci fixture: the ELF the recursion suite builds, no private + /// input, [`FIXTURE_EPOCH_LOG2`](super::proof_fixture::FIXTURE_EPOCH_LOG2). + pub(super) fn fixture() -> Self { + Self { + elf_bytes: super::proof_fixture::read_inner_elf(), + private_input: Vec::new(), + epoch_log2: super::proof_fixture::FIXTURE_EPOCH_LOG2, + label: "fibonacci fixture".to_string(), + } + } + + /// [`EpochInputs::fixture`] with the measurement overrides applied: + /// + /// - `LFM_CENSUS_ELF` — path to the inner guest ELF. + /// - `LFM_CENSUS_INPUT` — path to a file holding its private input. + /// - `LFM_CENSUS_EPOCH_LOG2` — epoch size, log2. + /// + /// The input override exists because a guest's epoch count is a property of + /// its INPUT, not just its ELF: the fibonacci guest reads its iteration + /// count from private input, so a run that needs a multi-epoch execution has + /// to be able to ask for one without a recompile. The ELF and epoch-size + /// overrides are what let the same harness build a real Ethereum-block + /// epoch, which is far too large to be a checked-in fixture. + /// + /// With none set this is byte-for-byte [`EpochInputs::fixture`]. + pub(super) fn from_env() -> Self { + let mut inputs = Self::fixture(); + if let Ok(p) = std::env::var("LFM_CENSUS_ELF") { + inputs.elf_bytes = + std::fs::read(&p).unwrap_or_else(|e| panic!("LFM_CENSUS_ELF {p}: {e}")); + inputs.label = p; + } + if let Ok(p) = std::env::var("LFM_CENSUS_INPUT") { + inputs.private_input = + std::fs::read(&p).unwrap_or_else(|e| panic!("LFM_CENSUS_INPUT {p}: {e}")); + } + if let Ok(v) = std::env::var("LFM_CENSUS_EPOCH_LOG2") { + inputs.epoch_log2 = v + .parse() + .unwrap_or_else(|e| panic!("LFM_CENSUS_EPOCH_LOG2 {v}: {e}")); + } + inputs + } +} + +/// [`real_epoch`] under supplied proof options — the wrap run's blowup axis. +/// +/// The options are the INNER proof's, so they change what the verifier has to do: +/// the query count, the LDE depth every Merkle walk climbs, and how many FRI +/// layers commit. What the epoch IS comes from [`EpochInputs::from_env`], which +/// is the fibonacci fixture unless a measurement run overrode it — so two runs +/// at different options stay comparable, and assembly ledger entry 10 still +/// holds: the trace-length profile travels with every number. +pub(super) fn real_epoch_with(opts: crate::ProofOptions) -> RealEpoch { + real_epoch_from(opts, EpochInputs::from_env()) +} + +/// [`real_epoch_with`] with the guest, its input and the epoch size supplied +/// rather than read from the environment. +/// +/// Only epoch 0 is built: the boundary starts from genesis provenance and the +/// label is `epoch_label(0)`, so a later epoch would need the previous one's +/// provenance carried in. That is a real limit of this harness and not an +/// oversight — the first epoch is what the compression work needs. +pub(super) fn real_epoch_from(opts: crate::ProofOptions, inputs: EpochInputs) -> RealEpoch { + use crate::tables::trace_builder::{Traces, build_initial_image_paged}; + use crate::tables::{MaxRowsConfig, bitwise, local_to_global, register}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use executor::elf::Elf; + use executor::vm::execution::Executor; + use stark::proof::view::MultiProofView; + use stark::verifier::IsStarkVerifier; + + let EpochInputs { + elf_bytes, + private_input, + epoch_log2, + label: guest_label, + } = inputs; + let elf = Elf::load(&elf_bytes).expect("the inner ELF must load"); + let epoch_size = 1usize << epoch_log2; + + let mut executor = Executor::new(&elf, private_input.clone()).expect("executor"); + let image = build_initial_image_paged(&elf, &private_input); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let logs = executor + .resume_with_limit(epoch_size) + .expect("resume") + .expect("the guest runs at least one epoch") + .to_vec(); + let is_final = executor.pc() == 0; + assert!(!is_final, "wanted an INTERMEDIATE epoch"); + + let mut traces = Traces::from_image_and_logs( + &elf, + &image, + ®ister_init, + &logs, + &MaxRowsConfig::default(), + &private_input, + is_final, + true, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("the epoch trace must build"); + + let label = local_to_global::epoch_label(0); + let mut provenance = + local_to_global::genesis_provenance(image.iter().map(|(a, v)| (a, v as u64))); + let boundary = + local_to_global::epoch_boundary(&mut provenance, label, &traces.touched_memory_cells); + bitwise::update_multiplicities( + &mut traces.bitwise, + &local_to_global::collect_bitwise_from_l2g(&boundary), + ); + + let reg_fini = register::fini_from_trace(&traces.register); + let table_counts = traces.table_counts(); + let public_output = traces.public_output_bytes.clone(); + let runtime_page_ranges = traces.runtime_page_ranges(); + + let airs = crate::VmAirs::new( + &elf, + &opts, + false, + &[], + &table_counts, + None, + is_final, + None, + None, + Some(( + register::compute_precomputed_commitment_with_fini(&opts, ®ister_init, ®_fini), + register::NUM_PREPROCESSED_COLS_WITH_FINI, + )), + ); + // The attestation fold's DECODE input, from PRODUCTION's own function — the + // same value `VmAirs::new` puts on the DECODE AIR, and the same one + // `recursion::check_attestation` recomputes from a trusted ELF. + let decode_root = crate::tables::decode::commitment_from_elf(&elf, &opts) + .expect("the DECODE commitment must compute"); + let l2g_air = crate::continuation::l2g_memory_air(&opts, label); + let mut l2g_trace = local_to_global::generate_local_to_global_trace(&boundary); + + let seed = || { + let mut t = DefaultTranscript::::new(&[]); + crate::statement::absorb_statement( + &mut t, + crate::statement::StatementKind::ContinuationEpoch { epoch_label: label }, + &elf_bytes, + &public_output, + &table_counts, + 0, + &runtime_page_ranges, + opts.fri_final_poly_log_degree, + ); + t + }; + + let proof = { + let mut pairs = airs.air_trace_pairs(&mut traces); + pairs.push((&l2g_air, &mut l2g_trace, &())); + let t = std::time::Instant::now(); + let proof = + crate::test_utils::multi_prove_ram(pairs, &mut seed()).expect("the epoch must prove"); + // The inner prove is the expensive half of a real-block run and is + // otherwise invisible inside the wrap's own timings, so it reports + // itself — with the guest and epoch size, since a number without them + // does not identify a workload. + eprintln!( + "inner epoch: {guest_label}, 2^{epoch_log2} cycles, {} cycles executed, \ + {} sub-proofs, proved in {:.1}s", + logs.len(), + proof.proofs.len(), + t.elapsed().as_secs_f64() + ); + proof + }; + let refs = { + let mut r = airs.air_refs(); + r.push(&l2g_air); + r + }; + let view = MultiProofView::Owned(&proof); + assert_eq!(refs.len(), view.len(), "one AIR per sub-proof"); + + // ---- production must ACCEPT it, or nothing below describes a real epoch. + let start_index = register_init[register::X254_INDEX] as u64; + let expected = crate::compute_expected_commit_bus_balance_view( + &refs, + view, + &public_output, + start_index, + &mut seed(), + ) + .expect("the COMMIT bus target must compute"); + assert!( + stark::verifier::Verifier::multi_verify_views(&refs, view, &mut seed(), &expected), + "production must accept the epoch this suite differentials against" + ); + + // ---- Phase A, transcribed from `multi_verify_views:1160-1227`. + let mut transcript = seed(); + let mut phase_a = Vec::new(); + for (idx, air) in refs.iter().enumerate() { + let v = view.get(idx); + if air.is_preprocessed() { + let prep = air.precomputed_commitment(); + transcript.append_bytes(&prep); + transcript.append_bytes(v.lde_trace_main_merkle_root()); + phase_a.push(( + Some(prep_source(prep, &opts, &elf, ®ister_init, ®_fini)), + *v.lde_trace_main_merkle_root(), + )); + } else { + transcript.append_bytes(v.lde_trace_main_merkle_root()); + phase_a.push((None, *v.lde_trace_main_merkle_root())); + } + } + let needs_lookup_challenges = refs.iter().any(|a| a.has_aux_trace()); + assert!(needs_lookup_challenges, "an epoch uses LogUp"); + let lookup_challenges: Vec = (0..stark::lookup::LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect(); + let z_alpha = (lookup_challenges[0], lookup_challenges[1]); + + // ---- one fork per table, and the rounds replayed on it. + let num_tables = refs.len(); + let tables = refs + .iter() + .enumerate() + .map(|(idx, air)| { + let v = view.get(idx); + let mut fork = transcript.clone(); + if num_tables > 1 { + fork.append_bytes(&(idx as u64).to_le_bytes()); + } + if let Some(root) = v.lde_trace_aux_merkle_root() { + fork.append_bytes(root); + } + if let Some(c) = v.bus_table_contribution() { + fork.append_field_element(&c); + } + host_table_forked(*air, v, idx, num_tables, &mut fork, &lookup_challenges) + }) + .collect(); + + // The legs' own reading of the same sub-proofs. Separate pass rather than + // part of `host_table_forked` because the challenge replay must run against + // a fork positioned exactly as production leaves it, and this reads nothing + // from the transcript at all. + let legs = refs + .iter() + .enumerate() + .map(|(idx, air)| { + super::epoch_verify_tests::build_table_legs(*air, view.get(idx), &lookup_challenges) + }) + .collect(); + + RealEpoch { + statement: super::statement_replay::EpochStatementShape { + public_output_len: public_output.len(), + table_counts: [ + table_counts.cpu as u64, + table_counts.lt as u64, + table_counts.memw as u64, + table_counts.memw_aligned as u64, + table_counts.load as u64, + table_counts.mul as u64, + table_counts.dvrm as u64, + table_counts.shift as u64, + table_counts.branch as u64, + table_counts.memw_register as u64, + table_counts.eq as u64, + table_counts.bytewise as u64, + table_counts.store as u64, + table_counts.cpu32 as u64, + ], + num_private_input_pages: 0, + fri_final_poly_log_degree: opts.fri_final_poly_log_degree, + page_ranges: runtime_page_ranges + .iter() + .map(|r| (r.base, r.count)) + .collect(), + }, + elf_digest: crate::statement::elf_digest(&elf_bytes), + public_output, + epoch_label: label, + phase_a, + register_init, + reg_fini, + pc_start: elf.entry_point, + // ★ EMPTY, and it is a claim about PRODUCTION rather than about this + // fixture: `prove_epoch` REJECTS an epoch with any PAGE config + // ("continuation epoch must have no PAGE configs (L2G bookend replaces + // PAGE)", `continuation.rs:695-702`) and both `build_epoch_airs` call + // sites pass `&[]`. The ELF-data page genesis roots the attestation folds + // are the GLOBAL proof's GlobalMemory AIRs' preprocessed commitments + // (`continuation.rs:997-1010`), never an epoch's. + page_commitments: Vec::new(), + reg_shape: super::programs::RegisterDerivationShape { + blowup: opts.blowup_factor as usize, + coset_offset: opts.coset_offset, + }, + expected_program_id: crate::recursion::program_id_from_digest( + &crate::statement::elf_digest(&elf_bytes), + elf.entry_point, + &decode_root, + &[], + ), + tables, + legs, + z_alpha, + start_index, + expected_bus_balance: expected, + } +} + +/// [`host_table`] for a sub-proof inside a multi-table epoch: the fork is +/// already positioned (separator, aux root and `L` absorbed), so the oracle +/// comes from `replay_rounds_after_round_1` on THAT transcript. +fn host_table_forked( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + index: usize, + num_tables: usize, + fork: &mut crypto::fiat_shamir::default_transcript::DefaultTranscript, + lookup_challenges: &[FEE], +) -> HostTable { + use stark::domain::new_verifier_domain; + use stark::verifier::IsStarkVerifier; + use stark::verifier::Verifier; + + let opts = air.options(); + let trace_length = view.trace_length(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + let domain = new_verifier_domain(air, trace_length); + let layout = Verifier::::ood_layout(air); + let challenges = Verifier::::replay_rounds_after_round_1( + air, + view, + &(), + &domain, + fork, + lookup_challenges.to_vec(), + &layout, + ); + + let nt = challenges.transition_coeffs.len(); + let beta = if nt > 1 { + challenges.transition_coeffs[1] + } else { + challenges.boundary_coeffs[0] + }; + // `γ` is the second term of the DEEP coefficient run, which starts at one — + // the same recovery `constraint_tests::deep_shape` makes. + let gamma = challenges.trace_term_coeffs[1][0]; + + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + let shape = TableChallengeShape { + index, + num_tables, + has_aux_root: view.lde_trace_aux_merkle_root().is_some(), + has_contribution: view.bus_table_contribution().is_some(), + log2_trace_length, + log2_blowup, + coset_offset: FE::from(opts.coset_offset), + ood_current_dims: (ood_c.width(), ood_c.height()), + ood_next_dims: (ood_n.width(), ood_n.height()), + num_parts: view.composition_poly_parts_ood_evaluation().len(), + fri: FriShape::from_options(opts, log2_trace_length + log2_blowup), + grinding_factor: opts.grinding_factor, + num_queries: opts.fri_number_of_queries, + }; + + HostTable { + shape, + precomputed_root: air.is_preprocessed().then(|| air.precomputed_commitment()), + main_root: *view.lde_trace_main_merkle_root(), + aux_root: view.lde_trace_aux_merkle_root().copied(), + contribution: view.bus_table_contribution(), + composition_root: *view.composition_poly_root(), + ood_current: ood_c.row_major_data().to_vec(), + ood_next: ood_n.row_major_data().to_vec(), + parts: view.composition_poly_parts_ood_evaluation().to_vec(), + fri_roots: view.fri_layers_merkle_roots().to_vec(), + fri_coeffs: view.fri_final_poly_coeffs().to_vec(), + nonce: view.nonce(), + needs_lookup_challenges: true, + beta, + z: challenges.z, + gamma, + zetas: challenges.zetas.clone(), + iotas: challenges.iotas.clone(), + } +} + +/// The whole epoch's Fiat-Shamir spine, as one program. +/// +/// Statement, then Phase A over every sub-proof, then a fork per table and its +/// rounds 2-4. This is the assembled verifier's skeleton: what hangs off each +/// fork (constraint evaluation, the query legs, the closure) consumes the cells +/// this returns. +/// +/// ## ⚠ The preprocessed commitments are HINTED here, and they must not stay so +/// +/// Production takes each preprocessed root from the AIR +/// (`verifier.rs:1187`), never from the proof, and rejects the sub-proof unless +/// the proof's copy matches. Only one of those roots has an in-machine +/// derivation today — REGISTER's, from the previous epoch's `reg_fini` +/// (`programs::register_derivation_program`). The others (BITWISE, DECODE, +/// KECCAK_RC, PAGE) are hinted, and PAGE's in particular CANNOT become a +/// program constant: it is a function of the inner ELF, which is per-proof arena +/// data. Baking it would make program identity proof-dependent. So each is a +/// derivation the assembly still owes; see the ledger entry this leg added. +fn epoch_challenge_program(e: &RealEpoch) -> LfmProgram { + epoch_program(e, false) +} + +/// The epoch program, with or without the verification LEGS hung off the spine. +/// +/// One emitter for both, deliberately. A second copy of the spine would be a +/// place for the assembled verifier's Fiat-Shamir to drift from the one +/// `the_epoch_challenge_spine_matches_production` checks against production — +/// and drift is exactly what the leg wiring must not introduce, since every leg +/// consumes the cells this spine bound. `with_legs = false` declares no leg +/// arenas and emits no verification, so the spine test's own arena-word count is +/// untouched. +pub(super) fn epoch_program(e: &RealEpoch, with_legs: bool) -> LfmProgram { + epoch_program_with(e, with_legs, false) +} + +/// The epoch program, optionally with the DECODE cell SPLIT — a deliberately +/// broken control, and the falsification the entry-7 ruling asked for. +/// +/// `split_decode = true` gives the attestation fold its own arena copy of the +/// DECODE root instead of the cell Phase A absorbed. Nothing about the program +/// then looks wrong: every assert still passes, the challenges are still +/// production's, and an honest host that fills both copies with the same 32 bytes +/// gets the same published `program_id`. That is exactly why the join has to be +/// denied STRUCTURALLY rather than by a differential — +/// [`a_split_decode_cell_forges_the_attestation`] runs the coherent forgery this +/// admits, and +/// [`the_assembled_verifier_declares_exactly_the_shape_words`] is what refuses it. +/// +/// The extra arena is declared LAST so no existing arena index moves. +fn epoch_program_with(e: &RealEpoch, with_legs: bool, split_decode: bool) -> LfmProgram { + use super::statement_replay::{EpochStatementVars, PhaseATable, absorb_epoch_statement}; + + let mut b = LfmBuilder::new(); + let n = e.tables.len(); + assert_eq!(e.legs.len(), n, "one leg reading per sub-proof"); + + // ---- arenas, in declaration order ---- + let stmt_halves = 8 + e.statement.public_output_len.div_ceil(4) + 2; + let a_stmt = b.declare_arena(stmt_halves as u32); + // ★ Only the ELF-DEPENDENT preprocessed roots are arena data (ledger entry + // 7). The options-only ones are interned as program text and the REGISTER one + // is derived in-machine, so neither takes a word here. + let num_arena_prep = e + .phase_a + .iter() + .filter(|(p, _)| p.is_some_and(PrepSource::is_arena)) + .count(); + let a_prep_roots = b.declare_arena(2 * num_arena_prep as u32); + let a_main_roots = b.declare_arena(2 * n as u32); + // The register boundary vectors, at production's width. `start_index` is slot + // 64 of INIT, and the REGISTER preprocessed root is COMPUTED from both — which + // is what ties the index to the chain (ledger entry 2): production has no + // arithmetic `start + len` check anywhere, it rebuilds the commitment from + // these vectors and rejects unless the absorbed root matches. + let num_reg = crate::tables::register::NUM_REGISTER_ADDRESSES as u32; + let a_reg_init = b.declare_arena(num_reg); + let a_reg_fini = b.declare_arena(num_reg); + // The attestation fold's own inputs. `elf_digest` is NOT here — it is the + // statement's, which is the join. `pc_start` has one consumer in an epoch + // verifier, and the page roots have none at all (a continuation epoch carries + // no PAGE sub-proof), so both are plain proof data the fold hashes. + let a_pc_start = b.declare_arena(2); + let a_page_roots = (!e.page_commitments.is_empty()) + .then(|| b.declare_arena(10 * e.page_commitments.len() as u32)); + let per_table: Vec = e + .tables + .iter() + .zip(&e.legs) + .map(|(h, leg)| Arenas { + main_root: a_main_roots, + aux_root: h.shape.has_aux_root.then(|| b.declare_arena(2)), + contribution: h.shape.has_contribution.then(|| b.declare_arena(1)), + composition_root: b.declare_arena(2), + ood_current: b + .declare_arena((h.shape.ood_current_dims.0 * h.shape.ood_current_dims.1) as u32), + ood_next: b.declare_arena((h.shape.ood_next_dims.0 * h.shape.ood_next_dims.1) as u32), + parts: b.declare_arena(h.shape.num_parts as u32), + fri_roots: b.declare_arena(2 * h.shape.fri.num_committed() as u32), + fri_coeffs: b.declare_arena(h.shape.fri.num_terminal_coeffs() as u32), + nonce: (h.shape.grinding_factor > 0).then(|| b.declare_arena(1)), + legs: with_legs.then(|| super::epoch_verify::declare_table_arenas(&mut b, &leg.verify)), + }) + .collect(); + // Last in declaration order, so turning the control on shifts no other arena. + let a_split_decode = split_decode.then(|| b.declare_arena(2)); + + // ---- the statement ---- + let stmt: Vec<_> = (0..stmt_halves as u32) + .map(|i| b.hint_felt(a_stmt, i)) + .collect(); + let out_halves = e.statement.public_output_len.div_ceil(4); + let (elf_digest, rest) = stmt.split_at(8); + let (public_output, epoch_label) = rest.split_at(out_halves); + + let mut t = TranscriptReplay::new(&[]); + absorb_epoch_statement( + &mut t, + &e.statement, + &EpochStatementVars { + elf_digest, + public_output, + epoch_label, + }, + ); + + // ---- ★ the preprocessed roots, each from the source its provenance admits + // + // Ledger entry 7, and entry 2 closes with it. `PrepSource` was decided + // host-side by recomputing production's candidate functions, so the split here + // is not a hardcoded sub-proof index: a preprocessed table with an unknown + // provenance would already have panicked. + let reg_init: Vec<_> = (0..num_reg).map(|r| b.hint_felt(a_reg_init, r)).collect(); + let reg_fini: Vec<_> = (0..num_reg).map(|r| b.hint_felt(a_reg_fini, r)).collect(); + // ★ LEDGER ENTRY 1. Production's boundary vectors are `Vec` and the TYPE + // is the whole enforcement; an arena is untyped felts, so without this the + // machine would derive a commitment over a value no production epoch can hold. + // The entry's stated default was "emit the check if the no->u32 argument is + // still unverified when assembly arrives" — it is, and assembly has arrived. + for cell in reg_init.iter().chain(®_fini) { + super::epoch::assert_u32(&mut b, *cell); + } + let reg_shape = e.reg_shape; + + let mut next_arena_prep = 0usize; + let mut decode_cells: Option = None; + let prep_cells: Vec> = e + .phase_a + .iter() + .map(|(prep, _)| match prep { + None => None, + Some(PrepSource::Constant(c)) => Some(RootCells::constant(&mut b, c)), + Some(PrepSource::Register(_)) => { + let digest = super::programs::emit_register_commitment( + &mut b, reg_shape, ®_init, ®_fini, + ); + Some(RootCells::from_digest(&mut b, digest)) + } + Some(PrepSource::ElfDependent(_)) => { + let cells = RootCells::hint(&mut b, a_prep_roots, 2 * next_arena_prep as u32); + next_arena_prep += 1; + // Every ELF-dependent root of a continuation EPOCH is DECODE (the + // page family lives in the global proof), and the attestation + // folds exactly one DECODE root — so a second one here would mean + // the fold's input is ambiguous, not that the fold needs a loop. + assert!( + decode_cells.is_none(), + "a continuation epoch has one ELF-dependent preprocessed root \ + (DECODE); a second one has no place in the program_id fold" + ); + decode_cells = Some(cells.clone()); + Some(cells) + } + }) + .collect(); + assert_eq!( + next_arena_prep, num_arena_prep, + "every declared preprocessed arena word must be read" + ); + + // ---- Phase A ---- + let main_cells: Vec = (0..n) + .map(|i| RootCells::hint(&mut b, a_main_roots, 2 * i as u32)) + .collect(); + let prep_halves: Vec>> = prep_cells + .iter() + .map(|c| c.as_ref().map(RootCells::halves)) + .collect(); + let main_halves: Vec> = main_cells.iter().map(RootCells::halves).collect(); + // The interned bytes, hoisted so Phase A can borrow them for the whole replay. + let prep_constants: Vec> = e + .phase_a + .iter() + .map(|(p, _)| match p { + Some(PrepSource::Constant(c)) => Some(*c), + _ => None, + }) + .collect(); + let tables: Vec = (0..n) + .map(|i| PhaseATable { + // A program-text root absorbs as literal BYTES — no splice arithmetic + // at all, which is the whole economy of interning it. A derived or + // supplied one absorbs as the cells its consumers share. + preprocessed_root: match (&prep_constants[i], &prep_halves[i]) { + (Some(bytes), _) => { + Some(super::statement_replay::PhaseAPreprocessed::Constant(bytes)) + } + (None, Some(halves)) => Some(super::statement_replay::PhaseAPreprocessed::Cells( + &halves[..], + )), + (None, None) => None, + }, + main_root: &main_halves[i][..], + }) + .collect(); + let (z, alpha) = super::statement_replay::replay_phase_a(&mut t, &mut b, &tables); + b.public(z.as_cell()); + b.public(alpha.as_cell()); + + // ---- ★ the attestation join: the DECODE cell Phase A absorbed, folded + // + // One cell, two consumers. Without this the DECODE root would be a free arena + // word — the machine would absorb whatever the prover offered and publish + // nothing that depended on it. + { + let pc_start: Vec<_> = (0..2).map(|i| b.hint_felt(a_pc_start, i)).collect(); + let page_cells: Vec<(Vec<_>, RootCells)> = e + .page_commitments + .iter() + .enumerate() + .map(|(k, _)| { + let base = 10 * k as u32; + let arena = a_page_roots.expect("a page arena exists when pages do"); + let base_halves: Vec<_> = (0..2).map(|j| b.hint_felt(arena, base + j)).collect(); + let root_halves: Vec<_> = + (0..8).map(|j| b.hint_felt(arena, base + 2 + j)).collect(); + ( + base_halves, + RootCells { + lanes: [ + [ + root_halves[0], + root_halves[1], + root_halves[2], + root_halves[3], + ], + [ + root_halves[4], + root_halves[5], + root_halves[6], + root_halves[7], + ], + ], + }, + ) + }) + .collect(); + let page_halves: Vec<(Vec<_>, Vec<_>)> = page_cells + .iter() + .map(|(base, root)| (base.clone(), root.halves())) + .collect(); + let page_refs: Vec<(&[_], &[_])> = page_halves + .iter() + .map(|(base, root)| (&base[..], &root[..])) + .collect(); + let decode = match a_split_decode { + // ★ THE BROKEN CONTROL: a second, independent reading of the DECODE + // root. The fold now attests to a value Phase A never absorbed. + Some(arena) => RootCells::hint(&mut b, arena, 0).halves(), + None => decode_cells + .as_ref() + .expect("a continuation epoch has a DECODE sub-proof") + .halves(), + }; + let id = super::programs::emit_program_id( + &mut b, + super::programs::ProgramIdShape { + num_pages: e.page_commitments.len(), + }, + elf_digest, + &pc_start, + &decode, + &page_refs, + ); + b.public(id[0]); + b.public(id[1]); + } + + // ---- one fork per table ---- + let mut contributions: Vec = Vec::new(); + for (i, h) in e.tables.iter().enumerate() { + let a = &per_table[i]; + let aux = a.aux_root.map(|id| RootCells::hint(&mut b, id, 0)); + let contribution = a.contribution.map(|id| b.hint_word(id, 0).as_ext()); + let composition = RootCells::hint(&mut b, a.composition_root, 0); + let ood_current: Vec<_> = (0..(h.shape.ood_current_dims.0 * h.shape.ood_current_dims.1) + as u32) + .map(|k| b.hint_word(a.ood_current, k).as_ext()) + .collect(); + let ood_next: Vec<_> = (0..(h.shape.ood_next_dims.0 * h.shape.ood_next_dims.1) as u32) + .map(|k| b.hint_word(a.ood_next, k).as_ext()) + .collect(); + let parts: Vec<_> = (0..h.shape.num_parts as u32) + .map(|k| b.hint_word(a.parts, k).as_ext()) + .collect(); + let fri_roots: Vec<_> = (0..h.shape.fri.num_committed()) + .map(|k| RootCells::hint(&mut b, a.fri_roots, 2 * k as u32)) + .collect(); + let fri_coeffs: Vec<_> = (0..h.shape.fri.num_terminal_coeffs() as u32) + .map(|k| b.hint_word(a.fri_coeffs, k).as_ext()) + .collect(); + let nonce = a.nonce.map(|id| b.hint_felt(id, 0)); + + if let Some(c) = contribution { + contributions.push(c); + } + let mut fork = fork_table(&t, h.shape.index, h.shape.num_tables); + let absorbs = TableAbsorbs { + aux_root: aux.as_ref(), + contribution, + composition_root: &composition, + ood_current: &ood_current, + ood_next: &ood_next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }; + let ch = emit_table_challenges(&mut b, &mut fork, &h.shape, &absorbs); + + // ---- ★ THE SEAM: the verification legs, on the cells just absorbed and + // the challenges just derived. `absorbs` is passed on by REFERENCE rather + // than rebuilt, so there is no second reading of the proof for a leg to + // disagree with the transcript about. + if let Some(leg_arenas) = &a.legs { + let leg = &e.legs[i]; + let out = super::epoch_verify::emit_table_verification( + &mut b, + &leg.verify, + &leg.analysis, + &ch, + &absorbs, + &super::epoch_verify::TableInputs { + // The precomputed root Phase A absorbed — the SAME cells, + // which is what makes production's explicit + // proof-copy-equals-AIR-copy check the absence of a second + // value here rather than a comparison. + precomputed_root: prep_cells[i].as_ref(), + main_root: &main_cells[i], + rap_challenges: &[z, alpha], + }, + leg_arenas, + ); + b.public(out.composition.as_cell()); + for v in &out.fri_terminal { + b.public(v.as_cell()); + } + } + b.public(ch.beta.as_cell()); + b.public(ch.z.as_cell()); + b.public(ch.gamma.as_cell()); + for zeta in &ch.zetas { + b.public(zeta.as_cell()); + } + for bits in &ch.iota_bits { + let felt = edsl::bits_to_felt(&mut b, bits); + b.public(felt.as_cell()); + } + } + + // ---- the LogUp closure, on the cells the forks already absorbed ---- + // + // Every `L` here is the cell its own fork bound into the transcript, and + // the output bytes are derived from the halves the statement absorbed — so + // the closure cannot be summing a different `L`, or folding a different + // output, from the one the challenges were drawn against. + let shape = super::logup::LogUpShape { + num_contributing_tables: contributions.len(), + num_output_bytes: e.statement.public_output_len, + }; + // ★ LEDGER ENTRY 2 CLOSES HERE. The carried commit index is not a word of its + // own and not even a second READ of one: it is the very cell the REGISTER + // preprocessed derivation consumed as INIT slot 64, so the COMMIT-bus target + // and the root Phase A absorbed are functions of one value. Production binds + // `start_index` exactly this way — it has no arithmetic `start + len` check + // anywhere, it rebuilds the commitment from the boundary vectors and rejects + // unless the absorbed root matches. + let start = reg_init[crate::tables::register::X254_INDEX]; + let bytes = super::epoch::emit_output_bytes(&mut b, public_output, shape.num_output_bytes); + let target = super::logup::emit_commit_bus_target(&mut b, &shape, z, alpha, start, &bytes); + let total = super::logup::emit_bus_closure(&mut b, &shape, &contributions, target); + b.public(total.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the epoch challenge program must be admissible"); + program +} + +/// The arenas [`epoch_challenge_program`] declares, in the same order. +fn epoch_arenas(e: &RealEpoch) -> Vec> { + epoch_arena_words(e, false) +} + +/// How many EPOCH-WIDE arenas [`epoch_program`] declares before the first +/// table's — statement, ELF-dependent preprocessed roots, main roots, the two +/// register boundary vectors, `pc_start`, and the page roots when there are any. +/// +/// Exposed rather than hardcoded because a test that walks to a per-table arena +/// by index silently tampers the WRONG arena when this changes, and reports a +/// pass: wiring ledger entry 7 moved it from 4 to 6. +pub(super) fn num_epoch_wide_arenas(e: &RealEpoch) -> usize { + 6 + usize::from(!e.page_commitments.is_empty()) +} + +/// The arenas [`epoch_program`] declares, in the same order. +pub(super) fn epoch_arena_words(e: &RealEpoch, with_legs: bool) -> Vec> { + let mut stmt: Vec = Vec::new(); + let halves = |bytes: &[u8]| -> Vec { + bytes + .chunks(4) + .map(|c| { + let mut w = [0u8; 4]; + w[..c.len()].copy_from_slice(c); + FE::from(u32::from_le_bytes(w) as u64) + }) + .collect() + }; + stmt.extend(halves(&e.elf_digest)); + stmt.extend(halves(&e.public_output)); + stmt.extend(halves(&e.epoch_label.to_le_bytes())); + + // Only the ELF-DEPENDENT roots take arena words; the rest are program text or + // derived in-machine. + let prep: Vec = e + .phase_a + .iter() + .filter_map(|(p, _)| match p { + Some(PrepSource::ElfDependent(c)) => Some(*c), + _ => None, + }) + .collect(); + let main: Vec = e.phase_a.iter().map(|(_, m)| *m).collect(); + + // The register boundary, at production's width. The carried commit index sits + // in slot 64 of INIT, and the REGISTER preprocessed root is derived from both + // vectors — so this arena is not padding around one word any more. + let reg = |v: &[u32]| -> Vec { + assert_eq!( + v.len(), + crate::tables::register::NUM_REGISTER_ADDRESSES, + "a register boundary vector is one word per register word address" + ); + v.iter() + .map(|w| base_word(FE::from(u64::from(*w)))) + .collect() + }; + assert_eq!( + e.register_init[crate::tables::register::X254_INDEX] as u64, + e.start_index, + "the carried commit index must BE slot 64 of the INIT vector, or the \ + COMMIT-bus target and the REGISTER derivation are reading two values" + ); + let mut out = vec![ + stmt.iter().map(|h| base_word(*h)).collect(), + super::proof_arena::commitments_to_arena(&prep), + super::proof_arena::commitments_to_arena(&main), + reg(&e.register_init), + reg(&e.reg_fini), + super::keccak_host::pack_stream(&e.pc_start.to_le_bytes()) + .into_iter() + .map(base_word) + .collect(), + ]; + if !e.page_commitments.is_empty() { + let mut pages: Vec = Vec::new(); + for (base, c) in &e.page_commitments { + pages.extend( + super::keccak_host::pack_stream(&base.to_le_bytes()) + .into_iter() + .map(base_word), + ); + pages.extend( + super::keccak_host::pack_stream(c) + .into_iter() + .map(base_word), + ); + } + out.push(pages); + } + for (h, leg) in e.tables.iter().zip(&e.legs) { + if let Some(r) = h.aux_root { + out.push(super::proof_arena::commitments_to_arena(&[r])); + } + if let Some(c) = h.contribution { + out.push(vec![ext_word(&c)]); + } + out.push(super::proof_arena::commitments_to_arena(&[ + h.composition_root + ])); + out.push(h.ood_current.iter().map(ext_word).collect()); + out.push(h.ood_next.iter().map(ext_word).collect()); + out.push(h.parts.iter().map(ext_word).collect()); + out.push(super::proof_arena::commitments_to_arena(&h.fri_roots)); + out.push(h.fri_coeffs.iter().map(ext_word).collect()); + if let Some(nc) = h.nonce { + out.push(vec![base_word(FE::from(nc))]); + } + if with_legs { + out.push(leg.opening_arena()); + out.push(leg.fri_arena()); + } + } + out +} + +/// A keccak digest published as two words starting at `at` — eight `u32` halves, +/// four per word, each four bytes little-endian. +fn published_digest(public: &[(u32, LfmWord)], at: usize) -> [u8; 32] { + use math::field::traits::IsPrimeField; + let mut out = [0u8; 32]; + for h in 0..8 { + let lane = public[at + h / 4].1[h % 4]; + let half = GoldilocksField::canonical(lane.value()) as u32; + out[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + out +} + +/// ★ THE RUN: the assembled verifier's Fiat-Shamir spine, executed against a +/// real continuation epoch proof that production accepts. +/// +/// This is what the single-table differential could not reach. Both defects +/// `the_single_table_fixture_is_blind_to_two_defects` pins are live here — the +/// epoch has many tables, so the fork's domain separator matters, and the +/// production AIRs have multi-row OOD blocks, so the column-major absorb +/// matters. +#[test] +fn the_epoch_challenge_spine_matches_production() { + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let arenas = epoch_arenas(&e); + let exec = execute(&program, &arenas, &TestPermutation).expect("the epoch spine must execute"); + + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the shared LogUp challenge z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the shared LogUp challenge alpha"); + + // ★ The attestation fold, published right after Phase A. Its DECODE input is + // the very cell Phase A absorbed, so this differential is simultaneously a + // check of the fold and of the join: had the fold read a second copy, this + // would still pass — which is why the split is denied STRUCTURALLY by + // `the_assembled_verifier_declares_exactly_the_shape_words` and demonstrated + // by `a_split_decode_cell_forges_the_attestation`. + assert_eq!( + published_digest(&exec.public_words, 2), + e.expected_program_id, + "the attestation program_id must equal production's \ + `program_id_from_digest` over the same inputs" + ); + + let mut cursor = 4usize; + let mut multi_row_ood = 0; + for (i, h) in e.tables.iter().enumerate() { + assert_eq!(pub_ext(cursor), h.beta, "beta of table {i}"); + assert_eq!(pub_ext(cursor + 1), h.z, "z of table {i}"); + assert_eq!(pub_ext(cursor + 2), h.gamma, "gamma of table {i}"); + cursor += 3; + for (k, want) in h.zetas.iter().enumerate() { + assert_eq!(pub_ext(cursor + k), *want, "zeta {k} of table {i}"); + } + cursor += h.zetas.len(); + for q in 0..h.shape.num_queries { + let w = exec.public_words[cursor + q].1; + let got = super::word::word_as_base(&w).expect("an index is a base felt"); + assert_eq!(got, FE::from(h.iotas[q] as u64), "iota {q} of table {i}"); + } + cursor += h.shape.num_queries; + if h.shape.ood_current_dims.1 > 1 || h.shape.ood_next_dims.1 > 1 { + multi_row_ood += 1; + } + println!( + " table {i:2}: ood_current {:?} ood_next {:?} parts {} fri_layers {} log2_trace {}", + h.shape.ood_current_dims, + h.shape.ood_next_dims, + h.shape.num_parts, + h.shape.fri.num_committed(), + h.shape.log2_trace_length + ); + } + // The closure's total, published last. Reaching it at all means the + // in-machine `assert_eq_ext` against the COMMIT-bus target already held. + assert_eq!( + word_as_ext(&exec.public_words[cursor].1).expect("the bus total is ext"), + e.expected_bus_balance, + "the LogUp closure must reach production's own COMMIT-bus target" + ); + cursor += 1; + assert_eq!( + cursor, + exec.public_words.len(), + "every published word must be checked" + ); + + // The blindness this fixture removes, asserted rather than hoped for. + assert!( + e.tables.len() > 1, + "the fork's domain separator needs more than one table to matter" + ); + // ★ MEASURED, not assumed: every one of the epoch's OOD blocks is ONE row + // tall, so column-major and row-major absorbs coincide on all of them. The + // current block's height IS `step_size` (`ood.rs:110-114`), and the phase + // already knows `step_size = 1` collapses production; the next block's is + // `num_eval_points − step_size`, which is 1 whenever an AIR has two + // transition offsets. So the absorb ORDER has no production witness at all, + // and closing it needs a synthetic AIR — see the ledger entry this leg added. + assert_eq!( + multi_row_ood, 0, + "an OOD block taller than one row appeared: the absorb-order blindness \ + recorded here is over, and the differential now covers it" + ); + // ---- THE MEASUREMENT ---- + // + // What this is and is NOT: the spine is the Fiat-Shamir half of the + // verifier — statement, Phase A, 24 forks, rounds 2-4 and the LogUp + // closure. The opening/DEEP/FRI-walk and constraint legs are NOT in this + // program, so these numbers say nothing about the composed per-epoch + // predictions (213,744 opening permutations at blowup 8, ~460k total). + // Those remain unconfirmed. This is the first per-epoch figure that is a + // RUN rather than a composition, and it is the cost of the part that had + // no per-epoch number at all. + let perms = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::KeccakF(_))) + .count(); + let hints = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::Hint { .. })) + .count(); + let arena_words: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + let bit_decs = program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::BitDec { .. })) + .count(); + // Attribution, not a guess: every EXTENSION value the transcript absorbs is + // three base felts, and each base felt is streamed BIG-endian, which costs + // one `felt_be_halves` — a `BitDec` plus its recomposition. So the absorbed + // ext count times three should account for nearly every `BitDec` here. + let ext_absorbs: usize = e + .tables + .iter() + .map(|h| { + h.ood_current.len() + + h.ood_next.len() + + h.parts.len() + + h.fri_coeffs.len() + + usize::from(h.contribution.is_some()) + }) + .sum(); + println!( + "\nepoch spine (min preset: blowup 2, {} quer{}/table, grinding {}):\n\ + \x20 sub-proofs {}\n\ + \x20 instructions {}\n\ + \x20 keccak perms {}\n\ + \x20 arena words {} ({} hinted)\n\ + \x20 published words {}\n\ + \x20 multi-row OOD {}\n\ + \x20 BitDec rows {}\n\ + \x20 ext values absorbed {} (x3 felts = {} big-endian streams, \ + {:.1}% of the BitDecs)", + e.tables[0].shape.num_queries, + if e.tables[0].shape.num_queries == 1 { + "y" + } else { + "ies" + }, + e.tables[0].shape.grinding_factor, + e.tables.len(), + program.instrs.len(), + perms, + arena_words, + hints, + exec.public_words.len(), + multi_row_ood, + bit_decs, + ext_absorbs, + 3 * ext_absorbs, + 100.0 * (3 * ext_absorbs) as f64 / bit_decs as f64 + ); +} + +/// ★ An ABSOLUTE structural guard (standing-decisions rule 7): no proof value +/// in the assembled spine is hinted twice. +/// +/// The two-consumer class hides exactly where a differential cannot look — a +/// value hinted once per consumer, with the host packing the same number into +/// both, passes every comparison against production and still lets a real +/// prover supply two different numbers. So this is a count over the emitted +/// program, not a comparison of two runs: every arena word is read by at most +/// one `Hint`, and the arenas whose values have two consumers (the roots, the +/// contributions, the statement's public output) are read exactly once. +/// +/// The exception this test used to carry is GONE: the register-boundary arena +/// had only its commit index read while the REGISTER derivation was unbuilt, and +/// wiring the derivation (ledger entries 7 and 2) makes every declared word live. +/// So the positive control is now exact — `declared` words, `declared` reads — +/// which is a strictly stronger statement than the one it replaces. +#[test] +fn the_spine_hints_each_proof_value_once() { + use std::collections::HashMap; + + let e = real_epoch(); + let program = epoch_challenge_program(&e); + + let mut hints: HashMap<(super::instr::ArenaId, u32), usize> = HashMap::new(); + for instr in &program.instrs { + if let super::instr::Instr::Hint { arena, index, .. } = instr { + *hints.entry((*arena, *index)).or_default() += 1; + } + } + let doubled: Vec<_> = hints.iter().filter(|(_, n)| **n > 1).collect(); + assert!( + doubled.is_empty(), + "these arena words are hinted more than once, which is the two-consumer \ + hazard the assembly exists to remove: {doubled:?}" + ); + + // Positive control: the count is nonzero and covers the whole proof, so a + // guard that simply found no hints would not pass for the wrong reason. + let declared: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + hints.len(), + declared, + "every declared arena word must be read exactly once" + ); +} + +/// Arena words the epoch program MUST declare, as arithmetic over the epoch's +/// shapes. +/// +/// Deliberately not derived from the emitter (standing-decisions rule 7's +/// refinement: a count taken from our own emitter is still a relative test). Every +/// term here comes from the production proof view `real_epoch` read, so the +/// comparison against the compiled program is absolute. +fn expected_arena_words(e: &RealEpoch, with_legs: bool) -> usize { + let num_reg = crate::tables::register::NUM_REGISTER_ADDRESSES; + let mut total = 8 + e.statement.public_output_len.div_ceil(4) + 2; + // ★ Two words per ELF-DEPENDENT preprocessed root and NOT ONE MORE. The + // options-only roots are program text and the REGISTER root is derived, so a + // program that hinted any of them — or that kept a second copy of DECODE for + // the attestation fold — declares more words than this. + total += 2 * e + .phase_a + .iter() + .filter(|(p, _)| p.is_some_and(PrepSource::is_arena)) + .count(); + total += 2 * e.tables.len(); + total += 2 * num_reg; + total += 2; + total += 10 * e.page_commitments.len(); + for (h, leg) in e.tables.iter().zip(&e.legs) { + let s = &h.shape; + total += 2 * usize::from(s.has_aux_root); + total += usize::from(s.has_contribution); + total += 2; + total += s.ood_current_dims.0 * s.ood_current_dims.1; + total += s.ood_next_dims.0 * s.ood_next_dims.1; + total += s.num_parts; + total += 2 * s.fri.num_committed(); + total += s.fri.num_terminal_coeffs(); + total += usize::from(s.grinding_factor > 0); + if with_legs { + total += leg.verify.opening_words() + leg.verify.fri_words(); + } + } + total +} + +/// ★ An ABSOLUTE guard on the arena SCHEMA — the structural half of the +/// attestation join (entry-7 ruling, condition (a)). +/// +/// The hinted-once guard denies a value being read twice from ONE word. It cannot +/// deny a value being supplied twice in TWO words, which is the whole two-consumer +/// hazard: an honest host fills both copies alike, every differential passes, and a +/// real prover supplies two different roots. What denies that is the schema itself +/// — the program declares exactly the words the epoch's shapes prescribe, so there +/// is nowhere for a second copy to live. +/// +/// Together the two guards are complete for this class: a second copy must either +/// re-read an existing word (hinted-once fails) or add one (this fails). A fold +/// that instead read some OTHER existing value would publish a `program_id` that is +/// not production's, which the spine differential catches. +#[test] +fn the_assembled_verifier_declares_exactly_the_shape_words() { + let e = real_epoch(); + for with_legs in [false, true] { + let program = epoch_program(&e, with_legs); + let declared: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + declared, + expected_arena_words(&e, with_legs), + "with_legs = {with_legs}: the arena schema must be exactly the epoch's \ + shapes and nothing more — a surplus word is where a second copy of a \ + joined value hides" + ); + } + + // Positive control on the guard itself: the split-cell control program DOES + // declare a surplus word, and this is the comparison that sees it. + let split = epoch_program_with(&e, false, true); + let split_declared: usize = split.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + split_declared, + expected_arena_words(&e, false) + 2, + "the split-cell control must declare exactly two surplus words, or it is \ + not the forgery this guard claims to deny" + ); +} + +/// ★ FALSIFICATION of the attestation join, as a COHERENT FORGERY rather than a +/// count (standing-decisions method rule 4). +/// +/// The attack: verify a real epoch proof of ELF X while attesting to the +/// `program_id` of a different ELF Y. A consumer who trusts Y's id accepts the +/// proof, and X is whatever the prover likes. +/// +/// On the SPLIT program this succeeds completely — every assert passes, the run +/// finishes, and the published id is the one computed from the substituted root, +/// not from the root the proof was made against. On the JOINED program the attack +/// is not merely rejected, it cannot be EXPRESSED: there is one cell, so changing +/// the fold's input changes what Phase A absorbed, which moves every challenge and +/// the run dies. Both halves are asserted, because "the joined program rejects it" +/// alone would be satisfied by a program that rejects everything. +#[test] +fn a_split_decode_cell_forges_the_attestation() { + let e = real_epoch(); + let honest = epoch_arena_words(&e, false); + + // A DECODE root for some other program. Any 32 bytes the honest arena does not + // carry will do; what matters is the id it produces. + let real_decode = e + .phase_a + .iter() + .find_map(|(p, _)| match p { + Some(PrepSource::ElfDependent(c)) => Some(*c), + _ => None, + }) + .expect("the epoch has a DECODE sub-proof"); + let mut substituted = real_decode; + substituted[0] ^= 0xa5; + substituted[31] ^= 0x5a; + let forged_id = + crate::recursion::program_id_from_digest(&e.elf_digest, e.pc_start, &substituted, &[]); + assert_ne!( + forged_id, e.expected_program_id, + "the substituted root must produce a different id, or this proves nothing" + ); + + // ---- (a) the SPLIT program: the forgery runs and publishes the forged id. + let split = epoch_program_with(&e, false, true); + let mut split_arenas = honest.clone(); + split_arenas.push(super::proof_arena::commitments_to_arena(&[substituted])); + let exec = execute(&split, &split_arenas, &TestPermutation).expect( + "the split-cell program must RUN on the forgery — that is the hazard, and \ + a rejection here would mean this control does not demonstrate it", + ); + assert_eq!( + published_digest(&exec.public_words, 2), + forged_id, + "the split program must attest to the SUBSTITUTED root while verifying a \ + proof made against the real one" + ); + // And it is genuinely a proof of the real epoch: the same program, given the + // honest root in the surplus arena, publishes the honest id. + let mut split_honest = honest.clone(); + split_honest.push(super::proof_arena::commitments_to_arena(&[real_decode])); + let exec_honest = execute(&split, &split_honest, &TestPermutation) + .expect("the split program must also run honestly"); + assert_eq!( + published_digest(&exec_honest.public_words, 2), + e.expected_program_id, + "the split program's two runs differ only in the surplus arena, so the \ + forgery is a free choice and not a broken proof" + ); + + // ---- (b) the JOINED program: the same substitution is inexpressible. + // + // There is no surplus arena to put it in, so the only way to move the fold's + // input is to move the cell Phase A absorbed — which moves every challenge + // derived after it. + let joined = epoch_program(&e, false); + let mut joined_arenas = honest.clone(); + joined_arenas[1] = super::proof_arena::commitments_to_arena(&[substituted]); + assert!( + execute(&joined, &joined_arenas, &TestPermutation).is_err(), + "with one cell, substituting the DECODE root must break the run: the \ + transcript absorbed it, so the challenges cannot survive it" + ); +} + +/// ★ LEDGER ENTRY 2, closed and falsified: the whole register boundary is bound, +/// not just the commit index. +/// +/// Production ties epoch N's carried commit index to the chain by REBUILDING the +/// REGISTER preprocessed commitment from epoch N−1's FINI vector and rejecting +/// unless the absorbed root matches — there is no arithmetic `start + len` check +/// anywhere (`lfm-team-lead-start-index-research.md`). So the machine's binding is +/// the derivation, and what must be true is that moving ANY word of either vector +/// makes the epoch unverifiable. +/// +/// Before the derivation was wired, 66 of the 67 INIT words were declared and never +/// read: moving them changed nothing at all. The positive control for that is +/// structural rather than historical — `the_spine_hints_each_proof_value_once` now +/// requires every declared word to be read, and it did not before. +/// +/// Slot 64 is the commit index and is tested separately by +/// [`the_closure_rejects_a_moved_index_or_output`]; the slots here are deliberately +/// elsewhere, including the first and last of each vector, because a derivation that +/// only really consumed a prefix would pass a test that only moved slot 64. +#[test] +fn the_derivation_binds_every_register_boundary_word() { + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let good = epoch_arenas(&e); + assert!( + execute(&program, &good, &TestPermutation).is_ok(), + "the untampered epoch must run" + ); + + let last = crate::tables::register::NUM_REGISTER_ADDRESSES - 1; + let x254 = crate::tables::register::X254_INDEX; + let mut moved = 0; + for (arena, what) in [(3usize, "INIT"), (4, "FINI")] { + for slot in [0usize, 1, 33, x254 + 1, last] { + let mut arenas = good.clone(); + let bumped = arenas[arena][slot][0] + FE::one(); + arenas[arena][slot] = base_word(bumped); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "{what} slot {slot} moved by one must not verify: the REGISTER \ + preprocessed root is derived from it, and the transcript absorbed \ + that root" + ); + moved += 1; + } + } + assert_eq!(moved, 10, "every planned vector must have been run"); +} + +/// ★ LEDGER ENTRY 1, in two halves — and the obvious formulation of this test is +/// VACUOUS, which is worth stating because I wrote it first. +/// +/// The tempting test is "set a boundary word to `2^32` and watch the assembled +/// epoch fail". It does fail — and it fails with the check REMOVED too, because a +/// wide value moves the derived REGISTER root, which moves every challenge drawn +/// after Phase A absorbs it. So that test says nothing about the width check at +/// all; it is the same rejection +/// `the_derivation_binds_every_register_boundary_word` already gets from moving a +/// word by one. +/// +/// What is not vacuous is the pair below, and together they are complete: +/// +/// 1. **What [`super::epoch::assert_u32`] does**, in isolation: the whole `u32` +/// range runs and everything at or above `2^32` is unprovable. Absolute — it is +/// a property of the check's own output, with no epoch involved. +/// 2. **That it is applied to every one of the 134 boundary cells**, structurally: +/// each register-arena `Hint` output is the INPUT of a 32-bit `BitDec`. A check +/// emitted over a prefix — the failure mode a value-tamper test cannot see, +/// since any single moved word rejects anyway — fails this. +#[test] +fn the_register_boundary_is_width_checked() { + // ---- (1) the check itself. + let drive = |v: u64| { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(1); + let cell = b.hint_felt(arena, 0); + super::epoch::assert_u32(&mut b, cell); + let program = compile(b.finish()); + validate(&program).expect("the width check must be admissible"); + execute(&program, &[vec![base_word(FE::from(v))]], &TestPermutation).is_ok() + }; + // ⚠ The bad values are CANONICAL felts, and that is not pedantry — it is the + // exact size of the gap. An arena word is a field element, so `FE::from(v)` + // reduces: `u64::MAX − 1` is the felt `2^32 − 3`, a perfectly good `u32`, and a + // test that used it would report the check broken when it is not (it did). The + // widening entry 1 names is therefore the interval `[2^32, p)` and nothing + // beyond — there is no felt at or above `p` to worry about. + const P_MINUS_1: u64 = 0xFFFF_FFFF_0000_0000; // Goldilocks p − 1 = 2^64 − 2^32 + for ok in [0u64, 1, 255, 1 << 31, (1u64 << 32) - 1] { + assert!(drive(ok), "{ok} is a u32 and must be admitted"); + } + for bad in [1u64 << 32, (1u64 << 32) + 1, 1 << 40, P_MINUS_1] { + assert!( + !drive(bad), + "{bad} is not a u32 and must be unprovable: production's boundary \ + vectors are Vec and the TYPE is their only enforcement" + ); + } + + // ---- (2) every boundary cell reaches it, in the assembled program. + use std::collections::HashSet; + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let num_reg = crate::tables::register::NUM_REGISTER_ADDRESSES; + + // The two register arenas are the ones whose declared length is + // NUM_REGISTER_ADDRESSES; identified by length rather than by index so that + // adding an epoch-wide arena cannot silently point this test at the wrong one. + let reg_arenas: Vec = program + .arena_schema + .lens + .iter() + .enumerate() + .filter(|(_, l)| **l as usize == num_reg) + .map(|(i, _)| i as super::instr::ArenaId) + .collect(); + assert_eq!( + reg_arenas.len(), + 2, + "expected exactly the INIT and FINI arenas to have the register width" + ); + + let mut boundary_cells: HashSet = HashSet::new(); + for instr in &program.instrs { + if let super::instr::Instr::Hint { arena, out, .. } = instr + && reg_arenas.contains(arena) + { + boundary_cells.insert(*out); + } + } + assert_eq!( + boundary_cells.len(), + 2 * num_reg, + "every declared boundary word must be read exactly once" + ); + + let decomposed: HashSet = program + .instrs + .iter() + .filter_map(|i| match i { + super::instr::Instr::BitDec { input, bits } if bits.len() == 32 => Some(*input), + _ => None, + }) + .collect(); + let unchecked: Vec<_> = boundary_cells.difference(&decomposed).collect(); + assert!( + unchecked.is_empty(), + "these register-boundary cells are never bit-decomposed, so their width \ + is unconstrained: {unchecked:?}" + ); +} + +/// ★ The closure's two joins, falsified by tampering. +/// +/// The COMMIT-bus target is a function of the carried commit index and of the +/// public output, and both reach it through cells another consumer already +/// used — `start_index` from the register-boundary arena the REGISTER +/// derivation will bind, the output bytes from the halves the STATEMENT +/// absorbed. Moving either must break the run. +#[test] +fn the_closure_rejects_a_moved_index_or_output() { + let e = real_epoch(); + let program = epoch_challenge_program(&e); + let good = epoch_arenas(&e); + assert!( + execute(&program, &good, &TestPermutation).is_ok(), + "the untampered epoch must run" + ); + + // The carried commit index. Production derives it from the previous epoch's + // FINI vector; a machine that let the prover pick it would let them + // renumber the whole output stream. + for delta in [1u64, 2, 7] { + let mut arenas = good.clone(); + arenas[3][crate::tables::register::X254_INDEX] = base_word(FE::from(e.start_index + delta)); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "start_index + {delta} must not close the bus" + ); + } + + // The public output. Moving a half moves both the statement the challenges + // were drawn against and the bytes the target folds, so this rejects + // whichever check notices first — but reject it must. + assert!( + !e.public_output.is_empty(), + "the fixture epoch must actually commit output, or this proves nothing" + ); + for half in 0..e.statement.public_output_len.div_ceil(4) { + let mut arenas = good.clone(); + let idx = 8 + half; + let bumped = arenas[0][idx][0] + FE::one(); + arenas[0][idx] = base_word(bumped); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "moving output half {half} must not verify" + ); + } +} diff --git a/prover/src/lfm/epoch_verify.rs b/prover/src/lfm/epoch_verify.rs new file mode 100644 index 000000000..ef63e893e --- /dev/null +++ b/prover/src/lfm/epoch_verify.rs @@ -0,0 +1,585 @@ +//! One sub-proof VERIFIED — the legs hung off the Fiat-Shamir spine. +//! +//! [`super::epoch`] replays production's challenge derivation and hands back +//! [`TableChallenges`]; every leg built so far took those same challenges as +//! arena words instead. This module is where the two meet: it takes the cells +//! the spine absorbed and the challenges the spine derived, and emits the four +//! checks a real verifier performs on one sub-proof. +//! +//! ```text +//! spine gives leg consumes +//! ----------- ------------ +//! ood_current / ood_next ──────► the reconstructed grid: constraints AND DEEP +//! parts ──────► the quotient's claimed value AND DEEP's h_sum +//! z ──────► the zerofier's ζ AND DEEP's row points +//! beta ──────► the β-power fold +//! gamma ──────► the DEEP batching challenge +//! zetas ──────► the FRI fold chain +//! iota_bits ──────► the Merkle walk, the query point, the FRI walk +//! contribution (L) ──────► the table offset AND the LogUp closure +//! every ROOT ──────► the authentication compare +//! ``` +//! +//! Nothing in that table is hinted twice. The roots arrive as +//! [`super::epoch::RootCells`] and become [`GroupCommitment`]s through +//! `from_lanes`, the OOD blocks become one grid through +//! [`super::epoch::emit_reconstruct_ood`], and the query index never exists as a +//! felt. What the arenas still carry, per sub-proof, is exactly the data a real +//! proof carries and a verifier cannot derive: the opened row pairs, the Merkle +//! paths, and the FRI layers' symmetric evaluations. +//! +//! ## What this module cannot see +//! +//! It verifies ONE sub-proof. It says nothing about the epoch's statement, about +//! Phase A, or about the LogUp closure across tables — those are the spine's and +//! [`super::logup`]'s. It also does not check the preprocessed commitments +//! against anything: production takes them from the AIR, and where they come +//! from in the assembled machine is [`TableInputs`]' caller's problem (assembly +//! ledger entry 7). + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, LfmBuilder}; +use super::constraints::{ + Analysis, BoundaryTerm, OodOperands, QuotientShape, emit_alpha_powers, emit_analyzed, + emit_quotient, emit_table_offset, +}; +use super::deep::{DeepInvariants, emit_deep_invariants}; +use super::epoch::{RootCells, TableAbsorbs, TableChallenges, emit_reconstruct_ood}; +use super::fri::{ + FriCommitments, FriQuery, FriShape, LayerCommitment, emit_query_fri, hint_layer_openings_from, +}; +use super::instr::ArenaId; +use super::sub_proof::{GroupCommitment, GroupOpening, SubProofShape, emit_query_from_bits}; + +/// The compile-time shape of one sub-proof's full verification. +/// +/// Every field is program SHAPE, in the sense `others/lfm-target-shape.md` fixes: +/// a value the AIR set and the proof options determine, never a value the proof +/// carries. The one exception worth naming is [`Self::quotient`]'s boundary list, +/// which production computes from the public inputs — see [`boundary_terms`] for +/// the rule it is built from and the premise that rule rests on. +#[derive(Clone, Debug)] +pub struct TableVerifyShape { + /// The trace/opening shape: DEEP columns, the committed groups, the tree + /// depth and the LDE domain. + pub sub: SubProofShape, + /// The FRI shape, which must describe the same LDE domain. + pub fri: FriShape, + /// The zerofier, the part count and the boundary constraints. + pub quotient: QuotientShape, + /// Where the aux columns start in a full-width `[main | aux]` row. + pub main_width: usize, + /// `AIR::max_bus_elements()` — how long the α-power chain is. Zero for an + /// AIR with no aux trace, which has no `Op::AlphaPow` to resolve. + pub num_alpha_powers: usize, + /// Queries the sub-proof carries. + pub num_queries: usize, +} + +impl TableVerifyShape { + /// Frame STEPS the constraint program indexes — `Op::Var{offset}` runs over + /// these, not over the OOD grid's rows. + /// + /// A frame step is `step_size` grid rows and production's own interpreter + /// reads only row 0 of each (`constraint_ir/interp.rs:240-242` asserts + /// `row == 0`), so the constraint leg's view of the grid is every + /// `step_size`-th row while DEEP's is all of it. The two coincide at + /// `step_size = 1`, which every production AIR has — carrying the stride + /// anyway is the same discipline `DeepShape::block` applies to the + /// coefficient run, and for the same reason. + pub fn num_frame_steps(&self) -> usize { + self.sub.deep.num_eval_points / self.sub.deep.step_size + } + + fn check(&self) { + assert_eq!( + self.sub.log2_lde_length, self.fri.log2_lde_length, + "both legs verify one sub-proof over one LDE domain" + ); + assert_eq!( + self.sub.merkle_depth, + self.fri.index_bits(), + "the FRI layers consume suffixes of the trace walk's decomposition" + ); + assert_eq!( + self.fri.num_queries, self.num_queries, + "the query count is one shape, declared once" + ); + assert_eq!( + self.sub.deep.num_composition_parts, self.quotient.num_composition_parts, + "the part count is one shape: DEEP folds the same parts the quotient \ + Horner claims" + ); + assert_eq!( + self.sub.deep.log2_trace_length, self.quotient.log2_trace_length, + "the trace length is one shape" + ); + assert_eq!( + self.num_frame_steps() * self.sub.deep.step_size, + self.sub.deep.num_eval_points, + "the OOD grid is a whole number of frame steps" + ); + assert!( + self.main_width <= self.sub.deep.num_total_cols, + "the aux columns start inside the row" + ); + } + + /// Arena words this sub-proof's trace openings occupy. + pub fn opening_words(&self) -> usize { + self.num_queries * self.sub.opening_words() + } + + /// Arena words this sub-proof's FRI openings occupy. + pub fn fri_words(&self) -> usize { + self.num_queries * self.fri.query_words() + } +} + +/// The two arenas one sub-proof's query verification reads, in declaration +/// order. +/// +/// Deliberately only two. Everything else a leg used to hint — the roots, the +/// challenges, the OOD grid, the claimed parts, the FRI layer roots and terminal +/// coefficients — reaches the legs as cells the spine already bound. +#[derive(Clone, Copy, Debug)] +pub struct TableQueryArenas { + /// Per query, per group: the row-pair values then the sibling digests (two + /// words per level). NO index word — the index is the transcript's. + pub openings: ArenaId, + /// Per query, per committed FRI layer: the symmetric evaluation then the + /// sibling digests. + pub fri: ArenaId, +} + +/// Declare the query arenas for one sub-proof. +pub fn declare_table_arenas(b: &mut LfmBuilder, shape: &TableVerifyShape) -> TableQueryArenas { + TableQueryArenas { + openings: b.declare_arena(shape.opening_words() as u32), + fri: b.declare_arena(shape.fri_words() as u32), + } +} + +/// The cells one sub-proof's verification takes from OUTSIDE its own arenas. +pub struct TableInputs<'a> { + /// The precomputed-columns root, when the AIR is preprocessed. + /// + /// Production never reads this from the proof: it takes + /// `air.precomputed_commitment()`, absorbs THAT, and rejects a proof whose + /// copy disagrees (`verifier.rs:1184-1209`). So the cells here are the ones + /// Phase A absorbed, and the equality production checks explicitly is, in + /// this machine, the absence of a second value. + pub precomputed_root: Option<&'a RootCells>, + /// The main trace root — the cells Phase A absorbed. + pub main_root: &'a RootCells, + /// The shared LogUp challenges, sampled once in Phase A and passed to every + /// table (`verifier.rs:1216-1227`). Never per-table. + pub rap_challenges: &'a [Ext], +} + +/// What one sub-proof's verification produced, for the epoch to compose. +pub struct TableVerifyOutput { + /// The recomputed composition at `z`, asserted equal to the claimed Horner + /// inside the program. + pub composition: Ext, + /// Per query, the FRI terminal value the chain arrived at. + pub fri_terminal: Vec, + /// The per-sub-proof DEEP invariants, exposed so a test can publish them. + pub deep: DeepInvariants, +} + +/// Emit one sub-proof's verification onto the spine's cells. +/// +/// `challenges` must be the output of [`super::epoch::emit_table_challenges`] on +/// THIS table's fork, and `absorbs` the very struct that call was given. Passing +/// a different one would be the two-consumer hazard reintroduced by hand, which +/// is why both are borrowed rather than rebuilt. +pub fn emit_table_verification( + b: &mut LfmBuilder, + shape: &TableVerifyShape, + analysis: &Analysis, + challenges: &TableChallenges, + absorbs: &TableAbsorbs<'_>, + inputs: &TableInputs<'_>, + arenas: &TableQueryArenas, +) -> TableVerifyOutput { + shape.check(); + assert_eq!( + challenges.iota_bits.len(), + shape.num_queries, + "one index per query" + ); + + // ---- the OOD grid, from the two blocks the transcript absorbed. + let grid = emit_reconstruct_ood(b, &shape.sub.deep, absorbs.ood_current, absorbs.ood_next); + + // ---- the LogUp uniforms, DERIVED (never hinted): the α powers from the one + // α Phase A sampled, and the per-row offset from the one `L` this table's + // fork absorbed and the closure sums. + let alpha_powers = if shape.num_alpha_powers > 0 { + let alpha = inputs + .rap_challenges + .get(stark::lookup::LOGUP_CHALLENGE_ALPHA) + .copied() + .expect("an AIR with bus elements has the shared LogUp challenges"); + emit_alpha_powers(b, alpha, shape.num_alpha_powers) + } else { + Vec::new() + }; + let table_offset = match absorbs.contribution { + Some(l) => emit_table_offset(b, l, shape.quotient.log2_trace_length), + // An AIR with no bus contribution has no `Op::TableOffset` to resolve; + // the pooled zero is a placeholder the lowering never reads. It is a + // program constant, so a prover cannot reach it either way. + None => b.felt_const(FE::zero()).as_ext(), + }; + + // The constraint program indexes FRAME STEPS; DEEP folds every grid row. + // Both views are of the one grid above, which is what makes the two legs + // agree by construction rather than by the host filling two arenas alike. + let steps = frame_step_view(&grid, shape.sub.deep.step_size); + assert_eq!( + steps.len(), + shape.num_frame_steps(), + "the strided view must have one entry per frame step" + ); + let ood = OodOperands { + steps, + main_width: shape.main_width, + rap_challenges: inputs.rap_challenges.to_vec(), + alpha_powers, + table_offset, + }; + + // ---- (1) the constraint evaluation and (2) the quotient check. + let evals = emit_analyzed(b, analysis, &ood); + let q = emit_quotient( + b, + &shape.quotient, + &ood, + challenges.z, + challenges.beta, + &evals, + absorbs.parts, + ); + b.assert_eq_ext(q.claimed, q.composition); + + // ---- (3) DEEP, over the same grid and the same parts. + let inv = emit_deep_invariants( + b, + &shape.sub.deep, + challenges.gamma, + challenges.z, + &grid, + absorbs.parts, + ); + + // ---- the committed matrices, in DEEP column order then the parts. + let groups = shape.sub.groups(); + let mut commitments: Vec = Vec::with_capacity(groups.len()); + let push = |root: &RootCells, out: &mut Vec| { + let g = groups[out.len()]; + out.push(GroupCommitment::from_lanes(root.lanes, g)); + }; + if let Some(prep) = inputs.precomputed_root { + push(prep, &mut commitments); + } + push(inputs.main_root, &mut commitments); + if let Some(aux) = absorbs.aux_root { + push(aux, &mut commitments); + } + push(absorbs.composition_root, &mut commitments); + assert_eq!( + commitments.len(), + groups.len(), + "one commitment per committed matrix: the sub-proof shape and the \ + supplied roots must describe the same proof" + ); + + // ---- the FRI commitments, likewise from the transcript's own cells. + let fri = FriCommitments { + layers: absorbs + .fri_roots + .iter() + .map(|r| LayerCommitment::from_lanes(r.lanes)) + .collect(), + zetas: challenges.zetas.clone(), + coeffs: absorbs.fri_coeffs.to_vec(), + }; + + // ---- (4) per query: authenticate, fold DEEP, then fold FRI. + let stride = shape.sub.opening_words(); + let mut fri_terminal = Vec::with_capacity(shape.num_queries); + for (qi, bits) in challenges.iota_bits.iter().enumerate() { + let mut cursor = (qi * stride) as u32; + let openings: Vec = groups + .iter() + .map(|g| { + let values = (0..g.num_values()) + .map(|_| { + let c = b.hint_word(arenas.openings, cursor); + cursor += 1; + c + }) + .collect(); + let siblings = (0..shape.sub.merkle_depth) + .map(|_| { + let lo = b.hint_word(arenas.openings, cursor); + let hi = b.hint_word(arenas.openings, cursor + 1); + cursor += 2; + [lo, hi] + }) + .collect(); + GroupOpening { values, siblings } + }) + .collect(); + assert_eq!( + cursor as usize, + (qi + 1) * stride, + "the emitter's cursor must agree with the declared query stride" + ); + + let out = emit_query_from_bits( + b, + &shape.sub, + challenges.gamma, + &inv, + &commitments, + bits.clone(), + &openings, + ); + let layers = hint_layer_openings_from(b, shape.fri, arenas.fri, qi); + fri_terminal.push(emit_query_fri( + b, + shape.fri, + &fri, + &FriQuery { + p0: out.deep.0, + p0_sym: out.deep.1, + point: out.point, + point_sym: out.point_sym, + bits: &out.bits, + }, + &layers, + )); + } + + TableVerifyOutput { + composition: q.composition, + fri_terminal, + deep: inv, + } +} + +/// The constraint frame's view of the reconstructed OOD grid: row 0 of each +/// evaluation STEP, which is every `step_size`-th grid row. +/// +/// This is assembly ledger entry 9, extracted so it can be differentialled. The +/// rule is production's, not ours: the verifier builds its frame with +/// `StarkTableView::into_frame(main_cols, step_size)` +/// (`verifier.rs:320-321`), which groups the `num_eval_points`-row grid into +/// `step_size`-row steps, and `Op::Var{offset, row}` resolves to +/// `frame.get_evaluation_step(offset).get_main_evaluation_element(0, col)` with +/// `row == 0` asserted (`constraint_ir/interp.rs:240-242`). So step `o`'s value is +/// grid row `o · step_size`, and the rows between are read by DEEP alone. +/// +/// A generic function rather than the two lines it replaces, because those two +/// lines had no witness: at `step_size = 1` the strided view and the whole grid +/// are the same vector, so an emitter that passed the whole grid to the +/// constraint fold — which is what the wave-5 sketch did — was indistinguishable. +/// `step_size_tests::the_frame_step_view_matches_productions_own_frame_assembly` +/// compares THIS function against `into_frame` at `step_size = 2`, where they +/// differ. +pub fn frame_step_view(grid: &[Vec], step_size: usize) -> Vec> { + assert!(step_size > 0, "a frame step is at least one row"); + assert!( + grid.len().is_multiple_of(step_size), + "the OOD grid is a whole number of frame steps: {} rows at step_size {step_size}", + grid.len() + ); + grid.iter().step_by(step_size).cloned().collect() +} + +/// Keccak permutations one sub-proof's committed leaves cost, per query. +/// +/// A leaf is NOT one permutation. It covers `ROWS_PER_LEAF · num_columns` +/// elements at 8 or 24 bytes each, and the sponge absorbs `⌊bytes/136⌋ + 1` rate +/// blocks — so the epoch's widest table (2,056 OOD columns) has a leaf worth +/// hundreds of permutations while a FRI layer's one-column leaf is worth one. +/// Predicting the leg's bill as "one leaf plus one per level" undercounts it by +/// the whole width of the trace, which is exactly the mistake this function +/// exists to not make. +pub fn leaf_permutations(shape: &SubProofShape) -> usize { + shape + .groups() + .iter() + .map(|g| super::keccak_host::num_blocks(g.leaf_bytes())) + .sum() +} + +/// Felts a keccak permutation absorbs — the 136-byte rate at 8 bytes per felt. +/// +/// `⌊bytes/136⌋` and `⌊felts/17⌋` are the same function because every element the +/// leaf hasher streams is 8 bytes wide (a base felt) or 24 (an extension element, +/// three felts), so `bytes = 8 · felts` with no remainder either way. +pub const KECCAK_RATE_FELTS: usize = 17; + +/// Felts an `LFM_HASH` invocation absorbs — **one digest cell**. +/// +/// ⚠ **This was 8 and is now 4, because the construction it was derived from no +/// longer exists.** The old value was "2 of 3 state cells" — the rate of the +/// overwrite-duplex `edsl::SpongeVar` used to be. Option B1 replaced that with a +/// **compress chain** that absorbs exactly one cell per step, so the rate is +/// [`super::hash::HASH_DIGEST_FELTS`]. It is written as that constant rather +/// than as a literal so it cannot outlive its own derivation a second time. +/// +/// **This is 4.25× worse than keccak's 17**, not the 2.125× the duplex gave, +/// and it is the one axis on which a field-native candidate loses: it pays more +/// permutations, each far cheaper. +/// +/// **The lever moved with the construction, and it is now a worse one.** Under +/// the duplex the rate followed from `HASH_STATE_FELTS = 12`, so widening the +/// state bought throughput and nothing else. Under the chain it follows from +/// `HASH_DIGEST_FELTS = 4` — the absorbed operand IS a digest — so the only way +/// to raise it is to widen the digest, which is the same constant the socket's +/// 64-bit collision bound rests on. Throughput and collision resistance are no +/// longer independent knobs. +/// +/// ✗ **What this models, and its remaining assumption.** A rate is the right +/// shape for a CHAIN — `state ← T(state, cell)`, one fresh cell per invocation — +/// which is what the machine now has. It is *not* the right shape for a 2-to-1 +/// tree over leaf data, where both inputs are fresh and the count is a tree size +/// rather than `felts / rate`. Which of the two a candidate uses for LEAVES is +/// the still-open O1 leaf-convention question, so a future answer there could +/// move this model's FORMULA and not merely this constant. +pub const LFM_HASH_RATE_FELTS: usize = super::hash::HASH_DIGEST_FELTS; + +/// Felts in one FRI-layer leaf: the symmetric evaluation PAIR, two extension +/// elements of three felts each. +/// +/// Named because it is the number that decides whether the FRI leaf term is +/// rate-sensitive: six felts fit one keccak block (rate 17) and do NOT fit one +/// block at the candidate's rate 4. +pub const FRI_LEAF_FELTS: usize = 6; + +/// Felts one query's opening of a group covers, the felt-side counterpart of +/// [`super::sub_proof::GroupShape::leaf_bytes`]. +pub fn group_leaf_felts(g: &super::sub_proof::GroupShape) -> usize { + g.num_values() * if g.is_ext { 3 } else { 1 } +} + +/// Permutations a sponge of `rate_felts` spends absorbing `felts`, under keccak's +/// own padding convention — `⌊n/rate⌋ + 1`, i.e. always at least one block and +/// always a padding block even when the length divides the rate. +/// +/// Carrying keccak's convention over to a candidate is deliberately +/// CONSERVATIVE: a candidate needs no trailing block, because it +/// domain-separates outside the absorbed data — in the capacity for a sponge, +/// and in the message tag for the B1 compress chain — so its true count lies +/// between `felts.div_ceil(rate)` and this. Using the same convention on both +/// sides is +/// what makes the rate-17 case reproduce [`leaf_permutations`] exactly, which is +/// the check that this felt-side reformulation is right at all. +pub fn blocks_at_rate(felts: usize, rate_felts: usize) -> usize { + felts / rate_felts + 1 +} + +/// [`leaf_permutations`] at an arbitrary sponge rate. +/// +/// Computed independently of [`leaf_permutations`] — through felts and a rate +/// rather than through bytes and `keccak_host::num_blocks` — precisely so that +/// asserting the two agree at [`KECCAK_RATE_FELTS`] is a real differential and +/// not two spellings of one function. Making either delegate to the other would +/// kill that test silently (standing-decisions rule 7). +pub fn leaf_permutations_at_rate(shape: &SubProofShape, rate_felts: usize) -> usize { + shape + .groups() + .iter() + .map(|g| blocks_at_rate(group_leaf_felts(g), rate_felts)) + .sum() +} + +/// FRI-layer LEAF permutations one query costs, at an arbitrary rate. +/// +/// Split out of [`super::fri::FriShape::permutations_per_query`] because it is +/// the half of that number which MOVES with the rate: a layer leaf is +/// [`FRI_LEAF_FELTS`] felts, and how many blocks that takes depends on the rate +/// like any other absorption. At keccak's 17 it is one block and this reduces to +/// `num_committed()`, which is what keeps the felt-side and byte-side closed +/// forms agreeing there. +/// +/// ⚠ It used to be folded into the rate-INVARIANT remainder, on the premise +/// that "a FRI layer leaf fits any rate at or above six". That premise was true +/// while the candidate's rate was 8 and is false now that it is 4 — six felts +/// take two blocks. The premise is gone rather than re-asserted; this function +/// is what replaced it. +pub fn fri_leaf_permutations_at_rate(fri: &FriShape, rate_felts: usize) -> usize { + fri.num_committed() * blocks_at_rate(FRI_LEAF_FELTS, rate_felts) +} + +/// [`query_permutations`] at an arbitrary sponge rate. +/// +/// **ABSORPTION is rate-sensitive; COMPRESSION is not.** That is the whole +/// decomposition, and it is a property of what each step does rather than of +/// which leg it belongs to: +/// +/// - **Leaves absorb**, so both kinds move with the rate — the trace groups' +/// ([`leaf_permutations_at_rate`]) and the FRI layers' +/// ([`fri_leaf_permutations_at_rate`]). +/// - A **Merkle parent compresses**, so it is one permutation at any rate: +/// `LfmHasher::compress` is "a single permutation of `[a ‖ b ‖ IV]` truncated +/// to the first cell" (`hash.rs:23-26`), and keccak's 64-byte parent likewise +/// sits inside one 136-byte block (`edsl.rs:151-155`). Both the trace trees' +/// parents and the FRI layers' path steps are of this kind. +pub fn query_permutations_at_rate(shape: &TableVerifyShape, rate_felts: usize) -> usize { + let groups = shape.sub.groups().len(); + let per_query = leaf_permutations_at_rate(&shape.sub, rate_felts) + + fri_leaf_permutations_at_rate(&shape.fri, rate_felts) + + groups * shape.sub.merkle_depth + + shape.fri.path_steps_per_query(); + shape.num_queries * per_query +} + +/// Keccak permutations one sub-proof's whole query verification costs, from +/// shape alone. +/// +/// Per query: every group's leaf ([`leaf_permutations`]), one permutation per +/// Merkle level per group (a parent hashes 64 bytes, one rate block), and the +/// FRI leg's own [`FriShape::permutations_per_query`]. A closed form over the +/// shapes, so comparing it against the emitted count is an absolute check and +/// not a difference of two of our own emitter passes. +pub fn query_permutations(shape: &TableVerifyShape) -> usize { + let groups = shape.sub.groups().len(); + let per_query = leaf_permutations(&shape.sub) + + groups * shape.sub.merkle_depth + + shape.fri.permutations_per_query(); + shape.num_queries * per_query +} + +/// The boundary constraints every production VM table carries, as program shape. +/// +/// `AIR::boundary_constraints` is a function of the public inputs, so it is not a +/// static property of the AIR and cannot be captured into a +/// `ConstraintArtifact`. What IS static — and is checked against every AIR of a +/// real epoch by `epoch_verify_tests::the_boundary_terms_are_program_shape` — is +/// that a table with an aux trace carries exactly the framework's `acc[0] = 0` on +/// its last aux column and nothing else: a zero VALUE at the trace generator's +/// zeroth power, neither of which depends on a challenge or on the proof. +/// +/// Building the list from that rule rather than from the call is what keeps the +/// emitted program independent of the proof it verifies; the test is what stops +/// the rule from silently ceasing to hold. Note which direction the risk runs: a +/// boundary term this rule MISSED would be a constraint the machine never +/// checks, so the test asserts set equality and not containment. +pub fn boundary_terms(has_aux_trace: bool, num_total_cols: usize) -> Vec { + if has_aux_trace { + vec![BoundaryTerm { + col: num_total_cols - 1, + point: FE::one(), + value: FEE::zero(), + }] + } else { + Vec::new() + } +} diff --git a/prover/src/lfm/epoch_verify_tests.rs b/prover/src/lfm/epoch_verify_tests.rs new file mode 100644 index 000000000..fc3c68547 --- /dev/null +++ b/prover/src/lfm/epoch_verify_tests.rs @@ -0,0 +1,1298 @@ +//! ★ The assembled epoch verifier — spine plus legs — run on a real +//! continuation epoch proof. +//! +//! [`super::epoch_tests`] built the Fiat-Shamir spine and checked all 111 of a +//! real 24-sub-proof epoch's challenges against production's own replay. Every +//! verification leg, meanwhile, was driven by its own isolation program with +//! HINTED challenges. This module hangs the legs off the spine: per sub-proof the +//! OOD grid is rebuilt from the two pruned blocks the transcript absorbed, the +//! constraint evaluation and quotient check run at the spine's `z` and `β`, and +//! each query's index bits go straight from `TableChallenges::iota_bits` into the +//! Merkle walk, the DEEP fold and the FRI chain. +//! +//! ## The oracle, and what is left of it +//! +//! There is deliberately LESS oracle here than in any leg suite, and that is the +//! point. A leg suite checks a computed value against production's own answer for +//! the same inputs. Here the checks are INSIDE the program: the quotient check is +//! `assert_eq_ext(claimed, composition)`, every Merkle walk ends in +//! `assert_word_eq_lanes` against a root the transcript absorbed, and the FRI +//! chain ends in `assert_eq_ext` against the terminal polynomial. A program that +//! executes at all has passed them. So the differential that remains is the +//! spine's — the 111 challenges, still checked — plus the fact of execution, and +//! the falsification tests below are what turn "it executed" into evidence, by +//! showing what does NOT execute. +//! +//! ## What this suite cannot see +//! +//! The preset. The fixture epoch is proved at the MIN preset (blowup 2, one +//! query per table, grinding factor 1), because that is what +//! `proof_fixture::fixture_options` gives and what keeps a 24-sub-proof epoch +//! provable in a unit test. Every per-query cost here is therefore ONE query's, +//! and the blowup-8 predictions the phase pinned (73 queries, 14,454 FRI +//! permutations per sub-proof) are reached by scaling, not by measurement — the +//! scaling factors are stated in [`the_assembled_epoch_verifier_runs`]'s output +//! rather than hidden in a comment. It also cannot see PAGE's preprocessed +//! commitment problem (ledger entry 7), which is about where a root COMES from +//! and not about what is done with it. + +use stark::config::Commitment; +use stark::constraint_ir::ConstraintArtifact; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::constraints::{Analysis, BoundaryTerm, QuotientShape, analyze}; +use super::deep::DeepShape; +use super::epoch_verify::{TableVerifyShape, boundary_terms}; +use super::executor::execute; +use super::fri::FriShape; +use super::hash::TestPermutation; +use super::sub_proof::{GroupShape, SubProofShape}; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type V = Verifier; + +/// Everything the verification legs read about one real sub-proof. +/// +/// The split against `epoch_tests::HostTable` is by CONSUMER, not by +/// convenience: that struct holds what the transcript absorbs, this one holds +/// what the legs open. Nothing appears in both — which is the arena-join +/// obligation showing up in the test fixture as well as in the emitted program. +pub(super) struct TableLegs { + pub(super) verify: TableVerifyShape, + pub(super) analysis: Analysis, + /// `[query][group]` — the row pair in leaf order, then the path. + openings: Vec, Vec)>>, + /// `[query][layer]` — `(pᵢ(−υ^(2ⁱ)), path)`. + fri_openings: Vec)>>, + /// Production's OWN boundary-constraint list for this AIR, kept so + /// [`the_boundary_terms_are_program_shape`] can compare the program-shape + /// rule against the call rather than against a belief about it. + production_boundary: Vec, + /// `AIR::has_aux_trace`, the rule's input. + has_aux_trace: bool, + /// Preprocessed-column count, zero when the AIR is not preprocessed. Which + /// sub-proofs are preprocessed is what assembly ledger entry 7 is about. + pub(super) num_precomputed_cols: usize, + /// The commitment production absorbs for this table, when preprocessed — + /// `air.precomputed_commitment()`, taken from the AIR and never from the + /// proof. + pub(super) precomputed_commitment: Option, +} + +/// Read one real sub-proof into the shapes and openings the legs consume. +/// +/// Every shape here is derived from the AIR and the proof OPTIONS. The one +/// parameter that is neither is `log2_trace_length` — a table's chunk length is +/// chosen by the prover's row counts — and it is program shape in the assembled +/// verifier for the reason the arena schema makes it one: the program is emitted +/// for a specific epoch shape, and a proof whose trace length disagreed would +/// not match the arenas it declares. +pub(super) fn build_table_legs( + air: &dyn AIR, + view: StarkProofView<'_, Gl, Ext3, ()>, + rap_challenges: &[FEE], +) -> TableLegs { + let opts = air.options(); + let layout = V::ood_layout(air); + let artifact = ConstraintArtifact::capture(air); + + let (main_width, aux_width) = air.trace_layout(); + let num_total_cols = main_width + aux_width; + let num_precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + + let trace_length = view.trace_length(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + let log2_lde_length = log2_trace_length + log2_blowup; + let claimed_parts = view.composition_poly_parts_ood_evaluation(); + + // The trace matrices in DEEP column order — precomputed, main, aux — as the + // proof carries them and `build_host_sub_proof` reads them. + let mut trace_groups = Vec::new(); + if num_precomputed > 0 { + trace_groups.push(GroupShape { + num_columns: num_precomputed, + is_ext: false, + }); + } + trace_groups.push(GroupShape { + num_columns: main_width - num_precomputed, + is_ext: false, + }); + if aux_width > 0 { + trace_groups.push(GroupShape { + num_columns: aux_width, + is_ext: true, + }); + } + + let deep = DeepShape { + step_size: layout.step_size(), + num_eval_points: artifact.shape.transition_offsets.len() * layout.step_size(), + num_total_cols, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: claimed_parts.len(), + log2_trace_length, + }; + // The grid the machine rebuilds and the blocks the proof carries must + // describe one table. Asserted rather than assumed because the machine's + // reconstruction is indexed by the SHAPE and filled from the BLOCKS: a width + // disagreement would silently scatter the next-row values into wrong columns. + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + assert_eq!( + ood_c.width(), + num_total_cols, + "the current-row OOD block is the full trace width" + ); + assert_eq!( + ood_c.height(), + deep.step_size, + "the current-row block's height IS step_size (ood.rs:110-114)" + ); + assert_eq!( + ood_n.width(), + deep.next_row_cols.len(), + "the next-row block is as wide as the transition window" + ); + assert_eq!( + ood_n.height(), + deep.num_eval_points - deep.step_size, + "the next-row block covers every evaluation point past the first step" + ); + + let sub = SubProofShape { + deep, + trace_groups, + merkle_depth: log2_lde_length as usize - 1, + log2_lde_length, + coset_offset: FE::from(opts.coset_offset), + }; + let has_aux_trace = air.has_aux_trace(); + let verify = TableVerifyShape { + quotient: QuotientShape { + log2_trace_length, + num_composition_parts: claimed_parts.len(), + boundary: boundary_terms(has_aux_trace, num_total_cols), + }, + fri: FriShape::from_options(opts, log2_lde_length), + main_width, + num_alpha_powers: if has_aux_trace { + artifact.shape.max_bus_elements as usize + } else { + 0 + }, + num_queries: opts.fri_number_of_queries, + sub, + }; + + // ---- the openings, per query, in the emitter's group order. + let openings = (0..view.deep_poly_openings_len()) + .map(|q| { + let o = view.deep_poly_opening(q); + let mut groups: Vec<(Vec, Vec)> = Vec::new(); + if num_precomputed > 0 { + let p = o + .precomputed_trace_polys() + .expect("a preprocessed air opens its precomputed columns"); + groups.push(( + p.evaluations() + .iter() + .chain(p.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + p.merkle_path().to_vec(), + )); + } + let m = o.main_trace_polys(); + groups.push(( + m.evaluations() + .iter() + .chain(m.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + m.merkle_path().to_vec(), + )); + if aux_width > 0 { + let a = o.aux_trace_polys().expect("an aux opening"); + groups.push(( + a.evaluations() + .iter() + .chain(a.evaluations_sym()) + .map(ext_word) + .collect(), + a.merkle_path().to_vec(), + )); + } + let c = o.composition_poly(); + groups.push(( + c.evaluations() + .iter() + .chain(c.evaluations_sym()) + .map(ext_word) + .collect(), + c.merkle_path().to_vec(), + )); + groups + }) + .collect(); + + let fri_openings = (0..view.query_list_len()) + .map(|q| { + let d = view.query(q); + d.layers_evaluations_sym() + .iter() + .enumerate() + .map(|(i, sym)| (*sym, d.layer_auth_path(i).to_vec())) + .collect() + }) + .collect(); + + // Production's own boundary list, for the premise check only. It takes the + // bus public inputs, which are PROOF data — which is exactly why the emitted + // program must not be built from this call. + let bus_public_inputs = view + .bus_table_contribution() + .map(stark::lookup::BusPublicInputs::from_contribution); + let generator = ::get_primitive_root_of_unity( + log2_trace_length as u64, + ) + .expect("a power-of-two trace length has a root of unity"); + let production_boundary = air + .boundary_constraints( + &(), + rap_challenges, + bus_public_inputs.as_ref(), + trace_length, + ) + .constraints + .iter() + .map(|c| BoundaryTerm { + col: if c.is_aux { main_width + c.col } else { c.col }, + point: generator.pow(c.step as u64), + value: c.value, + }) + .collect(); + + TableLegs { + verify, + analysis: analyze(&artifact), + openings, + fri_openings, + production_boundary, + has_aux_trace, + num_precomputed_cols: num_precomputed, + precomputed_commitment: air.is_preprocessed().then(|| air.precomputed_commitment()), + } +} + +impl TableLegs { + /// Per query, per group: the row-pair values then the sibling digests. + /// + /// NO index word, which is the whole difference from + /// `join_tests::HostSubProof::query_arena`: the assembled verifier's index is + /// the transcript's own bits, so an arena that carried one would be offering + /// the prover a second index. + pub(super) fn opening_arena(&self) -> Vec { + let mut out = Vec::new(); + for query in &self.openings { + for (values, siblings) in query { + out.extend(values.iter().copied()); + out.extend(super::proof_arena::commitments_to_arena(siblings)); + } + } + assert_eq!( + out.len(), + self.verify.opening_words(), + "the opening arena must fill exactly what the shape declares" + ); + out + } + + /// Per query, per committed layer: the symmetric evaluation then its path. + pub(super) fn fri_arena(&self) -> Vec { + let mut out = Vec::new(); + for query in &self.fri_openings { + for (sym, path) in query { + out.push(ext_word(sym)); + out.extend(super::proof_arena::commitments_to_arena(path)); + } + } + assert_eq!( + out.len(), + self.verify.fri_words(), + "the FRI arena must fill exactly what the shape declares" + ); + out + } +} + +/// ★ THE RUN: the whole epoch verifier — spine AND legs — on a real +/// continuation epoch proof that production accepts. +/// +/// What executing proves, stated precisely. Every check is an assert inside the +/// program, so reaching the end means: all 24 quotient identities held at the +/// spine's own `z` and `β`; every one of the 24 sub-proofs' opened row pairs +/// hashed to a leaf that walked to the root the transcript absorbed, at the index +/// the transcript sampled; every DEEP reconstruction fed a FRI chain that folded +/// to the terminal polynomial the transcript absorbed; and the LogUp closure +/// reached production's COMMIT-bus target. The 111 published challenges are +/// checked against production's replay on top, so the Fiat-Shamir the whole thing +/// hangs from is still differentialled. +#[test] +fn the_assembled_epoch_verifier_runs() { + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let exec = + execute(&program, &arenas, &TestPermutation).expect("the assembled verifier must execute"); + + // ---- the spine's differential, unchanged: production's own challenges. + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the shared LogUp challenge z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the shared LogUp challenge alpha"); + + // The attestation fold is published right after Phase A (two digest words), + // and its DECODE input is the cell Phase A absorbed — the join ledger entry 7 + // rests on. Its value is differentialled in the spine test; here it only has to + // be skipped, and skipped by NAME rather than by a literal. + let program_id_words = 2usize; + let mut cursor = 2 + program_id_words; + let mut checked = 2usize; + for (i, (h, leg)) in e.tables.iter().zip(&e.legs).enumerate() { + // The legs publish first: the recomputed composition, then a terminal + // value per query. + cursor += 1 + leg.verify.num_queries; + assert_eq!(pub_ext(cursor), h.beta, "beta of table {i}"); + assert_eq!(pub_ext(cursor + 1), h.z, "z of table {i}"); + assert_eq!(pub_ext(cursor + 2), h.gamma, "gamma of table {i}"); + cursor += 3; + checked += 3; + for (k, want) in h.zetas.iter().enumerate() { + assert_eq!(pub_ext(cursor + k), *want, "zeta {k} of table {i}"); + } + cursor += h.zetas.len(); + checked += h.zetas.len(); + for q in 0..h.shape.num_queries { + let w = exec.public_words[cursor + q].1; + let got = super::word::word_as_base(&w).expect("an index is a base felt"); + assert_eq!(got, FE::from(h.iotas[q] as u64), "iota {q} of table {i}"); + } + cursor += h.shape.num_queries; + checked += h.shape.num_queries; + } + assert_eq!( + checked, 111, + "the same 111 challenges the spine test checks must still be checked" + ); + assert_eq!( + word_as_ext(&exec.public_words[cursor].1).expect("the bus total is ext"), + e.expected_bus_balance, + "the LogUp closure must reach production's own COMMIT-bus target" + ); + assert_eq!( + cursor + 1, + exec.public_words.len(), + "every published word must be accounted for" + ); + + // ---- THE MEASUREMENT ---- + let spine = super::epoch_tests::epoch_program(&e, false); + let count = |p: &super::compiler::LfmProgram, f: fn(&super::instr::Instr) -> bool| { + p.instrs.iter().filter(|i| f(i)).count() + }; + let perms = |p: &_| count(p, |i| matches!(i, super::instr::Instr::KeccakF(_))); + let words = |p: &super::compiler::LfmProgram| -> usize { + p.arena_schema.lens.iter().map(|l| *l as usize).sum() + }; + + let queries = e.legs[0].verify.num_queries; + let opening_perms = perms(&program) - perms(&spine); + let legs_published: usize = e.legs.iter().map(|l| 1 + l.verify.num_queries).sum(); + println!( + "\n★ ASSEMBLED EPOCH VERIFIER (min preset: blowup 2, {queries} quer\ + {}/table, grinding {}):\n\ + \x20 spine +legs legs alone\n\ + \x20 instructions {:>10} {:>10} {:>10}\n\ + \x20 keccak perms {:>10} {:>10} {:>10}\n\ + \x20 arena words {:>10} {:>10} {:>10}\n\ + \x20 published {:>10} {:>10} {:>10}", + if queries == 1 { "y" } else { "ies" }, + e.tables[0].shape.grinding_factor, + spine.instrs.len(), + program.instrs.len(), + program.instrs.len() - spine.instrs.len(), + perms(&spine), + perms(&program), + opening_perms, + words(&spine), + words(&program), + words(&program) - words(&spine), + // The spine's own published count. Was `len - (x - x)` — a leftover that + // printed the assembled figure in the spine column. + exec.public_words.len() - legs_published, + exec.public_words.len(), + legs_published, + ); + + // ---- the constraint leg's share, from the analyses themselves. + // + // `Analysis::report` is the count of what the lowering pass DID, and + // `emit_analyzed` runs over the very analysis reported here — the module's own + // doc comment makes that a construction, not a coincidence — so summing the + // reports attributes the constraint evaluation inside the assembled program + // without a second emitter pass. `alu_rows` excludes constants because the + // builder interns them program-wide, so the sum is a lower bound on the + // constraint leg's instructions and not the whole of it. + let constraint_alu: usize = e.legs.iter().map(|l| l.analysis.report().alu_rows()).sum(); + let constraint_unfused: usize = e + .legs + .iter() + .map(|l| l.analysis.report().unfused_alu_rows()) + .sum(); + // The recombination half, measured in ISOLATION against its own plumbing + // baseline and compared against a number that did not come from this emitter + // (`others/lfm-constraint-lowering-design.md:604` splits the pinned 57,252 + // into 54,358 lowering + 2,894 recombination). That is what makes a + // two-pass difference admissible here — the comparison target is external. + let recombination: usize = e + .legs + .iter() + .map(|l| { + let plumb = |b: &mut super::builder::LfmBuilder| { + let n = 2 + + l.verify.sub.deep.num_composition_parts + + l.verify.num_frame_steps() * l.verify.sub.deep.num_total_cols + + l.analysis.report().nodes; + let a = b.declare_arena(n as u32); + let mut i = 0u32; + let mut take = |b: &mut super::builder::LfmBuilder| { + let c = b.hint_word(a, i).as_ext(); + i += 1; + c + }; + let z = take(b); + let beta = take(b); + let parts: Vec<_> = (0..l.verify.sub.deep.num_composition_parts) + .map(|_| take(b)) + .collect(); + let steps: Vec> = (0..l.verify.num_frame_steps()) + .map(|_| { + (0..l.verify.sub.deep.num_total_cols) + .map(|_| take(b)) + .collect() + }) + .collect(); + // One evaluation cell per constraint root, which is what + // `emit_analyzed` returns and `emit_quotient` folds. + let evals: Vec<_> = (0..l.analysis.program().roots.len()) + .map(|_| take(b)) + .collect(); + (z, beta, parts, steps, evals) + }; + let mut bare = super::builder::LfmBuilder::new(); + let _ = plumb(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut full = super::builder::LfmBuilder::new(); + let (z, beta, parts, steps, evals) = plumb(&mut full); + let ood = super::constraints::OodOperands { + steps, + main_width: l.verify.main_width, + rap_challenges: Vec::new(), + alpha_powers: Vec::new(), + table_offset: z, + }; + super::constraints::emit_quotient( + &mut full, + &l.verify.quotient, + &ood, + z, + beta, + &evals, + &parts, + ); + full.finish().instrs.len() - baseline + }) + .sum(); + println!( + "\x20 constraint leg inside the assembled verifier: {constraint_alu} ALU \ + rows lowering ({constraint_unfused} unfused) + {recombination} \ + recombination = {} over 24 sub-proofs [pinned: 54,358 + 2,894 = 57,252]\ + \n\x20 that is {:.1}% of the legs' {} instructions", + constraint_alu + recombination, + 100.0 * (constraint_alu + recombination) as f64 + / (program.instrs.len() - spine.instrs.len()) as f64, + program.instrs.len() - spine.instrs.len(), + ); + + // ---- the permutation bill, against a CLOSED FORM over the shapes. + // + // Not a difference of two emitter passes (which rule 7's refinement rules + // out) but arithmetic over byte widths: every group's leaf is + // `⌊bytes/136⌋ + 1` rate blocks, every Merkle level is one, and FRI's own + // per-query figure is the one the FRI leg pinned. Asserted, not printed, so + // a leg that silently stopped hashing a group would fail here. + let mut fri_perms = 0usize; + let mut leaf_perms = 0usize; + let mut walk_perms = 0usize; + for leg in &e.legs { + let groups = leg.verify.sub.groups().len(); + fri_perms += leg.verify.num_queries * leg.verify.fri.permutations_per_query(); + leaf_perms += + leg.verify.num_queries * super::epoch_verify::leaf_permutations(&leg.verify.sub); + walk_perms += leg.verify.num_queries * groups * leg.verify.sub.merkle_depth; + } + let predicted: usize = e + .legs + .iter() + .map(|l| super::epoch_verify::query_permutations(&l.verify)) + .sum(); + assert_eq!( + predicted, + leaf_perms + walk_perms + fri_perms, + "the closed form must decompose into exactly its three parts" + ); + assert_eq!( + opening_perms, predicted, + "the emitted permutation count must equal the closed form over the shapes" + ); + println!( + "\x20 leg permutations = {leaf_perms} leaves + {walk_perms} Merkle levels \ + + {fri_perms} FRI = {predicted} (closed form) = {opening_perms} (emitted)" + ); + println!( + "\x20 FRI layers committed across the epoch: {} | widest leaf: {} bytes", + e.legs + .iter() + .map(|l| l.verify.fri.num_committed()) + .sum::(), + e.legs + .iter() + .flat_map(|l| l.verify.sub.groups()) + .map(|g| g.leaf_bytes()) + .max() + .expect("the epoch has groups") + ); + + // ---- RECONCILIATION with the phase's pinned blowup-8 predictions. + // + // The pinned 213,744 came from `join_tests::join_leg_cost`, whose stated + // assumptions are: all 28 PRODUCTION AIRs, every trace at a UNIFORM + // 2^20, blowup 8, 73 queries, and NO FRI (the joined leg has none). The + // measurement above is: this epoch's 24 sub-proofs, at their REAL trace + // lengths, blowup 2, one query, FRI included. Three parameters differ, so + // the two numbers cannot be compared directly — they are projected onto each + // other one parameter at a time instead, which is also what says which + // assumption carries the difference. + let at_blowup_8 = |leg: &TableLegs, uniform_log2_trace: Option| -> TableVerifyShape { + let log2_trace = uniform_log2_trace.unwrap_or(leg.verify.sub.deep.log2_trace_length); + let log2_lde = log2_trace + 3; + let mut out = leg.verify.clone(); + out.sub.log2_lde_length = log2_lde; + out.sub.merkle_depth = log2_lde as usize - 1; + out.sub.deep.log2_trace_length = log2_trace; + out.quotient.log2_trace_length = log2_trace; + out.fri = FriShape { + log2_lde_length: log2_lde, + blowup_log: 3, + num_queries: 73, + ..leg.verify.fri + }; + out.num_queries = 73; + out + }; + let openings_only = |s: &TableVerifyShape| -> usize { + s.num_queries + * (super::epoch_verify::leaf_permutations(&s.sub) + + s.sub.groups().len() * s.sub.merkle_depth) + }; + + let real_lengths: Vec = e.legs.iter().map(|l| at_blowup_8(l, None)).collect(); + let uniform: Vec = e.legs.iter().map(|l| at_blowup_8(l, Some(20))).collect(); + let sum = |v: &[TableVerifyShape], f: &dyn Fn(&TableVerifyShape) -> usize| -> usize { + v.iter().map(f).sum() + }; + + // ---- THE HASH MATRIX'S PERMUTATION AXIS, at the production shape. + // + // A candidate hash moves two independent things: cells per permutation (its + // AIR's shape, which needs the AIR) and permutations per verify (the sponge's + // rate, which needs only arithmetic over these shapes). This block pins the + // second WITHOUT any candidate permutation existing, so the remaining unknown + // in a candidate's predicted column is one factor and not two. + // + // The differential: `query_permutations_at_rate` is written through felts and a + // rate, `query_permutations` through bytes and `keccak_host::num_blocks`. + // Neither delegates to the other, so their agreement at rate 17 is a real check + // on the felt-side reformulation — and the existing assert above already ties + // `query_permutations` to the EMITTED count, so the chain reaches the emitter. + use super::epoch_verify::{ + FRI_LEAF_FELTS, KECCAK_RATE_FELTS, LFM_HASH_RATE_FELTS, blocks_at_rate, + fri_leaf_permutations_at_rate, group_leaf_felts, query_permutations_at_rate, + }; + for s in &real_lengths { + assert_eq!( + query_permutations_at_rate(s, KECCAK_RATE_FELTS), + super::epoch_verify::query_permutations(s), + "the felt-side closed form must reproduce the byte-side one at keccak's rate" + ); + } + // ⚠ A FRI layer leaf is six felts. It fits ONE keccak block and it does NOT + // fit one block at the candidate's rate — which was 8 under the deleted + // three-cell duplex and is 4 under the B1 compress chain. So the FRI leaf + // term is rate-SENSITIVE and is no longer part of the invariant remainder. + // The old premise assertion here (`6 <= LFM_HASH_RATE_FELTS`) is gone: it + // was true at 8, is false at 4, and re-asserting it would have pinned the + // model to a construction that no longer exists. + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, KECCAK_RATE_FELTS), 1); + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, LFM_HASH_RATE_FELTS), 2); + + let keccak_p = sum(&real_lengths, &|s| { + query_permutations_at_rate(s, KECCAK_RATE_FELTS) + }); + let cand_p = sum(&real_lengths, &|s| { + query_permutations_at_rate(s, LFM_HASH_RATE_FELTS) + }); + // Decompose so the penalty is attributed rather than asserted in aggregate. + // The split is ABSORPTION vs COMPRESSION, not leaf vs rest: leaves of both + // kinds absorb and move with the rate, Merkle parents of both kinds + // compress and do not. + let absorb_at = |rate: usize| { + sum(&real_lengths, &|s| { + s.num_queries + * (super::epoch_verify::leaf_permutations_at_rate(&s.sub, rate) + + fri_leaf_permutations_at_rate(&s.fri, rate)) + }) + }; + let leaf_k = absorb_at(KECCAK_RATE_FELTS); + let leaf_c = absorb_at(LFM_HASH_RATE_FELTS); + let paths = keccak_p - leaf_k; + assert_eq!( + cand_p, + leaf_c + paths, + "only ABSORPTION may move with the rate; compression (Merkle parents, \ + trace trees and FRI path steps alike) must not" + ); + assert!( + cand_p > keccak_p, + "the candidate's smaller rate must COST permutations — if this ever fails, \ + the rate penalty reasoning in others/lfm-hash-matrix-scope.md is wrong" + ); + let widest = real_lengths + .iter() + .map(|s| { + s.sub + .groups() + .iter() + .map(group_leaf_felts) + .max() + .unwrap_or(0) + }) + .max() + .expect("the epoch has groups"); + println!( + "\n ★ HASH MATRIX — the PERMUTATION axis at blowup 8 / 73 queries, real \ + trace lengths (no candidate permutation exists yet; this is shape \ + arithmetic only):\n\ + \x20 keccak rate {KECCAK_RATE_FELTS:>2} felts/perm: {keccak_p:>9} permutations \ + ({leaf_k} absorbed + {paths} compressed)\n\ + \x20 LFM_HASH rate {LFM_HASH_RATE_FELTS:>2} felts/perm: {cand_p:>9} permutations \ + ({leaf_c} absorbed + {paths} compressed)\n\ + \x20 candidate/keccak = {:.4}x (absorption term alone {:.4}x; the \ + ceiling is 17/{LFM_HASH_RATE_FELTS} = {:.3}x and only absorption pays it)\n\ + \x20 absorption-bound share of the keccak bill: {:.1}% widest leaf: \ + {widest} felts\n\ + \x20 ⚠ the candidate rate is 4, not the 8 this model carried before \ + option B1 — the compress chain absorbs ONE cell per step, so both the \ + trace-group leaves and the 6-felt FRI-layer leaves take two blocks", + cand_p as f64 / keccak_p as f64, + leaf_c as f64 / leaf_k as f64, + KECCAK_RATE_FELTS as f64 / LFM_HASH_RATE_FELTS as f64, + 100.0 * leaf_k as f64 / keccak_p as f64, + ); + + println!( + "\n RECONCILIATION against the pinned blowup-8 predictions (projections \ + from shapes — this run is at the min preset and measures none of them):\n\ + \x20 openings only, 73 queries, UNIFORM 2^20 (deep-join's own \ + assumption, over this epoch's 24 sub-proofs): {} [pinned: 213,744 \ + over all 28 production AIRs]\n\ + \x20 openings only, 73 queries, this epoch's REAL trace lengths: {}\n\ + \x20 openings + FRI, 73 queries, real lengths: {}\n\ + \x20 FRI alone, 73 queries, real lengths: {} [pinned: 14,454 per \ + sub-proof at blowup 8, i.e. for a 2^20 table]", + sum(&uniform, &openings_only), + sum(&real_lengths, &openings_only), + sum(&real_lengths, &|s| super::epoch_verify::query_permutations( + s + )), + sum(&real_lengths, &|s: &TableVerifyShape| s.num_queries + * s.fri.permutations_per_query()), + ); + // The one sub-proof that IS a 2^20 table, so the per-sub-proof FRI figure the + // FRI leg pinned has something to be checked against. + let biggest = e + .legs + .iter() + .max_by_key(|l| l.verify.sub.deep.log2_trace_length) + .expect("the epoch has sub-proofs"); + let big8 = at_blowup_8(biggest, None); + println!( + "\x20 the epoch's 2^{} sub-proof at blowup 8: FRI {} permutations \ + ({} committed layers), openings {}", + big8.sub.deep.log2_trace_length, + big8.num_queries * big8.fri.permutations_per_query(), + big8.fri.num_committed(), + openings_only(&big8), + ); + println!("\x20 trace lengths in this epoch (log2): {:?}", { + let mut v: Vec = e + .legs + .iter() + .map(|l| l.verify.sub.deep.log2_trace_length) + .collect(); + v.sort_unstable(); + v + }); +} + +/// Where each per-table arena sits in the declaration order +/// `epoch_tests::epoch_arena_words` produces. +/// +/// Computed from the presence flags rather than hardcoded, because a table +/// without an aux root or without grinding shifts every arena behind it — which +/// is precisely the failure mode the per-field arena packing exists to prevent +/// and a hardcoded index would reintroduce in the TEST. +pub(super) struct ArenaIndex { + pub(super) openings: usize, + fri: usize, + parts: usize, + ood_current: usize, +} + +pub(super) fn arena_index(e: &super::epoch_tests::RealEpoch, table: usize) -> ArenaIndex { + // The epoch-wide arenas come first, and their COUNT comes from the emitter's + // own side rather than from a literal here: wiring ledger entry 7 added the + // second register vector, `pc_start` and (when non-empty) the page roots, and a + // literal `4` would have left every vector below tampering the wrong arena. + let mut at = super::epoch_tests::num_epoch_wide_arenas(e); + for (i, h) in e.tables.iter().enumerate() { + let aux = usize::from(h.shape.has_aux_root); + let contribution = usize::from(h.shape.has_contribution); + let nonce = usize::from(h.shape.grinding_factor > 0); + let composition = at + aux + contribution; + if i == table { + return ArenaIndex { + ood_current: composition + 1, + parts: composition + 3, + openings: composition + 6 + nonce, + fri: composition + 7 + nonce, + }; + } + // The table's last arena is `fri` at `composition + 7 + nonce`, so the + // next table starts one past it. Getting this stride wrong is how the + // first version of this test came to tamper an EMPTY arena two tables + // later — which is why the loop below checks every computed index + // against the arena lengths the shapes fix. + at = composition + 8 + nonce; + } + unreachable!("table index out of range"); +} + +/// ★ FALSIFICATION: run the attacks the wiring denies, and watch each fail. +/// +/// Every check the legs add is an `assert` inside the program, so "it executed" +/// is the whole positive result — which makes this test the entire negative half. +/// Each vector is a single arena word moved by one, and each must make the +/// program unexecutable. What each one proves is different, so they are labelled +/// rather than swept anonymously: +/// +/// - an OPENED VALUE: the leaf hash changes, so the walk reaches a root the +/// transcript never absorbed. This is also the two-consumer join — the same +/// cell is what DEEP folds, so there is no way to move one without the other. +/// - a MERKLE SIBLING, both words: a path that authenticates nothing. Both words +/// are hit deliberately; a past tamper suite in this phase touched only byte 0 +/// of every digest, so a digest's second word was never checked. +/// - a FRI SYMMETRIC EVALUATION and a FRI SIBLING: the layer walk, on the one +/// sub-proof of this epoch that actually folds (12 committed layers). +/// - a CLAIMED COMPOSITION PART: this one is absorbed, so it moves the +/// challenges as well — it must reject, and the interesting part is that it +/// cannot reject "only" the quotient check. +/// - an OOD CELL: likewise absorbed, and read by both the constraint fold and +/// DEEP. +#[test] +fn the_assembled_verifier_rejects_tampered_leg_data() { + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + let good = super::epoch_tests::epoch_arena_words(&e, true); + assert!( + execute(&program, &good, &TestPermutation).is_ok(), + "the untampered assembled verifier must run" + ); + + // The sub-proof that folds, so the FRI vectors reach the layer walk. + let folding = e + .legs + .iter() + .position(|l| l.verify.fri.num_committed() > 0) + .expect("this epoch has a sub-proof with committed FRI layers"); + + // ★ The index arithmetic above is a claim about the declaration order, and a + // WRONG index would make this whole test lie — it would tamper some other + // arena, still get a rejection, and report a pass. So the claim is checked + // against the arena LENGTHS, which the shapes fix independently. + for (t, leg) in e.legs.iter().enumerate() { + let ix = arena_index(&e, t); + assert_eq!( + good[ix.openings].len(), + leg.verify.opening_words(), + "table {t}: the arena at the computed openings index is not the \ + openings arena" + ); + assert_eq!( + good[ix.fri].len(), + leg.verify.fri_words(), + "table {t}: the arena at the computed FRI index is not the FRI arena" + ); + assert_eq!( + good[ix.parts].len(), + e.tables[t].parts.len(), + "table {t}: the arena at the computed parts index is not the parts arena" + ); + assert_eq!( + good[ix.ood_current].len(), + e.tables[t].ood_current.len(), + "table {t}: the arena at the computed OOD index is not the OOD arena" + ); + } + + let mut vectors: Vec<(String, usize, usize)> = Vec::new(); + // Trace openings: the first value and both words of the first sibling + // digest, on three tables including the folding one. + for &t in &[0usize, 1, folding] { + let ix = arena_index(&e, t); + let leg = &e.legs[t]; + let values = leg.verify.sub.groups()[0].num_values(); + vectors.push((format!("table {t}: opened value 0"), ix.openings, 0)); + vectors.push(( + format!("table {t}: last opened value of group 0"), + ix.openings, + values - 1, + )); + vectors.push((format!("table {t}: sibling lo"), ix.openings, values)); + vectors.push((format!("table {t}: sibling hi"), ix.openings, values + 1)); + vectors.push((format!("table {t}: claimed part 0"), ix.parts, 0)); + vectors.push((format!("table {t}: OOD cell 0"), ix.ood_current, 0)); + } + // FRI: the first layer's symmetric evaluation, then both words of its first + // sibling. + let fri_ix = arena_index(&e, folding).fri; + vectors.push(("FRI layer 0 sym".to_string(), fri_ix, 0)); + vectors.push(("FRI layer 0 sibling lo".to_string(), fri_ix, 1)); + vectors.push(("FRI layer 0 sibling hi".to_string(), fri_ix, 2)); + + for (label, arena, word) in &vectors { + let mut arenas = good.clone(); + let before = arenas[*arena][*word]; + arenas[*arena][*word][0] = before[0] + FE::one(); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "tampering {label} must make the assembled verifier unexecutable, \ + and did not" + ); + } + println!(" {} tamper vectors, all rejected", vectors.len()); +} + +/// ★ The boundary list the emitted program carries is a PROGRAM CONSTANT, and +/// this is the premise that makes it one. +/// +/// `AIR::boundary_constraints` takes the public inputs and the bus public inputs +/// — both proof data — so building the emitted list from that call would make the +/// program depend on the proof it verifies. `epoch_verify::boundary_terms` builds +/// it from a rule instead. The rule is only safe while it agrees with the call on +/// every AIR of a real epoch, so this compares them as SETS: a term the rule +/// missed would be a constraint the machine silently never checks. +#[test] +fn the_boundary_terms_are_program_shape() { + let e = super::epoch_tests::real_epoch(); + let mut with_boundary = 0usize; + for (i, leg) in e.legs.iter().enumerate() { + let rule = boundary_terms(leg.has_aux_trace, leg.verify.sub.deep.num_total_cols); + let want = &leg.production_boundary; + assert_eq!( + rule.len(), + want.len(), + "table {i}: the rule and production disagree about how many boundary \ + constraints the AIR has" + ); + for (r, w) in rule.iter().zip(want) { + assert_eq!(r.col, w.col, "table {i}: boundary column"); + assert_eq!(r.point, w.point, "table {i}: boundary point"); + assert_eq!(r.value, w.value, "table {i}: boundary value"); + } + if !want.is_empty() { + with_boundary += 1; + } + } + // Positive control: a suite where every AIR had an empty list would pass + // vacuously, and the rule's interesting branch would be untested. + assert!( + with_boundary > 0, + "no sub-proof carries a boundary constraint, so this proves nothing about \ + the rule's non-empty branch" + ); + println!( + " boundary premise: {with_boundary} of {} sub-proofs carry the \ + framework's acc[0] = 0 and nothing else", + e.legs.len() + ); +} + +/// ★ An ABSOLUTE structural guard over the ASSEMBLED verifier: no proof value is +/// hinted twice, legs included. +/// +/// This is the count that closes assembly obligation 3. The spine's own version +/// (`epoch_tests::the_spine_hints_each_proof_value_once`) could only say the +/// spine hinted nothing twice — the legs were not in the program, so their second +/// consumers had nothing to disagree with. Now they are, and the same absolute +/// property must hold over the whole thing: the OOD grid, the claimed parts, +/// every root and every challenge reach the legs as cells, never as a second +/// read. +#[test] +fn the_assembled_verifier_hints_each_proof_value_once() { + use std::collections::HashMap; + + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + + let mut hints: HashMap<(super::instr::ArenaId, u32), usize> = HashMap::new(); + for instr in &program.instrs { + if let super::instr::Instr::Hint { arena, index, .. } = instr { + *hints.entry((*arena, *index)).or_default() += 1; + } + } + let doubled: Vec<_> = hints.iter().filter(|(_, n)| **n > 1).collect(); + assert!( + doubled.is_empty(), + "these arena words are hinted more than once, which is the two-consumer \ + hazard the assembly exists to remove: {doubled:?}" + ); + + let declared: usize = program.arena_schema.lens.iter().map(|l| *l as usize).sum(); + assert_eq!( + hints.len(), + declared, + "every declared arena word must be read exactly once" + ); + // The legs are actually IN this program — without this the guard would pass + // just as happily over the spine alone. + let spine = super::epoch_tests::epoch_program(&e, false); + assert!( + declared + > spine + .arena_schema + .lens + .iter() + .map(|l| *l as usize) + .sum::(), + "the assembled program must declare more arena words than the spine, or \ + the legs are not wired and this guard is vacuous" + ); +} + +/// ★ The preprocessed-commitment inventory of a real epoch — the EVIDENCE +/// assembly ledger entry 7 was opened without. +/// +/// Entry 7 says five AIRs are preprocessed (BITWISE, DECODE, KECCAK_RC, REGISTER, +/// PAGE), that three are compile-time constants, that REGISTER has a derivation +/// and that PAGE cannot become a program constant because it is a function of the +/// inner ELF. That is a claim about the AIR SET. This test asks the real epoch +/// which of its sub-proofs are actually preprocessed, and how many columns each +/// commits, so the proposal that closes the entry is built on a census rather +/// than on a recollection. +/// +/// `VmAirs::air_refs` fixes the order (`lib.rs:610-625`): BITWISE, DECODE, COMMIT, +/// KECCAK, KECCAK_RND, KECCAK_RC, ECSM, ECDAS, REGISTER, then optional HALT, then +/// the chunked tables, then the PAGE tables, and this suite appends L2G_MEMORY. +/// So a preprocessed sub-proof at index 8 is REGISTER and one past the chunked +/// tables is a PAGE — which is what makes "which sub-proof is which AIR" program +/// shape rather than proof data. +#[test] +fn the_preprocessed_commitments_of_a_real_epoch() { + let e = super::epoch_tests::real_epoch(); + let preprocessed: Vec<(usize, usize)> = e + .legs + .iter() + .enumerate() + .filter(|(_, l)| l.num_precomputed_cols > 0) + .map(|(i, l)| (i, l.num_precomputed_cols)) + .collect(); + println!( + " {} of {} sub-proofs are preprocessed: {:?} (index, precomputed columns)", + preprocessed.len(), + e.legs.len(), + preprocessed + ); + // The REGISTER slot, checked by its column count rather than assumed from its + // index: the derivation commits OFFSET ‖ INIT ‖ FINI. + let register = e.legs.iter().position(|l| { + l.num_precomputed_cols == crate::tables::register::NUM_PREPROCESSED_COLS_WITH_FINI + }); + println!( + " the sub-proof whose preprocessed width is NUM_PREPROCESSED_COLS_WITH_FINI \ + ({}): index {:?}", + crate::tables::register::NUM_PREPROCESSED_COLS_WITH_FINI, + register + ); + // Every preprocessed sub-proof's commitment must actually be present, or the + // spine would be absorbing something it did not get from the AIR. + for (i, _) in &preprocessed { + assert!( + e.legs[*i].precomputed_commitment.is_some(), + "sub-proof {i} declares preprocessed columns but has no AIR commitment" + ); + } + assert!( + !preprocessed.is_empty(), + "an epoch with no preprocessed sub-proof cannot witness entry 7 at all" + ); + + // ---- ★ the PROVENANCE census, which is what entry 7 actually turns on. + // + // `epoch_tests::prep_source` decided each root's source by recomputing every + // candidate production has; reaching this line means every preprocessed root of + // a real epoch matched one, so nothing is hinted without a binding. What is + // asserted here is the SHAPE of the taxonomy — that the epoch is not all + // constants (which would make the derivation and the fold untested) and not all + // ELF-dependent (which would mean interning bought nothing). + let sources = super::epoch_tests::prep_source_census(&e); + println!( + " provenance: {} options-only (interned as program text), {} derived \ + in-machine (REGISTER), {} ELF-dependent (arena cell + attestation join)", + sources.0, sources.1, sources.2 + ); + assert_eq!( + sources.0 + sources.1 + sources.2, + preprocessed.len(), + "every preprocessed sub-proof must have exactly one classified source" + ); + assert!( + sources.0 > 0, + "no options-only root: the interning path is unexercised" + ); + assert_eq!( + sources.1, 1, + "exactly one derived root — the REGISTER commitment, from the epoch's own \ + register boundary" + ); + assert_eq!( + sources.2, 1, + "exactly one ELF-dependent root in a continuation epoch — DECODE. A second \ + would mean the attestation fold's input is ambiguous" + ); + + // ★ AND THE PAGE HALF OF ENTRY 7 IS NOT A FIXTURE ARTEFACT. `prove_epoch` + // rejects any epoch carrying a PAGE config ("continuation epoch must have no + // PAGE configs (L2G bookend replaces PAGE)", `continuation.rs:695-702`) and both + // `build_epoch_airs` call sites pass `&[]`. So no continuation epoch of any + // guest has a PAGE sub-proof, and the ELF-data page genesis roots the + // attestation folds are the GLOBAL proof's GlobalMemory AIRs' preprocessed + // commitments (`continuation.rs:997-1010`) — a different proof, out of an epoch + // verifier's scope. + // + // Asserted rather than remembered. The width test is unambiguous only because + // a continuation epoch's REGISTER always uses the WITH_FINI layout + // (`build_epoch_airs` always supplies `register_preprocessed`), and PAGE's + // width coincides with the non-FINI REGISTER one — so that premise is checked + // first. An ELF-data page root would in any case make `prep_source` panic, + // since its provenance is not in the classifier's candidate list. + assert!( + register.is_some(), + "a continuation epoch's REGISTER is preprocessed WITH FINI; without that \ + the width check below cannot tell a PAGE table from a REGISTER one" + ); + assert!( + e.legs + .iter() + .all(|l| l.num_precomputed_cols != crate::tables::page::NUM_PREPROCESSED_COLS), + "a sub-proof with PAGE's preprocessed width appeared: continuation epochs \ + are supposed to carry none, and the entry-7 taxonomy changes if they do" + ); +} + +/// ★ The composition and FRI-terminal CHECKS are in the program, counted. +/// +/// This guard exists because falsification found the hole it closes. Deleting +/// `assert_eq_ext(q.claimed, q.composition)` from the emitter fails NOTHING in +/// this suite: with honest data the two values ARE equal, so no differential and +/// no arena tamper can see the assert's absence. And no arena tamper ever will — +/// every input to the quotient identity (the OOD grid, the claimed parts, `z`, +/// `β`) is absorbed by the transcript, so moving any of them moves the challenges +/// and the run fails at the Merkle walk instead, for the wrong reason. +/// +/// What DOES witness the check is a mutation that makes the identity false while +/// leaving the transcript alone — emptying the boundary-term list does exactly +/// that, and three tests catch it. But "a mutation elsewhere catches it" is not +/// the same as "the check is present", so this counts the checks directly. +/// +/// `assert_eq_ext(a, b)` lowers to `esub` then `ediv(diff, ZERO)` +/// (`builder.rs:243-247`): division by the interned zero has a witness only when +/// the numerator vanishes, since `OUT · 0 = A` forces `A = 0`. So an extension +/// division whose DIVISOR is the pooled zero constant is an equality assertion, +/// and nothing else in the machine produces one — every other `ediv` here +/// inverts against the interned ONE. +/// +/// The expected count is arithmetic over the shapes, not a second emitter pass: +/// one composition check per sub-proof, plus per query one FRI terminal check +/// when the codeword folds and TWO when it does not (the zero-fold shape checks +/// `P` at both `υ` and `−υ`). +#[test] +fn the_assembled_verifier_contains_every_composition_and_terminal_check() { + use super::instr::{ExtOp, Instr}; + + let e = super::epoch_tests::real_epoch(); + let program = super::epoch_tests::epoch_program(&e, true); + let spine = super::epoch_tests::epoch_program(&e, false); + + let asserts = |p: &super::compiler::LfmProgram| -> usize { + // The interned all-zero word. `felt_const(0)` and `ext_const(0)` are the + // same word, and the builder interns program-wide, so there is one. + let zeros: Vec<_> = p + .instrs + .iter() + .filter_map(|i| match i { + Instr::Const { out, value, .. } if value.iter().all(|v| *v == FE::zero()) => { + Some(*out) + } + _ => None, + }) + .collect(); + assert_eq!( + zeros.len(), + 1, + "the zero word must be interned exactly once, or this count is \ + ambiguous" + ); + let zero = zeros[0]; + p.instrs + .iter() + .filter(|i| { + matches!( + i, + Instr::ExtAlu { + op: ExtOp::Div, + b, + .. + } if *b == zero + ) + }) + .count() + }; + + let expected: usize = e + .legs + .iter() + .map(|l| { + let terminal = if l.verify.fri.total_folds() > 0 { 1 } else { 2 }; + 1 + l.verify.num_queries * terminal + }) + .sum(); + assert_eq!( + asserts(&program) - asserts(&spine), + expected, + "the legs must add exactly one composition check per sub-proof plus the \ + FRI terminal checks the shapes call for" + ); + // Positive control: the count must be nonzero and the shapes must actually + // include both FRI branches, or the formula's second case is untested. + assert!(expected > 0); + assert!( + e.legs.iter().any(|l| l.verify.fri.total_folds() > 0) + && e.legs.iter().any(|l| l.verify.fri.total_folds() == 0), + "this epoch must exercise BOTH the folding and the zero-fold terminal \ + shapes, or the expected count is only half checked" + ); + println!( + " {} equality assertions added by the legs (24 composition + FRI \ + terminals)", + expected + ); +} + +/// The rate model's corrected pieces, WITHOUT a real epoch. +/// +/// The hash-matrix permutation-axis block that consumes these lives inside +/// `the_assembled_epoch_verifier_runs`, which needs `fibonacci.elf`. That is +/// exactly how `LFM_HASH_RATE_FELTS = 8` outlived the three-cell duplex it was +/// derived from: nothing that ran in a bare checkout touched it. This test does, +/// on shapes built by hand. +/// +/// What it pins is the correction itself — the constant's derivation, and that +/// the FRI-leaf term is rate-sensitive and still reduces to `num_committed()` at +/// keccak's rate, which is the identity that keeps the felt-side and byte-side +/// closed forms agreeing. +#[test] +fn the_candidate_rate_model_is_derived_not_remembered() { + use super::epoch_verify::{ + FRI_LEAF_FELTS, KECCAK_RATE_FELTS, LFM_HASH_RATE_FELTS, blocks_at_rate, + fri_leaf_permutations_at_rate, + }; + use super::fri::FriShape; + use super::hash::HASH_DIGEST_FELTS; + + // The chain absorbs ONE cell per step, so the rate IS the digest width. + // Written as the derivation, not as a literal, because the literal is what + // went stale. + assert_eq!(LFM_HASH_RATE_FELTS, HASH_DIGEST_FELTS); + assert_eq!(LFM_HASH_RATE_FELTS, 4, "was 8 under the deleted duplex"); + + // ⚠ The premise the old model folded the FRI leaf into: "a layer leaf fits + // one block at the candidate's rate". True at 8, FALSE at 4. + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, KECCAK_RATE_FELTS), 1); + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, 8), 1, "the old rate did fit"); + assert_eq!(blocks_at_rate(FRI_LEAF_FELTS, LFM_HASH_RATE_FELTS), 2); + + let fri = FriShape { + log2_lde_length: 20, + blowup_log: 3, + final_poly_log_degree: 3, + coset_offset: 3, + num_queries: 73, + }; + assert!(fri.num_committed() > 0, "the shape must exercise the term"); + + // At keccak's rate the new term reduces to the old `num_committed()`, which + // is why splitting it out did not move the rate-17 column. + assert_eq!( + fri_leaf_permutations_at_rate(&fri, KECCAK_RATE_FELTS), + fri.num_committed() + ); + // At the candidate's rate it doubles — the cost the old model hid. + assert_eq!( + fri_leaf_permutations_at_rate(&fri, LFM_HASH_RATE_FELTS), + 2 * fri.num_committed() + ); + + // A shape with nothing committed contributes nothing at any rate, so the + // correction cannot invent cost where there is no FRI leg. + let terminal = FriShape { + log2_lde_length: 6, + ..fri + }; + assert_eq!(terminal.num_committed(), 0); + for rate in [KECCAK_RATE_FELTS, LFM_HASH_RATE_FELTS] { + assert_eq!(fri_leaf_permutations_at_rate(&terminal, rate), 0); + } +} diff --git a/prover/src/lfm/executor.rs b/prover/src/lfm/executor.rs new file mode 100644 index 000000000..a800bc0dd --- /dev/null +++ b/prover/src/lfm/executor.rs @@ -0,0 +1,575 @@ +//! The LFM executor / witness generator. +//! +//! One `for` over the straight-line program, against write-once memory and +//! the host-supplied arenas. Produces per-chip **value-only** records — +//! addresses, selectors and multiplicities come from the program (they are +//! preprocessed data), so records carry values only, and the executor ignores +//! `mult` entirely: execution semantics never depend on it. +//! +//! Defense in depth the reference machine omits: double-writes and +//! read-before-write are checked at runtime here, independently of both the +//! compiler's tripwire panics and the admission validator. + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::compiler::LfmProgram; +use super::hash::{HASH_STATE_FELTS, LfmHasher}; +use super::instr::{Addr, BaseOp, ExtOp, HashMode, Instr, KeccakMode}; +use super::word::{LfmWord, base_word, ext_word}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LfmExecError { + DoubleWrite(u64), + ReadBeforeWrite(u64), + /// `x / 0` with `x ≠ 0` — this is also how a failed assertion surfaces. + DivByZero { + addr: u64, + }, + NonBooleanBit(u64), + /// A base-typed read found nonzero lanes 1–3 (the bus token would not + /// match any base write, so the AIR-side program would be unprovable). + NotBaseWord(u64), + /// An ext-typed read found a nonzero lane 3. + NotExtWord(u64), + /// A `KeccakF` input word lane held a value at or above `2^32`, so it is + /// not a `u32` half of a keccak lane. The chip recomposes each half from + /// four BITWISE-constrained byte columns, so no such value exists on the + /// AIR side — the program would be unprovable. + NotU32Half { + addr: u64, + lane: usize, + }, + /// A `KeccakF` input word's unused top lane (the state is 50 halves in 52 + /// slots) was nonzero; the bus pins those slots to zero as tuple constants. + KeccakSpareLaneNonZero { + addr: u64, + lane: usize, + }, + ArenaCountMismatch { + expected: usize, + found: usize, + }, + ArenaLenMismatch { + arena: u32, + expected: u32, + found: usize, + }, + ArenaOutOfBounds { + arena: u32, + index: u32, + }, + /// An `Instr::Hash` outside the selected hasher's domain, with the reason + /// the hasher gave (`LfmHasher::admits`). BLAKE3 raises both of its: a + /// `Permute` row, for which it has no socket, and a `Compress` input lane + /// at or above `2^32`, which its chip cannot decompose into bytes. In both + /// cases the program is unprovable under that hasher, so failing here — with + /// a reason — beats failing later inside the prover. + HasherRejected(&'static str), + Internal(&'static str), +} + +// ---- per-chip value records (values only; the program carries the rest) ---- + +#[derive(Debug, Clone)] +pub struct BaluRow { + pub a: FE, + pub b: FE, + pub c: FE, + pub out: FE, +} + +#[derive(Debug, Clone)] +pub struct XaluRow { + pub a: [FE; 3], + pub b: [FE; 3], + pub c: [FE; 3], + pub out: [FE; 3], +} + +#[derive(Debug, Clone)] +pub struct SelectRow { + pub bit: FE, + pub in_l: LfmWord, + pub in_r: LfmWord, + pub out_l: LfmWord, + pub out_r: LfmWord, +} + +#[derive(Debug, Clone)] +pub struct BitDecRow { + /// All 64 bit values, low-to-high (constrained witness columns). + pub bits: [FE; 64], + /// The canonicity gadget's witnesses: `z` = "top 32 bits all ones", + /// `ginv` = inverse of `(2^32 − 1) − top` when that is nonzero. + pub z: FE, + pub ginv: FE, +} + +#[derive(Debug, Clone)] +pub struct HashRow { + /// The 12 input columns: full state for `Permute`; `[a ‖ b ‖ 0⁴]` for the + /// two-to-one modes (lanes 8–11 are unconstrained on those rows — the AIR + /// injects the IV there via the mode selector). + pub ins: [FE; HASH_STATE_FELTS], + /// The full permuted state. + pub outs: [FE; HASH_STATE_FELTS], +} + +/// One `LFM_KECCAK` row. The 400 byte columns are derived from these two +/// states; the tag is the row ordinal (`layout::keccak::tag_for_row`), so it is +/// not recorded here — it is program data, not witness. +#[derive(Debug, Clone)] +pub struct KeccakRow { + pub mode: KeccakMode, + /// The state as received from memory. + pub state: [u64; 25], + /// The 136-byte rate block as received (all zero on `Permute` rows, where + /// the block columns are dead — nothing reads them). + pub block: [u8; 136], + /// What actually enters the permutation: `state` with `block` XORed into + /// its rate region on absorb rows, `state` unchanged on permute rows. + pub perm_in: [u64; 25], + pub output: [u64; 25], +} + +#[derive(Debug, Default)] +pub struct LfmRecords { + pub num_consts: usize, + pub balu: Vec, + pub xalu: Vec, + pub select: Vec, + pub bitdec: Vec, + pub hash: Vec, + pub keccak: Vec, + /// One word per Pack/Unpack row (the shared value columns). + pub lanes: Vec, + pub hint: Vec, + pub public: Vec, +} + +#[derive(Debug)] +pub struct LfmExecution { + pub records: LfmRecords, + /// The public output, in emission order: `(index, word)`. + pub public_words: Vec<(u32, LfmWord)>, + /// Final memory, exposed for tests and debugging. + pub memory: Vec>, +} + +struct Machine<'a> { + memory: Vec>, + arenas: &'a [Vec], +} + +impl Machine<'_> { + fn write(&mut self, addr: Addr, w: LfmWord) -> Result<(), LfmExecError> { + let slot = self + .memory + .get_mut(addr.0 as usize) + .ok_or(LfmExecError::Internal("address out of range"))?; + if slot.is_some() { + return Err(LfmExecError::DoubleWrite(addr.0)); + } + *slot = Some(w); + Ok(()) + } + + fn read_word(&self, addr: Addr) -> Result { + self.memory + .get(addr.0 as usize) + .cloned() + .flatten() + .ok_or(LfmExecError::ReadBeforeWrite(addr.0)) + } + + fn read_base(&self, addr: Addr) -> Result { + let w = self.read_word(addr)?; + super::word::word_as_base(&w).ok_or(LfmExecError::NotBaseWord(addr.0)) + } + + fn read_ext(&self, addr: Addr) -> Result { + let w = self.read_word(addr)?; + super::word::word_as_ext(&w).ok_or(LfmExecError::NotExtWord(addr.0)) + } +} + +pub fn execute( + program: &LfmProgram, + arenas: &[Vec], + hasher: &impl LfmHasher, +) -> Result { + let schema = &program.arena_schema.lens; + if arenas.len() != schema.len() { + return Err(LfmExecError::ArenaCountMismatch { + expected: schema.len(), + found: arenas.len(), + }); + } + for (i, (arena, &len)) in arenas.iter().zip(schema).enumerate() { + if arena.len() != len as usize { + return Err(LfmExecError::ArenaLenMismatch { + arena: i as u32, + expected: len, + found: arena.len(), + }); + } + } + + let mut m = Machine { + memory: vec![None; program.num_addrs as usize], + arenas, + }; + let mut records = LfmRecords::default(); + let mut public_words = Vec::new(); + + for instr in &program.instrs { + match instr { + Instr::Const { out, value, .. } => { + m.write(*out, *value)?; + records.num_consts += 1; + } + Instr::BaseAlu { + op, out, a, b, c, .. + } => { + let av = m.read_base(*a)?; + let bv = m.read_base(*b)?; + let cv = if *op == BaseOp::MulAdd { + m.read_base(*c)? + } else { + FE::zero() + }; + let ov = match op { + BaseOp::Add => &av + &bv, + BaseOp::Sub => &av - &bv, + BaseOp::Mul => &av * &bv, + BaseOp::Div => { + if bv == FE::zero() { + if av == FE::zero() { + FE::one() // the 0/0 = 1 convention + } else { + return Err(LfmExecError::DivByZero { addr: a.0 }); + } + } else { + &av * &bv.inv().map_err(|_| LfmExecError::Internal("base inv"))? + } + } + BaseOp::MulAdd => &av * &bv + &cv, + }; + m.write(*out, base_word(ov))?; + records.balu.push(BaluRow { + a: av, + b: bv, + c: cv, + out: ov, + }); + } + Instr::ExtAlu { + op, out, a, b, c, .. + } => { + let ae = m.read_ext(*a)?; + let (be, bv_base) = if *op == ExtOp::MulBase { + let bb = m.read_base(*b)?; + (FEE::zero(), Some(bb)) + } else { + (m.read_ext(*b)?, None) + }; + let ce = if *op == ExtOp::MulAdd { + m.read_ext(*c)? + } else { + FEE::zero() + }; + let oe = match op { + ExtOp::Add => &ae + &be, + ExtOp::Sub => &ae - &be, + ExtOp::Mul => &ae * &be, + ExtOp::Div => { + if be == FEE::zero() { + if ae == FEE::zero() { + FEE::one() // 0/0 = (1, 0, 0) + } else { + return Err(LfmExecError::DivByZero { addr: a.0 }); + } + } else { + &ae * &be.inv().map_err(|_| LfmExecError::Internal("ext inv"))? + } + } + ExtOp::MulAdd => &ae * &be + &ce, + ExtOp::MulBase => { + let bb = bv_base.ok_or(LfmExecError::Internal("mulbase"))?; + let [a0, a1, a2] = *ae.value(); + FEE::new([&a0 * &bb, &a1 * &bb, &a2 * &bb]) + } + }; + m.write(*out, ext_word(&oe))?; + let lanes = |e: &FEE| -> [FE; 3] { *e.value() }; + records.xalu.push(XaluRow { + a: lanes(&ae), + b: bv_base.map_or_else(|| lanes(&be), |bb| [bb, FE::zero(), FE::zero()]), + c: lanes(&ce), + out: lanes(&oe), + }); + } + Instr::Select { + bit, + out_l, + out_r, + in_l, + in_r, + .. + } => { + let bv = m.read_base(*bit).map_err(|e| match e { + LfmExecError::NotBaseWord(a) => LfmExecError::NonBooleanBit(a), + other => other, + })?; + let l = m.read_word(*in_l)?; + let r = m.read_word(*in_r)?; + let (ol, or) = if bv == FE::zero() { + (l, r) + } else if bv == FE::one() { + (r, l) + } else { + return Err(LfmExecError::NonBooleanBit(bit.0)); + }; + m.write(*out_l, ol)?; + m.write(*out_r, or)?; + records.select.push(SelectRow { + bit: bv, + in_l: l, + in_r: r, + out_l: ol, + out_r: or, + }); + } + Instr::BitDec { input, bits } => { + let v = m.read_base(*input)?; + let canon = GoldilocksField::canonical(v.value()); + let bit_vals: [FE; 64] = core::array::from_fn(|i| FE::from((canon >> i) & 1)); + let top = (canon >> 32) as u32; + let g = 0xFFFF_FFFFu64 - top as u64; + let (z, ginv) = if g == 0 { + (FE::one(), FE::zero()) + } else { + ( + FE::zero(), + FE::from(g) + .inv() + .map_err(|_| LfmExecError::Internal("bitdec ginv"))?, + ) + }; + for (i, (addr, _)) in bits.iter().enumerate() { + m.write(*addr, base_word(bit_vals[i]))?; + } + records.bitdec.push(BitDecRow { + bits: bit_vals, + z, + ginv, + }); + } + Instr::Hash { + mode, ins, outs, .. + } => { + let mut state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|_| FE::zero()); + let mut in_cols: [FE; HASH_STATE_FELTS] = core::array::from_fn(|_| FE::zero()); + if mode.num_input_cells() == 2 { + // Two cells, whatever they MEAN: two digests under Compress + // and Transcript, a chaining accumulator and four field + // elements under Leaf. What each cell is read AS belongs to + // the hasher and to the chip's lane split; what the executor + // owes is the memory reads the `LfmMem` receives claim, and + // those are the same two under all three. + let a = m.read_word(ins[0])?; + let b = m.read_word(ins[1])?; + state[0..4].clone_from_slice(&a); + state[4..8].clone_from_slice(&b); + state[8..12].clone_from_slice(&hasher.compress_iv()); + in_cols[0..4].clone_from_slice(&a); + in_cols[4..8].clone_from_slice(&b); + // lanes 8–11 of the IN columns stay zero on two-cell rows + } else { + for (cell, chunk) in ins.iter().zip(state.chunks_exact_mut(4)) { + chunk.clone_from_slice(&m.read_word(*cell)?); + } + in_cols = state; + } + // A hasher whose socket does not cover this row says so here, + // with a reason, rather than producing a witness no AIR accepts. + hasher + .admits(*mode, &state) + .map_err(LfmExecError::HasherRejected)?; + let out_state = match mode { + // Through `compress_out`/`transcript_out`, NOT `permute`: a + // hasher that overrides the two-to-one construction — + // BLAKE3 does, its IV entering through `h` rather than the + // capacity lanes, and its transcript domain differing from + // its Merkle one — must have both overrides reach the `OUT` + // columns. + HashMode::Compress | HashMode::Transcript => { + let a: LfmWord = core::array::from_fn(|i| state[i]); + let b: LfmWord = core::array::from_fn(|i| state[4 + i]); + if *mode == HashMode::Compress { + hasher.compress_out(&a, &b) + } else { + hasher.transcript_out(&a, &b) + } + } + HashMode::Leaf => { + let acc: LfmWord = core::array::from_fn(|i| state[i]); + let f: LfmWord = core::array::from_fn(|i| state[4 + i]); + hasher.leaf_out(&acc, &f) + } + HashMode::Permute => hasher.permute(state), + }; + if mode.num_output_cells() == 1 { + let digest: LfmWord = core::array::from_fn(|i| out_state[i]); + m.write(outs[0], digest)?; + } else { + for (cell, chunk) in outs.iter().zip(out_state.chunks_exact(4)) { + let w: LfmWord = core::array::from_fn(|i| chunk[i]); + m.write(*cell, w)?; + } + } + records.hash.push(HashRow { + ins: in_cols, + outs: out_state, + }); + } + Instr::KeccakF(op) => { + use super::layout::keccak as k; + // 13 words × 4 lanes → 50 u32 halves (+ 2 must-be-zero slots). + let mut halves = [0u32; k::NUM_HALVES]; + for (j, cell) in op.ins.iter().enumerate() { + let w = m.read_word(*cell)?; + for (l, lane) in w.iter().enumerate() { + let h = 4 * j + l; + let v = GoldilocksField::canonical(lane.value()); + if h >= k::NUM_HALVES { + if v != 0 { + return Err(LfmExecError::KeccakSpareLaneNonZero { + addr: cell.0, + lane: l, + }); + } + } else if v >= 1u64 << 32 { + return Err(LfmExecError::NotU32Half { + addr: cell.0, + lane: l, + }); + } else { + halves[h] = v as u32; + } + } + } + let state = super::keccak_adapter::halves_to_state(&halves); + + // Absorb: XOR the rate block into the state's first 136 bytes. + // Block byte k is byte k % 8 of lane k / 8, which is exactly + // state byte offset k — rate bytes are lane-major and + // little-endian within a lane, same as the byte columns. + let mut block = [0u8; k::RATE_BYTES]; + let mut perm_in = state; + if op.mode == KeccakMode::Absorb { + let mut bh = [0u32; k::BLOCK_HALVES]; + for (j, cell) in op.block.iter().enumerate() { + let w = m.read_word(*cell)?; + for (l, lane) in w.iter().enumerate() { + let h = 4 * j + l; + let v = GoldilocksField::canonical(lane.value()); + if h >= k::BLOCK_HALVES { + if v != 0 { + return Err(LfmExecError::KeccakSpareLaneNonZero { + addr: cell.0, + lane: l, + }); + } + } else if v >= 1u64 << 32 { + return Err(LfmExecError::NotU32Half { + addr: cell.0, + lane: l, + }); + } else { + bh[h] = v as u32; + } + } + } + for (h, half) in bh.iter().enumerate() { + block[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + for lane in 0..k::RATE_LANES { + let mut chunk = [0u8; 8]; + chunk.copy_from_slice(&block[lane * 8..lane * 8 + 8]); + perm_in[lane] ^= u64::from_le_bytes(chunk); + } + } + + let output = super::keccak_adapter::permute(perm_in); + for (cell, w) in op + .outs + .iter() + .zip(super::keccak_adapter::state_to_words(&output)) + { + m.write(*cell, w)?; + } + if let Some(rev) = &op.rev { + let words = super::keccak_adapter::reversed_digest_words(&output); + for (cell, w) in rev.outs.iter().zip(words) { + m.write(*cell, w)?; + } + } + records.keccak.push(KeccakRow { + mode: op.mode, + state, + block, + perm_in, + output, + }); + } + Instr::Hint { + arena, index, out, .. + } => { + let words = + m.arenas + .get(*arena as usize) + .ok_or(LfmExecError::ArenaOutOfBounds { + arena: *arena, + index: *index, + })?; + let w = *words + .get(*index as usize) + .ok_or(LfmExecError::ArenaOutOfBounds { + arena: *arena, + index: *index, + })?; + m.write(*out, w)?; + records.hint.push(w); + } + Instr::Pack { lanes, out, .. } => { + let mut word = [FE::zero(), FE::zero(), FE::zero(), FE::zero()]; + for (i, lane) in lanes.iter().enumerate() { + word[i] = m.read_base(*lane)?; + } + m.write(*out, word)?; + records.lanes.push(word); + } + Instr::Unpack { input, outs, .. } => { + let word = m.read_word(*input)?; + for (i, out) in outs.iter().enumerate() { + m.write(*out, base_word(word[i]))?; + } + records.lanes.push(word); + } + Instr::Public { addr, index } => { + let w = m.read_word(*addr)?; + records.public.push(w); + public_words.push((*index, w)); + } + } + } + + Ok(LfmExecution { + records, + public_words, + memory: m.memory, + }) +} diff --git a/prover/src/lfm/fixture.rs b/prover/src/lfm/fixture.rs new file mode 100644 index 000000000..00bf5e167 --- /dev/null +++ b/prover/src/lfm/fixture.rs @@ -0,0 +1,384 @@ +//! The Milestone-C inner-proof fixture: a host-side FRI commitment-opening +//! prover the machine verifies. +//! +//! Structurally real, deliberately small: coset LDE domains (offset 3, the +//! production pin), row-pair Merkle leaves, per-layer commitments, the +//! **unnormalized fold** convention (`(lo+hi) + inv_x·ζ·(lo−hi)`), a compress-chain +//! transcript over the machine's own hash, query indices sampled at a +//! power-of-two bound, and a terminal polynomial checked at the queried +//! points. What it is NOT: the production 25-AIR proof format — that lands +//! when the ecosystem hash decision unblocks the real machine-facing +//! pipeline (`crypto/stark` hardcodes keccak at its Merkle layer; the +//! measured 26-site migration seam is deliberately not touched here). +//! +//! Everything here mirrors `edsl.rs` bit-exactly; the emitted verifier +//! program (`programs::fri_toy_program`) consumes exactly the arena layout +//! `fixture_prove` produces. + +use math::field::traits::{IsFFTField, IsPrimeField}; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::edsl::SQUEEZE_MARK; +use super::hash::{HasherKind, LfmHasher}; +use super::word::{LfmWord, base_word, ext_word}; + +/// The fixed shape — compile-time constants of the emitted program. +pub mod shape { + /// log2 of the LDE domain size. + pub const LOG_LDE: usize = 5; // 32 points + pub const LDE_SIZE: usize = 1 << LOG_LDE; + /// Trace length 8 = LDE/blowup (blowup 4). + pub const TRACE_LEN: usize = 8; + /// Committed base columns (one machine word per row). + pub const NUM_COLS: usize = 4; + /// Two folds: 32 → 16 → 8, terminal degree < 2. + pub const NUM_LAYERS: usize = 2; + pub const TERMINAL_LEN: usize = 2; + pub const NUM_QUERIES: usize = 4; + /// Query index bits (indices sampled in [0, LDE/2)). + pub const QUERY_BITS: usize = LOG_LDE - 1; + /// The production coset offset. + pub const COSET_OFFSET: u64 = 3; + /// Words per query in the openings arena. + pub const WORDS_PER_QUERY: usize = 17; +} + +/// Host mirror of [`super::edsl::SpongeVar`] — the compress chain, state 1 +/// cell. +/// +/// Bit-exact by construction, not by coincidence: every operation here is the +/// same sequence of [`LfmHasher::transcript`] calls the emitted program makes +/// of `LFM_HASH`, in the same order, on the same operands. The two are rewritten +/// together for exactly that reason; a divergence would show up as a fixture +/// proof the machine rejects, which is a slow and confusing way to learn about +/// it. +/// +/// Parameterised by hasher because the transcript is: `Test` and `Poseidon` +/// hash a transcript step with their single domain, BLAKE3 with the `"LFMT"` +/// tag, and the host has to agree with whichever one the proof is under. +pub struct HostSponge { + state: LfmWord, + squeeze_index: u32, + hasher: HasherKind, +} + +impl Default for HostSponge { + fn default() -> Self { + Self::new() + } +} + +impl HostSponge { + /// The chain under the machine's default hasher. + pub fn new() -> Self { + Self::with_hasher(HasherKind::default()) + } + + pub fn with_hasher(hasher: HasherKind) -> Self { + HostSponge { + state: [FE::zero(); 4], + squeeze_index: 0, + hasher, + } + } + + /// `SQ(i) = [SQUEEZE_MARK, i, 0, 0]` — the advance operand. + pub fn squeeze_operand(i: u32) -> LfmWord { + [ + FE::from(u64::from(SQUEEZE_MARK)), + FE::from(u64::from(i)), + FE::zero(), + FE::zero(), + ] + } + + /// The state as it stands — for the KATs, which pin it after every step. + pub fn state(&self) -> LfmWord { + self.state + } + + pub fn absorb(&mut self, c: &LfmWord) { + self.state = self.hasher.transcript(&self.state, c); + } + + pub fn absorb2(&mut self, c0: &LfmWord, c1: &LfmWord) { + self.absorb(c0); + self.absorb(c1); + } + + /// Absorb a cell of four arbitrary FIELD ELEMENTS — the host mirror of + /// [`super::edsl::SpongeVar::absorb_felts`]. Data enters the transcript + /// through the leaf encoding, exactly as it enters a tree. + pub fn absorb_felts(&mut self, c: &LfmWord) { + let d = self.hasher.leaf(&leaf_chain_start(), c); + self.absorb(&d); + } + + /// Output the current state, then advance past it with `SQ(i)`. + pub fn squeeze_cell(&mut self) -> LfmWord { + let out = self.state; + let sq = Self::squeeze_operand(self.squeeze_index); + self.state = self.hasher.transcript(&self.state, &sq); + self.squeeze_index += 1; + out + } + + pub fn squeeze_ext(&mut self) -> FEE { + let c = self.squeeze_cell(); + FEE::new([c[0], c[1], c[2]]) + } + + pub fn squeeze_index(&mut self, nbits: usize) -> u64 { + let c = self.squeeze_cell(); + GoldilocksField::canonical(c[0].value()) & ((1 << nbits) - 1) + } +} + +/// Where a leaf chain starts — the host mirror of +/// [`super::edsl::leaf_chain_start`], and a chain START rather than a shape +/// HEADER for the reason stated there. +pub fn leaf_chain_start() -> LfmWord { + [FE::zero(); 4] +} + +/// The host's Merkle LEAF over a pair of DATA cells — the mirror of +/// [`super::edsl::leaf_hash_pair`]. +/// +/// Two hasher calls, in the machine's order: one `"LFML"` chain absorbing the +/// cells in sequence, four felts and one chaining step per call. Written beside +/// the tree rather than inside it because a tree's *leaves* are data and its +/// *nodes* are digests, and this is the one place that distinction becomes two +/// different hash domains. +pub fn host_leaf_hash_pair(hasher: HasherKind, c0: &LfmWord, c1: &LfmWord) -> LfmWord { + let acc = hasher.leaf(&leaf_chain_start(), c0); + hasher.leaf(&acc, c1) +} + +/// A binary Merkle tree over word digests. +/// +/// ⚠ Parameterised by hasher, and it has to be: the machine authenticates these +/// openings with `edsl::merkle_walk`, which compresses under whichever hasher +/// the proof is built with. A tree that hard-coded one hasher would produce +/// roots the machine cannot reproduce the moment the proof is under another — +/// the failure would surface as an authentication error deep in a query walk, +/// which is a slow way to learn about a fixture bug. +pub struct HostTree { + /// levels[0] = leaves … levels.last() = [root]. + pub levels: Vec>, +} + +impl HostTree { + pub fn build(hasher: HasherKind, leaves: Vec) -> Self { + assert!(leaves.len().is_power_of_two()); + let mut levels = vec![leaves]; + while levels.last().unwrap().len() > 1 { + let prev = levels.last().unwrap(); + let next: Vec = prev + .chunks_exact(2) + .map(|pair| hasher.compress(&pair[0], &pair[1])) + .collect(); + levels.push(next); + } + HostTree { levels } + } + + pub fn root(&self) -> LfmWord { + self.levels.last().unwrap()[0] + } + + /// Sibling digests along the path from leaf `index`, level 0 first. + pub fn open(&self, mut index: usize) -> Vec { + let mut siblings = Vec::new(); + for level in &self.levels[..self.levels.len() - 1] { + siblings.push(level[index ^ 1]); + index >>= 1; + } + siblings + } +} + +/// The fixture proof, already in the machine's arena layout: +/// arena 0 = `[main_root, l1_root, t0, t1]`; arena 1 = per-query openings +/// (`shape::WORDS_PER_QUERY` words each, order pinned by the emitter). +pub struct FriToyProof { + pub commitments: Vec, + pub openings: Vec, +} + +/// The committed columns: fixed low-degree polynomials evaluated over the +/// LDE coset. Deterministic — the honest witness. +pub fn fixture_columns() -> [Vec; shape::NUM_COLS] { + let omega = GoldilocksField::get_primitive_root_of_unity(shape::LOG_LDE as u64) + .expect("32nd root of unity"); + let offset = FE::from(shape::COSET_OFFSET); + core::array::from_fn(|k| { + // degree < TRACE_LEN coefficients, fixed per column. + let coeffs: Vec = (0..shape::TRACE_LEN) + .map(|j| FE::from(1_000 * (k as u64 + 1) + j as u64 + 1)) + .collect(); + (0..shape::LDE_SIZE) + .map(|i| { + let x = &offset * omega.pow(i as u64); + coeffs.iter().rev().fold(FE::zero(), |acc, c| acc * &x + c) + }) + .collect() + }) +} + +fn row_word(cols: &[Vec; shape::NUM_COLS], i: usize) -> LfmWord { + core::array::from_fn(|k| cols[k][i]) +} + +/// Runs the fixture prover over the honest columns, under the machine's +/// default hasher. +pub fn fixture_prove() -> FriToyProof { + fixture_prove_columns(&fixture_columns()) +} + +/// [`fixture_prove`] under an explicitly chosen hasher. +pub fn fixture_prove_with_hasher(hasher: HasherKind) -> FriToyProof { + fixture_prove_columns_with_hasher(&fixture_columns(), hasher) +} + +/// The prover proper, over arbitrary columns (tests tamper these). +pub fn fixture_prove_columns(cols: &[Vec; shape::NUM_COLS]) -> FriToyProof { + fixture_prove_columns_with_hasher(cols, HasherKind::default()) +} + +/// [`fixture_prove_columns`] under an explicitly chosen hasher. +/// +/// Every hash this performs — leaves, tree nodes and the transcript — goes +/// through `hasher`, so the proof it produces is one the machine can +/// authenticate when proved under the same choice, and only then. +pub fn fixture_prove_columns_with_hasher( + cols: &[Vec; shape::NUM_COLS], + hasher: HasherKind, +) -> FriToyProof { + let omega = GoldilocksField::get_primitive_root_of_unity(shape::LOG_LDE as u64) + .expect("32nd root of unity"); + let offset = FE::from(shape::COSET_OFFSET); + let half = shape::LDE_SIZE / 2; // 16 + + // Main tree: row-pair leaves. A leaf is DATA, so it hashes in the leaf + // domain — two LFML rows and an LFMC parent, mirroring the emitter. + let leaves: Vec = (0..shape::LDE_SIZE / 2) + .map(|l| host_leaf_hash_pair(hasher, &row_word(cols, 2 * l), &row_word(cols, 2 * l + 1))) + .collect(); + let main_tree = HostTree::build(hasher, leaves); + + let mut sponge = HostSponge::with_hasher(hasher); + sponge.absorb(&main_tree.root()); + let alpha = sponge.squeeze_ext(); + let zeta0 = sponge.squeeze_ext(); + + // g0 = α-combination of the columns, over the full LDE domain. + let g0: Vec = (0..shape::LDE_SIZE) + .map(|i| { + let row = row_word(cols, i); + row.iter().rev().fold(FEE::zero(), |acc, v| { + acc * &alpha + FEE::new([*v, FE::zero(), FE::zero()]) + }) + }) + .collect(); + + // Fold 0 (unnormalized): g1[j] = (g0[j]+g0[j+16]) + x_j⁻¹·ζ0·(g0[j]−g0[j+16]). + let g1: Vec = (0..half) + .map(|j| { + let x = &offset * omega.pow(j as u64); + let inv_x = x.inv().expect("nonzero domain point"); + let (lo, hi) = (&g0[j], &g0[j + half]); + (lo + hi) + (&zeta0 * (lo - hi)) * FEE::new([inv_x, FE::zero(), FE::zero()]) + }) + .collect(); + + // L1 tree co-locates fold partners: leaf j covers g1[j] and g1[j+8]. These + // are folded EXTENSION elements — arbitrary field data — so they are leaves + // in exactly the same sense the trace rows are. + let quarter = half / 2; // 8 + let l1_leaves: Vec = (0..quarter) + .map(|j| host_leaf_hash_pair(hasher, &ext_word(&g1[j]), &ext_word(&g1[j + quarter]))) + .collect(); + let l1_tree = HostTree::build(hasher, l1_leaves); + + sponge.absorb(&l1_tree.root()); + let zeta1 = sponge.squeeze_ext(); + + // Fold 1 over the size-16 domain c²·⟨ω²⟩: y_j = c²ω^{2j}. + let g2: Vec = (0..quarter) + .map(|j| { + let y = offset.square() * omega.pow(2 * j as u64); + let inv_y = y.inv().expect("nonzero domain point"); + let (lo, hi) = (&g1[j], &g1[j + quarter]); + (lo + hi) + (&zeta1 * (lo - hi)) * FEE::new([inv_y, FE::zero(), FE::zero()]) + }) + .collect(); + + // Terminal polynomial (degree < 2) over c⁴·⟨ω⁴⟩, from two points; the + // remaining points must agree — the honest-witness sanity check. + let y_a = offset.square().square(); + let y_b = &y_a * omega.pow(4u64); + let embed = |x: &FE| FEE::new([*x, FE::zero(), FE::zero()]); + let t1 = (&g2[1] - &g2[0]) * (embed(&(&y_b - &y_a))).inv().expect("distinct points"); + let t0 = &g2[0] - &t1 * embed(&y_a); + for (j, v) in g2.iter().enumerate() { + let y = &y_a * omega.pow(4 * j as u64); + debug_assert_eq!(*v, &t0 + &t1 * embed(&y), "terminal degree bound violated"); + } + + // t0 and t1 are the terminal polynomial's COEFFICIENTS — field data, not + // digests — so they enter the transcript through the leaf encoding. + sponge.absorb_felts(&ext_word(&t0)); + sponge.absorb_felts(&ext_word(&t1)); + + // Queries. + let mut openings = Vec::new(); + for _ in 0..shape::NUM_QUERIES { + let q0 = sponge.squeeze_index(shape::QUERY_BITS) as usize; // [0, 16) + let leaf_a = q0 >> 1; + let leaf_b = leaf_a + shape::LDE_SIZE / 4; // + 8 + + // Main leaf A: its two rows + path. + openings.push(row_word(cols, 2 * leaf_a)); + openings.push(row_word(cols, 2 * leaf_a + 1)); + openings.extend(main_tree.open(leaf_a)); + // Main leaf B. + openings.push(row_word(cols, 2 * leaf_b)); + openings.push(row_word(cols, 2 * leaf_b + 1)); + openings.extend(main_tree.open(leaf_b)); + // L1 leaf pair + path. + let j = q0 % quarter; + openings.push(ext_word(&g1[j])); + openings.push(ext_word(&g1[j + quarter])); + openings.extend(l1_tree.open(j)); + } + debug_assert_eq!(openings.len(), shape::NUM_QUERIES * shape::WORDS_PER_QUERY); + + FriToyProof { + commitments: vec![ + main_tree.root(), + l1_tree.root(), + ext_word(&t0), + ext_word(&t1), + ], + openings, + } +} + +/// A word with lane 0 bumped — the tamper helper. +pub fn bump_lane0(w: &LfmWord) -> LfmWord { + [&w[0] + FE::one(), w[1], w[2], w[3]] +} + +/// Re-exported for the emitter: ω and the coset offset as constants. +pub fn domain_constants() -> (FE, FE) { + let omega = GoldilocksField::get_primitive_root_of_unity(shape::LOG_LDE as u64) + .expect("32nd root of unity"); + (omega, FE::from(shape::COSET_OFFSET)) +} + +// Small helper so `base_word` isn't unused when records are built elsewhere. +#[allow(dead_code)] +fn _base(v: FE) -> LfmWord { + base_word(v) +} diff --git a/prover/src/lfm/framework_probe.rs b/prover/src/lfm/framework_probe.rs new file mode 100644 index 000000000..6368c2030 --- /dev/null +++ b/prover/src/lfm/framework_probe.rs @@ -0,0 +1,218 @@ +//! B0 de-risk probe (Milestone B entry gate). +//! +//! Every LFM chip stakes its instruction column group on one framework +//! pattern no in-tree chip exercises today: a **preprocessed column used as a +//! LogUp `Multiplicity`** (plus preprocessed bus values, which KECCAK_RC does +//! exercise). This probe round-trips a minimal sender/receiver pair through +//! the real `multi_prove` / `multi_verify_views`, with the sender's value and +//! multiplicity columns both preprocessed, and pins the tamper behavior: +//! a flipped preprocessed root is rejected by the prover (recommit mismatch) +//! and by the verifier (root equality), and a tampered witness value breaks +//! the bus balance. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::config::Commitment; +use stark::constraints::builder::EmptyConstraints; +use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, +}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::prover::{IsStarkProver, Prover}; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::commit::commit_columns; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type ProbeAir = AirWithBuses; +type DynAir<'a> = &'a dyn AIR; + +/// Scratch bus id, far above the live `BusId` range. +const PROBE_BUS: u64 = 63; +const PROBE_TAG: &[u8] = b"LFM_B0_PROBE_V1"; +const NUM_ROWS: usize = 256; + +fn fe(v: u64) -> FE { + FE::from(v) +} + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("probe options") +} + +fn values() -> Vec { + (0..NUM_ROWS as u64).map(|i| fe(1_000 + 3 * i)).collect() +} + +/// Sender: cols `[VAL (prep 0), MULT (prep 1), PAD (main 2)]` — both the bus +/// value AND the multiplicity read preprocessed columns. +fn sender_air(prep_root: Commitment, opts: &ProofOptions) -> ProbeAir { + let interactions = vec![BusInteraction::sender( + PROBE_BUS, + Multiplicity::Column(1), + vec![BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }], + )]; + AirWithBuses::new( + 3, + AuxiliaryTraceBuildData { interactions }, + opts, + 1, + EmptyConstraints, + ) + .with_name("B0_SEND") + .with_preprocessed(prep_root, 2) +} + +/// Receiver: cols `[VAL (main 0), MULT (main 1)]` — plain witness echo. +fn receiver_air(opts: &ProofOptions) -> ProbeAir { + let interactions = vec![BusInteraction::receiver( + PROBE_BUS, + Multiplicity::Column(1), + vec![BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }], + )]; + AirWithBuses::new( + 2, + AuxiliaryTraceBuildData { interactions }, + opts, + 1, + EmptyConstraints, + ) + .with_name("B0_RECV") +} + +fn sender_trace() -> TraceTable { + let mut data = Vec::with_capacity(NUM_ROWS * 3); + for v in values() { + data.extend([v, FE::one(), FE::zero()]); + } + TraceTable::new_main(data, 3, 1) +} + +fn receiver_trace() -> TraceTable { + let mut data = Vec::with_capacity(NUM_ROWS * 2); + for v in values() { + data.extend([v, FE::one()]); + } + TraceTable::new_main(data, 2, 1) +} + +fn prep_root(opts: &ProofOptions) -> Commitment { + commit_columns(&[values(), vec![FE::one(); NUM_ROWS]], opts) +} + +fn transcript() -> DefaultTranscript { + let mut t = DefaultTranscript::::new(&[]); + t.append_bytes(PROBE_TAG); + t +} + +fn prove( + sender: &ProbeAir, + receiver: &ProbeAir, +) -> Result, stark::prover::ProvingError> { + let mut st = sender_trace(); + let mut rt = receiver_trace(); + let pairs: Vec<(DynAir, &mut TraceTable, &())> = + vec![(sender, &mut st, &()), (receiver, &mut rt, &())]; + let mut t = transcript(); + Prover::multi_prove( + pairs, + &mut t, + #[cfg(feature = "disk-spill")] + Default::default(), + stark::residency_mode::ResidencyMode::Retain, + ) +} + +#[test] +fn b0_preprocessed_multiplicity_round_trips() { + let opts = options(); + let root = prep_root(&opts); + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + let proof = prove(&sender, &receiver).expect("prove with preprocessed multiplicity"); + + let refs: Vec = vec![&sender, &receiver]; + let mut vt = transcript(); + assert!( + Verifier::multi_verify_views(&refs, MultiProofView::Owned(&proof), &mut vt, &FEE::zero(),), + "honest proof must verify" + ); +} + +#[test] +fn b0_prover_rejects_mismatched_preprocessed_root() { + let opts = options(); + let mut root = prep_root(&opts); + root[0] ^= 1; + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + assert!( + prove(&sender, &receiver).is_err(), + "prover must reject a trace that does not recommit to the supplied root" + ); +} + +#[test] +fn b0_verifier_rejects_wrong_preprocessed_root() { + let opts = options(); + let root = prep_root(&opts); + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + let proof = prove(&sender, &receiver).expect("honest prove"); + + let mut bad_root = root; + bad_root[0] ^= 1; + let bad_sender = sender_air(bad_root, &opts); + let refs: Vec = vec![&bad_sender, &receiver]; + let mut vt = transcript(); + assert!( + !Verifier::multi_verify_views(&refs, MultiProofView::Owned(&proof), &mut vt, &FEE::zero(),), + "a supplied root differing from the proof's must reject" + ); +} + +#[test] +fn b0_tampered_witness_value_breaks_balance() { + let opts = options(); + let root = prep_root(&opts); + let sender = sender_air(root, &opts); + let receiver = receiver_air(&opts); + + // Receiver echoes one wrong value: prove succeeds locally (no constraint + // relates the two tables directly) but the bus no longer balances to 0. + let mut st = sender_trace(); + let mut rt = receiver_trace(); + rt.set_main(0, 0, fe(999_999)); + let pairs: Vec<(DynAir, &mut TraceTable, &())> = + vec![(&sender, &mut st, &()), (&receiver, &mut rt, &())]; + let mut t = transcript(); + let proof = Prover::multi_prove( + pairs, + &mut t, + #[cfg(feature = "disk-spill")] + Default::default(), + stark::residency_mode::ResidencyMode::Retain, + ) + .expect("locally consistent"); + + let refs: Vec = vec![&sender, &receiver]; + let mut vt = transcript(); + assert!( + !Verifier::multi_verify_views(&refs, MultiProofView::Owned(&proof), &mut vt, &FEE::zero(),), + "unbalanced bus must reject" + ); +} diff --git a/prover/src/lfm/fri.rs b/prover/src/lfm/fri.rs new file mode 100644 index 000000000..9adfb8d7e --- /dev/null +++ b/prover/src/lfm/fri.rs @@ -0,0 +1,639 @@ +//! FRI: the compile-time shape of the emitted verifier. +//! +//! Slice 1 of the FRI leg — the arithmetic only. `others/lfm-fri-verify-spec.md` +//! is the verified account of the production verify path this mirrors; §2 is +//! the section this file implements. +//! +//! ## Why the shape is a struct and not a runtime computation +//! +//! Production derives the fold layout at verify time from the AIR's options and +//! domain (`FriFoldLayout::new`, `fri/terminal.rs:45`). The machine cannot: it +//! is straight-line, so the layer count fixes how many walks and folds are +//! EMITTED. Every field below is therefore program shape in the sense of +//! `others/lfm-target-shape.md`, and a program that read any of it from an +//! arena would let the prover choose how much FRI to verify — the degenerate +//! case being "none". +//! +//! ## What this module is checked against +//! +//! `FriFoldLayout` is `pub(crate)` inside `crypto/stark`, so this mirror cannot +//! be differentialled against the struct itself. The oracle is production's +//! observable BEHAVIOUR instead — the vector lengths a real proof carries and +//! the verifier structurally enforces before its query loop +//! (`verifier.rs:426-448`): `fri_layers_merkle_roots.len() == num_committed` +//! and `fri_final_poly_coeffs.len() == 1 << effective_k`. That is a stronger +//! check than reading the struct would be, because those are the lengths the +//! verifier actually rejects on. +//! +//! ## What this cannot see +//! +//! It is arithmetic over a shape; it says nothing about whether the emitted +//! walk or fold is correct, only about how many of each there should be. It +//! also mirrors the CPU layout only — `fri/mod.rs` has cuda fast paths that +//! claim the same layout, unverified here and never run by the machine. + +use stark::proof::options::ProofOptions; + +use crate::tables::types::FE; + +use super::builder::{Bit, Ext, Felt, LfmBuilder}; +use super::edsl::{self, KeccakDigest}; +use super::instr::ArenaId; +use super::sub_proof::{self, GroupShape}; + +/// The compile-time shape of one sub-proof's FRI verification. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FriShape { + /// `log2` of the LDE (deep-composition) codeword length. + pub log2_lde_length: u32, + /// `log2` of the blowup factor. + pub blowup_log: u32, + /// The requested terminal log-degree, `ProofOptions::fri_final_poly_log_degree`. + pub final_poly_log_degree: u32, + /// The LDE coset offset. Carried rather than assumed: the emitter bakes + /// domain constants derived from it, and the standing deferral on + /// `coset_offset != 3` is about test COVERAGE, not about a hardcoded 3 — + /// so the value has to come from the options, and [`Self::from_options`] is + /// the only constructor that reads it. + pub coset_offset: u64, + /// Queries the sub-proof carries. + pub num_queries: usize, +} + +impl FriShape { + /// Derive the shape from the inner proof's own options. + /// + /// Every FRI-relevant parameter comes from `options` — including the coset + /// offset, which discharges the plumbing half of the `coset_offset != 3` + /// deferral recorded in `others/lfm-assembly-obligations.md`. + pub fn from_options(options: &ProofOptions, log2_lde_length: u32) -> Self { + Self { + log2_lde_length, + blowup_log: (options.blowup_factor as u32).trailing_zeros(), + final_poly_log_degree: options.fri_final_poly_log_degree as u32, + coset_offset: options.coset_offset, + num_queries: options.fri_number_of_queries, + } + } + + /// `log2` of the terminal codeword length, clamped to the full LDE for + /// traces too small to fold that far (`terminal.rs:46`'s `.min(lde_log)`). + pub fn terminal_log(self) -> u32 { + (self.blowup_log + self.final_poly_log_degree).min(self.log2_lde_length) + } + + /// Folds from the LDE codeword down to the terminal codeword. + pub fn total_folds(self) -> u32 { + self.log2_lde_length - self.terminal_log() + } + + /// Committed (Merkle-rooted) layers — one root, one auth path per query, + /// and one Merkle walk to emit, each. + /// + /// **`total_folds − 1`, not `total_folds`.** The final fold is performed + /// and never committed (`fri/mod.rs:114-118`), so a query folds once more + /// than it authenticates. This off-by-one is the readiest way to build a + /// verifier that looks right and checks one layer too few. + pub fn num_committed(self) -> usize { + self.total_folds().saturating_sub(1) as usize + } + + /// Folds a query performs: `num_committed + 1` whenever anything folds at + /// all, and 0 when the codeword is already terminal. + pub fn num_folds(self) -> usize { + self.total_folds() as usize + } + + /// Terminal codeword length. + pub fn terminal_len(self) -> usize { + 1usize << self.terminal_log() + } + + /// The terminal log-degree actually used — `min(k, trace_bits)`. Equals + /// `final_poly_log_degree` except under the clamp. + pub fn effective_k(self) -> u32 { + self.terminal_log() - self.blowup_log + } + + /// Coefficients the proof carries for the terminal polynomial. + pub fn num_terminal_coeffs(self) -> usize { + 1usize << self.effective_k() + } + + /// Merkle path length for committed layer `i`: that layer's codeword is + /// `2^(n−i−1)` long and its leaves are pairs, so the tree has `2^(n−i−2)` + /// leaves. + pub fn layer_path_len(self, layer: usize) -> usize { + (self.log2_lde_length as usize) + .checked_sub(layer + 2) + .expect("layer index must be below num_committed") + } + + /// Merkle path steps one query walks across every committed layer. + pub fn path_steps_per_query(self) -> usize { + (0..self.num_committed()) + .map(|i| self.layer_path_len(i)) + .sum() + } + + /// Keccak permutations one query costs: one leaf hash per committed layer + /// (a 48-byte pair, one rate block) plus one per path step (64 bytes, one + /// rate block). + pub fn permutations_per_query(self) -> usize { + self.num_committed() + self.path_steps_per_query() + } + + /// Index bits a query carries — `log2(lde) − 1`, which is both the TRACE + /// trees' Merkle depth and the bit width of `iota`. + /// + /// The FRI layers consume SUFFIXES of this one decomposition rather than + /// decompositions of their own, which is what makes the emitted walks + /// address the same query the trace openings did. Layer `i` reads `bits[i]` + /// as its leaf-ordering parity and `bits[i+1..]` as its walk, and + /// `bits[i+1..].len() = n − i − 2 = layer_path_len(i)` exactly — the layer + /// tree's depth is not a separate fact to keep in sync, it is what is left + /// of the index after the folds already performed. + pub fn index_bits(self) -> usize { + self.log2_lde_length as usize - 1 + } + + /// Arena words one query's FRI opening occupies: per committed layer the + /// symmetric evaluation (one word) and its path (two words per level). + pub fn query_words(self) -> usize { + self.num_committed() + 2 * self.path_steps_per_query() + } + + /// Keccak permutations the whole sub-proof's FRI costs. + pub fn permutations(self) -> usize { + self.num_queries * self.permutations_per_query() + } + + /// Invariants a caller cannot assemble their way out of. + pub fn check(self) { + assert!( + self.blowup_log >= 1, + "a blowup of 1 is not a low-degree extension" + ); + assert!( + self.log2_lde_length > self.blowup_log, + "the LDE must be strictly larger than the blowup: a trace of one \ + row has no FRI to do" + ); + assert!( + self.terminal_log() <= self.log2_lde_length, + "the terminal codeword cannot exceed the LDE" + ); + assert!( + self.effective_k() <= self.final_poly_log_degree, + "the clamp can only lower the terminal degree, never raise it" + ); + assert_eq!( + self.terminal_len(), + 1usize << (self.blowup_log + self.effective_k()), + "terminal_len must equal 2^(blowup_log + effective_k)" + ); + } +} + +// ============================ the emitter ============================ +// +// Slice 2: the per-layer walk, the fold chain, and the terminal check. The +// shape above says how many of each; this says what each one is. +// +// ## What the machine emits, against what production runs +// +// Production's `verify_query_and_sym_openings` (`verifier.rs:660-748`) is a loop +// over committed layers with a running `(v, index)` pair. Here the loop is +// unrolled at build time and `index` never exists as a value: every use of it is +// a use of some suffix of the query's bit decomposition. The three uses map as +// +// ```text +// production machine +// ---------- ------- +// iota % 2 (leaf order) bits[i] +// iota >> 1 (leaf position) bits[i+1..] (the walk) +// index >>= 1 (next layer) i += 1 (a host-side index) +// ``` +// +// so the halving that production performs per layer is, here, reading one bit +// further along a vector that was decomposed once — by the trace leg, for its +// own walk. That is the join: there is no second index in the program to +// disagree with the first. +// +// ## What this cannot see +// +// It emits ONE sub-proof's FRI. Nothing here says the terminal coefficients or +// the folding challenges are the ones the transcript produced — they arrive as +// arena values, exactly as `γ` and `ζ` do in [`super::sub_proof`], and binding +// them to a transcript replay is assembly's obligation, covered by the standing +// clause in `others/lfm-assembly-obligations.md`. It also says nothing about +// whether `p₀` is the DEEP value of the authenticated opening; that is the +// previous leg's join, consumed here as cells. + +/// The group shape of a FRI layer leaf: ONE extension column, so a leaf covers +/// `ROWS_PER_LEAF = 2` values and 48 bytes. +/// +/// Reusing [`super::sub_proof::emit_leaf_hash`] rather than writing a second +/// gadget is deliberate and was checked rather than assumed — see +/// `fri_tests::the_fri_leaf_is_byte_identical_to_productions_own_backends`, +/// which runs the machine's leaf against BOTH production backends on vectors +/// that differ in every one of the 48 bytes. +/// +/// It is worth stating why the shapes coincide at all, because the two +/// commitments are built by different code: a trace leaf applies +/// `reverse_index` INSIDE the leaf builder and concatenates column-by-column +/// across a row pair (`commitment.rs:81-91`), while a FRI layer leaf is +/// `evals.chunks_exact(2)` of an ALREADY bit-reversed single codeword +/// (`fri/mod.rs:96-99`). At one column those two descriptions produce the same +/// byte string from the same pair — the permutation a trace leaf applies is the +/// permutation a FRI codeword already carries — and at more than one column +/// they do not. So this constant is not "the trace shape with a 1 in it"; it is +/// the point where the two layouts happen to meet. +pub const FRI_LEAF_GROUP: GroupShape = GroupShape { + num_columns: 1, + is_ext: true, +}; + +/// One committed FRI layer's root, unpacked once per sub-proof. +/// +/// The hoist matters at production query counts for the same reason +/// [`super::sub_proof::GroupCommitment`]'s does: a root is a per-sub-proof value +/// and a 219-query proof would otherwise pay 219 redundant `Unpack`s per layer. +pub struct LayerCommitment { + /// The root's two words as lanes. + pub root_lanes: [[Felt; 4]; 2], +} + +impl LayerCommitment { + /// Read a layer root out of the arena and hoist its unpack. + pub fn hint(b: &mut LfmBuilder, arena: ArenaId, base: u32) -> Self { + let w0 = b.hint_word(arena, base); + let w1 = b.hint_word(arena, base + 1); + LayerCommitment { + root_lanes: [b.unpack(w0), b.unpack(w1)], + } + } + + /// A layer commitment over lanes the caller already holds. + /// + /// The assembled verifier's route: a FRI layer root is absorbed by the + /// transcript in Round 4 (right after its own `ζ`) and compared against here, + /// and those two consumers must read one cell. See + /// [`super::sub_proof::GroupCommitment::from_lanes`] for the same argument at + /// the trace trees. + pub fn from_lanes(root_lanes: [[Felt; 4]; 2]) -> Self { + LayerCommitment { root_lanes } + } +} + +/// A sub-proof's FRI data that does not depend on the query. +pub struct FriCommitments { + /// One per committed layer, in fold order. + pub layers: Vec, + /// The folding challenges `ζ₀ .. ζ_C` — `num_committed + 1` of them, or + /// none when nothing folds. The asymmetry is the whole off-by-one of this + /// leg: the first fold consumes the DEEP pair and is not committed, so + /// folds exceed layers by one (`fri/mod.rs:114-118`). + pub zetas: Vec, + /// The terminal polynomial's `2^effective_k` coefficients, low-to-high. + pub coeffs: Vec, +} + +/// One query's opening of one committed layer. +/// +/// There is deliberately no constructor that hints — like +/// [`super::sub_proof::GroupOpening`], the values are the caller's, so what the +/// walk authenticates is what the fold consumes. +pub struct LayerOpening { + /// `pᵢ(−υ^(2ⁱ))` — the conjugate the prover supplies. Its partner + /// `pᵢ(υ^(2ⁱ))` is not in the proof at all: the verifier computed it as the + /// previous fold's output, which is why a FRI layer opening is one value and + /// not two. + pub sym: Ext, + /// Sibling digests, LEAF LEVEL FIRST. + pub siblings: Vec, +} + +/// What the FRI leg needs from a query the trace legs already verified. +/// +/// Every field is a CELL the previous leg produced, never a fresh hint or a +/// re-derivation. [`super::sub_proof::QueryOutput`] is exactly this shape's +/// supplier. +pub struct FriQuery<'a> { + /// `p₀(υ)` — the DEEP reconstruction at the query point. + pub p0: Ext, + /// `p₀(−υ)`. + pub p0_sym: Ext, + /// `υ`. Not Merkle-checked here and not hinted: it is the point the + /// authenticated opening was folded at. + pub point: Felt, + /// `−υ`, needed only by the zero-fold shape. + pub point_sym: Felt, + /// The query index low-to-high, `shape.index_bits()` of them — the cells + /// the trace walk consumed. + pub bits: &'a [Bit], +} + +/// The arenas one sub-proof's FRI verification reads, in declaration order. +pub struct FriArenas { + /// Two words per committed layer root, in fold order. + pub roots: ArenaId, + /// `ζ₀ .. ζ_C`, one word each. Empty when nothing folds. + pub zetas: ArenaId, + /// The terminal polynomial's coefficients, low-to-high. + pub coeffs: ArenaId, + /// Per query, per committed layer: the symmetric evaluation, then the + /// sibling digests (two words per level). + pub queries: ArenaId, +} + +/// Declare the FRI arenas and hoist everything a query does not depend on. +pub fn declare_fri( + b: &mut LfmBuilder, + shape: FriShape, + num_queries: usize, +) -> (FriArenas, FriCommitments) { + shape.check(); + assert!(num_queries > 0, "a proof carries at least one query"); + let c = shape.num_committed(); + let num_zetas = if shape.total_folds() > 0 { c + 1 } else { 0 }; + + let roots = b.declare_arena(2 * c as u32); + let zetas = b.declare_arena(num_zetas as u32); + let coeffs = b.declare_arena(shape.num_terminal_coeffs() as u32); + let queries = b.declare_arena((num_queries * shape.query_words()) as u32); + + let layers = (0..c) + .map(|i| LayerCommitment::hint(b, roots, 2 * i as u32)) + .collect(); + let zeta_cells = (0..num_zetas as u32) + .map(|i| b.hint_word(zetas, i).as_ext()) + .collect(); + let coeff_cells = (0..shape.num_terminal_coeffs() as u32) + .map(|i| b.hint_word(coeffs, i).as_ext()) + .collect(); + + ( + FriArenas { + roots, + zetas, + coeffs, + queries, + }, + FriCommitments { + layers, + zetas: zeta_cells, + coeffs: coeff_cells, + }, + ) +} + +/// Hint one query's layer openings out of the query arena. +pub fn hint_layer_openings( + b: &mut LfmBuilder, + shape: FriShape, + arenas: &FriArenas, + query: usize, +) -> Vec { + hint_layer_openings_from(b, shape, arenas.queries, query) +} + +/// [`hint_layer_openings`] against a query arena the caller declared itself. +/// +/// The assembled verifier declares one arena per sub-proof and takes the roots, +/// the folding challenges and the terminal coefficients from the transcript +/// replay rather than from [`declare_fri`]'s three other arenas — so it needs +/// this one without the other three. +pub fn hint_layer_openings_from( + b: &mut LfmBuilder, + shape: FriShape, + arena: ArenaId, + query: usize, +) -> Vec { + let mut cursor = (query * shape.query_words()) as u32; + let openings: Vec = (0..shape.num_committed()) + .map(|layer| { + let sym = b.hint_word(arena, cursor).as_ext(); + cursor += 1; + let siblings: Vec = (0..shape.layer_path_len(layer)) + .map(|_| { + let lo = b.hint_word(arena, cursor); + let hi = b.hint_word(arena, cursor + 1); + cursor += 2; + [lo, hi] + }) + .collect(); + LayerOpening { sym, siblings } + }) + .collect(); + assert_eq!( + cursor as usize, + (query + 1) * shape.query_words(), + "the emitter's cursor must agree with the declared query stride" + ); + openings +} + +/// `P(x)` for the terminal polynomial — Horner over the coefficients the proof +/// carries, low-to-high. +/// +/// See [`emit_query_fri`] for why this is an evaluation and not a lookup into a +/// materialized codeword, which is what production does. +fn emit_terminal_eval(b: &mut LfmBuilder, fri: &FriCommitments, x: Felt) -> Ext { + edsl::horner_ext(b, x.as_ext(), &fri.coeffs) +} + +/// Emit one query's FRI verification: fold, authenticate each committed layer, +/// and check the terminal polynomial. +/// +/// Returns the terminal-layer value `v` — the quantity production compares +/// against its terminal codeword — so a caller can publish it. Nothing depends +/// on the caller doing so: the check is `assert_eq_ext` INSIDE the program, so a +/// query that failed would not execute. +/// +/// # The terminal check is an EVALUATION, not a codeword lookup — a deliberate +/// deviation from the spec +/// +/// `others/lfm-fri-verify-spec.md` §5 says to emit the FFT, on the strength of a +/// measurement (sim/24) that replacing production's terminal FFT with per-point +/// Horner cost +20M cycles in the RV64 guest verifier. That measurement is +/// sound and it does not transfer, because the two machines disagree about the +/// price of an array index. +/// +/// Production materializes the terminal codeword once per proof and then does +/// `terminal_codeword.get(index)` per query — one load. This machine is +/// straight-line with no addressable memory, so the same lookup is a `Select` +/// tree over `terminal_len` cells: `terminal_len − 1` `Select`s per query. At +/// blowup 8 (`terminal_len = 1024`, 73 queries) that is 74,679 `Select`s, +/// against which the FFT itself — `(terminal_len/2)·log₂(terminal_len) = 5,120` +/// butterflies at ~3 rows each — is the smaller half of the bill. +/// +/// Evaluating instead costs `2^effective_k − 1 = 127` ext `MulAdd`s per query +/// plus `total_folds` squarings for the point, and no FFT at all: 140 rows per +/// query, 10,220 at 73 queries, against ~90,000. The direction reverses because +/// the guest amortizes one FFT across queries while paying nothing per lookup, +/// and this machine pays nothing for the FFT it does not run and everything for +/// the lookup it cannot do. +/// +/// The two checks are the same check, and the argument is short. The terminal +/// codeword is `P` evaluated over the terminal coset in bit-reversed order +/// (`terminal.rs:134-155`), so position `index` holds +/// `P(terminal_offset · ω_T^{br(index)})`. With `index = iota >> C`, +/// `terminal_offset = coset_offset^(2^total_folds)` and `ω_T = g^(2^total_folds)`, +/// that point is exactly `υ^(2^total_folds)` — the bits of `iota` that survive +/// the shift are the bits `br` puts inside `ω_T`'s order. So the machine raises +/// the query point to `2^total_folds` and evaluates, which also makes the +/// terminal point BOUND to the query point by construction rather than by a +/// second derivation. `fri_tests::the_terminal_point_is_the_query_point_folded` +/// checks that identity against production's own FFT at every index of several +/// shapes; a wrong exponent, a missing coset offset or a dropped bit reversal +/// all fail it. +/// +/// # The zero-fold shape +/// +/// When `total_folds == 0` no challenge was ever drawn and the terminal codeword +/// IS `p₀` (`verifier.rs:683-690`), so the check becomes `terminal[2·iota] = p₀` +/// and `terminal[2·iota+1] = p₀ˢ`. Under evaluation the two branches unify: +/// `2^total_folds = 1`, the two positions are `υ` and `−υ`, and the shape simply +/// evaluates `P` twice instead of once. This is not a dead branch to pin — it is +/// the real proof fixture's own shape (`min` preset over a 2^4-step epoch), and +/// a real production path for any table small enough that its LDE is already +/// terminal. +pub fn emit_query_fri( + b: &mut LfmBuilder, + shape: FriShape, + fri: &FriCommitments, + q: &FriQuery<'_>, + openings: &[LayerOpening], +) -> Ext { + let c = shape.num_committed(); + assert_eq!( + q.bits.len(), + shape.index_bits(), + "the FRI leg reads suffixes of the trace walk's own decomposition, so \ + it needs all log2(lde) − 1 index bits" + ); + assert_eq!(fri.layers.len(), c, "one commitment per committed layer"); + assert_eq!(openings.len(), c, "one opening per committed layer"); + assert_eq!( + fri.coeffs.len(), + shape.num_terminal_coeffs(), + "the terminal polynomial carries 2^effective_k coefficients" + ); + + if shape.total_folds() == 0 { + assert!( + fri.zetas.is_empty(), + "a codeword that never folds draws no folding challenge" + ); + let at = emit_terminal_eval(b, fri, q.point); + b.assert_eq_ext(at, q.p0); + let at_sym = emit_terminal_eval(b, fri, q.point_sym); + b.assert_eq_ext(at_sym, q.p0_sym); + return q.p0; + } + assert_eq!( + fri.zetas.len(), + c + 1, + "folds exceed committed layers by one" + ); + + // `υ⁻¹`, once. Production batch-inverts across queries and REJECTS on a + // zero point (`verifier.rs:465`, fails closed on a malformed index); the + // machine's `Div` errors on a zero divisor, which is the same disposition — + // an unprovable program rather than a wrong answer. + let one = b.felt_const(FE::one()); + let inv = b.div(one, q.point); + + // Fold 0 consumes the DEEP pair and authenticates nothing: there is no + // layer under it, which is why `zetas` is one longer than `layers`. + let mut v = edsl::fri_fold(b, q.p0, q.p0_sym, fri.zetas[0], inv); + + // The point chain is one squaring per layer and nothing else — no bit + // reversal, no domain lookup, no coset offset past the first point + // (spec §6). And no parity branch, because the sign the odd slot introduces + // into `x⁻¹` is the same sign it introduces into `v − sym`, so the two + // cancel (spec §3). Parity is consulted ONLY for the leaf byte order below. + let mut inv_pow = inv; + for (i, opening) in openings.iter().enumerate() { + // `if index % 2 == 1 { [sym, v] } else { [v, sym] }` (`verifier.rs:637`) + // — the even codeword slot leads. `select(bit, l, r)` returns `(l, r)` + // at 0 and `(r, l)` at 1, so this IS that conditional. + let (first, second) = b.select(q.bits[i], v.as_cell(), opening.sym.as_cell()); + let leaf = sub_proof::emit_leaf_hash(b, FRI_LEAF_GROUP, &[first, second]); + let root = edsl::keccak_merkle_walk(b, leaf, &q.bits[i + 1..], &opening.siblings); + edsl::assert_word_eq_lanes(b, root[0], &fri.layers[i].root_lanes[0]); + edsl::assert_word_eq_lanes(b, root[1], &fri.layers[i].root_lanes[1]); + + // `evaluation_point_vec[i] = υ^(−2^(i+1))` — `inv.square()` then one + // squaring per layer (`verifier.rs:692-697`). + inv_pow = b.mul(inv_pow, inv_pow); + v = edsl::fri_fold(b, v, opening.sym, fri.zetas[i + 1], inv_pow); + } + + // `x = υ^(2^total_folds)`: where the fold chain has arrived, and the + // terminal codeword's point at position `iota >> C`. See the doc comment. + let mut x = q.point; + for _ in 0..shape.total_folds() { + x = b.mul(x, x); + } + let at = emit_terminal_eval(b, fri, x); + b.assert_eq_ext(at, v); + v +} + +/// A whole sub-proof, both legs: every query's openings authenticated and folded +/// to `p₀` ([`super::sub_proof::emit_sub_proof_with_bits`]), then that `p₀` +/// folded down FRI's layers to the terminal check. +/// +/// This is where the two legs become one program rather than two. Every seam is +/// a shared CELL, not a shared convention: `p₀`/`p₀ˢ` are the DEEP outputs, `υ` +/// is the point they were evaluated at, and the index bits are the ones the +/// trace walk selected on. Returns the per-query terminal values. +pub fn emit_sub_proof_with_fri( + b: &mut LfmBuilder, + sub: &super::sub_proof::SubProofShape, + shape: FriShape, + num_queries: usize, +) -> (super::sub_proof::SubProofArenas, FriArenas, Vec) { + assert_eq!( + sub.log2_lde_length, shape.log2_lde_length, + "both legs verify the same sub-proof over the same LDE domain" + ); + assert_eq!( + sub.merkle_depth, + shape.index_bits(), + "the FRI layers consume suffixes of the trace walk's decomposition, so \ + the two shapes must agree about how long it is" + ); + assert_eq!( + shape.num_queries, num_queries, + "the query count is one shape, declared once" + ); + + let (sub_arenas, queries) = super::sub_proof::emit_sub_proof_with_bits(b, sub, num_queries); + let (fri_arenas, fri) = declare_fri(b, shape, num_queries); + + let terminal = queries + .iter() + .enumerate() + .map(|(i, out)| { + let openings = hint_layer_openings(b, shape, &fri_arenas, i); + emit_query_fri( + b, + shape, + &fri, + &FriQuery { + p0: out.deep.0, + p0_sym: out.deep.1, + point: out.point, + point_sym: out.point_sym, + bits: &out.bits, + }, + &openings, + ) + }) + .collect(); + + (sub_arenas, fri_arenas, terminal) +} diff --git a/prover/src/lfm/fri_tests.rs b/prover/src/lfm/fri_tests.rs new file mode 100644 index 000000000..79b2c0ac3 --- /dev/null +++ b/prover/src/lfm/fri_tests.rs @@ -0,0 +1,1234 @@ +//! The FRI leg: the per-layer walk, the fold chain, and the terminal check. +//! +//! ## The instrument problem this suite had to solve first +//! +//! `join_tests::the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold` +//! pins the difficulty: the join fixture's sub-proof has `total_folds = 0`, so a +//! differential over it sees no fold, no per-layer walk and no terminal lookup. +//! The retiring FRI agent concluded the only witness was synthetic codewords +//! driven through production's commit and query phases. +//! +//! It is better than that, and the reason is one line of the fixture: the trace +//! is `boundaries.len().next_power_of_two()` rows +//! (`local_to_global.rs:269`). Ask for 512 boundaries instead of 4 and the same +//! production prover, the same AIR and the same verifier replay produce a proof +//! that FOLDS — real committed layer roots, real authentication paths, real +//! terminal coefficients, and the folding challenges out of production's own +//! `replay_rounds_after_round_1`. Nothing in this suite is synthetic. The layer +//! count is swept by asking for more rows. +//! +//! ## What this suite cannot see +//! +//! `k = 7` and `coset_offset = 3` in every configuration the prover can be +//! asked for, so — exactly as spec §7 says — nothing here distinguishes an +//! implementation that reads them from one that hardcodes them. That half is +//! discharged host-side by `join_tests::the_fold_layout_is_right_off_productions_constants`, +//! which sweeps `k ∈ {0, 6, 7, 63}` and the clamp regime over the shape +//! arithmetic. What IS now witnessed on real data is everything the shape feeds: +//! the `num_committed = total_folds − 1` off-by-one, the fold chain, the parity +//! branch, the walk depths and the terminal check. +//! +//! It also sees one sub-proof at a time. Nothing here says an epoch's sub-proofs +//! compose, and nothing here binds the terminal coefficients or the folding +//! challenges to a transcript — they arrive as arena values, and tying them to a +//! replay is assembly's obligation. + +use math::field::traits::IsPrimeField; +use math::polynomial::Polynomial; +use stark::config::Commitment; +use stark::proof::stark::MultiProof; +use stark::traits::AIR; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::{LfmProgram, compile}; +use super::constraint_tests::BoxedAir; +use super::executor::execute; +use super::fri::{ + FRI_LEAF_GROUP, FriQuery, FriShape, declare_fri, emit_query_fri, hint_layer_openings, +}; +use super::hash::TestPermutation; +use super::join_tests::{HostSubProof, build_host_sub_proof}; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +/// A base element as its image in the cubic extension — component 0, the +/// embedding `IsSubFieldOf` uses implicitly wherever production multiplies a +/// base by an ext. +fn embed(x: &FE) -> FEE { + FEE::new([*x, FE::zero(), FE::zero()]) +} + +// ============================================================================= +// A real proof that folds +// ============================================================================= + +/// Proves L2G_MEMORY over `num_boundaries` boundary claims at `blowup`. +/// +/// The same AIR, prover and options path as `constraint_tests::real_fixture` — +/// only the row count differs, and the row count is what decides whether FRI +/// folds. `num_boundaries` must be a power of two so the trace length is exactly +/// it (the generator pads to the next power of two, which would silently change +/// the shape this suite is measuring). +pub(super) fn folding_fixture( + num_boundaries: usize, + blowup: usize, +) -> (BoxedAir, MultiProof) { + use crate::tables::local_to_global::{ + CellBoundary, FiniClaim, InitClaim, generate_local_to_global_trace, + }; + use crate::test_utils::{EPOCH_TEST_LABEL, multi_prove_ram}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + assert!( + num_boundaries.is_power_of_two(), + "the trace is padded to a power of two, so a non-power-of-two row count \ + would not be the shape asked for" + ); + let opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(blowup as u8) + .expect("a power-of-two blowup is valid"); + let air = crate::continuation::l2g_memory_air(&opts, EPOCH_TEST_LABEL); + + let boundaries: Vec = (0..num_boundaries as u64) + .map(|i| CellBoundary { + address: 0x1000 + 8 * i, + init: InitClaim { + value: i + 1, + timestamp: 0, + originating_epoch: 0, + }, + fini: FiniClaim { + value: 2 * i + 3, + epoch: EPOCH_TEST_LABEL, + timestamp: 17 + i, + }, + }) + .collect(); + let mut trace = generate_local_to_global_trace(&boundaries); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("the L2G_MEMORY fixture must prove at any power-of-two row count"); + + (Box::new(air), proof) +} + +/// Everything the FRI leg reads about one real sub-proof. +struct HostFri { + shape: FriShape, + /// The trace-side host fixture over the SAME proof: the openings, the roots, + /// and production's own DEEP answers, which are this leg's `p₀`. + trace: HostSubProof, + /// One root per committed layer, in fold order. + layer_roots: Vec, + /// `ζ₀ .. ζ_C` from the verifier's replay. + zetas: Vec, + /// The terminal polynomial's coefficients, low-to-high. + coeffs: Vec, + /// `[query][layer]` — `(pᵢ(−υ^(2ⁱ)), path)`. + openings: Vec)>>, +} + +/// Build the FRI host fixture for a real proof of `num_boundaries` rows. +fn host_fri(num_boundaries: usize, blowup: usize) -> HostFri { + let (air, proof) = folding_fixture(num_boundaries, blowup); + host_fri_from(&*air, &proof) +} + +/// [`host_fri`] for a proof the caller already holds — needed where the test +/// also wants the AIR's verifier domain. +fn host_fri_from( + air: &dyn AIR, + proof: &MultiProof, +) -> HostFri { + use stark::proof::view::StarkProofView; + + let trace = build_host_sub_proof(air, proof); + let view = StarkProofView::Owned(&proof.proofs[0]); + let opts = air.options(); + let shape = FriShape::from_options(opts, trace.shape.log2_lde_length); + shape.check(); + + let openings = (0..view.query_list_len()) + .map(|q| { + let d = view.query(q); + d.layers_evaluations_sym() + .iter() + .enumerate() + .map(|(i, sym)| (*sym, d.layer_auth_path(i).to_vec())) + .collect() + }) + .collect(); + + HostFri { + shape, + layer_roots: view.fri_layers_merkle_roots().to_vec(), + zetas: trace.zetas.clone(), + coeffs: view.fri_final_poly_coeffs().to_vec(), + openings, + trace, + } +} + +impl HostFri { + /// The arenas the FRI-only program declares, for the given queries. + fn fri_arenas(&self, queries: &[usize]) -> Vec> { + vec![ + super::proof_arena::commitments_to_arena(&self.layer_roots), + self.zetas.iter().map(ext_word).collect(), + self.coeffs.iter().map(ext_word).collect(), + self.query_arena(queries), + ] + } + + /// Per query, per layer: the symmetric evaluation then its path. + fn query_arena(&self, queries: &[usize]) -> Vec { + let mut out = Vec::new(); + for &q in queries { + for (sym, path) in &self.openings[q] { + out.push(ext_word(sym)); + out.extend(super::proof_arena::commitments_to_arena(path)); + } + } + out + } + + /// The terminal codeword, rebuilt exactly as `terminal_codeword_from_coeffs` + /// does (`fri/terminal.rs:134-155`) out of production's own FFT and + /// bit-reverse permutation. + /// + /// That module is `pub(crate)` inside `crypto/stark`, so this is a mirror of + /// its three lines rather than a call to it. The mirror is what + /// [`the_terminal_point_is_the_query_point_folded`] tests the emitter's + /// evaluation against — and the mirror itself is checked, because the same + /// codeword must reproduce the values the PROVER folded to, which no reading + /// of these three lines could fake. + fn terminal_codeword(&self) -> Vec { + use math::fft::bit_reversing::in_place_bit_reverse_permute; + + let coset_offset = FE::from(self.shape.coset_offset); + let terminal_offset = coset_offset.pow(1u64 << self.shape.total_folds()); + let poly = Polynomial::new(&self.coeffs); + let blowup = self.shape.terminal_len() / self.coeffs.len(); + let mut natural = Polynomial::evaluate_offset_fft::( + &poly, + blowup, + Some(self.coeffs.len()), + &terminal_offset, + ) + .expect("the terminal coset is a power of two inside the two-adicity"); + in_place_bit_reverse_permute(&mut natural); + natural + } +} + +// ============================================================================= +// The owed check: is the leaf gadget reusable? +// ============================================================================= + +/// ★ The check the retiring agent OWED and never ran: the machine's leaf hash at +/// `GroupShape { num_columns: 1, is_ext: true }` is byte-identical to the FRI +/// layer leaf, run against BOTH production backends. +/// +/// The two sides genuinely use different types — the prover commits under +/// `PairKeccak256Backend` (`fri/mod.rs:100`) and the verifier authenticates +/// under `BatchedMerkleTreeBackend` (`verifier.rs:643`) — and the spec's claim is +/// that they are byte-identical. This asserts both, so a divergence shows up as +/// a named failure rather than as a mysterious walk that will not reach its root. +/// +/// ## The vectors, and the lesson they encode +/// +/// A tamper suite whose every vector differed in byte 0 is one of the holes this +/// phase found by falsifying its own guards. A leaf here is 48 bytes: two +/// extension elements, three components each, eight big-endian bytes each. So the +/// vectors are built to make **every one of the 48 byte positions carry a +/// distinct value**, with no component equal to another and none symmetric under +/// byte reversal. A wrong component order, a wrong element order, a +/// little-endian limb or a dropped high byte each move a different subset of the +/// 48, and all of them move at least one. +#[test] +fn the_fri_leaf_is_byte_identical_to_productions_own_backends() { + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + use stark::config::{BatchedMerkleTreeBackend, FriLayerMerkleTreeBackend}; + + // Six distinct components, each with six distinct nonzero bytes in + // descending positions, so no two of the 48 bytes agree and no component is + // a byte-reversal of itself or of another. + let component = |i: u64| FE::from(0x0102_0304_0506_0708u64 * (i + 1) + 0x11 * (i + 1)); + let ext = |base: u64| FEE::new([component(base), component(base + 1), component(base + 2)]); + let vectors: [(FEE, FEE); 4] = [ + (ext(0), ext(3)), + // Order-sensitivity: the same two elements swapped must hash differently + // (checked below), which is what says the leaf is ordered at all. + (ext(3), ext(0)), + // A zero element beside a maximal one: catches a gadget that skips or + // truncates a zero limb, and a canonicity slip at p−1. + ( + FEE::zero(), + FEE::new([ + FE::from(Gl::modulus_minus_one()), + FE::one(), + FE::from(0xFFFF_FFFF_0000_0000u64), + ]), + ), + // One bit apart in the LAST byte of the LAST component — the position a + // suite that only ever varied byte 0 would never reach. + (ext(9), FEE::new([component(12), component(13), FE::one()])), + ]; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let v0 = b.hint_word(arena, 0); + let v1 = b.hint_word(arena, 1); + let leaf = super::sub_proof::emit_leaf_hash(&mut b, FRI_LEAF_GROUP, &[v0, v1]); + b.public(leaf[0]); + b.public(leaf[1]); + let program = compile(b.finish()); + validate(&program).expect("the leaf program is admissible"); + + let mut digests = Vec::new(); + for (i, (a, c)) in vectors.iter().enumerate() { + let arenas = vec![vec![ext_word(a), ext_word(c)]]; + let exec = execute(&program, &arenas, &TestPermutation).expect("the leaf hash executes"); + let got = [exec.public_words[0].1, exec.public_words[1].1]; + + let batched = + as IsMerkleTreeBackend>::hash_data(&vec![*a, *c]); + let paired = as IsMerkleTreeBackend>::hash_data(&[*a, *c]); + assert_eq!( + batched, paired, + "vector {i}: the spec's claim is that the prover's pair backend and \ + the verifier's batched backend are byte-identical; they are not" + ); + assert_eq!( + got, + super::proof_arena::commitment_words(&batched), + "vector {i}: the machine's leaf must be the verifier's leaf — this \ + is the byte-level check the FRI leg was handed as unverified" + ); + digests.push(got); + } + assert_ne!( + digests[0], digests[1], + "swapping the two elements must change the leaf, or the emitted order is \ + not carried into the hash and the parity Select is decoration" + ); + println!( + "machine leaf == BatchedMerkleTreeBackend == PairKeccak256Backend on {} \ + vectors covering all 48 bytes", + vectors.len() + ); +} + +// ============================================================================= +// The emitter, against production +// ============================================================================= + +/// The FRI leg alone, driven by a hinted index and a hinted DEEP pair. +/// +/// Used where the point of the test is FRI rather than the join: the trace legs +/// cost ~5,000 instructions per query per group and would dominate a run whose +/// subject is the fold. The index still goes through one `bit_dec` and the point +/// still comes from [`super::sub_proof::emit_points_from_bits`], so the +/// machine's own derivation is under test rather than a supplied point. +/// +/// Arena order: the per-query `(index, p₀, p₀ˢ)` block, then the four +/// [`FriArenas`]. +fn fri_only_program(shape: FriShape, num_queries: usize) -> LfmProgram { + let mut b = LfmBuilder::new(); + let q = b.declare_arena(3 * num_queries as u32); + let (arenas, fri) = declare_fri(&mut b, shape, num_queries); + for i in 0..num_queries { + let index = b.hint_felt(q, 3 * i as u32); + let p0 = b.hint_word(q, 3 * i as u32 + 1).as_ext(); + let p0_sym = b.hint_word(q, 3 * i as u32 + 2).as_ext(); + let bits = b.bit_dec(index, shape.index_bits()); + let (point, point_sym) = super::sub_proof::emit_points_from_bits( + &mut b, + shape.log2_lde_length, + FE::from(shape.coset_offset), + &bits, + ); + let openings = hint_layer_openings(&mut b, shape, &arenas, i); + let v = emit_query_fri( + &mut b, + shape, + &fri, + &FriQuery { + p0, + p0_sym, + point, + point_sym, + bits: &bits, + }, + &openings, + ); + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the FRI program must be admissible"); + program +} + +impl HostFri { + /// The `(index, p₀, p₀ˢ)` arena [`fri_only_program`] reads. + fn deep_arena(&self, queries: &[usize]) -> Vec { + let mut out = Vec::new(); + for &q in queries { + out.push(base_word(FE::from(self.trace.iotas[q] as u64))); + out.push(ext_word(&self.trace.expected[q].0)); + out.push(ext_word(&self.trace.expected[q].1)); + } + out + } + + /// Every arena [`fri_only_program`] declares, in order. + fn all_arenas(&self, queries: &[usize]) -> Vec> { + let mut all = vec![self.deep_arena(queries)]; + all.extend(self.fri_arenas(queries)); + all + } +} + +/// ★ The premise of this suite, checked before anything is built on it: the +/// production prover FOLDS when the trace is big enough, and the layer count is +/// steerable by the row count. +/// +/// This is the finding that retires the leg's instrument problem. The FRI leg +/// was handed the conclusion that "the production instance exercises none of the +/// mechanism" and that synthetic codewords were the only witness. That was true +/// of the fixture as written and false of the fixture as available: the row count +/// is `boundaries.len().next_power_of_two()`, and `num_committed = trace_bits − 8`, +/// so 512 boundaries commit one layer and 2048 commit three. Everything below +/// therefore differentials against production data rather than against a +/// synthesized input, and the `saturating_sub(1)` off-by-one that +/// `join_tests::the_fold_layout_is_right_off_productions_constants` could only +/// catch host-side is now caught by an executed walk that fails to reach a real +/// root. +/// +/// The four shapes are the sweep the successor brief asked for — `num_committed` +/// over 0, 1, 2, 3 — and the zero row is the original fixture, unchanged. +#[test] +fn the_real_prover_folds_and_the_layer_count_follows_the_row_count() { + println!("rows n folds committed coeffs zetas queries terminal_len"); + for (rows, committed) in [(4usize, 0usize), (512, 1), (1024, 2), (2048, 3)] { + let h = host_fri(rows, 2); + println!( + "{rows:>5} {:>6} {:>6} {:>10} {:>7} {:>6} {:>8} {:>13}", + h.shape.log2_lde_length, + h.shape.total_folds(), + h.shape.num_committed(), + h.coeffs.len(), + h.zetas.len(), + h.trace.iotas.len(), + h.shape.terminal_len(), + ); + assert_eq!( + h.shape.num_committed(), + committed, + "{rows} rows must commit {committed} FRI layers" + ); + // The three structural lengths the verifier rejects on before its query + // loop (`verifier.rs:426-448`), asked of the real proof. + assert_eq!( + h.layer_roots.len(), + h.shape.num_committed(), + "committed roots" + ); + assert_eq!( + h.coeffs.len(), + h.shape.num_terminal_coeffs(), + "terminal coefficients" + ); + assert_eq!( + h.zetas.len(), + if h.shape.total_folds() > 0 { + h.shape.num_committed() + 1 + } else { + 0 + }, + "folds exceed committed layers by one, and nothing folds at all when \ + the codeword is already terminal" + ); + for (q, per_layer) in h.openings.iter().enumerate() { + assert_eq!(per_layer.len(), h.shape.num_committed(), "query {q} layers"); + for (i, (_, path)) in per_layer.iter().enumerate() { + assert_eq!( + path.len(), + h.shape.layer_path_len(i), + "query {q} layer {i}: the layer tree is one level shallower \ + per fold, so its path length is n − i − 2" + ); + } + } + } +} + +/// ★ The deviation from spec §5, justified numerically against production's own +/// FFT: the terminal codeword's value at position `iota >> C` is the terminal +/// polynomial evaluated at `υ^(2^total_folds)`. +/// +/// The emitter checks `P(υ^(2^total_folds)) = v` where production checks +/// `terminal_codeword[index] = v`. If those are not the same number the emitter +/// is wrong, and the machine's own assertions would not say so — they would +/// simply both be wrong together. So the identity is checked here, host-side, +/// over the real proofs at every one of their 219 indices: the codeword side +/// comes from production's `evaluate_offset_fft` + `in_place_bit_reverse_permute` +/// (`terminal.rs:150-155`) and the point side from production's own +/// `query_challenge_to_evaluation_point` raised by repeated squaring. +/// +/// A missing coset offset, a wrong exponent, a dropped bit reversal or the wrong +/// shift on `iota` each break it. The zero-fold shape is included, where the +/// claim is that the two positions `2·iota` and `2·iota+1` are `υ` and `−υ` — +/// the identity that lets one emitted shape serve production's two branches. +#[test] +fn the_terminal_point_is_the_query_point_folded() { + use stark::domain::new_verifier_domain; + use stark::verifier::{IsStarkVerifier, Verifier}; + type V = Verifier; + + for rows in [4usize, 512, 1024, 2048] { + let (air, proof) = folding_fixture(rows, 2); + let h = host_fri_from(&*air, &proof); + let codeword = h.terminal_codeword(); + assert_eq!(codeword.len(), h.shape.terminal_len()); + let domain = new_verifier_domain(&*air, proof.proofs[0].trace_length); + let c = h.shape.num_committed(); + + let mut distinct = std::collections::HashSet::new(); + for &iota in &h.trace.iotas { + let point = V::query_challenge_to_evaluation_point(iota, false, &domain); + let mut x = point; + for _ in 0..h.shape.total_folds() { + x = x.square(); + } + let at = Polynomial::new(&h.coeffs).evaluate(&embed(&x)); + + // The position production compares at, per branch. The two branches + // index DIFFERENTLY and conflating them is the mistake this test + // made on its first run: the folding branch walks `index` down from + // `iota` (`verifier.rs:735`), so it lands on `iota >> C`, while the + // zero-fold branch never has an `index` at all and reads the pair + // positions `2·iota` and `2·iota+1` directly (`verifier.rs:684-690`) + // — its terminal codeword IS the deep codeword, in which `iota` + // numbers pairs rather than elements. + let position = if h.shape.total_folds() == 0 { + iota * 2 + } else { + iota >> c + }; + assert_eq!( + at, + codeword[position], + "rows {rows} iota {iota}: P(υ^(2^{})) must be the terminal \ + codeword at position {position}", + h.shape.total_folds() + ); + if h.shape.total_folds() == 0 { + assert_eq!(x, point, "nothing folds, so the point is unchanged"); + assert_eq!( + Polynomial::new(&h.coeffs).evaluate(&embed( + &V::query_challenge_to_evaluation_point(iota, true, &domain) + )), + codeword[iota * 2 + 1], + "iota {iota}: the symmetric position must be −υ" + ); + } + distinct.insert(position); + } + println!( + "rows {rows:>5}: identity holds at all {} indices ({} distinct \ + terminal positions of {})", + h.trace.iotas.len(), + distinct.len(), + codeword.len() + ); + assert!( + distinct.len() > 1 || codeword.len() == 1, + "if every query landed on the same terminal position the check would \ + be one equation, not a sweep" + ); + } +} + +/// ★ The parity branch is REACHED, at every layer. +/// +/// The leaf order is `[sym, v]` at odd index and `[v, sym]` at even +/// (`verifier.rs:637-641`), selected on bit `i` of `iota` at layer `i`. An +/// implementation with the two arms swapped, or with no `Select` at all, is +/// invisible to a fixture whose indices all share a parity — the same +/// degenerate-parameter trap as the fold itself, one level down. This asserts the +/// real proof's 219 indices carry both parities at every committed layer, which +/// is what makes `no_tampered_fri_value_can_pass` able to catch the swap. +#[test] +fn the_real_indices_reach_both_leaf_parities_at_every_layer() { + for rows in [512usize, 1024, 2048] { + let h = host_fri(rows, 2); + for layer in 0..h.shape.num_committed() { + let (even, odd): (Vec<_>, Vec<_>) = h + .trace + .iotas + .iter() + .map(|iota| (iota >> layer) & 1) + .partition(|b| *b == 0); + assert!( + !even.is_empty() && !odd.is_empty(), + "rows {rows} layer {layer}: {} even and {} odd indices — a layer \ + reached by only one parity leaves the leaf-order Select \ + unexercised", + even.len(), + odd.len() + ); + } + println!( + "rows {rows:>5}: both parities present at all {} layers", + h.shape.num_committed() + ); + } +} + +/// ★ THE HEADLINE: the emitted FRI leg verifies every query of a real proof that +/// really folds, at three layer counts, and its terminal value is the one +/// production would have looked up. +/// +/// ## Why this is a strong check and not just an endpoint check +/// +/// The published value is only the LAST link. Every intermediate `v` is pinned +/// too, and not by an assertion this test writes — by the proof itself. At layer +/// `i` the machine hashes `{v, sym}` into a leaf and walks it to +/// `fri_layers_merkle_roots[i]`, a root the production prover committed to its +/// own folded codeword. So a `v` that were wrong at any layer could not reach +/// that root, and the run would not execute at all. The fold chain, the point +/// chain, the parity ordering, the walk depths and the layer-to-`ζ` alignment are +/// all inside that. +/// +/// What the published comparison adds is the terminal link, which no Merkle root +/// covers: the final fold is never committed (`fri/mod.rs:114-118`), so `v` at +/// the terminal layer is checked only against the coefficients. That is compared +/// here against production's own codeword at production's own position. +#[test] +fn the_fri_emitter_verifies_every_query_of_a_real_folding_proof() { + for rows in [4usize, 512, 1024, 2048] { + let h = host_fri(rows, 2); + let all: Vec = (0..h.trace.iotas.len()).collect(); + let program = fri_only_program(h.shape, all.len()); + let exec = execute(&program, &h.all_arenas(&all), &TestPermutation).expect( + "an honest FRI decommitment must authenticate every layer and reach \ + the terminal polynomial", + ); + + let codeword = h.terminal_codeword(); + let c = h.shape.num_committed(); + let mut nonzero = 0usize; + for (k, &q) in all.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("the fold output is ext"); + let iota = h.trace.iotas[q]; + let position = if h.shape.total_folds() == 0 { + iota * 2 + } else { + iota >> c + }; + assert_eq!( + v, codeword[position], + "rows {rows} query {q} (iota {iota}): the machine's terminal value \ + must be the terminal codeword at the position production compares \ + at" + ); + if v != FEE::zero() { + nonzero += 1; + } + } + assert_eq!( + nonzero, + all.len(), + "a vacuously zero fold would make the differential empty" + ); + println!( + "rows {rows:>5}: {} queries, {} committed layers, {} instructions, \ + {} permutations — every terminal value matches production's codeword", + all.len(), + c, + program.instrs.len(), + permutations(&program), + ); + } +} + +/// ★ Both legs as one program, on a real folding proof: the openings +/// authenticated, DEEP folded from the authenticated cells, and FRI folded from +/// DEEP's own output at DEEP's own point. +/// +/// This is the seam the leg exists to close. The FRI leg could be correct in +/// isolation and still verify a different query than the trace leg did — folding +/// `p₀` values it was handed while the walks authenticated some other index. Here +/// there is nothing to hand: `emit_sub_proof_with_fri` takes the `QueryOutput` +/// cells, so `p₀`, `υ` and the index bits are the same addresses in both legs by +/// construction. +/// +/// Run over a subset of queries because the trace side is ~50× the FRI side per +/// query at this shape; the coverage of the FRI mechanism itself is +/// [`the_fri_emitter_verifies_every_query_of_a_real_folding_proof`]'s, over all +/// 219. +#[test] +fn the_two_legs_verify_one_real_folding_proof_as_one_program() { + let h = host_fri(1024, 2); + let queries: Vec = (0..6).collect(); + assert_eq!( + h.shape.num_committed(), + 2, + "the 1024-row shape commits two layers" + ); + // The query count is program shape, so a subset run is a different shape and + // has to say so — `emit_sub_proof_with_fri` refuses to emit a query count + // that disagrees with the one in the shape it was handed. + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + + let mut b = LfmBuilder::new(); + let (_, _, terminal) = + super::fri::emit_sub_proof_with_fri(&mut b, &h.trace.shape, shape, queries.len()); + for v in &terminal { + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined program is admissible"); + + let mut arenas = h.trace.arenas(&queries); + arenas.extend(h.fri_arenas(&queries)); + let exec = execute(&program, &arenas, &TestPermutation) + .expect("the honest proof must authenticate, fold and reach the terminal"); + + let codeword = h.terminal_codeword(); + for (k, &q) in queries.iter().enumerate() { + let v = word_as_ext(&exec.public_words[k].1).expect("ext"); + assert_eq!( + v, + codeword[h.trace.iotas[q] >> h.shape.num_committed()], + "query {q}: the joined program's terminal value" + ); + } + println!( + "joined trace+DEEP+FRI over {} queries of a folding proof: {} \ + instructions, {} permutations", + queries.len(), + program.instrs.len(), + permutations(&program), + ); +} + +fn permutations(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|i| matches!(i, super::instr::Instr::KeccakF(_))) + .count() +} + +fn count_matching bool>(program: &LfmProgram, f: F) -> usize { + program.instrs.iter().filter(|i| f(i)).count() +} + +/// The marginal per-query cost of a shape, by emitting one query and two and +/// differencing — so no per-sub-proof plumbing (the hoisted root unpacks, the +/// coefficient hints) lands in the figure. +struct PerQuery { + perms: usize, + swaps: usize, + instrs: usize, +} + +fn marginal_fri(shape: FriShape) -> PerQuery { + let one = fri_only_program( + FriShape { + num_queries: 1, + ..shape + }, + 1, + ); + let two = fri_only_program( + FriShape { + num_queries: 2, + ..shape + }, + 2, + ); + let dec = + |p: &LfmProgram| count_matching(p, |i| matches!(i, super::instr::Instr::BitDec { .. })); + PerQuery { + perms: permutations(&two) - permutations(&one), + swaps: dec(&two) - dec(&one), + instrs: two.instrs.len() - one.instrs.len(), + } +} + +/// ★ MEASURED against the prediction pinned before the emitter existed. +/// +/// `join_tests::the_fri_sizing_prediction` recorded 174/186/198 permutations per +/// query and 38,106/20,460/14,454 per sub-proof at blowup 2/4/8, `trace_bits = +/// 20`, derived from spec §8. This counts the `LFM_KECCAK` rows the emitter +/// actually emits at those shapes and asserts the same numbers. +/// +/// The 2^20-row shape is emitted, not proved — a real proof at that size is a +/// prover run, not a test — but the quantity predicted IS the emitted +/// permutation count, so this is a measurement of the thing predicted rather +/// than a model of it. That the same formula holds on EXECUTED programs is +/// [`the_fri_emitter_verifies_every_query_of_a_real_folding_proof`]'s doing at +/// n = 10/11/12, where 219 queries produced exactly 1,971 / 4,161 / 6,570 +/// permutations against `219 × (C + Σ pathlen)` = 219 × 9 / 19 / 30. +/// +/// ## Two currencies that point opposite ways +/// +/// The other columns are reported because permutations alone hide where the rows +/// go, and because the two honest answers disagree. Rendering the two extension +/// values of a layer leaf into 48 big-endian bytes costs `6C` byteswaps per +/// query, each one `LFM_BITDEC` row plus 64 `LFM_BALU` rows — which makes +/// byteswapping the majority of the leg's INSTRUCTIONS. In main-trace CELLS the +/// same comparison inverts by two orders of magnitude, because a permutation +/// expands into 24 `KECCAK_RND` rounds of 1,480 columns while a byteswap carries +/// 322 cells. `others/lfm-target-shape.md`'s rule that rows of different chips +/// are not comparable is exactly this, so both are printed and neither is called +/// "the" cost. +#[test] +fn the_emitted_permutation_count_meets_the_pinned_prediction() { + const TRACE_BITS: u32 = 20; + let swap_cells = super::machine_tests::byteswap_cells(); + let perm_cells = super::machine_tests::permutation_cells(); + println!( + "blowup C Q perms/q predicted total predicted swaps/q instr/q hash cells/q swap cells/q" + ); + for (blowup_log, queries, predicted_per_query, predicted_total) in [ + (1u32, 219usize, 174usize, 38_106usize), + (2, 110, 186, 20_460), + (3, 73, 198, 14_454), + ] { + let shape = FriShape { + log2_lde_length: TRACE_BITS + blowup_log, + blowup_log, + final_poly_log_degree: 7, + coset_offset: 3, + num_queries: queries, + }; + shape.check(); + let per = marginal_fri(shape); + println!( + " 2^{blowup_log} {:>3} {:>3} {:>8} {:>10} {:>8} {:>10} {:>8} {:>8} {:>13} {:>13}", + shape.num_committed(), + queries, + per.perms, + predicted_per_query, + per.perms * queries, + predicted_total, + per.swaps, + per.instrs, + per.perms as u64 * perm_cells, + per.swaps as u64 * swap_cells, + ); + assert_eq!( + per.perms, predicted_per_query, + "blowup 2^{blowup_log}: emitted permutations per query against the \ + pinned prediction" + ); + assert_eq!( + per.perms * queries, + predicted_total, + "blowup 2^{blowup_log}: emitted permutations per sub-proof" + ); + // The same number, from the shape arithmetic rather than from the + // emitted program. Equal counts here mean the emitter walks the depths + // the shape says it should — the one place a wrong `layer_path_len` + // would show up as agreement between two wrongs is if BOTH came from the + // shape, and only one of these does. + assert_eq!( + per.perms, + shape.permutations_per_query(), + "the emitted program and the shape arithmetic must agree" + ); + // One byteswap per extension component per leaf value: two values, three + // components, per committed layer. + assert_eq!( + per.swaps, + 1 + 6 * shape.num_committed(), + "the index decomposition plus six component byteswaps per layer" + ); + // The inversion, asserted rather than left to the reader: byteswapping + // is the majority of the instructions and a rounding error in cells. + let swap_instrs = per.swaps * 65; + assert!( + swap_instrs * 2 > per.instrs, + "byteswapping should be the majority of the leg's instructions ({swap_instrs} of {})", + per.instrs + ); + assert!( + per.perms as u64 * perm_cells > 100 * per.swaps as u64 * swap_cells, + "and a rounding error in main-trace cells" + ); + } +} + +/// ★ ABSOLUTE (rule 7): the joined program contains ONE point derivation per +/// query and ONE decomposition of the index, and every term of the count comes +/// from a SHAPE rather than from a second emission. +/// +/// ## This test was wrong first, and how it was caught matters more than the fix +/// +/// Its first form measured the FRI leg's marginal `Select` count as +/// `selects(joined) − selects(trace_only)` and asserted the difference was +/// `C + 2 · path_steps`, reasoning that a second point derivation would add +/// `index_bits`. That is vacuous, and injecting the exact defect it denies — a +/// `QueryOutput` handing out a freshly derived point — left it GREEN. The reason +/// is rule 7's failure mode wearing a different hat: the defect lives in +/// `emit_sub_proof_with_bits`, which is what BOTH sides of the subtraction call, +/// so both gained `index_bits` selects and the difference never moved. +/// +/// **A difference of two counts taken from our own emitter is still a relative +/// test, however much it looks like a count.** The marginal-cost idiom this phase +/// uses everywhere is safe only when the RESULT is compared against a number that +/// did not come from the emitter — a pinned prediction, or a closed form over the +/// shapes: +/// +/// ```text +/// selects/query = index_bits (pow_bits, once per query) +/// + 2 · merkle_depth · num_groups (trace walks) +/// + num_committed (FRI leaf ordering) +/// + 2 · path_steps_per_query (FRI walks) +/// ``` +/// +/// `pow_bits` emits one `Select` per bit (`edsl.rs:257-262`) and each walk level +/// two, since a digest is two words and both must swap on the same bit +/// (`edsl.rs:164-169`). A second derivation makes the measured count exceed the +/// closed form by exactly `index_bits`, and nothing cancels it. Re-falsified in +/// that form: the injected defect now fails with "a surplus of 11 index bits". +#[test] +fn the_fri_join_adds_no_second_point_derivation() { + let h = host_fri(2048, 2); + let sub = &h.trace.shape; + let groups = sub.groups(); + let selects = + |p: &LfmProgram| count_matching(p, |i| matches!(i, super::instr::Instr::Select { .. })); + let decs = + |p: &LfmProgram| count_matching(p, |i| matches!(i, super::instr::Instr::BitDec { .. })); + + let emit = |n: usize| { + let mut b = LfmBuilder::new(); + super::fri::emit_sub_proof_with_fri( + &mut b, + sub, + FriShape { + num_queries: n, + ..h.shape + }, + n, + ); + compile(b.finish()) + }; + // Marginal, so the per-sub-proof plumbing is out of the figure — but the + // figure is then compared against the shapes, never against another emission. + let one = emit(1); + let two = emit(2); + let per_query_selects = selects(&two) - selects(&one); + let per_query_decs = decs(&two) - decs(&one); + + let expected_selects = h.shape.index_bits() + + 2 * sub.merkle_depth * groups.len() + + h.shape.num_committed() + + 2 * h.shape.path_steps_per_query(); + assert_eq!( + per_query_selects, + expected_selects, + "selects per query: {} index bits for the ONE point derivation, {} for \ + {} trace walks over {} levels, {} FRI leaf orderings, {} for {} FRI path \ + steps. A surplus of {} index bits is a second point derivation or a \ + second index decomposition", + h.shape.index_bits(), + 2 * sub.merkle_depth * groups.len(), + groups.len(), + sub.merkle_depth, + h.shape.num_committed(), + 2 * h.shape.path_steps_per_query(), + h.shape.path_steps_per_query(), + h.shape.index_bits(), + ); + + // One decomposition of the index, plus one byteswap per field element that + // enters a leaf: a base element is one, an extension element three. + let leaf_swaps: usize = groups + .iter() + .map(|g| g.num_values() * if g.is_ext { 3 } else { 1 }) + .sum(); + let expected_decs = 1 + leaf_swaps + 6 * h.shape.num_committed(); + assert_eq!( + per_query_decs, + expected_decs, + "decompositions per query: ONE for the index, {leaf_swaps} for the trace \ + leaves, {} for the FRI layer leaves. A surplus of one is a second index \ + decomposition", + 6 * h.shape.num_committed(), + ); + println!( + "per query: {per_query_selects} selects and {per_query_decs} \ + decompositions, both equal to the closed form over the shapes — one \ + point derivation, one index decomposition" + ); +} + +// ============================================================================= +// Falsification: what the leg denies +// ============================================================================= + +/// ★ No arena value the FRI leg reads can be moved without the run failing. +/// +/// The emitted checks are `assert_eq` inside the program, which lowers to +/// `diff / 0` — provable and executable only when `diff` is zero. So "the tamper +/// is caught" and "the run does not execute" are the same statement, and a +/// tamper that still executed would be a hole. +/// +/// The vectors sweep every KIND of value the leg reads, and the last one is the +/// only interesting attack: a COHERENT forgery in the sense of method rule 4 — +/// every value in it is a genuine value the production prover committed to, just +/// belonging to a different query. Nothing in it is malformed, no hash is +/// invented, and the leaf it builds is a leaf that really exists in the real +/// layer tree. What rejects it is only that the walk climbs at the index bits of +/// THIS query, so a real leaf at the wrong position cannot reach the root. +/// +/// ## What is deliberately absent +/// +/// There is no vector that tampers `p₀`. It is not an arena value here — it is a +/// cell the DEEP leg computed — and moving it is +/// `join_tests::no_tampered_value_can_move_the_fold_without_moving_the_root`'s +/// subject one leg back. A FRI-side tamper of `p₀` would only be possible in the +/// standalone driver, where it is hinted for isolation, and would prove nothing +/// about the joined program. +#[test] +fn no_tampered_fri_value_can_pass() { + const ROWS: usize = 2048; + let h = host_fri(ROWS, 2); + let c = h.shape.num_committed(); + assert_eq!(c, 3, "this suite wants several layers to tamper inside"); + + // Two queries whose layer-0 parities DIFFER, so the splice below moves a + // leaf between positions of opposite parity as well as of different index. + let a = (0..h.trace.iotas.len()) + .find(|&q| h.trace.iotas[q].is_multiple_of(2)) + .expect("an even index"); + let b = (0..h.trace.iotas.len()) + .find(|&q| !h.trace.iotas[q].is_multiple_of(2)) + .expect("an odd index"); + let queries = vec![a, b]; + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + let program = fri_only_program(shape, queries.len()); + let honest = h.all_arenas(&queries); + execute(&program, &honest, &TestPermutation).expect("the honest run must execute"); + + let stride = h.shape.query_words(); + // (label, arena, word) — arena order is the driver's: deep, roots, zetas, + // coeffs, queries. + let bump: Vec<(&str, usize, usize)> = vec![ + ("query index", 0, 0), + ("layer 0 root", 1, 0), + ("layer 0 root, second word", 1, 1), + ("layer 2 root", 1, 2 * (c - 1)), + ("zeta_0 (the DEEP fold's challenge)", 2, 0), + ("zeta_C (the uncommitted final fold)", 2, c), + ("terminal coefficient 0", 3, 0), + ("terminal coefficient 127", 3, h.coeffs.len() - 1), + ("layer 0 symmetric evaluation", 4, 0), + ("layer 0 sibling, leaf level", 4, 1), + ( + "layer 0 sibling, top level", + 4, + 2 * h.shape.layer_path_len(0) - 1, + ), + ("second query's layer 0 evaluation", 4, stride), + ]; + for (label, arena, word) in bump { + let mut tampered = honest.clone(); + tampered[arena][word][0] += FE::one(); + let err = execute(&program, &tampered, &TestPermutation).expect_err(&format!( + "moving the {label} must make the program unexecutable" + )); + println!(" {label:<40} rejected: {err:?}"); + } + + // The coherent forgery: query `a` presented with query `b`'s layer-0 + // decommitment. Every word is a real prover value. + let mut spliced = honest.clone(); + let (from, to) = (stride, 0usize); + let len = 1 + 2 * h.shape.layer_path_len(0); + let borrowed: Vec = spliced[4][from..from + len].to_vec(); + assert_ne!( + borrowed, + spliced[4][to..to + len], + "the two queries must actually have different layer-0 openings, or the \ + splice is a no-op and this vector proves nothing" + ); + spliced[4][to..to + len].copy_from_slice(&borrowed); + let err = execute(&program, &spliced, &TestPermutation).expect_err( + "a REAL leaf and a REAL path, at the wrong index, must still be rejected \ + — the walk climbs at this query's own bits", + ); + println!( + " {:<40} rejected: {err:?}", + "another query's real layer-0 opening" + ); +} + +/// ★ The three structural length checks production performs at RUNTIME are, in +/// this machine, impossible to fail — and that is worth demonstrating rather +/// than asserting. +/// +/// `verifier.rs:426-448` rejects on three lengths before its query loop, and the +/// comment there is emphatic about why: the per-query auth-path and +/// evaluation-sym vectors are **not** bound into the Fiat-Shamir transcript, so a +/// prover could send them EMPTY — making the fold loop run zero iterations and +/// accept the query vacuously — and that length check is the only thing pinning +/// them. +/// +/// In LFM there is no vector to send. `declare_fri` fixes each arena's length +/// from the shape, and the executor refuses an arena of any other length +/// (`ArenaLenMismatch`) before a single instruction runs. So the attack the +/// production comment describes is not defended against here, it is +/// unrepresentable: there is no encoding of "a proof with no FRI layers" that the +/// program for a 3-layer shape will accept. This test spells out each of the +/// three, including the vacuous-fold one. +#[test] +fn the_shape_pins_the_lengths_production_must_check_at_runtime() { + use super::executor::LfmExecError; + + let h = host_fri(2048, 2); + let queries = vec![0usize]; + let shape = FriShape { + num_queries: 1, + ..h.shape + }; + let program = fri_only_program(shape, 1); + let honest = h.all_arenas(&queries); + execute(&program, &honest, &TestPermutation).expect("the honest run must execute"); + + // (label, arena, what the truncation would buy a prover) + let attacks: [(&str, usize, &str); 3] = [ + ( + "no committed layer roots", + 1, + "production's `fri_layers_merkle_roots().len() != num_committed` check", + ), + ( + "fewer terminal coefficients", + 3, + "production's `fri_final_poly_coeffs().len() != 1 << effective_k` check", + ), + ( + "an EMPTY per-query decommitment — the vacuous fold", + 4, + "production's per-query `layers_auth_paths_len()` check, the one its \ + comment calls the only thing pinning these vecs", + ), + ]; + for (label, arena, mirrors) in attacks { + let mut truncated = honest.clone(); + truncated[arena].clear(); + let err = execute(&program, &truncated, &TestPermutation) + .expect_err(&format!("{label} must be refused")); + assert!( + matches!(err, LfmExecError::ArenaLenMismatch { .. }), + "{label} must be refused for its LENGTH, before any instruction \ + runs — got {err:?}" + ); + println!(" {label:<52} refused as {err:?}\n mirrors {mirrors}"); + } +} + +/// ★ The FRI leg PROVES and VERIFIES — method rule 2, discharged rather than +/// argued. +/// +/// Every other test here calls `execute`, which runs the executor and the +/// arena/assert semantics but never builds a trace or a proof. Rule 2 is explicit +/// that an execute-only test says nothing about the chips: where the executor +/// mirrors a computation the chip also does, only a prove+verify run sees the +/// chip. +/// +/// It is tempting to argue the coverage away — the FRI leg emits no instruction +/// the trace legs do not already emit, and `join_tests::the_join_proves_and_verifies` +/// proves those. That argument is probably true and is exactly the kind of thing +/// rule 5 says to check instead of assert, so this proves the JOINED program: the +/// openings authenticated, DEEP folded, and FRI folded to the terminal check, all +/// in one proved and verified run over a real folding sub-proof. +/// +/// One query, because the point is the chips rather than the sweep — the fold +/// mechanism's coverage is +/// [`the_fri_emitter_verifies_every_query_of_a_real_folding_proof`]'s, over all +/// 219 of three shapes. +#[test] +fn the_fri_leg_proves_and_verifies() { + use super::proof::{lfm_prove, verify_against}; + use super::registry::build_artifacts; + + let h = host_fri(512, 2); + assert_eq!( + h.shape.num_committed(), + 1, + "one committed layer is enough to put a leaf hash, a walk, a root compare \ + and both folds through the prover" + ); + let queries = [0usize]; + let opts = super::join_tests::prove_options(); + let shape = FriShape { + num_queries: queries.len(), + ..h.shape + }; + + let mut b = LfmBuilder::new(); + let (_, _, terminal) = + super::fri::emit_sub_proof_with_fri(&mut b, &h.trace.shape, shape, queries.len()); + for v in &terminal { + b.public(v.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined program is admissible"); + + let mut arenas = h.trace.arenas(&queries); + arenas.extend(h.fri_arenas(&queries)); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts) + .expect("the joined trace+DEEP+FRI program must prove"); + + // The proved run's published terminal value, against production's own + // codeword — so the proof is not merely valid but computes the right thing. + let codeword = h.terminal_codeword(); + assert_eq!( + word_as_ext(&proved.public_words[0].1).expect("ext"), + codeword[h.trace.iotas[queries[0]] >> h.shape.num_committed()], + "the PROVED run must publish the terminal codeword value production \ + would have looked up" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the joined FRI run must verify" + ); + println!( + "proved and verified: {} instructions, {} permutations, {} committed layer", + program.instrs.len(), + permutations(&program), + h.shape.num_committed(), + ); +} diff --git a/prover/src/lfm/hash.rs b/prover/src/lfm/hash.rs new file mode 100644 index 000000000..577edba0b --- /dev/null +++ b/prover/src/lfm/hash.rs @@ -0,0 +1,319 @@ +//! The LFM hash interface — the machine's swap surface. +//! +//! The ecosystem hash decision is open (Poseidon2 is broken; candidates are +//! Poseidon-original, RPO/XHash, Monolith and reduced-round Blake2s), so the +//! machine freezes only the *contract*: `Compress` and `Transcript` map two +//! digest cells to one — in different hash domains — `Permute` maps the +//! three-cell state to itself, and the `LFM_HASH` +//! bus tuples and opcode numbers are fixed. Whatever sits behind the trait is +//! the only thing a hash migration replaces. +//! +//! `TestPermutation` below is **NOT cryptographic**. It exists so the machine +//! can be built, executed and proved end-to-end before the hash decision +//! lands; it must never appear outside tests and pre-decision experiments. + +use crate::tables::types::FE; + +use super::instr::HashMode; +use super::word::LfmWord; + +/// Felts in the sponge state (three machine cells). +pub const HASH_STATE_FELTS: usize = 12; +/// Felts in a digest (one machine cell). +pub const HASH_DIGEST_FELTS: usize = 4; + +/// The machine's hash contract. `compress` has a default implementation as a +/// single permutation of `[a ‖ b ‖ IV]` truncated to the first cell, which is +/// the construction the chip's `Compress` mode implements; a real hash may +/// override it, but the bus contract (2 cells in, 1 cell out) is frozen. +/// +/// ⚠ **`permute` is not total for every candidate.** It is typed over arbitrary +/// Goldilocks elements, but a hasher built on 32-bit words can only accept +/// lane-restricted state, and a hasher may implement one socket and not the +/// other. [`LfmHasher::admits`] is where such a restriction is *declared* and +/// rejected; silently reducing an out-of-range input instead is the bug that +/// would make a host-side assertion pass while the chip proved something else. +pub trait LfmHasher { + /// The full state permutation (three cells → three cells). + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS]; + + /// The capacity cell injected into lanes 8–11 in `Compress` mode. + fn compress_iv(&self) -> LfmWord; + + /// The twelve `OUT` felts the chip writes on a `Compress` row. + /// + /// The default is the permute-and-truncate construction: all twelve lanes + /// of `permute(a ‖ b ‖ IV)`, of which the low four are the digest. The + /// executor records exactly this into the row's `OUT` columns, so a hasher + /// that overrides [`LfmHasher::compress`] must override this too — or the + /// trace would describe a permutation its own AIR does not constrain. + fn compress_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + let iv = self.compress_iv(); + let mut state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|_| FE::zero()); + state[0..4].clone_from_slice(a); + state[4..8].clone_from_slice(b); + state[8..12].clone_from_slice(&iv); + self.permute(state) + } + + /// Two digest cells → one digest cell. + fn compress(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + let out = self.compress_out(a, b); + [out[0], out[1], out[2], out[3]] + } + + /// One Fiat–Shamir transcript step: the same two-cells-in, one-cell-out + /// shape as [`LfmHasher::compress`], in the TRANSCRIPT hash domain. + /// + /// The default is `compress_out` — correct for a hasher with a single + /// domain, which is what `TestPermutation` and Poseidon are here. A hasher + /// that *has* domain separation overrides it, and BLAKE3 does: its socket + /// carries the domain tag in a message word, so a transcript step + /// and a Merkle parent over the same two cells are different digests. + /// + /// ⚠ The default is a real weakening for a single-domain hasher, and it is + /// deliberate rather than overlooked: under `Test` and `Poseidon` a + /// transcript step IS a Merkle parent, so those two hashers separate the + /// domains not at all. Neither is a production hash — `TestPermutation` is + /// explicitly non-cryptographic and Poseidon here is measurement-only — and + /// the machine's real hash is the one that separates them. A future + /// production candidate that reaches this default without overriding it is + /// shipping a transcript with no domain separation. + fn transcript_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + self.compress_out(a, b) + } + + /// [`LfmHasher::transcript_out`] truncated to the digest cell. + fn transcript(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + let out = self.transcript_out(a, b); + [out[0], out[1], out[2], out[3]] + } + + /// A Merkle LEAF: a chaining accumulator and one cell read as four arbitrary + /// FIELD ELEMENTS. + /// + /// **The accumulator is what makes the leaf a chain rather than a tree.** A + /// wide leaf is an arbitrary-width row pair, so its felts arrive four at a + /// time; carrying the running digest as this call's first operand absorbs + /// four felts AND chains in ONE hash, where folding a felts-only leaf digest + /// into the chain with a separate parent cost two (COMMIT.md §1.2). Leaf + /// absorption is the dominant term of a recursion tower node, which is why + /// the shape of this signature is worth the ripple. + /// + /// The default is a compress of the accumulator against the felts, which is + /// the natural reading for a **field-native** hasher: `TestPermutation` and + /// Poseidon take arbitrary Goldilocks elements directly, so a leaf needs no + /// encoding from them. + /// + /// BLAKE3 overrides it, and the override is the point of the whole mode: its + /// lanes must be `u32`, so each felt becomes a checked `lo`/`hi` pair inside + /// the socket, under the `"LFML"` tag. + /// + /// ⚠ Same weakening as [`LfmHasher::transcript_out`], recorded for the same + /// reason: a single-domain hasher does not separate a leaf from a parent, so + /// under `Test` and `Poseidon` the O5 second-preimage split is carried by + /// fixed tree depth alone, exactly as it was before this mode existed. + /// Neither is a production hash; the machine's real one separates them. + fn leaf_out(&self, acc: &LfmWord, felts: &LfmWord) -> [FE; HASH_STATE_FELTS] { + self.compress_out(acc, felts) + } + + /// [`LfmHasher::leaf_out`] truncated to the digest cell. + fn leaf(&self, acc: &LfmWord, felts: &LfmWord) -> LfmWord { + let out = self.leaf_out(acc, felts); + [out[0], out[1], out[2], out[3]] + } + + /// Rejects a hash instruction this hasher's chip cannot prove, naming why. + /// + /// Total for every candidate whose domain is the whole state under both + /// modes, which is why the default is `Ok`. It exists for the ones whose + /// domain is smaller: `HasherKind::Blake3` uses it for both of its + /// restrictions — it has no `permute` socket at all, and its `compress` + /// lanes must be `u32`. Returning an error here is what turns "the AIR + /// would reject this" into "the executor says so, with a reason". + fn admits(&self, mode: HashMode, state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + let _ = (mode, state); + Ok(()) + } +} + +/// A placeholder permutation: one round of `x ↦ (x + rc)³` followed by the +/// mixing matrix `M = I + J` (identity plus all-ones; eigenvalues 13 and 1, +/// so invertible over Goldilocks). Degree 3, one trace row per invocation. +/// +/// **NOT CRYPTOGRAPHIC — wiring placeholder only.** No diffusion analysis, no +/// round count, nothing: it is a stand-in with the right shape and degree +/// while the ecosystem hash decision is open. +pub struct TestPermutation; + +impl TestPermutation { + /// Fixed round "constants" — an odd multiplier walk; arbitrary, public. + pub fn round_constant(i: usize) -> FE { + FE::from(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1)) + } + + /// The compress-mode capacity constants as raw u64s (the chip bakes them + /// into its constraints via `const_base`; `FE::from` reduces identically). + pub fn compress_iv_raw() -> [u64; 4] { + core::array::from_fn(|i| 0xC0DE_0000_0000_0001u64.wrapping_add(i as u64)) + } +} + +impl LfmHasher for TestPermutation { + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + // t_i = (s_i + rc_i)^3 ; out_j = t_j + Σ_i t_i (M = I + J) + let t: Vec = state + .iter() + .enumerate() + .map(|(i, s)| { + let x = s + Self::round_constant(i); + &x * &x * x + }) + .collect(); + let sum: FE = t.iter().fold(FE::zero(), |acc, x| acc + x); + core::array::from_fn(|j| &t[j] + &sum) + } + + fn compress_iv(&self) -> LfmWord { + Self::compress_iv_raw().map(FE::from) + } +} + +/// Which permutation the `LFM_HASH` chip proves — a **construction-time** +/// choice, fixed before any trace exists. +/// +/// The chips bake their hasher's round constants into their constraints, so +/// execution, trace generation and the AIR set must all agree (`proof.rs` +/// enforces that by construction: one kind reaches all three). This enum is +/// what carries the agreement, and it is threaded rather than global so a +/// single process can prove under both. +/// +/// ⚠ **`Test` is the default and the machine's real hash is UNDECIDED.** The +/// default exists so every pre-decision call site keeps proving what it always +/// proved; it is not a statement that `TestPermutation` is the machine's hash. +/// The ecosystem hash decision is what the candidate columns feed. +/// +/// The discriminants are written out and `#[repr(u8)]` because [`as_tag`] feeds +/// `lfm_program_id`'s preimage: the wire value must never follow declaration +/// order, or inserting a variant would silently move every program digest. +/// +/// [`as_tag`]: HasherKind::as_tag +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[repr(u8)] +pub enum HasherKind { + /// [`TestPermutation`] — NOT cryptographic. One degree-3 round. + #[default] + Test = 0, + /// [`super::poseidon::PoseidonGoldilocks`] — Poseidon-original, width 12, + /// `x^7`, 8 full + 22 partial rounds. + Poseidon = 1, + /// [`super::blake3_socket::Blake3Permutation`] — BLAKE3 behind the Option-A + /// 2-to-1 compress socket, `compress` only. + /// + /// The one candidate here that is a real, standard, externally anchored + /// hash: at the default `SOCKET_ROUNDS = 7` a compress is literally + /// `blake3::hash(a ‖ b ‖ "LFMC")` truncated to 128 bits. It is also the one + /// with a restricted domain — no `permute` socket, and `u32` lanes — which + /// [`LfmHasher::admits`] enforces. + Blake3 = 2, +} + +impl HasherKind { + /// The stable one-byte tag bound into `lfm_program_id`. + /// + /// A new candidate takes the next unused value and never reuses a retired + /// one: a tag collision would give two different permutations one program + /// identity, which is the whole thing this binding exists to prevent. + pub const fn as_tag(self) -> u8 { + self as u8 + } +} + +impl LfmHasher for HasherKind { + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.permute(state), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.permute(state), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.permute(state), + } + } + + fn compress_iv(&self) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.compress_iv(), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.compress_iv(), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.compress_iv(), + } + } + + /// Delegated explicitly rather than left to the trait default: a candidate + /// that overrides `compress` must be honoured through this dispatch too. + fn compress(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.compress(a, b), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.compress(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.compress(a, b), + } + } + + /// Delegated explicitly, for the same reason `compress` is: BLAKE3 + /// overrides it, and a default that quietly permuted instead would write + /// twelve felts its own AIR pins to four. + fn compress_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.compress_out(a, b), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.compress_out(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.compress_out(a, b), + } + } + + /// Delegated explicitly, third time for the same reason: BLAKE3 is the one + /// candidate whose transcript domain differs from its compress domain, and + /// a dispatch that fell through to the trait default would hash a + /// transcript step under the MERKLE tag while its AIR proved the transcript + /// one — a host/chip disagreement, not a wrong answer the chip catches. + fn transcript_out(&self, a: &LfmWord, b: &LfmWord) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.transcript_out(a, b), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.transcript_out(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.transcript_out(a, b), + } + } + + fn transcript(&self, a: &LfmWord, b: &LfmWord) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.transcript(a, b), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.transcript(a, b), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.transcript(a, b), + } + } + + /// Delegated explicitly, fourth time for the same reason: BLAKE3's leaf mode + /// is an ENCODING, not just a tag, so a dispatch that fell through to the + /// trait default would hash four felts as a digest cell — a host answer no + /// chip proves. + fn leaf_out(&self, acc: &LfmWord, felts: &LfmWord) -> [FE; HASH_STATE_FELTS] { + match self { + HasherKind::Test => TestPermutation.leaf_out(acc, felts), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.leaf_out(acc, felts), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.leaf_out(acc, felts), + } + } + + fn leaf(&self, acc: &LfmWord, felts: &LfmWord) -> LfmWord { + match self { + HasherKind::Test => TestPermutation.leaf(acc, felts), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.leaf(acc, felts), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.leaf(acc, felts), + } + } + + fn admits(&self, mode: HashMode, state: &[FE; HASH_STATE_FELTS]) -> Result<(), &'static str> { + match self { + HasherKind::Test => TestPermutation.admits(mode, state), + HasherKind::Poseidon => super::poseidon::PoseidonGoldilocks.admits(mode, state), + HasherKind::Blake3 => super::blake3_socket::Blake3Permutation.admits(mode, state), + } + } +} diff --git a/prover/src/lfm/instr.rs b/prover/src/lfm/instr.rs new file mode 100644 index 000000000..5b3fe6040 --- /dev/null +++ b/prover/src/lfm/instr.rs @@ -0,0 +1,321 @@ +//! The LFM instruction set: eight algebra-shaped operations, no control flow. +//! +//! Addresses are dense indices assigned in emission order, so every operand +//! address is strictly below its destination (acyclicity by construction — +//! the validator re-checks it anyway). Every write carries its statically +//! known read count `mult`, backfilled by the compiler from the builder's +//! read counters. There is no pc, no branch, no computed address, no halt: +//! the program is a straight line and the dataflow is the execution. +//! +//! Assertions are deliberately not an instruction: `assert_eq(a, b)` lowers +//! to `diff = a - b; _ = diff / ZERO` under the division convention +//! `0/0 = 1, x/0 = error` — the AIR's division constraint `in2·out = in1` +//! with `in2 = 0` forces `in1 = 0`, and the executor errors on a nonzero +//! numerator. + +use crate::tables::types::FE; + +/// A write-once memory cell address (dense index into the address space). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Addr(pub u64); + +/// Identifies a host-supplied arena (id-addressed word sequence). +pub type ArenaId = u32; + +/// Base-field ALU operations. `MulAdd` computes `a·b + c` (the Horner step). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BaseOp { + Add, + Sub, + Mul, + Div, + MulAdd, +} + +/// Fp3 ALU operations, on word lanes 0–2. `MulAdd` is `a·b + c`; `MulBase` +/// multiplies an extension element by a base element (3 base muls, not 9). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExtOp { + Add, + Sub, + Mul, + Div, + MulAdd, + MulBase, +} + +/// The four hash-chiplet modes. `Compress`: two digest cells → one digest +/// cell. `Transcript`: the same shape in the Fiat–Shamir domain. `Leaf`: one +/// cell of four FIELD ELEMENTS → one digest cell. `Permute`: three state cells +/// → three state cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HashMode { + Compress, + /// One step of the Fiat–Shamir transcript chain. + /// + /// Structurally identical to [`HashMode::Compress`] — two cells in, one + /// cell out, same socket, same columns — and DIFFERENT in exactly one + /// thing: the hash domain. Under BLAKE3 the row's domain tag is the + /// message word `m[8]`, selected by the preprocessed mode columns, so a + /// transcript step cannot be replayed as a Merkle parent or the reverse. + /// Hashers with a single domain (`Test`, `Poseidon`) compute the same + /// function in both modes; the separation is a property of the hasher, not + /// of the machine. + Transcript, + /// A Merkle LEAF over four arbitrary field elements. + /// + /// **This mode implies felt-input semantics**, by decision rather than by + /// inference. The other modes read both their input cells as digests — four + /// `u32` lanes each; this one reads its FIRST cell that way, as a chaining + /// accumulator, and its SECOND as four Goldilocks elements, splitting each + /// into a checked `lo`/`hi` `u32` pair so eight halves fill the message + /// lanes above the accumulator. That is what lets FRI data — LDE evaluations + /// and folded extension elements, none of them `u32` — reach a hash whose + /// inputs must be `u32`, and it absorbs four felts per compression because + /// the chaining rides in the message rather than in a separate fold. + /// + /// It is also what retires obligation O5 — **under a hasher that separates + /// the domains.** BLAKE3 does: a leaf is `BLAKE3(…‖"LFML")` and a parent is + /// `BLAKE3(…‖"LFMC")`, so an internal node cannot be replayed as a leaf + /// whatever the tree's shape, where before that rested on every eDSL circuit + /// being fixed-depth — true, but enforced by nothing. + /// + /// ⚠ The mode is a machine-level shape, not a guarantee. A single-domain + /// hasher computes the same function in both modes, so under `Test` and + /// `Poseidon` O5 still rests on fixed depth exactly as it did before. The + /// separation is a property of the HASHER; see `LfmHasher::leaf_out`. + Leaf, + Permute, +} + +impl HashMode { + /// Whether this mode is the two-cells-in, one-cell-out shape — true for + /// `Compress` and `Transcript`, which differ only in hash domain. + /// + /// Every place that used to match `Compress` for a *shape* reason routes + /// through here, so adding a domain cannot silently take the permute arm. + pub const fn is_two_to_one(self) -> bool { + matches!(self, HashMode::Compress | HashMode::Transcript) + } + + /// Input cells this mode reads from memory: 2 or 3. + /// + /// The `LFM_HASH` bus receives are gated by exactly this, so a mode that + /// reads fewer cells must not receive the ones it does not read — a row + /// receiving a cell it never reads would claim a memory read it never makes. + /// + /// A `Leaf` reads TWO: its chaining accumulator, then the four felts it + /// absorbs. It read one until the leaf RATE put the accumulator in the + /// message rather than in a separate `"LFMC"` fold (COMMIT.md §1.2), which + /// is what took leaf absorption from 2 felts per compression to 4. + pub const fn num_input_cells(self) -> usize { + match self { + HashMode::Compress | HashMode::Transcript | HashMode::Leaf => 2, + HashMode::Permute => 3, + } + } + + /// Output cells this mode writes: 1 for every hashing mode, 3 for a + /// permutation. + pub const fn num_output_cells(self) -> usize { + match self { + HashMode::Permute => 3, + _ => 1, + } + } +} + +/// Operands of a [`Instr::KeccakF`]: 13 words of `u32`-half keccak state in, +/// 13 out, plus each output's static read count. +/// +/// The state's 25 `u64` lanes are not felt-representable (values in `[p, 2^64)` +/// exist), so they travel as 50 `u32` halves packed four to a word; the last +/// word's top two lanes are unused and must be zero. The permutation itself is +/// proved by the production `KECCAK_RND` / `KECCAK_RC` / `BITWISE` chips — +/// `LFM_KECCAK` only binds these words to the `Keccak` bus tokens. +#[derive(Debug, Clone)] +pub struct KeccakOperands { + pub mode: KeccakMode, + pub ins: [Addr; 13], + /// Rate-block words, read only in [`KeccakMode::Absorb`]; `Addr(0)` + /// placeholders otherwise (the receives are gated by the mode selector, so + /// the placeholders are never read). + pub block: [Addr; 9], + pub outs: [Addr; 13], + pub mults: [u64; 13], + /// When set, the row ALSO writes the byte-reversed first 32 bytes of the + /// output state as two words — the production transcript's `sample()`. + pub rev: Option, +} + +/// The reversed-digest outputs of a keccak row (see `layout::keccak::REV_ADDR0`). +#[derive(Debug, Clone)] +pub struct KeccakReversedDigest { + pub outs: [Addr; 2], + pub mults: [u64; 2], +} + +/// The adapter's two modes. +/// +/// `Permute` is the bare permutation (R1b). `Absorb` XORs a 136-byte rate block +/// into the state's rate region first — the sponge step, with the XOR done by +/// `BYTE_ALU[XOR]` lookups into the same BITWISE table the round chip uses. +/// Doing the XOR here rather than on the LFM side (bit-decompose, recombine) is +/// orders of magnitude cheaper: the adapter already owns byte-granular columns +/// and already talks to BITWISE. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeccakMode { + Permute, + Absorb, +} + +/// One LFM instruction. Operand-field conventions: +/// +/// - `c` on the ALU ops is meaningful iff the op is `MulAdd` (and is emitted +/// as address 0 otherwise — the corresponding bus receive is gated by the +/// `MulAdd` selector, so the placeholder is never read). +/// - `Hash` uses `ins[..num_input_cells()]` and `outs[..num_output_cells()]` +/// only — 2/1 for `Compress` and `Transcript`, 1/1 for `Leaf`, 3/3 for +/// `Permute`; the remaining slots are `Addr(0)` placeholders with `mults` +/// fixed to 0, and the validator checks both ends. +/// - `BitDec.bits` lists, low-to-high from bit 0, exactly the bit cells the +/// program consumes; bits beyond `bits.len()` exist as constrained witness +/// columns but get no memory cell. +#[derive(Debug, Clone)] +pub enum Instr { + Const { + out: Addr, + value: [FE; 4], + mult: u64, + }, + BaseAlu { + op: BaseOp, + out: Addr, + a: Addr, + b: Addr, + c: Addr, + mult: u64, + }, + ExtAlu { + op: ExtOp, + out: Addr, + a: Addr, + b: Addr, + c: Addr, + mult: u64, + }, + Select { + bit: Addr, + out_l: Addr, + out_r: Addr, + in_l: Addr, + in_r: Addr, + mult_l: u64, + mult_r: u64, + }, + BitDec { + input: Addr, + bits: Vec<(Addr, u64)>, + }, + Hash { + mode: HashMode, + ins: [Addr; 3], + outs: [Addr; 3], + mults: [u64; 3], + }, + Hint { + arena: ArenaId, + index: u32, + out: Addr, + mult: u64, + }, + /// Assemble a word from four base cells (unused lanes take the shared + /// zero-constant cell). The lane↔word coupling is enforced purely by the + /// `LFM_LANES` chip's bus tokens — no constraints. + Pack { + lanes: [Addr; 4], + out: Addr, + mult: u64, + }, + /// Split a word into four base cells — the only way a hash-state or + /// digest lane can reach the ALU (discovered as a real ISA gap in + /// Milestone C: challenges are squeezed as cells but consumed as felts). + Unpack { + input: Addr, + outs: [Addr; 4], + mults: [u64; 4], + }, + /// One `keccak-f[1600]` permutation over 13 words of `u32`-half state. + /// + /// Boxed: the 13-wide operand arrays are 312 bytes, four times the next + /// largest variant, and inlining them would quadruple every instruction in + /// the program vector. + KeccakF(Box), + Public { + addr: Addr, + index: u32, + }, +} + +impl Instr { + /// The addresses this instruction writes, in ascending order. + pub fn writes(&self) -> Vec { + match self { + Instr::Const { out, .. } + | Instr::BaseAlu { out, .. } + | Instr::ExtAlu { out, .. } + | Instr::Hint { out, .. } + | Instr::Pack { out, .. } => vec![*out], + Instr::Unpack { outs, .. } => outs.to_vec(), + Instr::KeccakF(k) => { + let mut v = k.outs.to_vec(); + if let Some(rev) = &k.rev { + v.extend_from_slice(&rev.outs); + } + v + } + Instr::Select { out_l, out_r, .. } => vec![*out_l, *out_r], + Instr::BitDec { bits, .. } => bits.iter().map(|(a, _)| *a).collect(), + Instr::Hash { mode, outs, .. } => outs[..mode.num_output_cells()].to_vec(), + Instr::Public { .. } => vec![], + } + } + + /// The addresses this instruction reads (meaningful operands only, per + /// the field conventions above). + pub fn reads(&self) -> Vec { + match self { + Instr::Const { .. } | Instr::Hint { .. } => vec![], + Instr::BaseAlu { op, a, b, c, .. } => { + if *op == BaseOp::MulAdd { + vec![*a, *b, *c] + } else { + vec![*a, *b] + } + } + Instr::ExtAlu { op, a, b, c, .. } => { + if *op == ExtOp::MulAdd { + vec![*a, *b, *c] + } else { + vec![*a, *b] + } + } + Instr::Select { + bit, in_l, in_r, .. + } => vec![*bit, *in_l, *in_r], + Instr::BitDec { input, .. } => vec![*input], + Instr::Hash { mode, ins, .. } => ins[..mode.num_input_cells()].to_vec(), + Instr::Pack { lanes, .. } => lanes.to_vec(), + Instr::Unpack { input, .. } => vec![*input], + Instr::KeccakF(k) => match k.mode { + KeccakMode::Permute => k.ins.to_vec(), + KeccakMode::Absorb => { + let mut v = k.ins.to_vec(); + v.extend_from_slice(&k.block); + v + } + }, + Instr::Public { addr, .. } => vec![*addr], + } + } +} diff --git a/prover/src/lfm/join_tests.rs b/prover/src/lfm/join_tests.rs new file mode 100644 index 000000000..5c9a57c68 --- /dev/null +++ b/prover/src/lfm/join_tests.rs @@ -0,0 +1,1902 @@ +//! The DEEP/Merkle join: DEEP across a full sub-proof, folding the SAME arena +//! cells the Merkle authentication authenticates. +//! +//! ## The oracle +//! +//! Two production functions, neither of them re-derived here: +//! `reconstruct_deep_composition_poly_evaluation_pair` for the fold, and the +//! proof's own committed roots for the authentication. The fixture is a real +//! proof of a real production AIR, produced by the production prover, and its +//! query indices come from a replay of the production verifier's transcript +//! rather than from a search. +//! +//! ## What this suite cannot see +//! +//! The FRI leg that consumes `DEEP(υ)` — nothing here checks that the +//! reconstructed value is the one the folding chain expects, only that it is +//! the value the production verifier would have computed. It also cannot see +//! whether the epoch's OTHER sub-proofs compose, since a sub-proof is verified +//! in isolation here. +//! +//! DEPTH it sees only as far as the fixtures go: six levels on the +//! preprocessed fixture (64 rows at blowup 2), two on the production one. +//! `join_leg_cost` emits at depth 22 but never runs it, and R1f's +//! `keccak_merkle_walk_authenticates_a_real_opening` remains the only executed +//! walk at production depth (20, main trace only). Six levels is enough to +//! distinguish a per-level walk from a two-level one; it is not enough to catch +//! something that only appears past a word boundary in the index. + +use math::field::traits::IsFFTField; +use stark::config::Commitment; +use stark::domain::new_verifier_domain; +use stark::proof::view::StarkProofView; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::constraint_tests::{deep_shape, open_sub_proof, real_fixture}; +use super::executor::execute; +use super::hash::TestPermutation; +use super::sub_proof::{ + GroupShape, ROWS_PER_LEAF, SubProofShape, emit_sub_proof, emit_sub_proof_with_bits, +}; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type V = Verifier; + +/// One committed matrix's data for one query, host side: the row pair in leaf +/// order and the path that authenticates it. +pub(super) struct HostGroupOpening { + /// `evaluations ‖ evaluations_sym`, as arena words. + values: Vec, + siblings: Vec, +} + +/// Everything the machine reads about one sub-proof, read off a real proof. +/// +/// Assembled once and shared, because `open_sub_proof` replays the whole +/// verifier transcript and the fixture proof is regenerated on every call. +pub(super) struct HostSubProof { + pub(super) shape: SubProofShape, + pub(super) gamma: FEE, + pub(super) zeta: FEE, + /// The OOD grid, row-major. + ood: Vec, + claimed_parts: Vec, + /// One root per group, in `SubProofShape::groups` order. + roots: Vec, + /// `[query][group]`. + openings: Vec>, + pub(super) iotas: Vec, + /// The FRI folding challenges, from the production verifier's own + /// `replay_rounds_after_round_1` (`verifier.rs:1461-1483`) — one per + /// committed layer plus the final-fold one. The FRI leg reads them; the + /// trace leg does not. + pub(super) zetas: Vec, + /// The production reconstruction's answer per query, `(regular, sym)`. + pub(super) expected: Vec<(FEE, FEE)>, + /// The same, asked of production with the PRECOMPUTED and MAIN slices + /// swapped — the alternative column order a fixture without a precomputed + /// group cannot distinguish. Empty when there is no precomputed group, or + /// when the two base groups are different widths (the swap would not be a + /// well-formed reading). + expected_base_swapped: Vec<(FEE, FEE)>, + /// Production's query points, kept so the machine's derivation can be + /// checked against them rather than against a local formula. + points: Vec<(FE, FE)>, +} + +fn host_sub_proof() -> &'static HostSubProof { + use std::sync::OnceLock; + static CELL: OnceLock = OnceLock::new(); + CELL.get_or_init(|| { + let (air, proof) = real_fixture(); + build_host_sub_proof(&*air, &proof) + }) +} + +pub(super) fn build_host_sub_proof( + air: &dyn stark::traits::AIR, + proof: &stark::proof::stark::MultiProof, +) -> HostSubProof { + let sp = open_sub_proof(air, proof); + let (deep, gamma) = deep_shape(&sp, air); + let view = StarkProofView::Owned(&proof.proofs[0]); + + let (main_width, aux_width) = air.trace_layout(); + let num_precomputed = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + let mut trace_groups = Vec::new(); + if num_precomputed > 0 { + trace_groups.push(GroupShape { + num_columns: num_precomputed, + is_ext: false, + }); + } + trace_groups.push(GroupShape { + num_columns: main_width - num_precomputed, + is_ext: false, + }); + if aux_width > 0 { + trace_groups.push(GroupShape { + num_columns: aux_width, + is_ext: true, + }); + } + + let blowup = air.options().blowup_factor as usize; + let lde_length = view.trace_length() * blowup; + let shape = SubProofShape { + deep: deep.clone(), + trace_groups, + merkle_depth: lde_length.trailing_zeros() as usize - 1, + log2_lde_length: lde_length.trailing_zeros(), + coset_offset: FE::from(air.options().coset_offset), + }; + + let mut roots = vec![]; + if num_precomputed > 0 { + roots.push( + *view + .lde_trace_precomputed_merkle_root() + .expect("a preprocessed air commits its precomputed columns"), + ); + } + roots.push(*view.lde_trace_main_merkle_root()); + if aux_width > 0 { + roots.push(*view.lde_trace_aux_merkle_root().expect("an aux root")); + } + roots.push(*view.composition_poly_root()); + + let domain = new_verifier_domain(air, view.trace_length()); + let layout = V::ood_layout(air); + let invariants = V::compute_query_invariant_deep_terms( + &sp.challenges, + view, + &sp.ood_full, + layout.next_row_cols(), + layout.step_size(), + ) + .expect("a real proof's invariant terms"); + let generator = ::get_primitive_root_of_unity(deep.log2_trace_length as u64) + .expect("root of unity"); + + let swap_is_well_formed = + num_precomputed > 0 && main_width - num_precomputed == num_precomputed; + let mut openings = Vec::new(); + let mut expected = Vec::new(); + let mut expected_base_swapped = Vec::new(); + let mut points = Vec::new(); + for (q, iota) in sp.challenges.iotas.iter().enumerate() { + let o = view.deep_poly_opening(q); + let mut groups: Vec = Vec::new(); + if num_precomputed > 0 { + let p = o.precomputed_trace_polys().expect("precomputed opening"); + groups.push(HostGroupOpening { + values: p + .evaluations() + .iter() + .chain(p.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + siblings: p.merkle_path().to_vec(), + }); + } + let m = o.main_trace_polys(); + groups.push(HostGroupOpening { + values: m + .evaluations() + .iter() + .chain(m.evaluations_sym()) + .map(|v| base_word(*v)) + .collect(), + siblings: m.merkle_path().to_vec(), + }); + if aux_width > 0 { + let a = o.aux_trace_polys().expect("aux opening"); + groups.push(HostGroupOpening { + values: a + .evaluations() + .iter() + .chain(a.evaluations_sym()) + .map(ext_word) + .collect(), + siblings: a.merkle_path().to_vec(), + }); + } + let c = o.composition_poly(); + groups.push(HostGroupOpening { + values: c + .evaluations() + .iter() + .chain(c.evaluations_sym()) + .map(ext_word) + .collect(), + siblings: c.merkle_path().to_vec(), + }); + openings.push(groups); + + let point = V::query_challenge_to_evaluation_point(*iota, false, &domain); + let point_sym = V::query_challenge_to_evaluation_point(*iota, true, &domain); + let empty_base: &[FE] = &[]; + let (want, want_sym) = V::reconstruct_deep_composition_poly_evaluation_pair( + &point, + &point_sym, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + o.precomputed_trace_polys() + .map(|p| p.evaluations()) + .unwrap_or(empty_base), + m.evaluations(), + o.aux_trace_polys().map(|a| a.evaluations()).unwrap_or(&[]), + c.evaluations(), + o.precomputed_trace_polys() + .map(|p| p.evaluations_sym()) + .unwrap_or(empty_base), + m.evaluations_sym(), + o.aux_trace_polys() + .map(|a| a.evaluations_sym()) + .unwrap_or(&[]), + c.evaluations_sym(), + ) + .expect("a real proof reconstructs"); + expected.push((want, want_sym)); + if swap_is_well_formed { + let p = o.precomputed_trace_polys().expect("precomputed opening"); + let swapped = V::reconstruct_deep_composition_poly_evaluation_pair( + &point, + &point_sym, + &generator, + &sp.challenges, + &invariants, + layout.next_row_cols(), + layout.step_size(), + m.evaluations(), + p.evaluations(), + o.aux_trace_polys().map(|a| a.evaluations()).unwrap_or(&[]), + c.evaluations(), + m.evaluations_sym(), + p.evaluations_sym(), + o.aux_trace_polys() + .map(|a| a.evaluations_sym()) + .unwrap_or(&[]), + c.evaluations_sym(), + ) + .expect("the swapped reading is well formed, so it reconstructs"); + expected_base_swapped.push(swapped); + } + points.push((point, point_sym)); + } + + let ood: Vec = (0..deep.num_eval_points) + .flat_map(|r| sp.ood_full.get_row(r)[..deep.num_total_cols].to_vec()) + .collect(); + + HostSubProof { + shape, + gamma, + zeta: sp.zeta, + ood, + claimed_parts: sp.claimed_parts.clone(), + roots, + openings, + iotas: sp.challenges.iotas.clone(), + zetas: sp.challenges.zetas.clone(), + expected, + expected_base_swapped, + points, + } +} + +impl HostSubProof { + /// The arenas [`emit_sub_proof`] declares, in its declaration order. + pub(super) fn arenas(&self, queries: &[usize]) -> Vec> { + vec![ + vec![ext_word(&self.gamma), ext_word(&self.zeta)], + self.ood.iter().map(ext_word).collect(), + self.claimed_parts.iter().map(ext_word).collect(), + super::proof_arena::commitments_to_arena(&self.roots), + self.query_arena(queries), + ] + } + + /// Per query: the index, then per group the row-pair values and the + /// sibling digests — the order the emitter's cursor walks. + pub(super) fn query_arena(&self, queries: &[usize]) -> Vec { + let mut out = Vec::new(); + for &q in queries { + out.push(base_word(FE::from(self.iotas[q] as u64))); + for group in &self.openings[q] { + out.extend(group.values.iter().copied()); + out.extend(super::proof_arena::commitments_to_arena(&group.siblings)); + } + } + out + } +} + +/// ★ Scrutinise the oracle before anything is built on it. +/// +/// Four separate premises the join rests on, each checked against the real +/// proof rather than assumed: that every group commits at the SAME depth (one +/// index addresses all four trees), that the depth is one below the LDE domain +/// (a leaf is a row pair), that the machine's point derivation reproduces +/// production's `query_challenge_to_evaluation_point` at every one of the +/// proof's indices, and that the symmetric point really is the negation. +#[test] +fn the_join_premises_hold_on_a_real_proof() { + let h = host_sub_proof(); + let s = &h.shape; + let groups = s.groups(); + + println!( + "sub-proof: {} groups {:?}, depth {}, log2(lde) {}, {} queries", + groups.len(), + groups + .iter() + .map(|g| (g.num_columns, g.is_ext)) + .collect::>(), + s.merkle_depth, + s.log2_lde_length, + h.iotas.len() + ); + + assert_eq!( + s.merkle_depth + 1, + s.log2_lde_length as usize, + "a leaf is a row pair, so the tree has one level fewer than the domain" + ); + assert_eq!( + ROWS_PER_LEAF, + stark::commitment::ROWS_PER_LEAF, + "the machine's leaf shape is a copy of the commitment layer's constant; \ + if that moves, every leaf hash and every DEEP index in this module goes \ + with it, and no differential would say so because both sides would move \ + together" + ); + for (q, per_group) in h.openings.iter().enumerate() { + for (g, opening) in per_group.iter().enumerate() { + assert_eq!( + opening.siblings.len(), + s.merkle_depth, + "query {q} group {g}: every tree must have the same depth, or \ + one index cannot address them all" + ); + assert_eq!( + opening.values.len(), + groups[g].num_values(), + "query {q} group {g}: width" + ); + } + } + + // The point derivation, run IN THE MACHINE at every one of the proof's + // indices and compared against production's own function. Recomputing the + // bit weights here instead would only check a host formula against + // production and leave the emitter unexamined — the same oracle mistake + // the method rules warn about, one level up. + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(1); + let index = b.hint_felt(arena, 0); + let bits = b.bit_dec(index, s.merkle_depth); + let (point, point_sym) = super::sub_proof::emit_query_points(&mut b, s, &bits); + b.public(point.as_cell()); + b.public(point_sym.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the point-derivation program is admissible"); + + for (q, iota) in h.iotas.iter().enumerate() { + let arenas = vec![vec![base_word(FE::from(*iota as u64))]]; + let exec = execute(&program, &arenas, &TestPermutation).expect("the derivation executes"); + assert_eq!( + exec.public_words[0].1[0], h.points[q].0, + "query {q}: the machine's point must be \ + query_challenge_to_evaluation_point(iota, false)" + ); + assert_eq!( + exec.public_words[1].1[0], h.points[q].1, + "query {q}: the machine's symmetric point must be \ + query_challenge_to_evaluation_point(iota, true)" + ); + } + println!( + "in-machine point derivation checked against production at all {} indices", + h.iotas.len() + ); +} + +/// ★ The headline differential: the machine's DEEP equals the production +/// verifier's, at every query of a full sub-proof, with every opened value +/// authenticated to the proof's own committed roots in the same run. +/// +/// The authentication is not a separate assertion here — it is `assert_word_eq` +/// inside the program, so a run in which any leaf failed to reach its root +/// would not execute at all. That the run produces DEEP values is already the +/// statement that the values it folded are the committed ones. +#[test] +fn the_join_matches_the_production_verifier_on_every_query() { + let h = host_sub_proof(); + let all: Vec = (0..h.iotas.len()).collect(); + + let mut b = LfmBuilder::new(); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, all.len()); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("the joined sub-proof program is admissible"); + + let exec = execute(&program, &h.arenas(&all), &TestPermutation) + .expect("an honest sub-proof must authenticate and fold"); + + let mut nonzero = 0usize; + for q in &all { + let (want, want_sym) = h.expected[*q]; + assert_eq!( + word_as_ext(&exec.public_words[2 * q].1).expect("ext"), + want, + "query {q}: DEEP at the regular point" + ); + assert_eq!( + word_as_ext(&exec.public_words[2 * q + 1].1).expect("ext"), + want_sym, + "query {q}: DEEP at the symmetric point" + ); + if want != FEE::zero() { + nonzero += 1; + } + } + assert_eq!( + nonzero, + all.len(), + "a vacuously zero reconstruction would make the differential empty" + ); + println!( + "joined sub-proof: {} queries, {} instructions, {} distinct indices", + all.len(), + program.instrs.len(), + { + let mut d = h.iotas.clone(); + d.sort_unstable(); + d.dedup(); + d.len() + } + ); +} + +// ============================================================================= +// Cost +// ============================================================================= + +/// Precomputed columns a table carries IN PRODUCTION. +/// +/// `test_utils::production_airs` builds the AIR objects without their +/// preprocessed commitments — the commitments need an ELF, a register file or a +/// page config, none of which a shape census has. `is_preprocessed()` is +/// therefore FALSE on five tables that are preprocessed in the real epoch +/// (`lib.rs`'s `VmAirs::new` wires BITWISE, DECODE, KECCAK_RC, REGISTER and +/// PAGE; `continuation.rs` wires GLOBAL_MEMORY), and reading the flag off these +/// objects would drop one opening group — one leaf hash and one path walk — +/// from each of them. +/// +/// The split is what matters here, not the commitment value: a preprocessed +/// table's columns `0..n` are committed in their own tree and the rest in the +/// main tree, so the same columns are hashed as TWO leaves instead of one. +fn production_num_precomputed( + label: &str, + air: &dyn stark::traits::AIR, +) -> usize { + use crate::tables::{bitwise, decode, keccak_rc, page, register}; + + let wired = match label { + "BITWISE" => bitwise::NUM_PRECOMPUTED_COLS, + "DECODE" => decode::NUM_PRECOMPUTED_COLS, + "KECCAK_RC" => keccak_rc::NUM_PRECOMPUTED_COLS, + "REGISTER" => register::NUM_PREPROCESSED_COLS, + "PAGE" => page::NUM_PREPROCESSED_COLS, + _ => 0, + }; + if air.is_preprocessed() { + // Already wired by the constructor (GLOBAL_MEMORY): trust the object. + assert_eq!( + wired, 0, + "{label} is wired preprocessed AND listed above; one of the two is stale" + ); + return air.num_precomputed_columns(); + } + wired +} + +/// The DEEP shape and the opening groups of a production AIR, as a sub-proof of +/// `log2_trace_length` rows at `blowup` would carry them. +fn shape_for( + air: &dyn stark::traits::AIR, + num_precomputed: usize, + log2_trace_length: u32, + log2_blowup: u32, +) -> SubProofShape { + use stark::constraint_ir::ConstraintArtifact; + + let artifact = ConstraintArtifact::capture(air); + let layout = V::ood_layout(air); + let (main_width, aux_width) = air.trace_layout(); + + let mut trace_groups = Vec::new(); + if num_precomputed > 0 { + trace_groups.push(GroupShape { + num_columns: num_precomputed, + is_ext: false, + }); + } + trace_groups.push(GroupShape { + num_columns: main_width - num_precomputed, + is_ext: false, + }); + if aux_width > 0 { + trace_groups.push(GroupShape { + num_columns: aux_width, + is_ext: true, + }); + } + + SubProofShape { + deep: super::deep::DeepShape { + step_size: layout.step_size(), + num_eval_points: artifact.shape.transition_offsets.len() * layout.step_size(), + num_total_cols: main_width + aux_width, + next_row_cols: layout.next_row_cols().to_vec(), + num_composition_parts: artifact.shape.composition_degree_multiplier as usize, + log2_trace_length, + }, + trace_groups, + merkle_depth: (log2_trace_length + log2_blowup) as usize - 1, + log2_lde_length: log2_trace_length + log2_blowup, + coset_offset: FE::from(3u64), + } +} + +fn count bool>( + program: &super::compiler::LfmProgram, + f: F, +) -> usize { + program.instrs.iter().filter(|i| f(i)).count() +} + +fn permutations(program: &super::compiler::LfmProgram) -> usize { + count(program, |i| matches!(i, super::instr::Instr::KeccakF(_))) +} + +/// Byte swaps — one `LFM_BITDEC` row each. Every field element that enters a +/// leaf hash needs one; nothing else in this leg decomposes, except the one +/// index decomposition per query. +fn bit_decs(program: &super::compiler::LfmProgram) -> usize { + count(program, |i| matches!(i, super::instr::Instr::BitDec { .. })) +} + +/// Marginal per-query cost of one shape, by emitting one query and two and +/// differencing — so no per-sub-proof plumbing (the invariants, the OOD grid, +/// the hoisted root unpacks) leaks into the figure. +struct PerQuery { + instrs: usize, + perms: usize, + swaps: usize, +} + +fn marginal(shape: &SubProofShape) -> PerQuery { + let mut one = LfmBuilder::new(); + emit_sub_proof(&mut one, shape, 1); + let one = compile(one.finish()); + let mut two = LfmBuilder::new(); + emit_sub_proof(&mut two, shape, 2); + let two = compile(two.finish()); + PerQuery { + instrs: two.instrs.len() - one.instrs.len(), + perms: permutations(&two) - permutations(&one), + swaps: bit_decs(&two) - bit_decs(&one), + } +} + +/// The DEEP fold alone, both points, with no authentication — the same +/// measurement `constraint_tests::deep_leg_cost` reports, repeated here so the +/// two halves of the joined leg can be compared on one line. +fn deep_only_rows(shape: &SubProofShape) -> usize { + use super::deep::{DeepOpening, emit_deep_invariants, emit_deep_point}; + + let d = &shape.deep; + let plumb = |b: &mut LfmBuilder| { + let n = 2 + + d.num_eval_points * d.num_total_cols + + d.num_composition_parts + + 2 * (d.num_total_cols + d.num_composition_parts) + + 2; + let arena = b.declare_arena(n as u32); + let mut i = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(arena, i).as_ext(); + i += 1; + c + }; + let g = take(b); + let z = take(b); + let steps: Vec> = (0..d.num_eval_points) + .map(|_| (0..d.num_total_cols).map(|_| take(b)).collect()) + .collect(); + let parts: Vec<_> = (0..d.num_composition_parts).map(|_| take(b)).collect(); + let openings: Vec<(Vec<_>, Vec<_>)> = (0..2) + .map(|_| { + ( + (0..d.num_total_cols).map(|_| take(b)).collect(), + (0..d.num_composition_parts).map(|_| take(b)).collect(), + ) + }) + .collect(); + let points: Vec<_> = (0..2) + .map(|_| super::builder::Felt(take(b).addr())) + .collect(); + (g, z, steps, parts, openings, points) + }; + + let mut bare = LfmBuilder::new(); + let _ = plumb(&mut bare); + let baseline = bare.finish().instrs.len(); + + let mut inv_only = LfmBuilder::new(); + let (g, z, steps, parts, _, _) = plumb(&mut inv_only); + let _ = emit_deep_invariants(&mut inv_only, d, g, z, &steps, &parts); + let invariant_rows = inv_only.finish().instrs.len() - baseline; + + let mut full = LfmBuilder::new(); + let (g, z, steps, parts, openings, points) = plumb(&mut full); + let inv = emit_deep_invariants(&mut full, d, g, z, &steps, &parts); + for (k, (trace, qparts)) in openings.into_iter().enumerate() { + emit_deep_point( + &mut full, + d, + g, + &inv, + &DeepOpening { + point: points[k], + trace, + parts: qparts, + }, + ); + } + full.finish().instrs.len() - baseline - invariant_rows +} + +/// ★ What the joined leg costs, per query and per epoch, and how the bill +/// splits between folding the values and authenticating them. +/// +/// Measured by emitting one query and two and differencing, so the marginal +/// figure carries no per-sub-proof plumbing. Three currencies, because the +/// sizing rule in `others/lfm-target-shape.md` says rows of different chips are +/// not comparable: instructions, keccak permutations, and main-trace CELLS — +/// the last being the only one in which a byteswap and a permutation can be +/// added together. +/// +/// ### What this instrument cannot see +/// +/// The trace LENGTH of each table in a real epoch. It is workload-dependent and +/// enters only through the Merkle depth (`log2(N·blowup) − 1`), which the walk +/// is linear in, so the line below is parameterised on one uniform length +/// rather than measured. It also cannot see FRI, whose own layer openings are a +/// separate authentication bill this leg does not carry, nor the query COUNT, +/// which is a proof-options property. +#[test] +fn join_leg_cost() { + /// Queries at blowup 8 — a proof-options property, stated not measured. + const QUERIES: usize = 73; + const LOG2_BLOWUP: u32 = 3; + const LOG2_TRACE: u32 = 20; + + let swap_cells = super::machine_tests::byteswap_cells(); + let perm_cells = super::machine_tests::permutation_cells(); + + let opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(1 << LOG2_BLOWUP) + .expect("a power-of-two blowup is valid"); + let airs = crate::test_utils::production_airs(&opts); + assert_eq!(airs.len(), crate::test_utils::NUM_PRODUCTION_AIRS); + + println!( + "\nJoined DEEP+authentication, per query, at log2(N) = {LOG2_TRACE}, \ + blowup 2^{LOG2_BLOWUP} (Merkle depth {})", + LOG2_TRACE + LOG2_BLOWUP - 1 + ); + println!( + "{:<14} {:>5} {:>4} {:>9} {:>8} {:>7} {:>8} {:>12}", + "table", "cols", "grp", "instr/qry", "of it DEEP", "perm", "swaps", "cells/qry" + ); + + let mut total_instr = 0usize; + let mut total_deep = 0usize; + let mut total_perm = 0usize; + let mut total_swaps = 0usize; + for (label, air) in &airs { + let num_precomputed = production_num_precomputed(label, &**air); + let shape = shape_for(&**air, num_precomputed, LOG2_TRACE, LOG2_BLOWUP); + let per = marginal(&shape); + let deep = deep_only_rows(&shape); + + total_instr += per.instrs; + total_deep += deep; + total_perm += per.perms; + total_swaps += per.swaps; + println!( + "{:<14} {:>5} {:>4} {:>9} {:>8} {:>7} {:>8} {:>12}", + label, + shape.deep.num_total_cols, + shape.groups().len(), + per.instrs, + deep, + per.perms, + per.swaps, + per.perms as u64 * perm_cells + per.swaps as u64 * swap_cells, + ); + } + + let total_cells = total_perm as u64 * perm_cells + total_swaps as u64 * swap_cells; + println!( + "\nOne query, all {} AIRs: {total_instr} instructions ({total_deep} of \ + them the DEEP fold, {:.1}%), {total_perm} permutations, \ + {total_swaps} byteswaps.", + airs.len(), + 100.0 * total_deep as f64 / total_instr as f64, + ); + println!( + "In main-trace CELLS: {} hashing, {} byteswapping — hashing is {:.1}x \ + the swap bill.", + total_perm as u64 * perm_cells, + total_swaps as u64 * swap_cells, + (total_perm as u64 * perm_cells) as f64 / (total_swaps as u64 * swap_cells) as f64, + ); + println!( + "At {QUERIES} queries: {} instructions, {} permutations, {} cells.", + total_instr * QUERIES, + total_perm * QUERIES, + total_cells * QUERIES as u64, + ); + + // ---- what a SHARED commitment would cost, exactly, under one assumption - + // + // `others/lfm-team-lead-shared-commitment-ruling.md` parks the lever and + // pins a prediction of 55-70k permutations, noting that leaf WIDENING under + // a shared tree is unmeasured and could offset the walk saving. It can be + // settled without building anything: a permutation count is a function of + // the shape alone -- `ceil(leaf_bytes / 136)` absorbs plus one per level -- + // so the only thing being assumed is the SHAPE (one tree per sub-proof + // whose leaf is the four matrices' row pairs concatenated in matrix order). + // Nothing about the arithmetic is estimated. + // + // Widening cannot offset much, and the reason is structural: absorbs scale + // with total bytes, which do not change when the matrices share a leaf, + // while walks scale with the number of TREES, which is what collapses. The + // only bytes lost are the per-leaf padding of the groups that disappear. + const RATE_BYTES: usize = 136; + let mut shared_perm = 0usize; + for (label, air) in &airs { + let num_precomputed = production_num_precomputed(label, &**air); + let shape = shape_for(&**air, num_precomputed, LOG2_TRACE, LOG2_BLOWUP); + let leaf_bytes: usize = shape.groups().iter().map(GroupShape::leaf_bytes).sum(); + shared_perm += leaf_bytes.div_ceil(RATE_BYTES) + shape.merkle_depth; + } + println!( + "\nOne shared tree per sub-proof instead of four: {shared_perm} \ + permutations per query against {total_perm} ({:.0}% collapse), \ + {} per epoch at {QUERIES} queries.", + 100.0 * (1.0 - shared_perm as f64 / total_perm as f64), + shared_perm * QUERIES, + ); + println!( + "Absorbs are {} of the shared figure and walks {}; widening costs \ + nothing here because total leaf BYTES do not change when matrices \ + share a leaf -- only the padding of the vanished leaves.", + shared_perm - airs.len() * (LOG2_TRACE + LOG2_BLOWUP - 1) as usize, + airs.len() * (LOG2_TRACE + LOG2_BLOWUP - 1) as usize, + ); +} + +// ============================================================================= +// Falsification: the join, and the two attacks it denies +// ============================================================================= + +use super::builder::{Bit, Cell, Ext, Felt}; +use super::deep::{DeepOpening, emit_deep_invariants, emit_deep_point}; +use super::proof::{lfm_prove, verify_against}; +use super::registry::build_artifacts; +use super::sub_proof::{ + GroupCommitment, GroupOpening, emit_group_authentication, emit_query_points, +}; + +pub(super) fn prove_options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// How a control program differs from the joined one. Each variant is an +/// attack surface the join closes, built so the attack can be RUN rather than +/// argued about. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Control { + /// DEEP folds a second arena instead of the authenticated cells — the + /// "two parallel copies" shape. + SplitValues, + /// The query point is hinted instead of derived from the walk's own index + /// bits, so the leaf may be authenticated at one index and folded at + /// another point. + HintedPoint, +} + +/// A one-query program in one of the control shapes. +/// +/// Deliberately NOT a variant of [`emit_sub_proof`]: the production emitter has +/// no switch that could produce these, and giving it one would be a runtime +/// off-switch on a soundness obligation. This is a test artifact that exists to +/// be attacked. +/// +/// Arenas: the joined program's five, plus one extra carrying whatever the +/// control decouples. +fn control_program_source( + shape: &SubProofShape, + control: Control, +) -> super::builder::LfmProgramSource { + let mut b = LfmBuilder::new(); + let groups = shape.groups(); + + let uniforms = b.declare_arena(2); + let ood = b.declare_arena((shape.deep.num_eval_points * shape.deep.num_total_cols) as u32); + let parts_arena = b.declare_arena(shape.deep.num_composition_parts as u32); + let roots = b.declare_arena(2 * groups.len() as u32); + let queries = b.declare_arena(shape.query_words() as u32); + let extra = b.declare_arena(match control { + // A second copy of every folded value, both points. + Control::SplitValues => { + 2 * (shape.deep.num_total_cols + shape.deep.num_composition_parts) as u32 + } + // The two points. + Control::HintedPoint => 2, + }); + + let gamma = b.hint_word(uniforms, 0).as_ext(); + let zeta = b.hint_word(uniforms, 1).as_ext(); + let mut next = 0u32; + let ood_steps: Vec> = (0..shape.deep.num_eval_points) + .map(|_| { + (0..shape.deep.num_total_cols) + .map(|_| { + let c = b.hint_word(ood, next).as_ext(); + next += 1; + c + }) + .collect() + }) + .collect(); + let claimed_parts: Vec = (0..shape.deep.num_composition_parts as u32) + .map(|j| b.hint_word(parts_arena, j).as_ext()) + .collect(); + let commitments: Vec = groups + .iter() + .enumerate() + .map(|(i, g)| GroupCommitment::hint(&mut b, roots, 2 * i as u32, *g)) + .collect(); + let inv = emit_deep_invariants(&mut b, &shape.deep, gamma, zeta, &ood_steps, &claimed_parts); + + let mut cursor = 0u32; + let index = b.hint_felt(queries, cursor); + cursor += 1; + let openings: Vec = groups + .iter() + .map(|g| { + let values: Vec = (0..g.num_values()) + .map(|_| { + let c = b.hint_word(queries, cursor); + cursor += 1; + c + }) + .collect(); + let siblings: Vec<[Cell; 2]> = (0..shape.merkle_depth) + .map(|_| { + let lo = b.hint_word(queries, cursor); + let hi = b.hint_word(queries, cursor + 1); + cursor += 2; + [lo, hi] + }) + .collect(); + GroupOpening { values, siblings } + }) + .collect(); + + let bits: Vec = b.bit_dec(index, shape.merkle_depth); + for (commitment, opening) in commitments.iter().zip(&openings) { + emit_group_authentication(&mut b, commitment, opening, &bits); + } + + let (point, point_sym) = match control { + Control::HintedPoint => ( + Felt(b.hint_word(extra, 0).addr()), + Felt(b.hint_word(extra, 1).addr()), + ), + Control::SplitValues => emit_query_points(&mut b, shape, &bits), + }; + + let read = |b: &mut LfmBuilder, k: usize, point: Felt| -> DeepOpening { + let width = shape.deep.num_total_cols + shape.deep.num_composition_parts; + let base = (k * width) as u32; + let (trace, parts): (Vec, Vec) = match control { + Control::SplitValues => ( + (0..shape.deep.num_total_cols) + .map(|c| b.hint_word(extra, base + c as u32).as_ext()) + .collect(), + (0..shape.deep.num_composition_parts) + .map(|j| { + b.hint_word(extra, base + (shape.deep.num_total_cols + j) as u32) + .as_ext() + }) + .collect(), + ), + Control::HintedPoint => { + let mut trace = Vec::new(); + for (opening, g) in openings.iter().zip(&groups).take(shape.trace_groups.len()) { + for c in 0..g.num_columns { + trace.push(opening.values[k * g.num_columns + c].as_ext()); + } + } + let parts_opening = openings.last().expect("parts"); + let np = shape.deep.num_composition_parts; + ( + trace, + (0..np) + .map(|j| parts_opening.values[k * np + j].as_ext()) + .collect(), + ) + } + }; + DeepOpening { + point, + trace, + parts, + } + }; + let regular = read(&mut b, 0, point); + let symmetric = read(&mut b, 1, point_sym); + let got = emit_deep_point(&mut b, &shape.deep, gamma, &inv, ®ular); + let got_sym = emit_deep_point(&mut b, &shape.deep, gamma, &inv, &symmetric); + b.public(got.as_cell()); + b.public(got_sym.as_cell()); + b.finish() +} + +impl HostSubProof { + /// The values one query folds, in the order a [`Control::SplitValues`] + /// program reads them: the regular point's trace then parts, then the + /// symmetric point's. + fn split_values(&self, q: usize) -> Vec { + let groups = self.shape.groups(); + let mut out = Vec::new(); + for k in 0..ROWS_PER_LEAF { + for (opening, g) in self.openings[q] + .iter() + .zip(&groups) + .take(self.shape.trace_groups.len()) + { + out.extend(&opening.values[k * g.num_columns..(k + 1) * g.num_columns]); + } + let parts = self.openings[q].last().expect("parts"); + let np = self.shape.deep.num_composition_parts; + out.extend(&parts.values[k * np..(k + 1) * np]); + } + out + } +} + +/// ★ The joined program authenticates and folds under a real PROOF, not just +/// an execution. +/// +/// Method rule 2: the executor mirrors the ALU it is checking, so nothing run +/// so far says the CHIPS agree. One query, because the whole point of this test +/// is the chips and 219 of them would only repeat the same rows. +#[test] +fn the_join_proves_and_verifies() { + let h = host_sub_proof(); + let opts = prove_options(); + let queries = [0usize]; + + let mut b = LfmBuilder::new(); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, queries.len()); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &h.arenas(&queries), &opts) + .expect("the joined sub-proof must prove"); + + let (want, want_sym) = h.expected[queries[0]]; + assert_eq!( + word_as_ext(&proved.public_words[0].1).expect("ext"), + want, + "the proved run must publish the production reconstruction" + ); + assert_eq!( + word_as_ext(&proved.public_words[1].1).expect("ext"), + want_sym + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the joined run must verify" + ); +} + +/// ★ The join, stated as the property it exists for: there is no arena the +/// prover can move that changes the folded value without breaking the +/// authentication. +/// +/// Every opened value of one query is tampered, one at a time, and each vector +/// is run BOTH ways. Incoherent (claim the real root) must not execute. +/// Coherent (also claim the root the tampered leaf really folds to, so nothing +/// in the run is inconsistent) must execute — and then publish a DEEP value +/// that is not the production one, against a root that is not the committed +/// one. A prover who wants the wrong fold must pay with the wrong root. +/// +/// Run over BOTH fixtures. The three-matrix one is the real production table; +/// the four-matrix one is the only fixture in which the precomputed group's own +/// leaf and path are ever tampered, and its 64-row trace is the only executed +/// walk in this file deeper than two levels. +#[test] +fn no_tampered_value_can_move_the_fold_without_moving_the_root() { + sweep_tampers(host_sub_proof(), "L2G_MEMORY (3 matrices, depth 2)"); + sweep_tampers( + preprocessed_sub_proof(), + "PREPROCESSED_FIXTURE (4 matrices, depth 6)", + ); +} + +fn sweep_tampers(h: &HostSubProof, label: &str) { + use super::proof_arena::{commitments_to_arena, walk_to_root}; + + let q = 0usize; + let groups = h.shape.groups(); + + let mut b = LfmBuilder::new(); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, 1); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + let honest = execute(&program, &h.arenas(&[q]), &TestPermutation).expect("honest"); + + // Sweep every value slot of every group, so no vector class (first group, + // first column, regular point) is silently the only one tested. + let mut vectors = 0usize; + for (g, group) in groups.iter().enumerate() { + for slot in 0..group.num_values() { + let mut arenas = h.arenas(&[q]); + let word_of_slot = { + // Offset of this group's value `slot` inside the query arena. + let mut off = 1usize; + for prior in groups.iter().take(g) { + off += prior.num_values() + 2 * h.shape.merkle_depth; + } + off + slot + }; + arenas[4][word_of_slot][0] += FE::one(); + + // Incoherent: the real roots, a moved leaf. + let err = execute(&program, &arenas, &TestPermutation) + .err() + .unwrap_or_else(|| { + panic!("{label}: group {g} slot {slot}: a moved value must not authenticate") + }); + + // Coherent: recompute the leaf the tampered values really give and + // the root that leaf really reaches, using PRODUCTION's hashers. + let leaf = tampered_leaf(h, q, g, slot); + let forged = walk_to_root(leaf, h.iotas[q], &h.openings[q][g].siblings); + assert_ne!( + forged, h.roots[g], + "{label}: group {g} slot {slot}: the tamper must move the root, or the \ + vector is vacuous" + ); + let mut coherent_roots = h.roots.clone(); + coherent_roots[g] = forged; + arenas[3] = commitments_to_arena(&coherent_roots); + let forged_run = execute(&program, &arenas, &TestPermutation).unwrap_or_else(|e| { + panic!("{label}: group {g} slot {slot}: the coherent forgery must execute: {e:?}") + }); + // Which of the two points moves is not incidental: a leaf holds + // the row PAIR, its first half is the regular point and its second + // the symmetric, and folding the halves into the wrong point is a + // mistake no root check would catch. Asserting exactly one moved, + // and which, is what pins that split. + let moved = [ + forged_run.public_words[0].1 != honest.public_words[0].1, + forged_run.public_words[1].1 != honest.public_words[1].1, + ]; + let regular_half = slot < group.num_columns; + assert_eq!( + moved, + [regular_half, !regular_half], + "{label}: group {g} slot {slot}: a value in the leaf's {} half must move \ + DEEP at {} and nothing else", + if regular_half { "first" } else { "second" }, + if regular_half { + "the regular point" + } else { + "-v" + }, + ); + if vectors == 0 { + println!("first incoherent rejection: {err:?}"); + } + vectors += 1; + } + } + println!("{label}: {vectors} tamper vectors, every value slot of every group, both ways round"); + + // ---- the index, which this leg binds to the POINT as well as the leaf -- + // + // R1f authenticated a leaf at an index; here the same bits also derive the + // evaluation point, so moving the index has to move the reconstruction as + // well as the walk. A padding-heavy table can have several indices that + // authenticate (identical rows give identical leaves), which would make the + // walk half of this vector vacuous — so that is asserted, not assumed. + for level in 0..h.shape.merkle_depth { + let bad = h.iotas[q] ^ (1 << level); + let mut arenas = h.arenas(&[q]); + arenas[4][0] = base_word(FE::from(bad as u64)); + + let mut moved_a_root = false; + let mut coherent_roots = h.roots.clone(); + for (g, group) in groups.iter().enumerate() { + let words = &h.openings[q][g].values; + let leaf = if group.is_ext { + type ExtBackend = stark::config::BatchedMerkleTreeBackend; + let v: Vec = words.iter().map(|w| FEE::new([w[0], w[1], w[2]])).collect(); + ExtBackend::hash_data_from_slices(&v, &[]) + } else { + type BaseBackend = stark::config::BatchedMerkleTreeBackend; + let v: Vec = words.iter().map(|w| w[0]).collect(); + BaseBackend::hash_data_from_slices(&v, &[]) + }; + coherent_roots[g] = walk_to_root(leaf, bad, &h.openings[q][g].siblings); + moved_a_root |= coherent_roots[g] != h.roots[g]; + } + assert!( + moved_a_root, + "{label}: flipping index bit {level} left every root unchanged — the fixture's \ + trees are degenerate at this index and the walk half of this vector \ + tests nothing" + ); + execute(&program, &arenas, &TestPermutation) + .err() + .unwrap_or_else(|| { + panic!("{label}: index bit {level}: a moved index must not authenticate") + }); + + arenas[3] = commitments_to_arena(&coherent_roots); + let forged = execute(&program, &arenas, &TestPermutation).unwrap_or_else(|e| { + panic!("{label}: index bit {level}: coherent forgery must execute: {e:?}") + }); + assert_ne!( + forged.public_words[0].1, honest.public_words[0].1, + "{label}: index bit {level}: the index derives the evaluation point, so a \ + forged walk at another index must also fold at another point" + ); + } + println!( + "{label}: {} index vectors, one per level", + h.shape.merkle_depth + ); + + // ---- a sibling, at every level ------------------------------------- + for level in 0..h.shape.merkle_depth { + let mut siblings = h.openings[q][0].siblings.clone(); + siblings[level][0] ^= 1; + let mut arenas = h.arenas(&[q]); + let base = 1 + groups[0].num_values(); + arenas[4][base..base + 2 * h.shape.merkle_depth] + .copy_from_slice(&commitments_to_arena(&siblings)); + execute(&program, &arenas, &TestPermutation) + .err() + .unwrap_or_else(|| { + panic!("{label}: sibling level {level}: a moved path must not authenticate") + }); + } + println!( + "{label}: {} sibling vectors, one per level", + h.shape.merkle_depth + ); + + /// The leaf hash a tampered opening really produces, under production's own + /// backend rather than a local model. + fn tampered_leaf(h: &HostSubProof, q: usize, g: usize, slot: usize) -> Commitment { + type BaseBackend = stark::config::BatchedMerkleTreeBackend; + type ExtBackend = stark::config::BatchedMerkleTreeBackend; + let group = h.shape.groups()[g]; + let words = &h.openings[q][g].values; + if group.is_ext { + let mut v: Vec = words.iter().map(|w| FEE::new([w[0], w[1], w[2]])).collect(); + v[slot] = &v[slot] + FEE::new([FE::one(), FE::zero(), FE::zero()]); + ExtBackend::hash_data_from_slices(&v, &[]) + } else { + let mut v: Vec = words.iter().map(|w| w[0]).collect(); + v[slot] += FE::one(); + BaseBackend::hash_data_from_slices(&v, &[]) + } + } +} + +/// ★ The two attacks the join denies, RUN against control programs that permit +/// them. +/// +/// A join is a negative claim — "these cannot disagree" — and a negative claim +/// is only worth what its counterexample is worth. So each control is the +/// joined program with exactly one link cut, fed inputs that are honest +/// everywhere else, and each one accepts a reconstruction the production +/// verifier would not have produced. That is the thing the joined program has +/// to refuse, and the test above shows it does. +#[test] +fn the_controls_show_what_the_join_denies() { + let h = host_sub_proof(); + let q = 0usize; + + // ---- Control 1: DEEP folds a parallel copy. ------------------------- + let program = compile(control_program_source(&h.shape, Control::SplitValues)); + validate(&program).expect("admissible"); + let mut arenas = h.arenas(&[q]); + arenas.push(h.split_values(q)); + let clean = execute(&program, &arenas, &TestPermutation) + .expect("the control must accept honest inputs"); + assert_eq!( + word_as_ext(&clean.public_words[0].1).expect("ext"), + h.expected[q].0, + "the control must agree with production before it is attacked, or the \ + attack below proves nothing" + ); + + let mut attacked = arenas.clone(); + attacked[5][0][0] += FE::one(); + let forged = execute(&program, &attacked, &TestPermutation).expect( + "SplitValues: authenticating one set of values and folding another is \ + exactly what this control permits", + ); + assert_ne!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + h.expected[q].0, + "the attack must actually move the reconstruction" + ); + println!("SplitValues control: forged fold accepted against honest roots"); + + // ---- Control 2: the query point is hinted. -------------------------- + // Two queries with DIFFERENT indices: authenticate one, fold at the + // other's point. + let other = (0..h.iotas.len()) + .find(|&i| h.iotas[i] != h.iotas[q]) + .expect("the fixture must carry two distinct query indices"); + let program = compile(control_program_source(&h.shape, Control::HintedPoint)); + validate(&program).expect("admissible"); + let mut arenas = h.arenas(&[q]); + arenas.push(vec![base_word(h.points[q].0), base_word(h.points[q].1)]); + let clean = execute(&program, &arenas, &TestPermutation).expect("honest"); + assert_eq!( + word_as_ext(&clean.public_words[0].1).expect("ext"), + h.expected[q].0 + ); + + let mut attacked = arenas.clone(); + attacked[5] = vec![base_word(h.points[other].0), base_word(h.points[other].1)]; + let forged = execute(&program, &attacked, &TestPermutation).expect( + "HintedPoint: a hinted point is not tied to the authenticated index, \ + which is what this control permits", + ); + assert_ne!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + h.expected[q].0, + "folding query {q}'s values at query {other}'s point must give a \ + different answer" + ); + assert_ne!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + h.expected[other].0, + "and it must not accidentally be the other query's answer either" + ); + println!( + "HintedPoint control: query {q}'s leaf authenticated, folded at query \ + {other}'s point, accepted" + ); +} + +// ============================================================================= +// The degenerate parameter this leg introduced: the precomputed group +// ============================================================================= + +/// A PREPROCESSED sub-proof, so the four-group shape is exercised. +/// +/// L2G_MEMORY — the fixture everything above runs on — is not preprocessed, and +/// neither is any AIR a single-table proof can cheaply be built from: the real +/// preprocessed tables are BITWISE (2^20 rows), DECODE, KECCAK_RC, REGISTER and +/// PAGE. So on that fixture the precomputed group is ABSENT, DEEP's column +/// order `precomputed ‖ main ‖ aux` degenerates to `main ‖ aux`, and an emitter +/// that put main first would pass every test in this file. That is the same +/// hazard as `step_size = 1` and it needs the same answer: a case production +/// does not produce. +/// +/// Built the way `tests::bitwise_tests` builds its preprocessed receiver — a +/// small `AirWithBuses` whose commitment comes from the prover's own +/// `compute_precomputed_commitment_for_testing`, so the precomputed root in the +/// proof and the one the AIR declares are computed by the same code the real +/// tables use. Widths are 2 and 2 so the two base groups can be SWAPPED, which +/// the falsification half needs. +fn preprocessed_fixture() -> ( + super::constraint_tests::BoxedAir, + stark::proof::stark::MultiProof, +) { + use crate::tables::types::{BusId, alu_op}; + use crate::test_utils::multi_prove_ram; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, + }; + use stark::prover::IsStarkProver; + use stark::trace::TraceTable; + use stark::traits::AIR; + + /// Columns 0..3 are precomputed (x, y, x&y); 3..6 are the multiplicity + /// block (a copy of x&y, a spare, and the bus multiplicity). The copy is + /// there so the table carries a real TRANSITION constraint: `EmptyConstraints` + /// leaves a single coefficient in the run and `open_sub_proof` recovers + /// `beta` from its second element. + const NUM_COLS: usize = 6; + const NUM_PRECOMPUTED: usize = 3; + /// 64 rows, not the 4 the other fixture uses. Trace length only enters this + /// leg through the Merkle depth, and at 4 rows every executed walk in the + /// suite is two levels deep — enough to hide a level-count error. 64 rows at + /// blowup 2 gives depth 6, which is the only executed multi-level walk over + /// all four committed matrices. + const NUM_ROWS: usize = 64; + + let opts = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup=2 is valid"); + + let build = |commitment: Option| { + let air = AirWithBuses::::new( + NUM_COLS, + AuxiliaryTraceBuildData { + interactions: vec![BusInteraction::receiver( + BusId::ByteAlu, + // The multiplicity is the LAST column, past the precomputed + // block — the production split (`0..n` precomputed, the + // rest multiplicities). + Multiplicity::Column(5), + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 2, + packing: Packing::Direct, + }, + ], + )], + }, + &opts, + 1, + CopiedColumn, + ) + .with_name("PREPROCESSED_FIXTURE"); + match commitment { + Some(c) => air.with_preprocessed(c, NUM_PRECOMPUTED), + None => air, + } + }; + + // Distinct rows, so the committed leaves are distinct and the tree is not + // the degenerate one R1f warns about. + let make_trace = || { + let mut data = vec![FE::zero(); NUM_ROWS * NUM_COLS]; + for r in 0..NUM_ROWS { + let x = 5u64 + r as u64; + let y = 3u64 + 2 * r as u64; + data[r * NUM_COLS] = FE::from(x); + data[r * NUM_COLS + 1] = FE::from(y); + data[r * NUM_COLS + 2] = FE::from(x & y); + data[r * NUM_COLS + 3] = FE::from(x & y); + data[r * NUM_COLS + 5] = FE::one(); + } + TraceTable::::new_main(data, NUM_COLS, 1) + }; + + let trace = make_trace(); + let commitment = as IsStarkProver< + Gl, + Ext3, + (), + stark::config::KeccakStarkHash, + >>::compute_precomputed_commitment_for_testing( + &trace, &build(None), NUM_PRECOMPUTED + ) + .expect("the precomputed columns commit"); + + let air = build(Some(commitment)); + let mut trace = make_trace(); + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("the preprocessed fixture must prove"); + (Box::new(air), proof) +} + +/// `main[3] == main[2]` — one transition constraint, satisfied by the fixture +/// trace, spanning the precomputed/multiplicity boundary. +struct CopiedColumn; + +impl + stark::constraints::builder::ConstraintSet for CopiedColumn +{ + fn eval>(&self, b: &mut B) { + let precomputed_and = b.main(0, 2); + let copied_and = b.main(0, 3); + b.emit_base(0, copied_and - precomputed_and); + } +} + +fn preprocessed_sub_proof() -> &'static HostSubProof { + use std::sync::OnceLock; + static CELL: OnceLock = OnceLock::new(); + CELL.get_or_init(|| { + let (air, proof) = preprocessed_fixture(); + build_host_sub_proof(&*air, &proof) + }) +} + +/// ★ The four-group shape, and the witness that the group ORDER is +/// load-bearing. +/// +/// Both halves, as the degenerate-parameter rule requires. The machine +/// reproduces the production reconstruction on a proof that HAS a precomputed +/// group — and production's own reconstruction, handed the same two base +/// groups in the opposite order, gives a DIFFERENT answer. Without the second +/// half this test would pass against a main-first emitter, which is the exact +/// failure mode it exists to prevent. +#[test] +fn the_precomputed_group_comes_first_and_that_is_checkable() { + let h = preprocessed_sub_proof(); + let groups = h.shape.groups(); + + assert_eq!( + groups.len(), + 4, + "the point of this fixture is a sub-proof with all four committed \ + matrices; got {groups:?}" + ); + assert_eq!(h.shape.trace_groups[0].num_columns, 3, "precomputed width"); + assert!(!h.shape.trace_groups[0].is_ext); + assert_eq!(h.shape.trace_groups[1].num_columns, 3, "main width"); + assert_eq!( + h.shape.trace_groups[0].num_columns, h.shape.trace_groups[1].num_columns, + "the two base groups must be the same width, or the swap below is not \ + a well-formed alternative reading" + ); + println!( + "preprocessed fixture: groups {:?}, depth {}, {} queries", + groups + .iter() + .map(|g| (g.num_columns, g.is_ext)) + .collect::>(), + h.shape.merkle_depth, + h.iotas.len() + ); + + // ---- half one: the machine agrees with production. ------------------- + let queries: Vec = (0..h.iotas.len().min(16)).collect(); + let mut b = LfmBuilder::new(); + let (_, outs) = emit_sub_proof(&mut b, &h.shape, queries.len()); + for (p, s) in &outs { + b.public(p.as_cell()); + b.public(s.as_cell()); + } + let program = compile(b.finish()); + validate(&program).expect("admissible"); + let exec = execute(&program, &h.arenas(&queries), &TestPermutation) + .expect("the four-group sub-proof must authenticate and fold"); + for (k, q) in queries.iter().enumerate() { + assert_eq!( + word_as_ext(&exec.public_words[2 * k].1).expect("ext"), + h.expected[*q].0, + "query {q}: DEEP at the regular point" + ); + assert_eq!( + word_as_ext(&exec.public_words[2 * k + 1].1).expect("ext"), + h.expected[*q].1, + "query {q}: DEEP at the symmetric point" + ); + } + + // ---- half two: the swapped reading DISAGREES. ------------------------ + // + // Asked of production's own reconstruction, not of a model of it: hand it + // the main slice where the precomputed one belongs and vice versa. If that + // came out equal, the order would be unobservable and this fixture would be + // no witness at all. + let swapped = &h.expected_base_swapped; + assert_eq!( + swapped.len(), + h.expected.len(), + "the swapped reading must have been computed for this fixture" + ); + let mut differs = 0usize; + for q in &queries { + if swapped[*q].0 != h.expected[*q].0 || swapped[*q].1 != h.expected[*q].1 { + differs += 1; + } + } + assert_eq!( + differs, + queries.len(), + "swapping the precomputed and main slices must change the \ + reconstruction at every query, or the column order is not observable \ + on this fixture and it witnesses nothing" + ); + println!( + "column order is load-bearing: the swapped reading differs at all {} \ + checked queries", + queries.len() + ); +} + +/// ⚠ The FRI leg's instrument problem, pinned: the proof fixture carries ZERO +/// committed FRI layers, so a differential over it cannot see the fold loop, +/// the per-layer walks, or the terminal check. +/// +/// `FriFoldLayout::new(lde_log, blowup_log, k)` sets +/// `terminal_log = min(blowup_log + k, lde_log)` and +/// `num_committed = (lde_log - terminal_log) - 1`. The fixture is the `min` +/// preset — blowup 2 (`blowup_log = 1`), `fri_final_poly_log_degree = 7` — over +/// an epoch of 2^4 steps, so its sub-proof has `log2(lde) = 3` and +/// `terminal_log = min(8, 3) = 3`: no folds at all, and `query_phase` returns +/// the empty-decommitment branch. +/// +/// This is the degenerate-parameter rule in its most extreme form. Not "one +/// value hides a difference between two implementations" but "the production +/// instance exercises none of the mechanism". The assertions below are still +/// exactly true of THIS fixture, and this test still earns its place: the day +/// the fixture grows and starts folding, the change is announced rather than +/// silently altering what the FRI tests cover. +/// +/// ## ⚠ CORRECTION — the conclusion drawn from this was wrong +/// +/// This test's original text went on to say that no amount of care with real +/// data could repair the gap, and that the FRI leg's primary instrument had to +/// be SYNTHETIC codewords. That is false, and the counterexample is one line of +/// the fixture: the trace is `boundaries.len().next_power_of_two()` +/// (`local_to_global.rs:269`), and `num_committed = trace_bits − 8`. So the same +/// construction with 512, 1024 or 2048 boundaries yields real production proofs +/// with one, two or three committed layers — real roots, real paths, real +/// terminal coefficients, real folding challenges — in under a second each. +/// `fri_tests::the_real_prover_folds_and_the_layer_count_follows_the_row_count` +/// is that sweep, and the FRI leg is differentialled entirely against real +/// proofs. Nothing in it is synthetic. +/// +/// The lesson is narrower than the one first drawn here. "The production +/// instances all share a degenerate parameter" was a claim about the fixtures on +/// hand, not about the prover, and the two are not the same claim. Worth +/// checking which one is being made before concluding that real data cannot +/// reach a mechanism. +/// +/// The zero-layer case is not merely an artifact to route around, either — it +/// is a real production path (small tables fold no further than their terminal) +/// and the emitted verifier handles it as a first-class shape. +#[test] +fn the_fixture_carries_no_fri_layers_so_it_cannot_witness_the_fold() { + let (_air, proof) = real_fixture(); + assert_eq!( + proof.proofs.len(), + 1, + "the join fixture is a single sub-proof" + ); + let p = &proof.proofs[0]; + println!( + "fixture sub-proof: fri_layers_merkle_roots = {}, fri_final_poly_coeffs = {}, \ + query decommitments = {}", + p.fri_layers_merkle_roots.len(), + p.fri_final_poly_coeffs.len(), + p.deep_poly_openings.len(), + ); + assert_eq!( + p.fri_layers_merkle_roots.len(), + 0, + "the fixture is expected to carry no committed FRI layers; if it now \ + folds, the FRI leg's coverage story changed and its synthetic sweep \ + should be re-justified against what the real proof now exercises" + ); + // The coefficient count is `2^effective_k` with + // `effective_k = terminal_log - blowup_log = 3 - 1`. Checking it is what + // says the layout arithmetic above is read correctly rather than merely + // asserted: a wrong reading of `FriFoldLayout` would land on a different + // power of two here. + assert_eq!( + p.fri_final_poly_coeffs.len(), + 4, + "terminal codeword encodes a degree-<2^2 polynomial at this shape" + ); +} + +/// ★ The bits handed to a later leg are the cells the WALK ITSELF consumed. +/// +/// The FRI leg reuses a query's index per layer (leaf position `index >> 1`, +/// partner `index ^ 1`, halving each layer). Were it to decompose its own copy +/// it would authenticate at one index and fold at another — the gap this module +/// closes, reopened one level up. +/// +/// ## Why this is not the obvious test +/// +/// The obvious test compares the program `emit_sub_proof` emits against the one +/// `emit_sub_proof_with_bits` emits and asserts they are identical. That test is +/// VACUOUS and I wrote it before catching it: `emit_sub_proof` is implemented by +/// delegating to `emit_sub_proof_with_bits`, so the two sides are the same +/// program by construction and any defect lands on both and cancels. Injecting a +/// second `bit_dec` — the precise failure this is meant to deny — left it green. +/// +/// What discriminates is an ABSOLUTE property rather than a relative one: every +/// returned bit must be consumed by a `Select`. The walk selects sibling order +/// on each bit and `pow_bits` selects the point factors on the same bits, so a +/// bit the emitter actually used is necessarily read by one. A freshly +/// decomposed second copy would be read by nothing. +#[test] +fn the_exposed_bits_are_the_cells_the_walk_consumed() { + let h = host_sub_proof(); + const QUERIES: usize = 3; + + let mut b = LfmBuilder::new(); + let (_, out) = emit_sub_proof_with_bits(&mut b, &h.shape, QUERIES); + let src = b.finish(); + assert_eq!(out.len(), QUERIES); + + // Every address any Select reads as its selector. + let selector_bits: std::collections::HashSet = src + .instrs + .iter() + .filter_map(|i| match i { + super::instr::Instr::Select { bit, .. } => Some(bit.0), + _ => None, + }) + .collect(); + assert!( + !selector_bits.is_empty(), + "the walk and the point derivation both select on bits; an empty set \ + means this test is looking at the wrong instruction" + ); + + for (q, output) in out.iter().enumerate() { + assert_eq!( + output.bits.len(), + h.shape.merkle_depth, + "query {q}: one bit per Merkle level" + ); + for (level, bit) in output.bits.iter().enumerate() { + assert!( + selector_bits.contains(&bit.0.0), + "query {q} level {level}: the returned bit is read by no Select, \ + so it is not a cell the walk or the point derivation used — a \ + second decomposition of the index has been handed out" + ); + } + } +} + +// ==================== FRI slice 1: the fold layout ==================== + +use super::fri::FriShape; + +/// ★ The shape mirror against production's observable BEHAVIOUR on the real +/// proof — the vector lengths the verifier structurally enforces. +/// +/// `FriFoldLayout` is `pub(crate)` inside `crypto/stark`, so the mirror cannot +/// be compared against the struct. It is compared against what a real proof +/// actually carries instead, which is the better oracle: `verifier.rs:426-448` +/// rejects on exactly these two lengths before its query loop runs, and the +/// spec notes they are the ONLY thing pinning vectors Fiat-Shamir does not bind. +#[test] +fn the_fri_shape_predicts_the_real_proofs_vector_lengths() { + let (_air, proof) = real_fixture(); + let h = host_sub_proof(); + let opts = prove_options(); + let shape = FriShape::from_options(&opts, h.shape.log2_lde_length); + shape.check(); + + let p = &proof.proofs[0]; + println!( + "FRI shape: lde 2^{}, blowup 2^{}, k {}, terminal_log {}, total_folds {}, \ + committed {}, coeffs {}", + shape.log2_lde_length, + shape.blowup_log, + shape.final_poly_log_degree, + shape.terminal_log(), + shape.total_folds(), + shape.num_committed(), + shape.num_terminal_coeffs(), + ); + assert_eq!( + p.fri_layers_merkle_roots.len(), + shape.num_committed(), + "committed layer count must match what the proof carries" + ); + assert_eq!( + p.fri_final_poly_coeffs.len(), + shape.num_terminal_coeffs(), + "terminal coefficient count must match 2^effective_k" + ); + assert_eq!( + shape.coset_offset, opts.coset_offset, + "the shape must take its coset offset from the options, not a literal" + ); +} + +/// ★ The synthetic sweep §7 requires, and the reason it is needed. +/// +/// Production pins `k = 7` and `coset_offset = 3` in every configuration, so no +/// real proof distinguishes an implementation that reads `k` from one that +/// hardcodes 7, and none reaches the clamp (`trace_bits <= 7`) at all. In LFM +/// these are not dead emitted branches — shape is compile-time, so they are +/// host-side arithmetic — which is exactly why they are testable here for free, +/// with no proving. +/// +/// Each row is `(trace_bits, blowup_log, k)` with its expected +/// `(total_folds, num_committed, effective_k, terminal_len)`, derived by hand +/// from `terminal.rs:45-54` rather than from this module. +/// +/// ## ★ Falsified, and the result is the leg's blindness finding made concrete +/// +/// Deleting the `saturating_sub(1)` from `FriShape::num_committed` — the +/// off-by-one that makes a verifier authenticate one layer FEWER than the proof +/// commits — fails this test and +/// [`the_fri_sizing_prediction`], and **passes** +/// [`the_fri_shape_predicts_the_real_proofs_vector_lengths`]. The fixture has +/// `total_folds = 0`, so `0` and `0.saturating_sub(1)` are the same number and +/// the real proof cannot tell the two implementations apart. +/// +/// So the most soundness-relevant constant in this leg is invisible to the only +/// real data available. That is not an argument for a better fixture; it is the +/// reason these synthetic rows are the primary instrument rather than a +/// supplement. +#[test] +fn the_fold_layout_is_right_off_productions_constants() { + /// `(total_folds, num_committed, effective_k, terminal_len)`. + type Layout = (u32, usize, u32, usize); + // (trace_bits, blowup_log, k) -> Layout + let cases: [(u32, u32, u32, Layout); 10] = [ + // The production point, at three blowups. k = 7 throughout. Note + // `total_folds = trace_bits - k` is INDEPENDENT of the blowup: the + // blowup enters `n` and `terminal_log` identically and cancels. That + // cancellation is what makes the scout's `num_committed = trace_bits - 8` + // invariant hold across every preset, and mis-expanding it (subtracting + // the blowup twice) is how the first version of this table was wrong. + (20, 1, 7, (13, 12, 7, 256)), + (20, 2, 7, (13, 12, 7, 512)), + (20, 3, 7, (13, 12, 7, 1024)), + // k = 0: fold all the way down to one coefficient per coset. + (10, 1, 0, (10, 9, 0, 2)), + // k = 6, one below production. + (10, 1, 6, (4, 3, 6, 128)), + // The CLAMP regime, trace_bits <= k: terminal_log pins to lde_log, so + // nothing folds and effective_k drops below the requested k. + (7, 1, 7, (0, 0, 7, 256)), + (4, 1, 7, (0, 0, 4, 32)), + (2, 3, 7, (0, 0, 2, 32)), + // k = 63 — far past any real trace, so the clamp always wins. + (5, 1, 63, (0, 0, 5, 64)), + // A single fold — `trace_bits = k + 1` — commits NOTHING, because the + // last fold is never committed. The row that catches the off-by-one. + (8, 1, 7, (1, 0, 7, 256)), + ]; + for (trace_bits, blowup_log, k, expected) in cases { + let shape = FriShape { + log2_lde_length: trace_bits + blowup_log, + blowup_log, + final_poly_log_degree: k, + coset_offset: 3, + num_queries: 1, + }; + shape.check(); + let got = ( + shape.total_folds(), + shape.num_committed(), + shape.effective_k(), + shape.terminal_len(), + ); + assert_eq!( + got, expected, + "trace_bits {trace_bits} blowup 2^{blowup_log} k {k}: \ + (total_folds, committed, effective_k, terminal_len)" + ); + // Folds exceed committed layers by exactly one whenever anything folds. + if shape.total_folds() > 0 { + assert_eq!( + shape.num_folds(), + shape.num_committed() + 1, + "the final fold is never committed" + ); + } + } +} + +/// ★ The sizing prediction, PINNED BEFORE MEASURING — the leg's +/// measurements-vs-prediction target. +/// +/// Derived from `others/lfm-fri-verify-spec.md` §8: `pathlen(i) = n − i − 2`, +/// so a query walks `Σ pathlen(i)` steps and pays one permutation per step plus +/// one leaf hash per committed layer. The blowup-2 row reproduces the spec's +/// own worked example (162 steps, 174 permutations, 38,106 total), which is +/// what says the formula is being read as written rather than re-derived. +/// +/// Recorded as a test rather than a comment so that the emitter, when it +/// arrives, is measured against a number that was fixed beforehand. +#[test] +fn the_fri_sizing_prediction() { + println!("blowup n C Q steps/q perms/q total"); + // (blowup_log, queries, expected steps/q, perms/q, total) at trace_bits = 20. + for (blowup_log, queries, steps, perms, total) in [ + (1u32, 219usize, 162usize, 174usize, 38_106usize), + (2, 110, 174, 186, 20_460), + (3, 73, 186, 198, 14_454), + ] { + let shape = FriShape { + log2_lde_length: 20 + blowup_log, + blowup_log, + final_poly_log_degree: 7, + coset_offset: 3, + num_queries: queries, + }; + shape.check(); + println!( + " 2^{blowup_log} {:>3} {:>3} {:>4} {:>8} {:>8} {:>9}", + shape.log2_lde_length, + shape.num_committed(), + shape.num_queries, + shape.path_steps_per_query(), + shape.permutations_per_query(), + shape.permutations(), + ); + assert_eq!( + shape.path_steps_per_query(), + steps, + "blowup 2^{blowup_log} steps" + ); + assert_eq!( + shape.permutations_per_query(), + perms, + "blowup 2^{blowup_log} perms/query" + ); + assert_eq!(shape.permutations(), total, "blowup 2^{blowup_log} total"); + } +} diff --git a/prover/src/lfm/keccak_adapter.rs b/prover/src/lfm/keccak_adapter.rs new file mode 100644 index 000000000..4404f4e37 --- /dev/null +++ b/prover/src/lfm/keccak_adapter.rs @@ -0,0 +1,516 @@ +//! R1a probe adapter: host the production keccak table family in a foreign AIR set. +//! +//! The production keccak family is three chips: `KECCAK` (the core, in +//! `tables::keccak`), `KECCAK_RND` (24 rounds of the permutation, in +//! `tables::keccak_rnd`) and `KECCAK_RC` (the round-constant fixed table). Only +//! the core is VM-coupled — it reads and writes the 25 state lanes through +//! timestamped `MEMW` tokens, so it cannot be lifted into the LFM machine as-is. +//! `KECCAK_RND` and `KECCAK_RC` are pure: they speak only the `Keccak`, +//! `KeccakRc` and BITWISE buses. +//! +//! This module is the minimal chip that replaces the core: it opens the +//! `Keccak` bus with a request token and closes it with the reply token, +//! reproducing exactly the two tokens `tables::keccak::bus_interactions` emits +//! (see `keccak.rs:264-325`) and nothing else. No memory, no timestamps, no +//! address range checks. Feeding those two tokens is the entire contract the +//! round chip needs, so the unchanged `KECCAK_RND` + `KECCAK_RC` + `BITWISE` +//! AIRs prove real `keccak-f[1600]` permutations driven by this adapter. +//! +//! # Token layout +//! +//! Both tokens are 203 bus elements: `[tag_lo, tag_hi, round, state[200]]`, +//! `round = 0` on the request (sent) and `round = 24` on the reply (received). +//! The 200 state elements are traversed **column-major over lanes**: element +//! `3 + 8 * (5x + y) + b` is byte `b` (LSB-first) of lane `x + 5y`, so lanes are +//! visited in the order 0, 5, 10, 15, 20, 1, 6, ... That asymmetry is inherited +//! from the production sender's `for x { for y { for b } } }` loop over +//! `cols::input_state(x, y, b) = INPUT_STATE + (x + 5y) * 8 + b`; emitting the +//! lanes in natural order instead would leave the bus unbalanced. +//! +//! # Constraints +//! +//! The adapter carries no polynomial constraints. Byte-ness of the 400 state +//! columns is enforced transitively rather than locally: every one of the 200 +//! IN bytes is an operand of at least one `BYTE_ALU[XOR]` lookup in the round +//! chip's θ column-parity chain (which covers all 25 lanes) and again in its θ +//! final XOR, and every OUT byte is the *result* of a `BYTE_ALU[XOR]` lookup +//! (χ, or ι for lane 0). A non-byte value in any of those columns finds no row +//! in the BITWISE table and breaks the bus balance. The tag columns are pure +//! labels and are deliberately unconstrained here. +//! +//! The production LFM adapter will add what this probe omits: binding the state +//! columns to `LfmMem` words (so the permutation's input and output are the +//! machine's data, not free witness), and sourcing the tag from preprocessed +//! program data. +//! +//! # Tag uniqueness is soundness-critical +//! +//! Nothing but the tag binds a request token to its reply token. Two rows +//! carrying the same tag let a malicious prover swap their output states: the +//! two `(tag, 24, ·)` receives and the two `(tag, 24, ·)` sends still form the +//! same multiset, so the bus balances and the proof verifies. The probe test +//! `duplicate_tag_output_swap_accepts_demonstrating_hazard` pins exactly this. +//! This probe uses distinct per-row constant tags; the production LFM adapter +//! will carry tags as preprocessed program data with registrar-vouched +//! uniqueness, so a prover cannot choose them at all. + +use stark::lookup::{BusInteraction, BusValue, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use crate::tables::bitwise::{BitwiseOperation, BitwiseOperationType}; +use crate::tables::keccak_rnd::KeccakRoundOperation; +use crate::tables::types::BusId; +use crate::tables::types::{ + FE, GoldilocksExtension, GoldilocksField, VmTable, dword_wl, zeroed_fe_vec, +}; + +use super::layout; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +/// Column layout: `[TAG_LO, TAG_HI, IN[200], OUT[200], MU]`. +pub mod cols { + /// Low 32 bits of the tag (matches the core chip's `TIMESTAMP_0`, a `DWordWL`). + pub const TAG_LO: usize = 0; + /// High 32 bits of the tag (matches the core chip's `TIMESTAMP_1`). + pub const TAG_HI: usize = 1; + /// Input state, 200 bytes, lane-major: `IN + (x + 5y) * 8 + b`. + pub const IN: usize = 2; + /// Output state, 200 bytes, same indexing as [`IN`]. + pub const OUT: usize = IN + 200; // 202 + /// Is-real column; the multiplicity of both `Keccak` bus tokens. + pub const MU: usize = OUT + 200; // 402 + + pub const NUM_COLUMNS: usize = MU + 1; // 403 + + /// Column holding byte `b` of input lane `x + 5y`. + #[inline] + pub const fn in_byte(x: usize, y: usize, b: usize) -> usize { + IN + (x + 5 * y) * 8 + b + } + + /// Column holding byte `b` of output lane `x + 5y`. + #[inline] + pub const fn out_byte(x: usize, y: usize, b: usize) -> usize { + OUT + (x + 5 * y) * 8 + b + } +} + +/// One permutation: `output = keccak_f1600(input)`, labelled by `tag`. +#[derive(Debug, Clone, Copy)] +pub struct KeccakAdapterOperation { + /// Binds the request token to its reply token. MUST be unique across rows. + pub tag: u64, + pub input: [u64; 25], +} + +/// The two `Keccak` bus tokens, mirroring `tables::keccak::bus_interactions` +/// interactions 2 and 3 with the memory-coupled columns dropped. +#[allow(clippy::needless_range_loop)] +pub fn bus_interactions() -> Vec { + let mut interactions = Vec::with_capacity(2); + + let tag_values = || { + vec![ + BusValue::Packed { + start_column: cols::TAG_LO, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TAG_HI, + packing: Packing::Direct, + }, + ] + }; + + // Request: send (tag, 0, input_state[200]). + { + let mut values = tag_values(); + values.push(BusValue::constant(0)); + for x in 0..5 { + for y in 0..5 { + for b in 0..8 { + values.push(BusValue::Packed { + start_column: cols::in_byte(x, y, b), + packing: Packing::Direct, + }); + } + } + } + interactions.push(BusInteraction::sender( + BusId::Keccak, + Multiplicity::Column(cols::MU), + values, + )); + } + + // Reply: receive (tag, 24, output_state[200]). + { + let mut values = tag_values(); + values.push(BusValue::constant(24)); + for x in 0..5 { + for y in 0..5 { + for b in 0..8 { + values.push(BusValue::Packed { + start_column: cols::out_byte(x, y, b), + packing: Packing::Direct, + }); + } + } + } + interactions.push(BusInteraction::receiver( + BusId::Keccak, + Multiplicity::Column(cols::MU), + values, + )); + } + + interactions +} + +/// One row per permutation; padding rows are all-zero (so `MU = 0` and they +/// send nothing). +pub fn generate_adapter_trace(ops: &[KeccakAdapterOperation]) -> TraceTable { + let n_rows = ops.len().next_power_of_two().max(4); + let mut trace = TraceTable::new_main( + zeroed_fe_vec(n_rows * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + + for (row, op) in ops.iter().enumerate() { + let [lo, hi] = dword_wl(op.tag); + table.set_fe(row, cols::TAG_LO, lo); + table.set_fe(row, cols::TAG_HI, hi); + + let output = permute(op.input); + for (lane, (&in_lane, &out_lane)) in op.input.iter().zip(output.iter()).enumerate() { + table.set_dword_bl(row, cols::IN + lane * 8, in_lane); + table.set_dword_bl(row, cols::OUT + lane * 8, out_lane); + } + + table.set_fe(row, cols::MU, FE::one()); + } + + trace +} + +/// `keccak-f[1600]` as the VM defines it — the same primitive the production +/// trace builder replays (`trace_builder.rs:635`). +pub fn permute(input: [u64; 25]) -> [u64; 25] { + let mut state = input; + executor::vm::instruction::execution::keccak_f1600(&mut state); + state +} + +// ========================================================================= +// Machine-word view of the state (the LFM_KECCAK chip's u32-half convention) +// ========================================================================= + +/// Half `h` of a state: the low (`h` even) or high (`h` odd) 32 bits of lane +/// `h / 2`. +/// +/// A keccak lane is a `u64` and so is *not* felt-representable — values in +/// `[p, 2^64)` exist — which is why machine-side keccak state travels as `u32` +/// halves, one per felt lane. +#[inline] +pub fn half_of(state: &[u64; 25], h: usize) -> u32 { + (state[h / 2] >> (32 * (h % 2))) as u32 +} + +/// A state as [`layout::keccak::NUM_WORDS`] machine words, four halves each. +/// +/// The top `WORD_SLOTS − NUM_HALVES` lanes of the last word are unused and set +/// to zero; the `LFM_KECCAK` chip pins them as bus tuple constants, so a +/// nonzero value there cannot balance. +pub fn state_to_words(state: &[u64; 25]) -> [LfmWord; layout::keccak::NUM_WORDS] { + core::array::from_fn(|j| { + core::array::from_fn(|l| { + let h = 4 * j + l; + if h < layout::keccak::NUM_HALVES { + FE::from(u64::from(half_of(state, h))) + } else { + FE::zero() + } + }) + }) +} + +/// The `BYTE_ALU[XOR]` lookups the `LFM_KECCAK` chip's absorb rows send: one +/// per rate byte, `PERM_IN[k] = STATE[k] ⊕ BLOCK[k]`. +/// +/// Permute rows send none — their XOR interactions are gated by `MODE_ABSORB`. +pub fn absorb_bitwise_ops(rows: &[super::executor::KeccakRow]) -> Vec { + use super::instr::KeccakMode; + let mut out = Vec::new(); + for r in rows.iter().filter(|r| r.mode == KeccakMode::Absorb) { + for k in 0..layout::keccak::RATE_BYTES { + let state_byte = (r.state[k / 8] >> (8 * (k % 8))) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + state_byte, + r.block[k], + )); + } + } + out +} + +/// The byte-REVERSED first 32 bytes of a state, as two machine words — the +/// value the production transcript's `sample()` both returns and re-absorbs. +/// +/// Reversed byte `j` is digest byte `31 − j`, so both the byte order within a +/// half and the order of the halves flip. The `LFM_KECCAK` chip produces this +/// with two extra bus sends over the SAME output byte columns; this is the host +/// mirror. +pub fn reversed_digest_words(state: &[u64; 25]) -> [LfmWord; 2] { + let mut digest = [0u8; 32]; + for (lane, chunk) in state[..4].iter().zip(digest.chunks_exact_mut(8)) { + chunk.copy_from_slice(&lane.to_le_bytes()); + } + digest.reverse(); + core::array::from_fn(|w| { + core::array::from_fn(|l| { + let h = 4 * w + l; + let mut half = [0u8; 4]; + half.copy_from_slice(&digest[4 * h..4 * h + 4]); + FE::from(u64::from(u32::from_le_bytes(half))) + }) + }) +} + +/// Reassembles the 25 `u64` lanes from 50 `u32` halves. +pub fn halves_to_state(halves: &[u32; layout::keccak::NUM_HALVES]) -> [u64; 25] { + core::array::from_fn(|lane| { + u64::from(halves[2 * lane]) | (u64::from(halves[2 * lane + 1]) << 32) + }) +} + +/// The `KECCAK_RND` operations matching `ops`: one per permutation, expanding to +/// 24 trace rows each. The round chip keys its rows on `timestamp`, which is our +/// tag. Its `output` field is dead (the trace builder recomputes the state round +/// by round) but is filled with the true value anyway. +pub fn round_operations(ops: &[KeccakAdapterOperation]) -> Vec { + ops.iter() + .map(|op| KeccakRoundOperation { + timestamp: op.tag, + input: op.input, + output: permute(op.input), + }) + .collect() +} + +/// BITWISE lookups the `KECCAK_RND` rows of `ops` send: exactly `24 * 1028` per +/// permutation. +/// +/// This is the per-round half of `trace_builder::collect_bitwise_from_keccak`, +/// forked rather than called: the original also emits the 105 address-shaped +/// lookups (1 `BYTE_ALU[AND]` alignment check, 4 `ARE_BYTES` on the address +/// bytes, 100 `IS_HALF` on the lane pointers) that belong to the dropped core +/// chip. Calling it with a synthetic address and subtracting would depend on +/// those counts staying fixed; forking the loop keeps the coupling explicit. +/// +/// The θ/ρ halfword shifts no longer emit HWSL lookups (120 sends/round removed: +/// 20 in θ, 100 in ρ) — they are enforced by inline μ-gated linear identities on +/// the round chip, matching main's `keccak_rnd::bus_interactions` / +/// `KeccakRndConstraints` and `trace_builder::collect_bitwise_from_keccak`. Per +/// round: 1148 − 120 = 1028. +#[allow(clippy::needless_range_loop)] +pub fn bitwise_ops_for(ops: &[KeccakAdapterOperation]) -> Vec { + use executor::vm::instruction::execution::{KECCAK_RC, KECCAK_RHO}; + + let mut out = Vec::with_capacity(ops.len() * 24 * 1028); + + for op in ops { + let mut state = op.input; + for round in 0..24 { + // --- theta: Cxz chain BYTE_ALU[XOR] (160) --- + let mut cxz = [[[0u8; 8]; 4]; 5]; + for x in 0..5 { + for b in 0..8 { + let v0 = ((state[x] >> (b * 8)) & 0xFF) as u8; + let v1 = ((state[x + 5] >> (b * 8)) & 0xFF) as u8; + cxz[x][0][b] = v0 ^ v1; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + v0, + v1, + )); + } + for stage in 1..4usize { + let y = stage + 1; + for b in 0..8 { + let prev = cxz[x][stage - 1][b]; + let sv = ((state[x + 5 * y] >> (b * 8)) & 0xFF) as u8; + cxz[x][stage][b] = prev ^ sv; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + prev, + sv, + )); + } + } + } + + // theta: ARE_BYTES on Cxz_left (20 pairs). The rotate-C-by-1 halfword + // shift no longer emits an HWSL lookup — it is enforced by an inline + // μ-gated linear identity on the round chip (matching main's + // `KeccakRndConstraints`; the HWSL sends were dropped from + // `keccak_rnd::bus_interactions`). Cxz_right is range-checked by + // IS_BIT polynomial constraints on the round chip (spec d75944ee). + let mut rotated_c = [[0u8; 8]; 5]; + for x in 0..5 { + let c = cxz[x][3]; + for hw in 0..4 { + let halfword = (c[hw * 2] as u16) | ((c[hw * 2 + 1] as u16) << 8); + let shifted = halfword << 1; // u16 wraps + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (shifted & 0xFF) as u8, + ((shifted >> 8) & 0xFF) as u8, + )); + } + let mut left_bytes = [0u8; 8]; + let mut right_bits = [0u8; 4]; + for hw in 0..4 { + let halfword = (c[hw * 2] as u16) | ((c[hw * 2 + 1] as u16) << 8); + let shifted = halfword << 1; + left_bytes[hw * 2] = (shifted & 0xFF) as u8; + left_bytes[hw * 2 + 1] = ((shifted >> 8) & 0xFF) as u8; + right_bits[hw] = (halfword >> 15) as u8; + } + for b in 0usize..8 { + let right_contribution = if b.is_multiple_of(2) { + right_bits[(b / 2 + 3) % 4] + } else { + 0 + }; + rotated_c[x][b] = left_bytes[b].wrapping_add(right_contribution); + } + } + + // theta: Dxz BYTE_ALU[XOR] (40) + let mut d_bytes = [[0u8; 8]; 5]; + for x in 0..5 { + for b in 0..8 { + let a = cxz[(x + 4) % 5][3][b]; + let rb = rotated_c[(x + 1) % 5][b]; + d_bytes[x][b] = a ^ rb; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + a, + rb, + )); + } + } + + // theta final: BYTE_ALU[XOR] (200) + let mut theta_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let lane = state[x + 5 * y]; + let mut d_lane = 0u64; + for b in 0..8 { + d_lane |= (d_bytes[x][b] as u64) << (b * 8); + } + theta_lanes[x + 5 * y] = lane ^ d_lane; + for b in 0..8 { + let s = ((lane >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + s, + d_bytes[x][b], + )); + } + } + } + + // rho: ARE_BYTES (200 pairs). The ρ halfword shifts no longer emit + // HWSL lookups — enforced by inline μ-gated identities on the round + // chip (matching main's `KeccakRndConstraints`). + for x in 0..5 { + for y in 0..5 { + let rho_offset = KECCAK_RHO[x][y] as usize; + let rnc_val = (rho_offset % 16) as u8; + let theta_lane = theta_lanes[x + 5 * y]; + for hw in 0..4 { + let halfword = ((theta_lane >> (hw * 16)) & 0xFFFF) as u16; + let (shifted, carry) = if rnc_val == 0 { + (halfword, 0u16) + } else { + (halfword << rnc_val, halfword >> (16 - rnc_val)) + }; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (shifted & 0xFF) as u8, + (carry & 0xFF) as u8, + )); + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((shifted >> 8) & 0xFF) as u8, + ((carry >> 8) & 0xFF) as u8, + )); + } + } + } + + // pi + let mut pi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let rotated = theta_lanes[x + 5 * y].rotate_left(KECCAK_RHO[x][y]); + let dst_x = y; + let dst_y = (2 * x + 3 * y) % 5; + pi_lanes[dst_x + 5 * dst_y] = rotated; + } + } + + // chi: BYTE_ALU[AND] (200) + BYTE_ALU[XOR] (200) + let mut chi_lanes = [0u64; 25]; + for x in 0..5 { + for y in 0..5 { + let not_next = !pi_lanes[(x + 1) % 5 + 5 * y]; + let next2 = pi_lanes[(x + 2) % 5 + 5 * y]; + let and_val = not_next & next2; + chi_lanes[x + 5 * y] = pi_lanes[x + 5 * y] ^ and_val; + for b in 0..8 { + let not_byte = ((not_next >> (b * 8)) & 0xFF) as u8; + let n2_byte = ((next2 >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + not_byte, + n2_byte, + )); + let pi_byte = ((pi_lanes[x + 5 * y] >> (b * 8)) & 0xFF) as u8; + let and_byte = ((and_val >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + pi_byte, + and_byte, + )); + } + } + } + + // iota: BYTE_ALU[XOR] (8) + let rc_val = KECCAK_RC[round]; + for b in 0..8 { + let chi_byte = ((chi_lanes[0] >> (b * 8)) & 0xFF) as u8; + let rc_byte = ((rc_val >> (b * 8)) & 0xFF) as u8; + out.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + chi_byte, + rc_byte, + )); + } + + chi_lanes[0] ^= rc_val; + state = chi_lanes; + } + } + + out +} diff --git a/prover/src/lfm/keccak_host.rs b/prover/src/lfm/keccak_host.rs new file mode 100644 index 000000000..4d5886c44 --- /dev/null +++ b/prover/src/lfm/keccak_host.rs @@ -0,0 +1,198 @@ +//! Host side of the machine's keccak256: the byte-stream packing convention +//! and the padding the emitter bakes in as program constants. +//! +//! The machine has no bytes — its cells are felts — so a byte stream reaches +//! `edsl::keccak256` pre-packed as `u32` halves, four bytes each, little-endian. +//! That is the same convention the state itself uses (`keccak_adapter`), which +//! is what lets a rate block be assembled with plain `Pack` instructions and no +//! per-byte arithmetic. + +use crate::tables::types::FE; + +use super::layout::keccak::RATE_BYTES; + +/// Bytes carried by one `u32`-half felt. +pub const BYTES_PER_HALF: usize = 4; + +/// Packs a byte stream into `u32`-half felts, four bytes each, little-endian. +/// +/// The final half is zero-padded when `bytes.len()` is not a multiple of four. +/// The emitter relies on exactly that: where a half straddles the end of the +/// message it adds the padding constant to the stream half, and addition equals +/// bitwise-or only because the stream half's high bytes are known zero. +/// [`assert_high_bytes_zero`] is the executable statement of that obligation. +pub fn pack_stream(bytes: &[u8]) -> Vec { + bytes + .chunks(BYTES_PER_HALF) + .map(|chunk| { + let mut half = [0u8; BYTES_PER_HALF]; + half[..chunk.len()].copy_from_slice(chunk); + FE::from(u64::from(u32::from_le_bytes(half))) + }) + .collect() +} + +/// Number of halves [`pack_stream`] produces for `len_bytes`. +pub const fn num_stream_halves(len_bytes: usize) -> usize { + len_bytes.div_ceil(BYTES_PER_HALF) +} + +/// The keccak256 padded length: `pad10*1` always adds at least one byte, so the +/// message grows to the next multiple of the rate even when it already is one. +pub const fn padded_len(len_bytes: usize) -> usize { + (len_bytes / RATE_BYTES + 1) * RATE_BYTES +} + +/// Number of rate blocks the emitter absorbs for `len_bytes`. +pub const fn num_blocks(len_bytes: usize) -> usize { + padded_len(len_bytes) / RATE_BYTES +} + +/// The `pad10*1` byte at padded position `pos` for a message of `len_bytes`: +/// `0x01` at the first padding position, `0x80` at the last of the final block, +/// and `0x81` when they coincide. +pub fn pad_byte(len_bytes: usize, pos: usize) -> u8 { + debug_assert!(pos >= len_bytes && pos < padded_len(len_bytes)); + let mut v = 0u8; + if pos == len_bytes { + v |= 0x01; + } + if pos == padded_len(len_bytes) - 1 { + v |= 0x80; + } + v +} + +/// The padding contribution to half `h` of the padded message — the value the +/// emitter adds to (or uses in place of) the stream half. +/// +/// Returns `0` for halves that lie entirely inside the message. +pub fn pad_half(len_bytes: usize, h: usize) -> u64 { + let mut acc = 0u64; + for j in 0..BYTES_PER_HALF { + let pos = h * BYTES_PER_HALF + j; + if pos >= len_bytes && pos < padded_len(len_bytes) { + acc |= u64::from(pad_byte(len_bytes, pos)) << (8 * j); + } + } + acc +} + +/// Checks the packing obligation for the half that straddles the end of the +/// message: its bytes at or beyond `len_bytes` must be zero, or the emitter's +/// `stream_half + pad_half` would carry instead of merging. +pub fn assert_high_bytes_zero(stream: &[FE], len_bytes: usize) { + use math::field::traits::IsPrimeField; + let tail = len_bytes % BYTES_PER_HALF; + if tail == 0 { + return; + } + let h = len_bytes / BYTES_PER_HALF; + let v = crate::tables::types::GoldilocksField::canonical(stream[h].value()); + assert_eq!( + v >> (8 * tail), + 0, + "stream half {h} must be zero above byte {tail}: pack_stream guarantees it" + ); +} + +/// `keccak256` over `bytes`, as the production hasher computes it. The machine +/// program's public output is compared against this. +pub fn keccak256(bytes: &[u8]) -> [u8; 32] { + use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; + use digest::Digest; + let mut h = Keccak256::new(); + h.update(bytes); + h.finalize().into() +} + +// ===================== Host model of DefaultTranscript ===================== + +/// Bytes in one squeeze; the duplex output buffer hands them out 8 at a time. +pub const SQUEEZE_LEN: usize = 32; + +/// Host mirror of the production `DefaultTranscript` (post-#841), tracking the +/// same state the machine emitter tracks at emit time. +/// +/// This exists so the emitter's static consumption schedule — which candidate +/// comes from which squeeze, and where absorbs invalidate the buffer — can be +/// derived and tested without a machine proof. It is checked against the real +/// `DefaultTranscript` in `machine_tests`. +#[derive(Clone)] +pub struct TranscriptModel { + /// Bytes absorbed since the last finalize (the hasher's pending input). + segment: Vec, + buf: [u8; SQUEEZE_LEN], + /// Bytes already handed out of `buf`; `SQUEEZE_LEN` means empty. + pos: usize, +} + +impl TranscriptModel { + pub fn new(data: &[u8]) -> Self { + Self { + segment: data.to_vec(), + buf: [0u8; SQUEEZE_LEN], + pos: SQUEEZE_LEN, + } + } + + /// Absorbing invalidates the buffer: a later challenge must depend on this + /// input, so bytes squeezed before it are dropped. + pub fn append(&mut self, bytes: &[u8]) { + self.pos = SQUEEZE_LEN; + self.segment.extend_from_slice(bytes); + } + + /// Finalize, reverse, re-absorb the reversed bytes, return them. Also + /// invalidates the buffer. + pub fn sample(&mut self) -> [u8; SQUEEZE_LEN] { + let mut digest = keccak256(&self.segment); + digest.reverse(); + self.segment = digest.to_vec(); + self.pos = SQUEEZE_LEN; + digest + } + + /// Next big-endian 64-bit candidate, refilling with one squeeze when fewer + /// than 8 bytes remain. + pub fn next_u64(&mut self) -> u64 { + if self.pos + 8 > SQUEEZE_LEN { + self.buf = self.sample(); + self.pos = 0; + } + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(&self.buf[self.pos..self.pos + 8]); + self.pos += 8; + u64::from_be_bytes(bytes) + } + + /// Whether the next `next_u64` would refill (the emitter needs this to know + /// where to place a keccak row). + pub fn would_refill(&self) -> bool { + self.pos + 8 > SQUEEZE_LEN + } + + pub fn pos(&self) -> usize { + self.pos + } +} + +/// THE IDENTITY THE MACHINE EMITTER RELIES ON. +/// +/// The four big-endian candidates carved out of a reversed digest are exactly +/// the ORIGINAL digest's first four `u64` lanes, in reverse lane order — so the +/// machine never has to reverse anything to read candidates. +/// +/// Candidate `i` is `Σ_{k<8} reversed[8i+k]·2^(8(7−k))`, and `reversed[j]` is +/// `digest[31−j]`, so substituting `m = 7−k` gives +/// `Σ_{m<8} digest[24−8i+m]·2^(8m)` — the LITTLE-endian `u64` at digest byte +/// offset `24−8i`, i.e. keccak state lane `3−i`. The big-endian read and the +/// byte reversal cancel exactly. +/// +/// Consequence: candidates come straight off the plain digest words (state +/// lanes, already `u32` halves on the bus), and the reversed digest is needed +/// only for the RE-ABSORB. Verified by `be_candidates_are_plain_state_lanes`. +pub fn candidate_from_state(state: &[u64; 25], index: usize) -> u64 { + debug_assert!(index < 4); + state[3 - index] +} diff --git a/prover/src/lfm/keccak_probe.rs b/prover/src/lfm/keccak_probe.rs new file mode 100644 index 000000000..b8a8dd208 --- /dev/null +++ b/prover/src/lfm/keccak_probe.rs @@ -0,0 +1,292 @@ +//! R1a probe: prove real `keccak-f[1600]` permutations through the UNCHANGED +//! production `KECCAK_RND` + `KECCAK_RC` + `BITWISE` AIRs, driven by +//! [`super::keccak_adapter`] instead of the VM-coupled `KECCAK` core chip. +//! +//! This is the entry gate for hosting the keccak table family inside the LFM +//! recursion machine's AIR set: it establishes that the family's only coupling +//! to the VM is the core chip's two `Keccak` bus tokens, and that a chip owning +//! nothing but those tokens is a sufficient driver. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::constraints::builder::EmptyConstraints; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::MultiProofView; +use stark::prover::{IsStarkProver, Prover}; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField, VmTable}; +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; +use crate::test_utils::{create_bitwise_air, create_keccak_rc_air, create_keccak_rnd_air}; + +use super::keccak_adapter::{self, KeccakAdapterOperation, cols}; + +type F = GoldilocksField; +type E = GoldilocksExtension; +type AdapterAir = AirWithBuses; +type DynAir<'a> = &'a dyn AIR; + +const PROBE_TAG: &[u8] = b"LFM_R1A_KECCAK_PROBE_V1"; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("probe options") +} + +fn transcript() -> DefaultTranscript { + let mut t = DefaultTranscript::::new(&[]); + t.append_bytes(PROBE_TAG); + t +} + +fn adapter_air(opts: &ProofOptions) -> AdapterAir { + AirWithBuses::new( + cols::NUM_COLUMNS, + AuxiliaryTraceBuildData { + interactions: keccak_adapter::bus_interactions(), + }, + opts, + 1, + EmptyConstraints, + ) + .with_name("KECCAK_ADAPTER") +} + +/// Three permutations, distinct nontrivial inputs, distinct tags. +/// +/// Tags are `row + 1` here. The production LFM adapter will source them from +/// preprocessed program data — see the tag-uniqueness note on +/// [`super::keccak_adapter`] and `duplicate_tag_output_swap_accepts_demonstrating_hazard`. +fn probe_ops() -> Vec { + (0..3u64) + .map(|i| { + let mut input = [0u64; 25]; + for (lane, slot) in input.iter_mut().enumerate() { + *slot = (lane as u64) + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(i.wrapping_mul(0xD1B5_4A32_D192_ED03)) + ^ 0x0123_4567_89AB_CDEF; + } + KeccakAdapterOperation { tag: i + 1, input } + }) + .collect() +} + +/// The four traces, in AIR order: adapter, KECCAK_RND, KECCAK_RC, BITWISE. +fn build_traces(ops: &[KeccakAdapterOperation]) -> [TraceTable; 4] { + let adapter = keccak_adapter::generate_adapter_trace(ops); + let rnd = keccak_rnd::generate_keccak_rnd_trace(&keccak_adapter::round_operations(ops)); + + let mut rc = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut rc, ops.len()); + + let mut hist = bitwise::BitwiseHistogram::new(); + hist.add_ops(&keccak_adapter::bitwise_ops_for(ops)); + let mut bw = bitwise::generate_bitwise_trace(); + hist.fill_multiplicities(&mut bw); + + [adapter, rnd, rc, bw] +} + +/// Prove the four-AIR set over `traces`, which the caller may have tampered +/// with after generation. +fn prove_traces( + opts: &ProofOptions, + adapter: &AdapterAir, + traces: &mut [TraceTable; 4], +) -> Result, stark::prover::ProvingError> { + let rnd_air = create_keccak_rnd_air(opts); + let rc_air = create_keccak_rc_air(opts).with_preprocessed( + keccak_rc::preprocessed_commitment(opts), + keccak_rc::NUM_PRECOMPUTED_COLS, + ); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + + let [t0, t1, t2, t3] = traces; + let pairs: Vec<(DynAir, &mut TraceTable, &())> = vec![ + (adapter, t0, &()), + (&rnd_air, t1, &()), + (&rc_air, t2, &()), + (&bw_air, t3, &()), + ]; + let mut t = transcript(); + Prover::multi_prove( + pairs, + &mut t, + #[cfg(feature = "disk-spill")] + Default::default(), + Default::default(), + ) +} + +fn verify_proof( + opts: &ProofOptions, + adapter: &AdapterAir, + proof: &stark::proof::stark::MultiProof, +) -> bool { + let rnd_air = create_keccak_rnd_air(opts); + let rc_air = create_keccak_rc_air(opts).with_preprocessed( + keccak_rc::preprocessed_commitment(opts), + keccak_rc::NUM_PRECOMPUTED_COLS, + ); + let bw_air = create_bitwise_air(opts).with_preprocessed( + bitwise::preprocessed_commitment(opts), + bitwise::NUM_PRECOMPUTED_COLS, + ); + let refs: Vec = vec![adapter, &rnd_air, &rc_air, &bw_air]; + let mut vt = transcript(); + Verifier::multi_verify_views(&refs, MultiProofView::Owned(proof), &mut vt, &FEE::zero()) +} + +/// Prove + verify, optionally corrupting the adapter trace in between. +/// +/// `Err` means the prover refused. The adapter carries no constraints, so +/// tampering with its values is expected to reach the verifier and be caught +/// there (`Ok(false)`); the reject tests assert that stronger outcome rather +/// than accepting either failure, so a prover-side refusal would show up as a +/// change in behavior instead of hiding behind a passing test. +fn round_trip(mutate: impl FnOnce(&mut TraceTable)) -> Result { + let opts = options(); + let adapter = adapter_air(&opts); + let ops = probe_ops(); + let mut traces = build_traces(&ops); + mutate(&mut traces[0]); + match prove_traces(&opts, &adapter, &mut traces) { + Ok(proof) => Ok(verify_proof(&opts, &adapter, &proof)), + Err(e) => Err(format!("{e:?}")), + } +} + +/// Assert the prover accepted the tampered trace and the verifier rejected it. +fn assert_proves_but_fails_verification(what: &str, mutate: impl FnOnce(&mut TraceTable)) { + match round_trip(mutate) { + Ok(true) => panic!("{what} must break the Keccak bus balance, but the proof verified"), + Ok(false) => {} + Err(e) => panic!("{what} should reach the verifier, but the prover refused first: {e}"), + } +} + +#[test] +fn adapter_probe_proves_real_permutations() { + let ops = probe_ops(); + let traces = build_traces(&ops); + + // The adapter's OUT columns must be the real permutation, byte for byte. + for (row, op) in ops.iter().enumerate() { + let expected = keccak_adapter::permute(op.input); + for (lane, &value) in expected.iter().enumerate() { + for b in 0..8 { + assert_eq!( + traces[0].main_table.get_row(row)[cols::OUT + lane * 8 + b], + FE::from(u64::from((value >> (b * 8)) as u8)), + "OUT byte ({lane}, {b}) of row {row}" + ); + } + } + } + + // Known-answer vector: keccak-f[1600] of the all-zero state. Same constant + // the executor pins in `executor/src/tests/keccak_tests.rs`. + let zero_out = keccak_adapter::permute([0u64; 25]); + assert_eq!( + zero_out[0], 0xF1258F7940E1DDE7, + "keccak_f1600(0) lane 0 must match the published vector" + ); + + // The BITWISE feed is exactly the per-round half of the production + // collector: 1028 lookups per round, no address-shaped lookups and no HWSL + // (the θ/ρ shifts are inline μ-gated identities on the round chip). + assert_eq!( + keccak_adapter::bitwise_ops_for(&ops).len(), + ops.len() * 24 * 1028, + "per-permutation BITWISE lookup count" + ); + + // The round-trip through the AIRs is the real check: KECCAK_RND enforces + // all 24 rounds, and the bus only balances if the adapter's OUT state is + // what those rounds actually produce from its IN state. + assert_eq!( + round_trip(|_| {}), + Ok(true), + "honest keccak adapter proof must verify" + ); +} + +#[test] +fn tampered_output_byte_rejects() { + assert_proves_but_fails_verification("a flipped OUT byte", |t| { + let old = t.main_table.get_row(1)[cols::out_byte(2, 3, 4)]; + t.main_table + .set_fe(1, cols::out_byte(2, 3, 4), old + FE::one()); + }); +} + +#[test] +fn tampered_input_byte_rejects() { + assert_proves_but_fails_verification("a flipped IN byte", |t| { + let old = t.main_table.get_row(0)[cols::in_byte(4, 1, 7)]; + t.main_table + .set_fe(0, cols::in_byte(4, 1, 7), old + FE::one()); + }); +} + +#[test] +fn padding_row_multiplicity_rejects() { + // Row 3 is padding (3 real ops, height 4). Turning it real makes the + // adapter send a (tag=0, round=0, all-zero state) request and receive a + // (tag=0, round=24, all-zero state) reply that no KECCAK_RND row answers. + assert_proves_but_fails_verification("an is-real padding row", |t| { + t.main_table.set_fe(3, cols::MU, FE::one()) + }); +} + +/// DOCUMENTS A HAZARD — this test asserts that a forgery SUCCEEDS. +/// +/// Nothing in the bus contract binds a request token to its reply token except +/// the tag. Given two permutations sharing a tag, a prover can hand back each +/// one's output as the other's: the reply multiset `{(tag, 24, A_out), +/// (tag, 24, B_out)}` is unchanged by the swap, so the `Keccak` bus still +/// balances and the proof verifies even though neither adapter row states a +/// true permutation. +/// +/// This is why [`super::keccak_adapter`] requires unique tags, and why the +/// production LFM adapter must carry them as preprocessed program data with +/// registrar-vouched uniqueness rather than as prover-chosen witness. If this +/// test ever starts FAILING, something began binding request to reply and the +/// tag-uniqueness obligation should be re-derived before it is relaxed. +#[test] +fn duplicate_tag_output_swap_accepts_demonstrating_hazard() { + let opts = options(); + let adapter = adapter_air(&opts); + + let mut ops = probe_ops(); + ops.truncate(2); + ops[1].tag = ops[0].tag; // the whole point: duplicate tag + + let mut traces = build_traces(&ops); + assert_ne!( + keccak_adapter::permute(ops[0].input), + keccak_adapter::permute(ops[1].input), + "the two outputs must differ or the swap is a no-op" + ); + + // Swap the two rows' 200 OUT bytes. + for col in cols::OUT..cols::MU { + let a = traces[0].main_table.get_row(0)[col]; + let b = traces[0].main_table.get_row(1)[col]; + traces[0].main_table.set_fe(0, col, b); + traces[0].main_table.set_fe(1, col, a); + } + + let proof = prove_traces(&opts, &adapter, &mut traces).expect("locally consistent"); + assert!( + verify_proof(&opts, &adapter, &proof), + "documents the tag-uniqueness obligation: with duplicate tags the swapped \ + outputs still balance the bus, so the verifier cannot catch the forgery" + ); +} diff --git a/prover/src/lfm/layout.rs b/prover/src/lfm/layout.rs new file mode 100644 index 000000000..37d8ded21 --- /dev/null +++ b/prover/src/lfm/layout.rs @@ -0,0 +1,276 @@ +//! Instruction column-group layouts — the single source of truth shared by +//! the compiler's group emission (Milestone A), the admission validator, and +//! the chips' preprocessed column constants (Milestone B). +//! +//! Each chip's instruction fields (addresses, opcode selectors, +//! multiplicities, pooled constants) form its *instruction column group*: +//! the leading, preprocessed columns of that chip's trace, committed once per +//! program and supplied at verify time. Value columns follow after +//! `PREP_WIDTH` and are defined by the chips. + +/// `LFM_CONST` — pooled constants and immediates. +pub mod const_ { + pub const ADDR: usize = 0; + pub const V0: usize = 1; // .. V3 = 4 + pub const MULT: usize = 5; + pub const PREP_WIDTH: usize = 6; +} + +/// `LFM_BALU` — Goldilocks ALU. +pub mod balu { + pub const A_ADDR: usize = 0; + pub const B_ADDR: usize = 1; + pub const C_ADDR: usize = 2; + pub const OUT_ADDR: usize = 3; + pub const SEL_ADD: usize = 4; + pub const SEL_SUB: usize = 5; + pub const SEL_MUL: usize = 6; + pub const SEL_DIV: usize = 7; + pub const SEL_MULADD: usize = 8; + pub const MULT: usize = 9; + pub const PREP_WIDTH: usize = 10; + pub const NUM_SELECTORS: usize = 5; +} + +/// `LFM_XALU` — Fp3 ALU (word lanes 0–2, `w³ = 2`). +pub mod xalu { + pub const A_ADDR: usize = 0; + pub const B_ADDR: usize = 1; + pub const C_ADDR: usize = 2; + pub const OUT_ADDR: usize = 3; + pub const SEL_ADD: usize = 4; + pub const SEL_SUB: usize = 5; + pub const SEL_MUL: usize = 6; + pub const SEL_DIV: usize = 7; + pub const SEL_MULADD: usize = 8; + pub const SEL_MULBASE: usize = 9; + pub const MULT: usize = 10; + pub const PREP_WIDTH: usize = 11; + pub const NUM_SELECTORS: usize = 6; +} + +/// `LFM_SELECT` — conditional cell swap. +pub mod select { + pub const BIT_ADDR: usize = 0; + pub const INL_ADDR: usize = 1; + pub const INR_ADDR: usize = 2; + pub const OUTL_ADDR: usize = 3; + pub const OUTR_ADDR: usize = 4; + pub const MULT_L: usize = 5; + pub const MULT_R: usize = 6; + pub const IS_REAL: usize = 7; + pub const PREP_WIDTH: usize = 8; +} + +/// `LFM_BITDEC` — canonical 64-bit decomposition. Per-bit pairs +/// `(BIT_ADDR_i, MULT_i)` at `2 + 2i` / `3 + 2i`. +pub mod bitdec { + pub const IN_ADDR: usize = 0; + pub const IS_REAL: usize = 1; + pub const NUM_BITS: usize = 64; + pub const fn bit_addr(i: usize) -> usize { + 2 + 2 * i + } + pub const fn bit_mult(i: usize) -> usize { + 3 + 2 * i + } + pub const PREP_WIDTH: usize = 2 + 2 * NUM_BITS; // 130 +} + +/// `LFM_HASH` — the hash chiplet (frozen tuple contract). +/// +/// Four mode selectors, all preprocessed, at most one of them set: +/// +/// | selector | shape | domain | +/// |---|---|---| +/// | `MODE_C` | 2 cells → 1 | Merkle parent / 2-to-1 compress | +/// | `MODE_T` | 2 cells → 1 | a Fiat–Shamir transcript step | +/// | `MODE_L` | **1 cell → 1** | a **leaf** over four arbitrary FIELD ELEMENTS† | +/// | `MODE_P` | 3 cells → 3 | the full permutation | +/// +/// Being preprocessed is what makes them trustworthy: a row's mode is fixed by +/// its position in the committed instruction group, so a prover chooses neither +/// which domain a row hashes in nor which input semantics it has. +/// +/// † The mode is a shape the machine offers; whether a leaf and a parent are +/// actually different FUNCTIONS is the hasher's business. BLAKE3 separates them +/// by tag, and a single-domain hasher does not — see `LfmHasher::leaf_out`. +pub mod hash { + pub const IN_ADDR0: usize = 0; + pub const IN_ADDR1: usize = 1; + pub const IN_ADDR2: usize = 2; + pub const OUT_ADDR0: usize = 3; + pub const OUT_ADDR1: usize = 4; + pub const OUT_ADDR2: usize = 5; + pub const MODE_C: usize = 6; + pub const MODE_P: usize = 7; + /// The transcript-domain selector. + /// + /// A FRESH column, not a repurposed `MODE_P`: `MODE_P` is pinned to zero + /// under BLAKE3 but still carries its own meaning under `Test` and + /// `Poseidon`, and one preprocessed column meaning two things under two + /// hashers is worse than the column it saves. + /// + /// It sits INSIDE the selector run rather than after the multiplicities, + /// because the admission validator's one-hot check reads the selectors as a + /// contiguous span (`NUM_SELECTORS` from `MODE_C`). A selector parked past + /// the mults would be outside that span and silently unchecked, which is + /// the sort of gap that only shows up when someone forges a row. + pub const MODE_T: usize = 8; + /// The LEAF-domain selector, and the machine's felt-input mode. + /// + /// **`MODE_L` implies felt-input semantics** — that is a decision, not an + /// inference. A leaf row reads ONE cell of four arbitrary Goldilocks + /// elements and hashes them as eight checked `u32` halves under the `"LFML"` + /// tag, which is what lets FRI data reach a hash whose inputs are `u32` + /// lanes. It is a constraint rather than a convention: a `MODE_L` row that + /// skipped the canonicity block would be unprovable. + /// + /// Placed inside the selector run for the reason [`MODE_T`] gives, which is + /// the same mistake caught once already and spec'd since so it is not made + /// a third time. + pub const MODE_L: usize = 9; + /// Mode selectors, contiguous from [`MODE_C`]: exactly one is set on a real + /// row. + pub const NUM_SELECTORS: usize = 4; + pub const MULT0: usize = 10; + pub const MULT1: usize = 11; + pub const MULT2: usize = 12; + pub const PREP_WIDTH: usize = 13; +} + +/// `LFM_KECCAK` — the keccak-f[1600] adapter: binds 13 machine words of state +/// to the production `KECCAK_RND` family's two `Keccak`-bus tokens. +/// +/// A keccak lane is a `u64`, which is **not** felt-representable (values in +/// `[p, 2^64)` exist), so machine-side keccak state travels as `u32` halves, +/// one half per felt lane: 25 lanes = 50 halves = 13 words of 4 lanes, with the +/// last word's top two lanes unused (tuple constants — see `chips::keccak`). +pub mod keccak { + /// Low 32 bits of the row's tag (matches `KECCAK_RND`'s `DWordWL` timestamp). + pub const TAG_LO: usize = 0; + /// High 32 bits of the row's tag. + pub const TAG_HI: usize = 1; + /// Machine words per keccak state: `ceil(50 / 4)`. + pub const NUM_WORDS: usize = 13; + /// `u32` halves per keccak state: `2 × 25`. + pub const NUM_HALVES: usize = 50; + /// Half slots the words provide: `4 × NUM_WORDS`. The top + /// `WORD_SLOTS − NUM_HALVES = 2` are unused and pinned to zero on the bus. + pub const WORD_SLOTS: usize = 4 * NUM_WORDS; + /// Sponge rate for keccak256: 136 bytes = 17 lanes = 34 halves. + pub const RATE_BYTES: usize = 136; + pub const RATE_LANES: usize = 17; + pub const BLOCK_HALVES: usize = RATE_BYTES / 4; // 34 + /// Machine words per rate block: `ceil(34 / 4)`. The top + /// `4 * BLOCK_WORDS − BLOCK_HALVES = 2` half slots are unused. + pub const BLOCK_WORDS: usize = BLOCK_HALVES.div_ceil(4); // 9 + + pub const IN_ADDR0: usize = 2; // ..IN_ADDR12 = 14 + pub const OUT_ADDR0: usize = IN_ADDR0 + NUM_WORDS; // 15 ..27 + pub const MULT0: usize = OUT_ADDR0 + NUM_WORDS; // 28 ..40 + pub const BLOCK_ADDR0: usize = MULT0 + NUM_WORDS; // 41 ..49 + /// One-hot mode selectors. Their sum is the row's is-real flag, so a + /// padding row (both zero) emits no bus tokens. + pub const MODE_PERM: usize = BLOCK_ADDR0 + BLOCK_WORDS; // 50 + pub const MODE_ABSORB: usize = MODE_PERM + 1; // 51 + /// The production transcript's `sample()` finalizes, REVERSES the 32 digest + /// bytes, absorbs the reversed bytes, and returns them — the returned + /// challenge and the next segment's prefix are the same 32 bytes. Reversal + /// is free here: the bus recomposes each `u32` half from byte columns + /// anyway, so a second send with the coefficients (and half order) flipped + /// costs two interactions and four preprocessed columns, and NO value + /// columns. `sample()` is byte-for-byte identical pre- and post-#841, so + /// this primitive is independent of which transcript revision is targeted. + pub const REV_ADDR0: usize = MODE_ABSORB + 1; // 52 + pub const REV_ADDR1: usize = REV_ADDR0 + 1; // 53 + pub const REV_MULT0: usize = REV_ADDR1 + 1; // 54 + pub const REV_MULT1: usize = REV_MULT0 + 1; // 55 + pub const PREP_WIDTH: usize = REV_MULT1 + 1; // 56 + + /// Machine words in a keccak256 digest: 32 bytes = 8 halves. + pub const DIGEST_WORDS: usize = 2; + + pub const fn rev_addr(word: usize) -> usize { + REV_ADDR0 + word + } + pub const fn rev_mult(word: usize) -> usize { + REV_MULT0 + word + } + + pub const fn in_addr(word: usize) -> usize { + IN_ADDR0 + word + } + pub const fn out_addr(word: usize) -> usize { + OUT_ADDR0 + word + } + /// Write-multiplicity of output word `word`. + pub const fn mult(word: usize) -> usize { + MULT0 + word + } + /// Address of rate-block word `word` (absorb rows only). + pub const fn block_addr(word: usize) -> usize { + BLOCK_ADDR0 + word + } + + /// The tag a keccak row carries, as a function of its row index. + /// + /// SOUNDNESS: the tag is the *only* thing binding a row's request token to + /// its reply token, so tags must be unique across real rows — with a + /// duplicate, a prover can swap two permutations' outputs and the bus still + /// balances (pinned empirically by `keccak_probe`'s + /// `duplicate_tag_output_swap_accepts_demonstrating_hazard`). Making the tag + /// the row ordinal gives uniqueness by construction, and putting it in the + /// *preprocessed* group means the prover cannot choose it at all. The + /// admission validator re-checks uniqueness independently — this function is + /// the compiler's rule, not the guarantee. + pub const fn tag_for_row(row: usize) -> u64 { + row as u64 + 1 + } +} + +/// `LFM_LANES` — word ↔ lane conversion (Pack / Unpack). Pack rows receive +/// four lane cells and send the assembled word; Unpack rows receive a word +/// and send its four lanes as base cells. The shared value columns appear in +/// both tuples, which IS the semantics — the chip has no constraints. +pub mod lanes { + pub const WORD_ADDR: usize = 0; + pub const LANE_ADDR0: usize = 1; // ..LANE_ADDR3 = 4 + pub const MODE_PACK: usize = 5; + pub const MODE_UNPACK: usize = 6; + pub const WORD_MULT: usize = 7; // write-mult of the word (Pack rows only) + pub const LANE_MULT0: usize = 8; // ..LANE_MULT3 = 11 (Unpack rows only) + pub const PREP_WIDTH: usize = 12; +} + +/// `LFM_HINT` — arena ingestion. +pub mod hint { + pub const OUT_ADDR: usize = 0; + pub const MULT: usize = 1; + pub const PREP_WIDTH: usize = 2; +} + +/// `LFM_PUBLIC` — attestation output. +pub mod public { + pub const IN_ADDR: usize = 0; + pub const INDEX: usize = 1; + pub const IS_REAL: usize = 2; + pub const PREP_WIDTH: usize = 3; +} + +/// `LFM_RANGE` — the fixed 2^16 lookup table (program-independent; its group +/// is materialized at commitment time, Milestone B). +pub mod range { + pub const VALUE: usize = 0; + pub const PREP_WIDTH: usize = 1; + pub const NUM_ROWS: usize = 1 << 16; +} + +/// Minimum padded height for any instruction column group (the in-tree +/// `.next_power_of_two().max(4)` convention). +pub const MIN_GROUP_ROWS: usize = 4; + +/// Pads a real row count to its committed height. +pub fn padded_rows(real_rows: usize) -> usize { + real_rows.next_power_of_two().max(MIN_GROUP_ROWS) +} diff --git a/prover/src/lfm/lde.rs b/prover/src/lfm/lde.rs new file mode 100644 index 000000000..b9158a64b --- /dev/null +++ b/prover/src/lfm/lde.rs @@ -0,0 +1,173 @@ +//! Low-degree extension, emitted as machine instructions. +//! +//! A preprocessed column reaches its Merkle commitment as evaluations on the +//! LDE domain, and production gets there in two steps — +//! `Polynomial::interpolate_fft` then `evaluate_polynomial_on_lde_domain` +//! (`tables/register.rs::commit_register_columns` is one caller of exactly this +//! pair). Any leg that must DERIVE a preprocessed commitment rather than read +//! it has to emit that transform, because the values are what the derivation +//! binds; hinting the extended column would hand the prover a degree of freedom +//! the protocol does not give them. +//! +//! Everything here is shape-static. The domain size, the blowup, the coset +//! offset and therefore every twiddle are compile-time constants of the emitted +//! program: host-side `for` loops unroll and nothing loop-shaped reaches the +//! machine, exactly as in [`super::edsl`]. +//! +//! ## Why cosets rather than one big transform +//! +//! Production zero-pads the `n` coefficients to `n·blowup` and runs a single +//! transform of that size. Emitting that shape would cost +//! `(n·blowup)/2 · log₂(n·blowup)` butterflies. Splitting the output domain +//! into its `blowup` cosets of the size-`n` subgroup instead costs +//! `blowup · (n/2 · log₂ n)` butterflies plus `blowup · n` scaling +//! multiplications. At two `LFM_BALU` rows per butterfly and one per scaling +//! (see [`butterfly`]) that is `n·log₂n + blowup·(n + n·log₂n)` rows per column +//! against `n·log₂n + n + n·blowup·log₂(n·blowup)` — **9,088 against 11,264** +//! at the register shape (`n = 128`, blowup 8), and the gap widens with the +//! blowup. +//! +//! Both are arithmetic, not measurement, and both are PER COLUMN; +//! `machine_tests::register_derivation_cost` asserts the emitted total (two +//! columns, 18,176 rows at that shape) against the first formula. The two +//! schemes evaluate the same polynomial on the same points, and the +//! differential tests against production are what says so. +//! +//! ## What this cannot see +//! +//! The emitter is validated by differential tests against production's own +//! `interpolate_fft`/`evaluate_polynomial_on_lde_domain` pair over the field +//! elements those functions produce. It says nothing about domains whose size +//! exceeds Goldilocks' two-adicity (`root_of_unity` panics there rather than +//! emitting a wrong program), and nothing about extension-field columns — +//! preprocessed columns are base-field throughout. + +use math::fft::bit_reversing::reverse_index; +use math::field::traits::IsFFTField; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::builder::{Felt, LfmBuilder}; + +/// The `2^log_n`-th root of unity production's FFT uses. +/// +/// `get_primitive_root_of_unity` is the same entry point `LayerTwiddles` builds +/// its twiddles from, and it is defined by repeated squaring of the field's +/// two-adic generator, so `root_of_unity(k + 1)² == root_of_unity(k)`. That +/// nesting is what lets the coset decomposition below index the big domain with +/// the small domain's root. +pub fn root_of_unity(log_n: u32) -> FE { + GoldilocksField::get_primitive_root_of_unity(log_n as u64) + .expect("the LDE domain must fit Goldilocks' two-adicity") +} + +/// One radix-2 butterfly: `(u + w·x, u − w·x)`. +/// +/// `w` is a program constant, so the subtracting half is `mul_add` against the +/// interned constant `−w` rather than a separate negation — two `LFM_BALU` rows +/// per butterfly, not three. At `w = 1` (every level-1 butterfly, and the first +/// of every later block) there is no multiplication at all. +fn butterfly(b: &mut LfmBuilder, u: Felt, x: Felt, w: FE) -> (Felt, Felt) { + if w == FE::one() { + (b.add(u, x), b.sub(u, x)) + } else { + let pos = b.felt_const(w); + let neg = b.felt_const(-w); + (b.mul_add(pos, x, u), b.mul_add(neg, x, u)) + } +} + +/// Decimation-in-time radix-2 transform: `out[m] = Σᵢ input[i]·root^(i·m)`. +/// +/// `input` is in natural order (the bit-reverse the algorithm needs is a +/// host-side index permutation and costs nothing). Pass `root = ω_n` for the +/// forward direction and `root = ω_n⁻¹` for the inverse — the inverse's `1/n` +/// is NOT applied here, so callers that follow it with a scaling pass fold the +/// factor into their own constants. +fn dit(b: &mut LfmBuilder, input: &[Felt], root: FE) -> Vec { + let n = input.len(); + let log_n = n.trailing_zeros(); + let mut a: Vec = (0..n).map(|i| input[reverse_index(i, n as u64)]).collect(); + for s in 1..=log_n { + let m = 1usize << s; + let step = root.pow(n / m); + for k in (0..n).step_by(m) { + let mut w = FE::one(); + for j in 0..m / 2 { + let (hi, lo) = butterfly(b, a[k + j], a[k + j + m / 2], w); + a[k + j] = hi; + a[k + j + m / 2] = lo; + w *= step; + } + } + } + a +} + +/// Emit the low-degree extension of a column given by its `n` evaluations on +/// the size-`n` subgroup, onto `coset_offset · ⟨ω_{n·blowup}⟩`. +/// +/// The result is in NATURAL domain order — output `j` is the value at +/// `coset_offset · ω_{n·blowup}^j` — which is the layout +/// `stark::commitment::commit_bit_reversed` consumes (it applies the +/// bit-reversal itself). +/// +/// ## The decomposition +/// +/// With `c = iFFT(values)` the polynomial's coefficients and +/// `s_k = coset_offset · ω_{n·blowup}^k`, domain index `j = k + blowup·m` +/// carries the point `s_k · ω_n^m`, so the `k`-th coset is one size-`n` forward +/// transform of `c` scaled by `s_k^i`. The interpolation's `1/n` rides along in +/// those scaling constants, which is why the inverse pass emits no scaling of +/// its own. +pub fn coset_lde( + b: &mut LfmBuilder, + values: &[Felt], + blowup: usize, + coset_offset: FE, +) -> Vec { + let n = values.len(); + assert!( + n.is_power_of_two(), + "the interpolation domain is a subgroup" + ); + assert!( + blowup.is_power_of_two() && blowup > 0, + "blowup is a power of two" + ); + + let log_n = n.trailing_zeros(); + let omega_n = root_of_unity(log_n); + let omega_big = root_of_unity(log_n + blowup.trailing_zeros()); + + // `n · coefficients`: the 1/n is folded into the per-coset scaling below. + let scaled_coeffs = dit( + b, + values, + omega_n.inv().expect("a root of unity is invertible"), + ); + + let n_inv = FE::from(n as u64) + .inv() + .expect("the domain size is nonzero in Goldilocks"); + + let mut out: Vec> = vec![None; n * blowup]; + for k in 0..blowup { + let s = coset_offset * omega_big.pow(k); + let mut weight = n_inv; + let coset_coeffs: Vec = (0..n) + .map(|i| { + let c = b.felt_const(weight); + let scaled = b.mul(scaled_coeffs[i], c); + weight *= s; + scaled + }) + .collect(); + for (m, value) in dit(b, &coset_coeffs, omega_n).into_iter().enumerate() { + out[k + blowup * m] = Some(value); + } + } + out.into_iter() + .map(|v| v.expect("every domain index is written exactly once")) + .collect() +} diff --git a/prover/src/lfm/leaf_kats.rs b/prover/src/lfm/leaf_kats.rs new file mode 100644 index 000000000..165aa89e0 --- /dev/null +++ b/prover/src/lfm/leaf_kats.rs @@ -0,0 +1,211 @@ +//! LEAF-mode KATs for the LFM `"LFML"` domain, at 6 and 7 rounds. +//! +//! GENERATED — do not hand-edit. Rendered by +//! `thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py` from +//! `gate-oracle/blake3_oracle.py`, a Python BLAKE3 written **before any Rust +//! existed**. These vectors are a specification the implementation is checked +//! against, not a recording of what the implementation happened to do. +//! +//! A leaf row hashes FOUR arbitrary Goldilocks elements AND chains an +//! accumulator, in ONE compression (COMMIT.md §1.2). The accumulator is a digest +//! cell and fills lanes 0–3; each felt occupies two lanes above it as checked +//! `u32` halves, `[lo0, hi0, …, lo3, hi3]`. So the message is +//! `LE32(acc ‖ halves) ‖ "LFML"` — 52 bytes, still one BLAKE3 block, so the +//! crate-KAT anchor survives the widening. + +/// One leaf row: the chaining accumulator, four felts, the twelve lanes they +/// become, and the digest at each round count. +pub struct LeafVector { + pub name: &'static str, + pub acc: [u32; 4], + pub felts: [u64; 4], + pub lanes: [u32; 12], + /// Digest at 6 rounds (the A6R variant; no library computes it). + pub digest_6: [u32; 4], + /// Digest at 7 rounds — `blake3::hash(LE32(lanes) ‖ "LFML")[..16]`. + pub digest_7: [u32; 4], +} + +pub const LEAF_VECTORS: [LeafVector; 6] = [ + LeafVector { + name: "zeros", + acc: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + felts: [0u64, 0u64, 0u64, 0u64], + lanes: [ + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + ], + digest_6: [0x9D79DC29, 0xFC6E166E, 0x30387614, 0xF6B51296], + digest_7: [0xB30DB92A, 0xC648E66E, 0x85368146, 0x30A98B38], + }, + LeafVector { + name: "boundary_mix", + acc: [0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF], + felts: [0u64, 1u64, 18446744069414584320u64, 4294967296u64], + lanes: [ + 0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000001, + 0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000001, + ], + digest_6: [0xA101443C, 0xA70F5A93, 0xAD973E8C, 0x17C8F7BA], + digest_7: [0x0E214E2C, 0x5D16CE5C, 0xA4DE74CF, 0x9FA39D59], + }, + LeafVector { + name: "all_p_minus_1", + acc: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + felts: [ + 18446744069414584320u64, + 18446744069414584320u64, + 18446744069414584320u64, + 18446744069414584320u64, + ], + lanes: [ + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, + 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, + ], + digest_6: [0x5FDFAEF4, 0x0F63DEEE, 0xCC4EC296, 0x675289C3], + digest_7: [0xAE3CB971, 0x6F0EDEE7, 0x75BBD078, 0xC07D12D6], + }, + LeafVector { + name: "ramp", + acc: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + felts: [ + 72623859790382856u64, + 1230066625199609624u64, + 2387509390608836392u64, + 3544952156018063160u64, + ], + lanes: [ + 0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10, 0x05060708, 0x01020304, 0x15161718, + 0x11121314, 0x25262728, 0x21222324, 0x35363738, 0x31323334, + ], + digest_6: [0x7E8EC742, 0x478136B7, 0xDC4010C2, 0xA7B85A1F], + digest_7: [0x6F6562C3, 0x6755528E, 0xBD65A6F0, 0xA9B1551D], + }, + LeafVector { + name: "u32_edges", + acc: [0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF], + felts: [4294967295u64, 4294967296u64, 18446744065119617025u64, 1u64], + lanes: [ + 0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF, 0xFFFFFFFF, 0x00000000, 0x00000000, + 0x00000001, 0x00000001, 0xFFFFFFFE, 0x00000001, 0x00000000, + ], + digest_6: [0x0E17BDDF, 0x1E0CA3B6, 0x7F8B414F, 0xDDF551B2], + digest_7: [0x77E4CDFD, 0x92CC8E05, 0x1BBC4BD0, 0x64B4D8D2], + }, + LeafVector { + name: "acc_ignored_control", + acc: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + felts: [0u64, 0u64, 0u64, 0u64], + lanes: [ + 0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + ], + digest_6: [0xE4AE2501, 0x1FCF9DAB, 0x85643F4E, 0xE24B3793], + digest_7: [0xB8094093, 0xA7EBC1A4, 0xB7955183, 0x0BA8929B], + }, +]; + +/// A boundary felt and the halves it must decompose into. +pub struct BoundaryFelt { + pub name: &'static str, + pub felt: u64, + pub lo: u32, + pub hi: u32, +} + +/// The six canonical boundary cases, `p − 1` included — the tight one. +pub const BOUNDARY_FELTS: [BoundaryFelt; 6] = [ + BoundaryFelt { + name: "zero", + felt: 0u64, + lo: 0x00000000, + hi: 0x00000000, + }, + BoundaryFelt { + name: "one", + felt: 1u64, + lo: 0x00000001, + hi: 0x00000000, + }, + BoundaryFelt { + name: "u32_max", + felt: 4294967295u64, + lo: 0xFFFFFFFF, + hi: 0x00000000, + }, + BoundaryFelt { + name: "two_pow_32", + felt: 4294967296u64, + lo: 0x00000000, + hi: 0x00000001, + }, + BoundaryFelt { + name: "p_minus_2_32", + felt: 18446744065119617025u64, + lo: 0x00000001, + hi: 0xFFFFFFFE, + }, + BoundaryFelt { + name: "p_minus_1", + felt: 18446744069414584320u64, + lo: 0x00000000, + hi: 0xFFFFFFFF, + }, +]; + +/// Values the leaf mode must REJECT rather than reduce. Each has `hi` maximal +/// and `lo >= 1`, so each aliases a canonical felt — which is exactly the +/// collision the canonicity block exists to prevent. +pub struct NonCanonical { + pub name: &'static str, + pub value: u64, + pub lo: u32, + pub hi: u32, +} + +pub const NON_CANONICAL: [NonCanonical; 3] = [ + NonCanonical { + name: "p", + value: 18446744069414584321u64, + lo: 0x00000001, + hi: 0xFFFFFFFF, + }, + NonCanonical { + name: "p_plus_1", + value: 18446744069414584322u64, + lo: 0x00000002, + hi: 0xFFFFFFFF, + }, + NonCanonical { + name: "two_pow_64_minus_1", + value: 18446744073709551615u64, + lo: 0xFFFFFFFF, + hi: 0xFFFFFFFF, + }, +]; + +/// The eight-felt `FriToyV0` leaf: ONE `LFML` chain, two rows, no fold. +pub struct FriLeafVector { + pub felts: [u64; 8], + pub digest_6: [u32; 4], + pub digest_7: [u32; 4], + /// Compressions the whole leaf costs — 3 before the accumulator moved into + /// the message, 2 after (COMMIT.md §1.4.1: the RATE, measured). + pub compresses: usize, +} + +pub const FRI_LEAF: FriLeafVector = FriLeafVector { + felts: [ + 18446744069414584320u64, + 0u64, + 1u64, + 4294967296u64, + 12345678901234567u64, + 4294967295u64, + 18446744065119617025u64, + 999u64, + ], + digest_6: [0x8578A6BC, 0x9160F074, 0x3F4C82B9, 0x98C5C775], + digest_7: [0x9C36DE23, 0xCD397230, 0x2013BF3D, 0xD72A0346], + compresses: 2, +}; diff --git a/prover/src/lfm/leaf_tests.rs b/prover/src/lfm/leaf_tests.rs new file mode 100644 index 000000000..e43df5982 --- /dev/null +++ b/prover/src/lfm/leaf_tests.rs @@ -0,0 +1,879 @@ +//! The `"LFML"` leaf mode (option C): its vectors, its canonicity gate, and the +//! milestone it unblocks — `FriToyV0` proving under BLAKE3. +//! +//! ## What pins what +//! +//! 1. **The felt boundary** — halves and canonicity — against +//! [`super::leaf_kats`]'s boundary table, `p − 1` and the non-canonical +//! aliases included. +//! 2. **The step function** against the `blake3` crate: at 7 rounds a leaf is +//! `blake3::hash(LE32(lanes) ‖ "LFML")` truncated, so the leaf domain +//! inherits the socket's external anchor rather than claiming a new one. +//! 3. **The chip**: M9 (mode confusion, six ordered pairs) and M10 (a `MODE_L` +//! row that skips canonicity), which are what make "`MODE_L` implies +//! felt-input semantics" a constraint rather than a convention. +//! 4. **The assembled program**: `FriToyV0` proves and verifies under BLAKE3 — +//! the F3.4 milestone — with a NEGATIVE leg showing the canonicity gate does +//! work in the real proof and not only in a unit test. +//! +//! Every rejection test is paired with an honest-path assertion. + +use math::field::traits::IsPrimeField; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::blake3_socket::{ + self, FELTS_PER_LEAF, SOCKET_ROUNDS, TAG_LFMC, TAG_LFML, TAG_LFMT, cols, felt_halves, + is_canonical, leaf_digest_rounds, leaf_lanes, word_of, +}; +use super::hash::HasherKind; +use super::instr::{HashMode, Instr}; +use super::leaf_kats::{BOUNDARY_FELTS, FRI_LEAF, LEAF_VECTORS, NON_CANONICAL}; +use super::proof::{lfm_prove_with_hasher, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::word::LfmWord; + +const KIND: HasherKind = HasherKind::Blake3; + +/// Goldilocks `p`. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +fn felts_of(vals: &[u64; 4]) -> LfmWord { + core::array::from_fn(|i| FE::from(vals[i])) +} + +// ========================================================================= +// L2/L3/L4 — the felt boundary +// ========================================================================= + +/// Every boundary felt splits into the halves the spec pins, `p − 1` included — +/// the tight case, where `hi` is maximal and `lo` is exactly zero. +#[test] +fn every_boundary_felt_round_trips_through_its_halves() { + for v in BOUNDARY_FELTS.iter() { + let (lo, hi) = felt_halves(v.felt).unwrap_or_else(|| panic!("{} is canonical", v.name)); + assert_eq!((lo, hi), (v.lo, v.hi), "halves of {}", v.name); + assert!(is_canonical(lo, hi), "{} must pass the predicate", v.name); + assert_eq!( + u64::from(lo) + (u64::from(hi) << 32), + v.felt, + "halves must recompose {}", + v.name + ); + } +} + +/// ★ Non-canonical values are REJECTED, never reduced — and the test says why +/// it matters: each one ALIASES a canonical felt, so reducing would give one +/// field element two leaf digests. +#[test] +fn non_canonical_values_are_rejected_not_reduced() { + for v in NON_CANONICAL.iter() { + assert!(!is_canonical(v.lo, v.hi), "{} is non-canonical", v.name); + assert_eq!(felt_halves(v.value), None, "{} must be refused", v.name); + + // The alias, spelled out: hi maximal makes 2^32·hi = p − 1 ≡ −1, so the + // pair encodes `lo − 1`, which has its own ordinary encoding. THAT is + // the collision the canonicity block prevents. + let aliased = (u128::from(v.lo) + (u128::from(v.hi) << 32)) % u128::from(P); + assert_eq!(aliased, u128::from(v.lo) - 1); + let (clo, chi) = felt_halves(aliased as u64).expect("the alias target is canonical"); + assert_ne!( + (clo, chi), + (v.lo, v.hi), + "{}: two half-pairs for one felt is exactly the hazard", + v.name + ); + } +} + +/// L4 — the predicate IS `v < p`, over every boundary and a dense sweep. +/// +/// The spec ran 300,007 cases; this runs the same boundaries plus a sweep near +/// the wrap, which is where a predicate that is merely *nearly* right fails. +#[test] +fn the_canonicity_predicate_is_exactly_less_than_p() { + let check = |v: u64| { + let (lo, hi) = (v as u32, (v >> 32) as u32); + assert_eq!( + is_canonical(lo, hi), + v < P, + "predicate disagrees with v < p at {v:#x}" + ); + }; + for v in [ + 0u64, + 1, + u64::from(u32::MAX), + 1 << 32, + P - 2, + P - 1, + P, + P + 1, + u64::MAX, + ] { + check(v); + } + // The whole neighbourhood of the wrap, both sides. + for d in 0..2_000u64 { + check(P.wrapping_sub(d)); + check(P.wrapping_add(d)); + } + // A stride across the space, so the sweep is not only local. + for k in 0..20_000u64 { + check(k.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + } +} + +// ========================================================================= +// L1 — the leaf digest, and its crate anchor +// ========================================================================= + +/// Every leaf vector reproduces at BOTH round counts, lanes included. +#[test] +fn every_leaf_vector_reproduces_at_both_round_counts() { + for v in LEAF_VECTORS.iter() { + let (acc, felts) = (word_of(&v.acc), felts_of(&v.felts)); + assert_eq!( + blake3_socket::leaf_row_lanes(&acc, &felts).expect("canonical"), + v.lanes, + "lanes of {}", + v.name + ); + assert_eq!( + leaf_lanes(&felts).expect("canonical"), + v.lanes[4..], + "the halves of {} sit ABOVE the accumulator", + v.name + ); + assert_eq!( + leaf_digest_rounds(&acc, &felts, 6).expect("canonical"), + v.digest_6, + "6-round leaf {}", + v.name + ); + assert_eq!( + leaf_digest_rounds(&acc, &felts, 7).expect("canonical"), + v.digest_7, + "7-round leaf {}", + v.name + ); + } +} + +/// ★ The accumulator REACHES the digest — the property the whole RATE rests on. +/// +/// A row that absorbed its felts and dropped the chaining value would still +/// satisfy every canonicity and halves constraint, and would still be a +/// perfectly good hash of the felts; what it would not be is a CHAIN, and a wide +/// leaf built out of it would bind only its last four felts. The table carries +/// `acc_ignored_control` for exactly this: same felts as `zeros`, different +/// accumulator. +#[test] +fn the_accumulator_changes_the_leaf_digest() { + let find = |name: &str| { + LEAF_VECTORS + .iter() + .find(|v| v.name == name) + .expect("vector present") + }; + let (zeros, control) = (find("zeros"), find("acc_ignored_control")); + assert_eq!( + zeros.felts, control.felts, + "the control varies ONLY the acc" + ); + assert_ne!(zeros.acc, control.acc); + assert_ne!(zeros.digest_6, control.digest_6); + assert_ne!(zeros.digest_7, control.digest_7); +} + +/// ★ **The external anchor, direct.** At 7 rounds a leaf is literally +/// `blake3::hash(LE32(acc)‖LE32(lo0)‖LE32(hi0)‖…‖LE32(hi3)‖"LFML")` truncated to +/// 16 bytes. +/// +/// The message is rebuilt from the byte-level specification rather than from +/// `socket_message`, so the word-level and byte-level forms can disagree. This +/// is the property option C was chosen to preserve and the RATE had to keep: +/// putting the felt encoding INSIDE the socket keeps the message layout +/// byte-identical to a digest-mode compress, and 52 bytes is still ONE block, so +/// the crate stays a direct KAT for the leaf domain. Carrying the accumulator in +/// the chaining value `h` instead would have made the row a chunk continuation +/// and thrown this away for the same rate. +#[test] +fn seven_rounds_is_blake3_of_the_leaf_message() { + for v in LEAF_VECTORS.iter() { + let mut msg = Vec::with_capacity(52); + for lane in v.lanes.iter() { + msg.extend_from_slice(&lane.to_le_bytes()); + } + msg.extend_from_slice(b"LFML"); + assert_eq!(msg.len(), 52, "a leaf row is one 52-byte block"); + assert!(msg.len() < 64, "and one block is what the anchor needs"); + + let full = blake3::hash(&msg); + let want: [u32; 4] = core::array::from_fn(|i| { + u32::from_le_bytes(full.as_bytes()[4 * i..4 * i + 4].try_into().unwrap()) + }); + assert_eq!(want, v.digest_7, "leaf {} must be blake3::hash", v.name); + } +} + +/// The tag word is the ASCII, little-endian, and the three live domains are +/// pairwise distinct as VALUES — the cheap check that a typo cannot pass. +#[test] +fn the_leaf_tag_is_lfml_and_the_three_domains_are_distinct() { + assert_eq!(TAG_LFML, u32::from_le_bytes(*b"LFML")); + assert_eq!(TAG_LFML.to_le_bytes(), *b"LFML"); + assert_ne!(TAG_LFML, TAG_LFMC); + assert_ne!(TAG_LFML, TAG_LFMT); + assert_ne!(TAG_LFMC, TAG_LFMT); +} + +/// L5 — the three domains produce three different digests from the SAME twelve +/// lanes. Distinct tag values are necessary; distinct digests are the property. +#[test] +fn the_three_domains_differ_on_the_same_lanes() { + for v in LEAF_VECTORS.iter() { + for rounds in [6, 7] { + let d: Vec<[u32; 4]> = [TAG_LFMC, TAG_LFMT, TAG_LFML] + .iter() + .map(|t| blake3_socket::socket_digest_lanes(&v.lanes, rounds, *t)) + .collect(); + assert_ne!(d[0], d[1], "{} @{rounds}: LFMC == LFMT", v.name); + assert_ne!(d[0], d[2], "{} @{rounds}: LFMC == LFML", v.name); + assert_ne!(d[1], d[2], "{} @{rounds}: LFMT == LFML", v.name); + } + } +} + +/// L6 — an eight-felt leaf is exactly TWO compressions: one `LFML` chain of two +/// rows, absorbing four felts and chaining in each. +/// +/// ★ **This is the RATE, measured rather than asserted.** It was three — two +/// felts-only leaf rows and an `LFMC` parent folding them — which is 2 felts per +/// compression. Moving the accumulator into the message makes the fold +/// unnecessary and takes it to 4, and leaf absorption is ~70% of a recursion +/// tower node's bill (COMMIT.md §1.4.1). The count is pinned in the vector table +/// so the saving cannot quietly regress. +#[test] +fn an_eight_felt_leaf_is_one_chain_of_two_rows() { + let lo: LfmWord = core::array::from_fn(|i| FE::from(FRI_LEAF.felts[i])); + let hi: LfmWord = core::array::from_fn(|i| FE::from(FRI_LEAF.felts[4 + i])); + let start = super::fixture::leaf_chain_start(); + for (rounds, want) in [(6, FRI_LEAF.digest_6), (7, FRI_LEAF.digest_7)] { + let d0 = word_of(&leaf_digest_rounds(&start, &lo, rounds).expect("canonical")); + let chained = leaf_digest_rounds(&d0, &hi, rounds).expect("canonical"); + assert_eq!(chained, want, "the 8-felt leaf at {rounds} rounds"); + } + assert_eq!(FRI_LEAF.compresses, 2, "8 felts / RATE 4 = 2 compressions"); + + // The HOST path agrees with the reference — `host_leaf_hash_pair` is what + // the fixture builds its trees with, so a divergence here is a fixture that + // the machine cannot authenticate. + let host = super::fixture::host_leaf_hash_pair(KIND, &lo, &hi); + let want = if SOCKET_ROUNDS == 7 { + FRI_LEAF.digest_7 + } else { + FRI_LEAF.digest_6 + }; + assert_eq!(host, word_of(&want)); +} + +// ========================================================================= +// H6 — the lane-identity gate, both directions +// ========================================================================= +// +// The gate is per LANE RANGE, not per mode, and COMMIT.md §1.4.2 asks for a +// control in the WA1/WA2 style because one direction is a soundness break: gate +// lanes 0–3 on `digest_mu` and a leaf row's accumulator carries no identity at +// all, so the prover picks the chain's message words freely and the whole leaf +// chain unbinds. Two tests, one per direction, because a gate that is wrong +// EITHER way passes the other test. + +/// ★★ **H6, the soundness direction: a leaf row's ACCUMULATOR lanes are +/// constrained.** +/// +/// Shaped like WA9 — not "the tampered row is rejected", which would pass for a +/// set that rejected it incidentally, but "the violated set IS the accumulator +/// lane's own identity". That carries both legs: with the identity the row is +/// rejected, and without it every other constraint still evaluates to zero on +/// this row, so it would be accepted. If the gate were `digest_mu` here, this +/// row would satisfy everything. +#[test] +fn h6_a_leaf_rows_accumulator_lanes_carry_the_identity() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let base = leaf_row(&acc, &felts); + assert_eq!( + super::blake3_socket_tests::violations(&base), + Vec::::new(), + "HONEST CONTROL: a leaf row must satisfy every constraint" + ); + + for lane in 0..cols::NUM_ACC_LANES { + // Move the accumulator FELT and leave its byte columns honest. The + // message the row hashes is the bytes, so this is the forgery that + // matters: a prover claiming one accumulator in `IN` while the mixing + // core consumes another. Only the identity ties the two together. + let mut forged = base.clone(); + forged[cols::IN0 + lane] += FE::one(); + assert_eq!( + super::blake3_socket_tests::violations(&forged), + vec![blake3_socket::LANE_IDX + lane], + "lane {lane} of the accumulator must be pinned to its bytes, and by \ + ITS identity — anything else and the dropped-leg claim fails" + ); + } +} + +/// ★ **H6, the other direction: the identity must NOT hold on a leaf row's FELT +/// lanes.** +/// +/// Lanes 4–11 are the felts' `lo`/`hi` halves, so `IN` and the message word are +/// deliberately different field elements there; the halves binding is what +/// relates them. Gating those on the full `mu` would make every leaf row +/// unprovable — the failure the original eight-lane comment warned about, which +/// survives the widening in exactly this narrowed form. +/// +/// Asserted as arithmetic on an honest row rather than by building a broken +/// chip: if `IN(felt i) == lo_i` for every felt, the claim is vacuous and this +/// test says so. +#[test] +fn h6_the_felt_lanes_do_not_satisfy_the_lane_identity() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let row = leaf_row(&acc, &felts); + + let mut discriminated = 0; + for i in 0..FELTS_PER_LEAF { + let lo_lane = cols::leaf_lo_lane(i); + let bytes: u64 = (0..4) + .map(|b| { + GoldilocksField::canonical(row[cols::lane_byte(lo_lane, b)].value()) << (8 * b) + }) + .sum(); + let felt = GoldilocksField::canonical(row[cols::leaf_felt(i)].value()); + // The felt is `lo + 2^32·hi`, so it equals its low half only when the + // high half is zero. The vector is chosen so that is not always true. + if felt != bytes { + discriminated += 1; + } + } + assert!( + discriminated > 0, + "the vector must contain a felt wider than 32 bits, or this test is \ + vacuous and the gate could be `mu` without anyone noticing" + ); +} + +// ========================================================================= +// M9 / M10 — the chip-level controls the leaf spec pre-committed +// ========================================================================= + +/// A hash row in `mode` over `felts`/`lanes`, exactly as the trace filler builds +/// one — for the controls, which need to force a mismatch the filler cannot. +fn leaf_row(acc: &LfmWord, felts: &LfmWord) -> Vec { + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[cols::MODE_L] = FE::one(); + row[cols::IN0..cols::IN0 + 4].copy_from_slice(acc); + row[cols::leaf_felt(0)..cols::leaf_felt(0) + FELTS_PER_LEAF].copy_from_slice(felts); + for (k, iv) in super::blake3::BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = leaf_digest_rounds(acc, felts, SOCKET_ROUNDS).expect("canonical"); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness(&mut row); + row +} + +/// **M9 — mode confusion, all six ordered pairs.** A row in one domain whose +/// witness computes another domain's digest must be rejected. +/// +/// Three tags means six ordered confusions, and the leaf domain adds four of +/// them. `L5` above shows the three functions differ; this shows the CHIP +/// notices, which is a different claim. +#[test] +fn m9_no_domain_can_compute_another_domains_digest() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let halves = leaf_lanes(&felts).expect("canonical"); + // Two digest cells for the two-to-one modes. A leaf row's lanes are its own + // (accumulator ‖ halves) and cannot be shared with a digest row's: lanes + // 8–11 read the third input cell, which the unread-`IN` pins hold at zero on + // every digest row. So each mode is built with the lanes it actually has, + // and the digest is taken over THOSE — the confusion under test is the + // TAG's, not the lanes'. + let (a, b) = ( + blake3_socket::lanes_of(&acc).expect("a digest cell is u32 lanes"), + [halves[0], halves[1], halves[2], halves[3]], + ); + + for (mode, own) in [ + (HashMode::Compress, TAG_LFMC), + (HashMode::Transcript, TAG_LFMT), + (HashMode::Leaf, TAG_LFML), + ] { + for other in [TAG_LFMC, TAG_LFMT, TAG_LFML] { + let mut row = vec![FE::zero(); cols::NUM_COLUMNS]; + row[super::blake3_socket_tests::mode_col(mode)] = FE::one(); + let lanes = if mode == HashMode::Leaf { + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&acc); + row[cols::leaf_felt(0)..cols::leaf_felt(0) + FELTS_PER_LEAF] + .copy_from_slice(&felts); + blake3_socket::leaf_row_lanes(&acc, &felts).expect("canonical") + } else { + row[cols::IN0..cols::IN0 + 4].copy_from_slice(&word_of(&a)); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(&word_of(&b)); + blake3_socket::digest_row_lanes(&a, &b) + }; + for (k, iv) in super::blake3::BLAKE3_IV.iter().take(4).enumerate() { + row[cols::S8 + k] = FE::from(u64::from(*iv)); + } + let digest = blake3_socket::socket_digest_lanes(&lanes, SOCKET_ROUNDS, other); + row[cols::OUT0..cols::OUT0 + 4].copy_from_slice(&word_of(&digest)); + blake3_socket::fill_socket_witness_tagged(&mut row, other); + + let violated = super::blake3_socket_tests::violations(&row); + if other == own { + assert_eq!( + violated, + Vec::::new(), + "HONEST CONTROL: {mode:?} in its own domain must be accepted" + ); + } else { + assert!( + !violated.is_empty(), + "{mode:?} computing the {other:#010x} domain must be rejected" + ); + } + } + } +} + +/// **M10 — `MODE_L` implies felt-input semantics, as a CONSTRAINT.** +/// +/// A leaf row that skips the canonicity block must be rejected. Two ways to +/// skip it, and both are tried: zero the witnesses, and install the +/// non-canonical alias with a witness that would satisfy every constraint the +/// canonicity block does not impose. +#[test] +fn m10_a_leaf_row_cannot_skip_canonicity() { + let acc = word_of(&LEAF_VECTORS[1].acc); + let felts = felts_of(&LEAF_VECTORS[1].felts); + let base = leaf_row(&acc, &felts); + assert_eq!( + super::blake3_socket_tests::violations(&base), + Vec::::new(), + "HONEST CONTROL: a canonical leaf row satisfies every constraint" + ); + + // (a) blank the canonicity witnesses. `canon_b` pins `Z` from `G`, so a + // zeroed `Z` is only satisfiable when `G` is invertible AND `GINV` matches; + // blanking both breaks it. + let mut blanked = base.clone(); + for i in 0..FELTS_PER_LEAF { + blanked[cols::canon_z(i)] = FE::zero(); + blanked[cols::canon_ginv(i)] = FE::zero(); + } + assert!( + !super::blake3_socket_tests::violations(&blanked).is_empty(), + "a leaf row with no canonicity witness must be rejected" + ); + + // (b) ★ THE ALIAS. Re-encode felt 0 as (lo + 1, hi = 2^32 − 1), which is the + // SAME field element — the binding constraint is satisfied — and let the + // witness be otherwise consistent. Only canonicity can catch this, and the + // test asserts it is `canon-c` that does. + let target = felts_of(&[0, 0, 0, 0]); + let mut alias = leaf_row(&acc, &target); + // felt 0's lo half becomes 1 and its hi half 2^32 − 1. ⚠ Located through + // `leaf_lo_lane`/`leaf_hi_lane` rather than as lanes 0 and 1: the felts sit + // ABOVE the accumulator now, and a literal 0/1 here would silently corrupt + // the accumulator instead and test nothing (COMMIT.md §1.4.4 H4). + for (lane, v) in [ + (cols::leaf_lo_lane(0), 1u32), + (cols::leaf_hi_lane(0), u32::MAX), + ] { + for byte in 0..4 { + alias[cols::lane_byte(lane, byte)] = FE::from(u64::from((v >> (8 * byte)) as u8)); + } + } + // The witness the alias would need: hi is maximal, so G = 0 and Z = 1. + alias[cols::canon_z(0)] = FE::one(); + alias[cols::canon_ginv(0)] = FE::zero(); + + let violated = super::blake3_socket_tests::violations(&alias); + // canon-c for felt 0 — located from the arm's own indices rather than by a + // literal, so growing the framing cannot silently point this at another + // constraint. + const CANON_C_FELT0: usize = + blake3_socket::LEAF_IDX + blake3_socket::LEAF_CONSTRAINTS_PER_FELT - 1; + assert!( + violated.contains(&CANON_C_FELT0), + "the alias must be caught by canon-c (idx {CANON_C_FELT0}), got {violated:?}" + ); + + // And the alias really is the same field element, so nothing ELSE could + // have caught it — that is what makes canonicity load-bearing rather than + // redundant with the binding constraint. + assert_eq!( + (u128::from(1u32) + (u128::from(u32::MAX) << 32)) % u128::from(P), + 0, + "the alias encodes felt 0" + ); +} + +// ========================================================================= +// ★ The F3.4 milestone: FriToyV0 under BLAKE3 +// ========================================================================= + +fn fri_arenas(inner: &super::fixture::FriToyProof) -> Vec> { + vec![inner.commitments.clone(), inner.openings.clone()] +} + +/// ★★ **`FriToyV0` proves and verifies under BLAKE3.** The milestone the whole +/// campaign was for: a real verification program, over real FRI data — LDE +/// evaluations and folded extension elements, none of them `u32` — proved under +/// the machine's real hash. +/// +/// This replaces `blake3_socket_tests::fri_toy_is_still_blocked_by_o1…`, whose +/// own doc required a prove+verify rather than an execute when O1 closed. +#[test] +fn fri_toy_proves_and_verifies_under_blake3() { + let opts = options(); + let program = super::programs::fri_toy_program(); + let inner = super::fixture::fixture_prove_with_hasher(KIND); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&inner), &opts, KIND) + .expect("FriToyV0 must prove under BLAKE3"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest BLAKE3 proof of FriToyV0 must verify" + ); + // The attested output is the inner proof's identity: both roots. + assert_eq!(proved.public_words[0].1, inner.commitments[0]); + assert_eq!(proved.public_words[1].1, inner.commitments[1]); +} + +/// The same program under the other two hashers — B1 and option C both changed +/// shared constructions, so all three must stay green. +#[test] +fn fri_toy_proves_and_verifies_under_every_hasher() { + let opts = options(); + let program = super::programs::fri_toy_program(); + for kind in [HasherKind::Test, HasherKind::Poseidon, HasherKind::Blake3] { + let inner = super::fixture::fixture_prove_with_hasher(kind); + let artifacts = build_artifacts_with_hasher(&program, &opts, kind); + let proved = lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&inner), &opts, kind) + .unwrap_or_else(|e| panic!("prove under {kind:?}: {e:?}")); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest proof of FriToyV0 must verify under {kind:?}" + ); + } +} + +/// ⚠ **The NEGATIVE leg — the criterion most likely to be skipped.** +/// +/// A fixture whose tree was built under a DIFFERENT hasher must not +/// authenticate. This is what shows the leaf digests are load-bearing in the +/// assembled program: every opened row is authenticated by re-deriving its leaf, +/// so a leaf computed by another hash breaks the walk. +#[test] +fn fri_toy_rejects_a_fixture_built_under_another_hasher() { + let opts = options(); + let program = super::programs::fri_toy_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let mismatched = super::fixture::fixture_prove_with_hasher(HasherKind::Test); + assert!( + lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&mismatched), &opts, KIND).is_err(), + "a Test-hashed fixture must not authenticate under BLAKE3" + ); + + // HONEST CONTROL: the matching fixture does prove, so the rejection is about + // the hasher and not about the program. + let matching = super::fixture::fixture_prove_with_hasher(KIND); + assert!( + lfm_prove_with_hasher(&program, &artifacts, &fri_arenas(&matching), &opts, KIND).is_ok() + ); +} + +/// The leaf mode is what closed O1 for this program: the fixture's committed +/// values are still not `u32`-laned, and it proves anyway. +/// +/// The old tripwire asserted the opposite conclusion from the same premise. It +/// is kept as a positive statement because the premise is what makes the +/// milestone meaningful — proving over `u32`-shaped data would have proved +/// nothing about the leaf mode. +#[test] +fn the_fixture_data_is_still_not_u32_and_that_is_the_point() { + let over = super::fixture::fixture_columns() + .iter() + .flatten() + .filter(|v| GoldilocksField::canonical(v.value()) >= 1u64 << 32) + .count(); + assert!( + over > 0, + "if the fixture became u32-laned the milestone would be vacuous" + ); + + // Every leaf row in the emitted program is a `Leaf`, and every Merkle-walk + // step is a `Compress` — the split the O5 retirement rests on. + let program = super::programs::fri_toy_program(); + let leaves = program + .instrs + .iter() + .filter(|i| matches!(i, Instr::Hash { mode, .. } if *mode == HashMode::Leaf)) + .count(); + assert_eq!( + leaves, 26, + "4 queries × 3 data leaves × 2 LFML rows, plus the two terminal \ + coefficients the transcript absorbs as data" + ); +} + +// ========================================================================= +// D1 — the unread input cells, on EVERY arm +// ========================================================================= + +/// Evaluates a hash row against `kind`'s constraint set and returns the +/// violated indices. +fn violations_under(kind: HasherKind, row: &[FE]) -> Vec { + use math::field::element::FieldElement; + use stark::constraints::builder::{ConstraintSet, ProverEvalFolder}; + use stark::frame::Frame; + use stark::table::TableView; + use stark::traits::TransitionEvaluationContext; + + use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + + let set = super::chips::hash::HashConstraints { kind }; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new( + vec![row.to_vec()], + vec![vec![]], + )]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +/// A `MODE_L` row for `kind` whose THIRD input cell carries `extra`. +/// +/// Everything downstream is derived from the cells the mode reads, by each arm's +/// own rule, so the row is internally consistent whatever `extra` is: `extra = 0` +/// is the honest row a trace filler would write, and any other `extra` is the +/// forgery a prover controlling the whole trace would actually submit. Building +/// both the same way is what makes "only the pins fire" a meaningful assertion — +/// a half-built forgery would trip the round constraints instead and prove +/// nothing about the pins. +fn leaf_row_with_third_cell( + kind: HasherKind, + acc: &LfmWord, + felts: &LfmWord, + extra: &LfmWord, +) -> Vec { + use super::hash::{HASH_STATE_FELTS, LfmHasher}; + + let mut row = vec![FE::zero(); super::chips::hash::num_columns(kind)]; + row[cols::MODE_L] = FE::one(); + row[cols::IN0..cols::IN0 + 4].copy_from_slice(acc); + row[cols::IN0 + 4..cols::IN0 + 8].copy_from_slice(felts); + row[cols::IN0 + 8..cols::IN0 + 12].copy_from_slice(extra); + let iv = kind.compress_iv(); + row[cols::S8..cols::S8 + iv.len()].copy_from_slice(&iv); + + match kind { + // BLAKE3 reads an accumulator and four felts and nothing else, so its + // output does not depend on the third cell at all — which is exactly why + // only a pin can notice junk there. + HasherKind::Blake3 => { + let out = kind.leaf_out(acc, felts); + row[cols::OUT0..cols::OUT0 + out.len()].copy_from_slice(&out); + blake3_socket::fill_socket_witness(&mut row); + } + // The field-native arms permute the state — which is the two cells the + // mode reads and the IV, never the third cell. + HasherKind::Test => { + let mut state = [FE::zero(); HASH_STATE_FELTS]; + state[0..4].copy_from_slice(acc); + state[4..8].copy_from_slice(felts); + state[8..12].copy_from_slice(&iv); + let out = kind.permute(state); + row[cols::OUT0..cols::OUT0 + out.len()].copy_from_slice(&out); + } + HasherKind::Poseidon => { + // The filler reads `IN`/`S` back out of the row and writes every + // round intermediate AND `OUT`, so the whole witness follows the + // junk rather than only the final output. + super::trace::fill_poseidon_witness(&mut row); + } + } + row +} + +/// ★★ **D1 — a leaf row's UNREAD input cell is pinned on every arm.** +/// +/// `MODE_L` reads two cells; the third receives nothing from `LfmMem`, so unless +/// a constraint pins it, it is four free felts. +/// +/// ⚠ **The break this regression-tests was on the SECOND cell**, back when a +/// leaf read one: `Test`'s and `Poseidon`'s round 0 reads `A_i = IN_i` for +/// `i < 8`, so on those arms the four free felts were consumed by the +/// permutation the AIR proves and `leaf(c)` stopped being a function of `c` — a +/// Fiat–Shamir break for any program that absorbs data through `absorb_felts`. +/// It shipped that way and an adversarial review executed it: Poseidon proved +/// AND verified with attacker junk in those columns. The leaf RATE closed that +/// hole structurally by making the second cell a cell the mode READS (it carries +/// the felts now, with the accumulator in the first), which is why this test +/// moved up to the third cell rather than being deleted: what it guards is the +/// derivation, and the derivation is what stops the NEXT mode repeating the +/// defect. +/// +/// It runs on all three arms because the defect was that one arm had the pin and +/// two did not. +/// +/// **Shaped like WA9**: it does not merely show the junk row is rejected, it +/// shows the pins are what rejects it — the violated set is exactly those four +/// constraints, so a set without them accepts the row. Necessary, not just +/// present. +#[test] +fn d1_the_unread_input_pins_are_load_bearing_under_every_hasher() { + let acc = word_of(&LEAF_VECTORS[3].acc); + let felts = felts_of(&LEAF_VECTORS[3].felts); + let zero: LfmWord = [FE::zero(); 4]; + + for kind in [HasherKind::Test, HasherKind::Poseidon, HasherKind::Blake3] { + // HONEST CONTROL FIRST: the pin must not reject honest rows. It cannot — + // every arm's `leaf_out` leaves the unread cell zero — but a fix that + // rejected everything would pass the negative leg on its own. + let honest = leaf_row_with_third_cell(kind, &acc, &felts, &zero); + assert_eq!( + violations_under(kind, &honest), + Vec::::new(), + "{kind:?}: an honest leaf row must still satisfy every constraint" + ); + + // ★ The forgery, built the way an attacker would: junk in the cell the + // mode does not read, and the rest of the row made CONSISTENT with it — + // a prover controls the whole trace, so they would never leave a + // detectable inconsistency behind. That is what makes the assertion + // below exact: with the row otherwise honest, the pins are the only + // constraints that can fire, so `== 4` says the PIN caught it rather + // than something downstream noticing the junk by accident. + let junk: LfmWord = core::array::from_fn(|j| FE::from(0x9E37_79B9_u64 + j as u64)); + let forged = leaf_row_with_third_cell(kind, &acc, &felts, &junk); + + // ⚠ The third cell reaches NO arm's output: `S_i = MODE_P·IN_i + …` + // gates it on the permute selector, and no hashing mode's state includes + // it. So on a leaf row this junk is inert on all three arms and the pin + // is hygiene rather than a live soundness fix — which was NOT true of + // the second cell before the RATE made it a read cell, and is why the + // assertion below is about the pins being the only thing that fires. + assert_eq!( + forged[cols::OUT0], + honest[cols::OUT0], + "{kind:?}: the third cell must not reach the digest" + ); + + // ★ THE WA9 SHAPE. Not "the row is rejected" — that would pass for a + // constraint set that rejected it for some incidental reason, and would + // say nothing about whether the pins are needed. What is asserted is + // that the violated set IS EXACTLY the pins for the cell that was + // forged, which carries both legs at once: + // + // - WITH the pins, the row is rejected; + // - WITHOUT them — delete those four constraints and every other + // constraint in the set still evaluates to zero on this row — it is + // ACCEPTED. That is the dropped-leg, and it is what makes the pins + // load-bearing rather than merely present. + // + // On `Test` and `Poseidon` that acceptance was the shipped behaviour and + // an executed Fiat–Shamir break; on BLAKE3 the row is inert either way, + // which is why the same assertion means "hygiene" there and "soundness" + // on the two arms whose round 0 reads `IN4..8`. + let base = super::chips::hash::unread_input_pin_base(kind); + let expected: Vec = (base..base + 4).collect(); + let violated = violations_under(kind, &forged); + assert_eq!( + violated, expected, + "{kind:?}: the violated set must be EXACTLY the four pins for the \ + forged cell — anything else and the dropped-leg claim does not hold" + ); + } +} + +/// The pins are derived from `HashMode::num_input_cells`, not written per arm — +/// so a mode added later cannot acquire free columns by an arm forgetting it. +/// +/// Structural, and deliberately so: the test above shows the pins fire on the +/// three arms that exist, this shows a fourth arm could not miss them. +#[test] +fn d1_the_pins_come_from_one_derivation() { + use super::chips::hash::{MODE_SELECTORS, NUM_UNREAD_INPUT_PINS}; + + // Every selector is in the table exactly once, and the table agrees with the + // layout's contiguous one-hot span. + assert_eq!(MODE_SELECTORS.len(), super::layout::hash::NUM_SELECTORS); + let mut cols_seen: Vec = MODE_SELECTORS.iter().map(|(c, _)| *c).collect(); + cols_seen.sort_unstable(); + let span: Vec = (super::layout::hash::MODE_C + ..super::layout::hash::MODE_C + super::layout::hash::NUM_SELECTORS) + .collect(); + assert_eq!(cols_seen, span, "the selectors are the one-hot span"); + + // Four pins per unread cell, over the two cells some mode does not read. + let unread: usize = (1..3) + .filter(|slot| { + MODE_SELECTORS + .iter() + .any(|(_, m)| m.num_input_cells() <= *slot) + }) + .count(); + assert_eq!(NUM_UNREAD_INPUT_PINS, 4 * unread); + + // ⚠ And the counts the derivation runs over. `Leaf` reads TWO cells under + // the RATE — accumulator and felts — which is what empties slot 1's set and + // takes the pins from 8 to 4. An emitter that assumed some mode always + // under-reads slot 1 panicked on exactly this (COMMIT.md §1.4.4 H2), so the + // count is asserted rather than assumed. + assert_eq!(HashMode::Leaf.num_input_cells(), 2); + assert_eq!(HashMode::Compress.num_input_cells(), 2); + assert_eq!(HashMode::Transcript.num_input_cells(), 2); + assert_eq!(HashMode::Permute.num_input_cells(), 3); + assert_eq!(unread, 1, "only the third cell is under-read now"); + assert_eq!(NUM_UNREAD_INPUT_PINS, 4); +} diff --git a/prover/src/lfm/logup.rs b/prover/src/lfm/logup.rs new file mode 100644 index 000000000..ff06a5e9f --- /dev/null +++ b/prover/src/lfm/logup.rs @@ -0,0 +1,150 @@ +//! The LogUp closure: `Σ_tables L = expected_bus_balance`. +//! +//! Every other leg verifies one sub-proof. This is the only one that is about +//! the epoch as a whole: each table exposes the total of its LogUp terms, `L`, +//! and the bus balances when those totals sum to the target. Production's check +//! is `verifier.rs`'s final block — `Σ L over tables with trace interactions`, +//! compared against an `expected_bus_balance` the caller supplies. +//! +//! # The target is computed, not zero +//! +//! It would be zero if every bus participant were an in-trace table. One is +//! not: the COMMIT output bus has a receiver the verifier computes rather than +//! proves, so the target is that missing positive remainder +//! (`lib.rs`'s `compute_commit_bus_offset`): +//! +//! ```text +//! expected = Σ_i 1 / (z − (BusId::Commit + (start + i)·α + byte_i·α²)) +//! ``` +//! +//! over the public output BYTES. So half this leg is a per-byte gadget over the +//! epoch's public output, not a comparison against a constant. +//! +//! # Reciprocals, and why the machine's division convention matters here +//! +//! Production batch-inverts the fingerprints and REJECTS on a zero divisor — +//! `inplace_batch_inverse(...).ok()?`, a fingerprint collision. The machine's +//! `x/0` is an error and `0/0` is one, so `1/fingerprint` is unprovable at a +//! collision and provable everywhere else: the convention already matches, but +//! only because the numerator is the constant one. The DEEP leg had to invert +//! against an interned one for exactly this reason and a direct divide would +//! have accepted what production rejects; the same care applies here. +//! +//! # What is shape and what is data +//! +//! Which tables carry a bus contribution is `AIR::has_trace_interaction()` — +//! AIR shape, so a program constant. The number of public output bytes is shape +//! too, because it fixes the gadget's length; the byte VALUES are data. `start` +//! is the carried commit index, data. + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Ext, Felt, LfmBuilder}; + +/// The compile-time shape of one epoch's LogUp closure. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LogUpShape { + /// Sub-proofs whose `L` enters the sum — those with trace interactions. + /// SHAPE: a program that read this off the proof would let the prover + /// choose which tables are on the bus. + pub num_contributing_tables: usize, + /// Public output bytes the COMMIT-bus target folds over. + pub num_output_bytes: usize, +} + +/// The `BusId::Commit` discriminant, as the fingerprint's constant term. +/// +/// Mirrored rather than imported so this module does not depend on the VM's bus +/// enum; [`bus_id_matches_production`] pins the two together. +pub const COMMIT_BUS_ID: u64 = crate::tables::types::BusId::Commit as u64; + +/// The COMMIT-bus target: `Σ_i 1/(z − (busId + (start+i)·α + byte_i·α²))`. +/// +/// `bytes` are the public output bytes as base cells, one byte per cell, in +/// order — the same order `compute_commit_bus_offset` enumerates them. `start` +/// is the carried commit index (`x254`): zero for a monolithic proof or a first +/// epoch, nonzero for an epoch continuing a prior one. +/// +/// The index `start + i` is derived by ADDING ONE per byte rather than by +/// hinting each index, so a prover cannot renumber the output: `i` is position, +/// and position is program text. +pub fn emit_commit_bus_target( + b: &mut LfmBuilder, + shape: &LogUpShape, + z: Ext, + alpha: Ext, + start: Felt, + bytes: &[Felt], +) -> Ext { + assert_eq!( + bytes.len(), + shape.num_output_bytes, + "the output length is shape and fixes the gadget's size" + ); + if shape.num_output_bytes == 0 { + // `compute_commit_bus_offset` short-circuits to zero on empty output. + return b.ext_const(&FEE::zero()); + } + + let one = b.ext_const(&FEE::one()); + let bus_id = b.ext_const(&FEE::from(COMMIT_BUS_ID)); + let alpha_sq = b.emul(alpha, alpha); + let one_base = b.felt_const(FE::one()); + + let mut acc: Option = None; + let mut index = start; + for (i, byte) in bytes.iter().enumerate() { + // linear = busId + index·α + byte·α². + let index_term = b.emul_base(alpha, index); + let byte_term = b.emul_base(alpha_sq, *byte); + let linear = b.eadd(bus_id, index_term); + let linear = b.eadd(linear, byte_term); + let fingerprint = b.esub(z, linear); + // Inverted against the interned one: a collision is `1/0`, which is + // unprovable, matching production's rejection. A direct divide of a + // vanishing numerator would instead give `0/0 = 1`. + let term = b.ediv(one, fingerprint); + acc = Some(match acc { + None => term, + Some(a) => b.eadd(a, term), + }); + if i + 1 < bytes.len() { + index = b.add(index, one_base); + } + } + acc.expect("a nonempty output folds at least one term") +} + +/// The closure: sum the per-table contributions and assert the bus balances. +/// +/// `contributions` are the `L` cells — and they must be the SAME cells the +/// constraint leg divided by `N` to get its per-row offset (see +/// [`super::constraints::emit_table_offset`]). A program that hinted `L` here +/// and `L/N` there would let the prover pick both, and this assert would be a +/// statement about numbers bound to no trace. +/// +/// Returns the published sum, so a verifier sees what balanced rather than only +/// that something did. +pub fn emit_bus_closure( + b: &mut LfmBuilder, + shape: &LogUpShape, + contributions: &[Ext], + target: Ext, +) -> Ext { + assert_eq!( + contributions.len(), + shape.num_contributing_tables, + "the contributing-table count is shape and is never read off the proof" + ); + let mut total = match contributions.first() { + Some(first) => *first, + // No table carries a bus interaction: production skips the check + // entirely, so the honest total is zero and the target must be too. + None => b.ext_const(&FEE::zero()), + }; + for c in contributions.iter().skip(1) { + total = b.eadd(total, *c); + } + b.assert_eq_ext(total, target); + total +} diff --git a/prover/src/lfm/logup_tests.rs b/prover/src/lfm/logup_tests.rs new file mode 100644 index 000000000..75cb5d17f --- /dev/null +++ b/prover/src/lfm/logup_tests.rs @@ -0,0 +1,1431 @@ +//! The LogUp closure, and the join it inherits. +//! +//! ## The oracles +//! +//! Two, both production's own. `compute_commit_bus_offset` (`lib.rs`) for the +//! COMMIT-bus target, and `Verifier::multi_verify` for the balance itself — the +//! fixture is a real sender/receiver pair whose bus genuinely closes, and +//! production accepting it at target zero is what says so. Nothing here asserts +//! a balance this file computed. +//! +//! ## What this suite cannot see +//! +//! Whether `L` is bound to a table's aux TRACE. That binding is the circular +//! accumulator constraint plus the `acc[0] = 0` boundary, and it belongs to the +//! constraint leg; this suite checks only that the closure consumes the same +//! `L` that leg divides by `N`. +//! +//! Most fixtures here are two or three tables rather than the twenty-odd a +//! continuation epoch carries, so they exercise the SUM but not its length. One +//! is not: [`a_zero_row_fixed_table_carries_some_zero_not_none`] proves and +//! verifies a real epoch and runs the closure over all twenty-four of its +//! contributions. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use stark::proof::stark::MultiProof; +use stark::proof::view::StarkProofView; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::executor::execute; +use super::hash::TestPermutation; +use super::logup::{COMMIT_BUS_ID, LogUpShape, emit_bus_closure, emit_commit_bus_target}; +use super::validator::validate; +use super::word::{LfmWord, base_word, ext_word, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +fn options() -> stark::proof::options::ProofOptions { + stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The mirrored bus discriminant really is the VM's. +/// +/// A copied protocol constant is the one thing a differential cannot catch, +/// because both sides of it move together — the same reason the join suite pins +/// `ROWS_PER_LEAF` against `crypto/stark`'s. +#[test] +fn bus_id_matches_production() { + assert_eq!( + COMMIT_BUS_ID, + crate::tables::types::BusId::Commit as u64, + "the COMMIT fingerprint's constant term is the VM's own bus id" + ); +} + +// ============================================================================= +// The COMMIT-bus target +// ============================================================================= + +/// Emit the target gadget alone and run it. +fn run_target(bytes: &[u8], start: u64, z: FEE, alpha: FEE) -> Option { + let shape = LogUpShape { + num_contributing_tables: 0, + num_output_bytes: bytes.len(), + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena((3 + bytes.len()) as u32); + let z_cell = b.hint_word(arena, 0).as_ext(); + let alpha_cell = b.hint_word(arena, 1).as_ext(); + let start_cell = b.hint_felt(arena, 2); + let byte_cells: Vec<_> = (0..bytes.len() as u32) + .map(|i| b.hint_felt(arena, 3 + i)) + .collect(); + let target = + emit_commit_bus_target(&mut b, &shape, z_cell, alpha_cell, start_cell, &byte_cells); + b.public(target.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the target program is admissible"); + + let words: Vec = std::iter::once(ext_word(&z)) + .chain(std::iter::once(ext_word(&alpha))) + .chain(std::iter::once(base_word(FE::from(start)))) + .chain(bytes.iter().map(|v| base_word(FE::from(*v as u64)))) + .collect(); + execute(&program, &[words], &TestPermutation) + .ok() + .map(|e| word_as_ext(&e.public_words[0].1).expect("ext")) +} + +/// ★ The machine's COMMIT-bus target equals production's, over a sweep of +/// lengths and carried start indices. +/// +/// The lengths are not decorative. `start` advances by one per byte inside the +/// gadget, so a formula that reset it, or that folded the bytes in reverse, +/// agrees with production only at length one — and the empty case is a separate +/// short-circuit in production that a nonempty-only test would never reach. +#[test] +fn the_commit_bus_target_matches_production() { + let z = FEE::new([FE::from(7u64), FE::from(11u64), FE::from(13u64)]); + let alpha = FEE::new([FE::from(5u64), FE::from(3u64), FE::from(2u64)]); + + let mut checked = 0usize; + for len in [0usize, 1, 2, 3, 7, 8, 33] { + // Distinct byte values, so a gadget that mixed up index and value would + // not accidentally agree. + let bytes: Vec = (0..len).map(|i| (17 * i + 3) as u8).collect(); + for start in [0u64, 1, 254, 1_000_000] { + let want = crate::compute_commit_bus_offset(&bytes, start, &z, &alpha) + .expect("no collision on this fixture"); + let got = run_target(&bytes, start, z, alpha) + .unwrap_or_else(|| panic!("len {len} start {start}: the target must execute")); + assert_eq!(got, want, "len {len}, start {start}"); + if len > 0 { + assert_ne!( + got, + FEE::zero(), + "len {len} start {start}: a zero target would make the \ + comparison vacuous" + ); + } + checked += 1; + } + } + println!("commit-bus target: {checked} (length, start) combinations vs production"); +} + +/// ★ A fingerprint COLLISION is rejected, not silently folded. +/// +/// Production batch-inverts and returns `None` on a zero divisor. The machine +/// divides the interned ONE by the fingerprint, so a collision is `1/0` — an +/// error, hence unprovable. Had the term been written as a direct division with +/// a vanishing numerator instead, the `0/0 = 1` convention would have accepted +/// exactly the proof production rejects, which is the mistake the DEEP leg +/// documents and this test exists to keep from recurring. +#[test] +fn a_fingerprint_collision_is_unprovable() { + // fingerprint_0 = z − (busId + start·α + byte·α²). With α = 1, start = 0 + // and byte = 0 that is z − busId, so z = busId collides exactly. + let alpha = FEE::one(); + let z = FEE::from(COMMIT_BUS_ID); + let bytes = [0u8]; + + assert!( + crate::compute_commit_bus_offset(&bytes, 0, &z, &alpha).is_none(), + "the fixture must be a genuine collision for production too, or this \ + test is checking the machine against nothing" + ); + assert!( + run_target(&bytes, 0, z, alpha).is_none(), + "a colliding fingerprint must make the run unexecutable" + ); + + // And the same shape one step away from the collision still works, so the + // rejection is the collision and not the shape. + let z_ok = z + FEE::one(); + let want = crate::compute_commit_bus_offset(&bytes, 0, &z_ok, &alpha).expect("no collision"); + assert_eq!(run_target(&bytes, 0, z_ok, alpha).expect("executes"), want); + println!("collision rejected; the neighbouring non-colliding z still folds"); +} + +// ============================================================================= +// The closure, over a bus that really balances +// ============================================================================= + +/// A sender/receiver pair over one bus, proved together. +/// +/// Modelled on `tests::bitwise_tests`' pair: the sender emits one AND lookup, +/// the receiver answers it. Their contributions are equal and opposite, so the +/// bus closes at zero — and `multi_verify` accepting at target zero is what +/// establishes that, rather than any arithmetic here. +fn balanced_pair() -> (Vec, MultiProof) { + use crate::tables::types::{BusId, alu_op}; + use crate::test_utils::multi_prove_ram; + use stark::constraints::builder::EmptyConstraints; + use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, + }; + use stark::trace::TraceTable; + + const X: u64 = 5; + const Y: u64 = 3; + const NUM_ROWS: usize = 4; + + type Air = AirWithBuses; + let opts = options(); + + // Columns: 0 = x, 1 = y, 2 = and, 3 = multiplicity/flag. Same layout both + // sides, so one trace builder serves. + // Both sides fingerprint the SAME tuple; only the sender/receiver sign + // differs, which is what makes the two contributions cancel. + let values = || { + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 2, + packing: Packing::Direct, + }, + ] + }; + + let sender = Air::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(3), + values(), + )], + }, + &opts, + 1, + EmptyConstraints, + ) + .with_name("SENDER"); + let receiver = Air::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![BusInteraction::receiver( + BusId::ByteAlu, + Multiplicity::Column(3), + values(), + )], + }, + &opts, + 1, + EmptyConstraints, + ) + .with_name("RECEIVER"); + + let make_trace = || { + let mut data = vec![FE::zero(); NUM_ROWS * 4]; + data[0] = FE::from(X); + data[1] = FE::from(Y); + data[2] = FE::from(X & Y); + data[3] = FE::one(); + TraceTable::::new_main(data, 4, 1) + }; + let mut sender_trace = make_trace(); + let mut receiver_trace = make_trace(); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&sender, &mut sender_trace, &()), + (&receiver, &mut receiver_trace, &()), + ]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("the balanced pair must prove"); + (vec![Box::new(sender), Box::new(receiver)], proof) +} + +type BoxedAir = Box>; + +/// ★ The machine's closure accepts a bus production says balances, and rejects +/// every single-word move away from it. +/// +/// The oracle is `multi_verify` at target zero. It is checked FIRST: if the +/// fixture's bus did not actually close, the machine agreeing with it would say +/// nothing. +#[test] +fn the_closure_matches_a_bus_that_really_balances() { + let (airs, proof) = balanced_pair(); + let air_refs: Vec<&dyn AIR> = + airs.iter().map(|a| &**a).collect(); + + assert!( + Verifier::multi_verify( + &air_refs, + &proof, + &mut DefaultTranscript::::new(&[]), + &FEE::zero(), + ), + "production must accept this pair at target zero, or the fixture is not \ + a balanced bus and nothing below means anything" + ); + + let contributions: Vec = (0..proof.proofs.len()) + .map(|i| { + StarkProofView::Owned(&proof.proofs[i]) + .bus_table_contribution() + .expect("both tables carry a contribution") + }) + .collect(); + assert_eq!(contributions.len(), 2); + assert!( + contributions.iter().all(|c| *c != FEE::zero()), + "both contributions must be nonzero, else the sum is vacuously zero: {contributions:?}" + ); + assert_eq!( + contributions[0] + contributions[1], + FEE::zero(), + "the pair is equal and opposite" + ); + + let shape = LogUpShape { + num_contributing_tables: 2, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let cells: Vec<_> = (0..2u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + let total = emit_bus_closure(&mut b, &shape, &cells, zero); + b.public(total.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the closure program is admissible"); + + let honest: Vec = contributions.iter().map(ext_word).collect(); + let exec = execute(&program, std::slice::from_ref(&honest), &TestPermutation) + .expect("a balanced bus must close in the machine too"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + FEE::zero() + ); + + // Falsification: move either contribution, in any lane. + let mut vectors = 0usize; + for table in 0..2usize { + for lane in 0..3usize { + let mut arenas = vec![honest.clone()]; + arenas[0][table][lane] += FE::one(); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "table {table} lane {lane}: an unbalanced bus must not close" + ); + vectors += 1; + } + } + println!("closure: balanced pair accepted, {vectors} single-lane moves rejected"); +} + +// ============================================================================= +// The join: one `L`, two consumers +// ============================================================================= + +use super::constraint_tests::{RealSubProof, real_sub_proof}; +use super::constraints::{ + OodOperands, emit_alpha_powers, emit_constraint_evals, emit_quotient, emit_table_offset, + hint_ood_frame, ood_frame_words, +}; + +/// Where the per-row offset comes from. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Offset { + /// `L/N` derived in-machine from the closure's own `L`. What production + /// does, and what [`emit_table_offset`] exists to enforce. + Derived, + /// `L/N` hinted as its own arena word, independent of the `L` the closure + /// sums. The shape this leg exists to forbid — a test artifact, built to be + /// attacked. + HintedSeparately, +} + +/// One sub-proof's composition check AND the LogUp closure, over the same +/// proof, with the offset wired either way. +/// +/// Arenas 0-2 are the constraint leg's (frame, uniforms, parts); arena 3 is the +/// closure's expected balance, plus — in the split shape only — the separately +/// hinted offset. +fn composition_and_closure_source( + sp: &RealSubProof, + offset: Offset, +) -> super::builder::LfmProgramSource { + let mut b = LfmBuilder::new(); + + let frame_arena = b.declare_arena(ood_frame_words(&sp.artifact)); + let (steps, _) = hint_ood_frame(&mut b, &sp.artifact, frame_arena, 0); + + let num_uniforms = (sp.rap_challenges.len() + 3) as u32; + let uniform_arena = b.declare_arena(num_uniforms); + let mut next = 0u32; + let mut take = |b: &mut LfmBuilder| { + let c = b.hint_word(uniform_arena, next).as_ext(); + next += 1; + c + }; + let rap_challenges: Vec<_> = (0..sp.rap_challenges.len()).map(|_| take(&mut b)).collect(); + let alpha_powers = emit_alpha_powers( + &mut b, + rap_challenges[stark::lookup::LOGUP_CHALLENGE_ALPHA], + sp.alpha_powers.len(), + ); + let contribution = take(&mut b); + let zeta = take(&mut b); + let beta = take(&mut b); + + let parts_arena = b.declare_arena(sp.claimed_parts.len() as u32); + let claimed_parts: Vec<_> = (0..sp.claimed_parts.len() as u32) + .map(|i| b.hint_word(parts_arena, i).as_ext()) + .collect(); + + let closure_arena = b.declare_arena(match offset { + Offset::Derived => 1, + Offset::HintedSeparately => 2, + }); + let target = b.hint_word(closure_arena, 0).as_ext(); + + let table_offset = match offset { + Offset::Derived => emit_table_offset(&mut b, contribution, sp.quotient.log2_trace_length), + Offset::HintedSeparately => b.hint_word(closure_arena, 1).as_ext(), + }; + + let ood = OodOperands { + steps, + main_width: sp.main_width, + rap_challenges, + alpha_powers, + table_offset, + }; + let (evals, _) = emit_constraint_evals(&mut b, &sp.artifact, &ood); + let q = emit_quotient( + &mut b, + &sp.quotient, + &ood, + zeta, + beta, + &evals, + &claimed_parts, + ); + b.assert_eq_ext(q.claimed, q.composition); + + let shape = LogUpShape { + num_contributing_tables: 1, + num_output_bytes: 0, + }; + let total = emit_bus_closure(&mut b, &shape, &[contribution], target); + b.public(total.as_cell()); + b.finish() +} + +/// Arenas for the program above. `delta` moves the `L` the CLOSURE sums (and +/// the target with it, so the closure itself still balances); the offset stays +/// truthful, which is what a forger would want. +fn join_arenas(sp: &RealSubProof, offset: Offset, delta: FEE) -> Vec> { + let mut arenas = sp.arenas(); + let forged = sp.contribution + delta; + // Slot of `contribution` inside the uniform arena. + let slot = sp.rap_challenges.len(); + arenas[1][slot] = ext_word(&forged); + let mut closure = vec![ext_word(&forged)]; + if offset == Offset::HintedSeparately { + closure.push(ext_word(&sp.table_offset)); + } + arenas.push(closure); + arenas +} + +/// ★ The join, stated as the property it exists for: a prover cannot feed the +/// bus a contribution the constraint leg did not accept. +/// +/// The honest run passes both halves. Moving `L` — while keeping the closure +/// self-consistent by moving its target too, which is exactly what a forger +/// would do — must break the CONSTRAINT half, because the offset is derived +/// from the very cell that moved. +/// +/// The control is the same program with the offset hinted separately. It +/// accepts the forgery: the accumulator sees a truthful `L/N` and wraps, the +/// closure sees a fabricated `L` and balances, and the bus statement is about a +/// number attached to no trace. That is what the derivation denies, and it is +/// run here rather than argued. +#[test] +fn the_closure_cannot_sum_a_contribution_the_constraints_rejected() { + let sp = real_sub_proof(); + assert_ne!( + sp.contribution, + FEE::zero(), + "the fixture's table must carry a real bus contribution, or moving it \ + is not a tamper" + ); + + let joined = compile(composition_and_closure_source(&sp, Offset::Derived)); + validate(&joined).expect("the joined program is admissible"); + let split = compile(composition_and_closure_source( + &sp, + Offset::HintedSeparately, + )); + validate(&split).expect("the control is admissible"); + + // Honest, both shapes. + for (label, program, offset) in [ + ("joined", &joined, Offset::Derived), + ("control", &split, Offset::HintedSeparately), + ] { + let exec = execute( + program, + &join_arenas(&sp, offset, FEE::zero()), + &TestPermutation, + ) + .unwrap_or_else(|e| panic!("{label}: the honest run must execute: {e:?}")); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + sp.contribution, + "{label}: the published total is the contribution that was checked" + ); + } + + // Forge, several deltas and several lanes, so the vector class is not one + // value in one coordinate. + let deltas = [ + FEE::one(), + FEE::new([FE::zero(), FE::one(), FE::zero()]), + FEE::new([FE::zero(), FE::zero(), FE::from(7u64)]), + FEE::new([FE::from(3u64), FE::from(5u64), FE::from(9u64)]), + ]; + for delta in deltas { + assert!( + execute( + &joined, + &join_arenas(&sp, Offset::Derived, delta), + &TestPermutation + ) + .is_err(), + "joined: a forged contribution must break the constraint half \ + (delta {delta:?})" + ); + let forged = execute( + &split, + &join_arenas(&sp, Offset::HintedSeparately, delta), + &TestPermutation, + ) + .unwrap_or_else(|e| panic!("control: the split shape is what PERMITS this forgery: {e:?}")); + assert_eq!( + word_as_ext(&forged.public_words[0].1).expect("ext"), + sp.contribution + delta, + "control: the forgery publishes its own fabricated contribution" + ); + } + println!( + "join: {} forged contributions rejected by the derivation, all accepted \ + by the split control", + deltas.len() + ); +} + +/// ★ The two-consumer rule, as an ABSOLUTE property of the emitted program. +/// +/// Method rule 7: a relative test dies the moment its two sides unify, so this +/// asserts nothing about variants and everything about the program itself — +/// which cells are arena hints and which are computed. `L/N` and every alpha +/// power must be COMPUTED, because a hinted one is a claim about `L` or `α` +/// that no other constraint checks; `L` itself and the raw challenges must be +/// hints, or the test would pass vacuously against an emitter that simply +/// dropped them. +/// +/// This survives any refactor of how the offset is produced. It only fails if +/// something starts reading these values out of an arena again, which is +/// exactly the regression it exists to catch. +#[test] +fn the_derived_uniforms_are_not_arena_words() { + use super::instr::Instr; + + let sp = real_sub_proof(); + assert!( + !sp.alpha_powers.is_empty(), + "the fixture must exercise LogUp alpha powers" + ); + + let mut b = LfmBuilder::new(); + let uniform_arena = b.declare_arena((sp.rap_challenges.len() + 1) as u32); + let rap: Vec<_> = (0..sp.rap_challenges.len() as u32) + .map(|i| b.hint_word(uniform_arena, i).as_ext()) + .collect(); + let contribution = b + .hint_word(uniform_arena, sp.rap_challenges.len() as u32) + .as_ext(); + let alpha_powers = emit_alpha_powers( + &mut b, + rap[stark::lookup::LOGUP_CHALLENGE_ALPHA], + sp.alpha_powers.len(), + ); + let table_offset = emit_table_offset(&mut b, contribution, sp.quotient.log2_trace_length); + let source = b.finish(); + + let hinted: std::collections::HashSet = source + .instrs + .iter() + .filter_map(|i| match i { + Instr::Hint { out, .. } => Some(out.0), + _ => None, + }) + .collect(); + + // Positive control: the things that SHOULD be arena words are. + assert!( + hinted.contains(&contribution.addr().0), + "L itself is proof data and must be hinted, or this test is checking \ + an emitter that reads nothing" + ); + for (i, c) in rap.iter().enumerate() { + assert!( + hinted.contains(&c.addr().0), + "rap challenge {i} is hinted in this isolated slice" + ); + } + + // The property: derived values are not arena words. + assert!( + !hinted.contains(&table_offset.addr().0), + "L/N must be COMPUTED from L; a hinted offset lets a prover satisfy \ + every accumulator while the closure sums a different L" + ); + for (i, p) in alpha_powers.iter().enumerate() { + // alpha^1 IS the alpha cell, which is legitimately hinted here; every + // other power must be computed (alpha^0 is an interned constant). + if i == stark::lookup::LOGUP_CHALLENGE_ALPHA { + continue; + } + assert!( + !hinted.contains(&p.addr().0), + "alpha power {i} must be COMPUTED from alpha; a hinted power is a \ + claim about alpha that nothing checks, and the LogUp fingerprints \ + are built out of exactly these" + ); + } + println!( + "absolute check: L/N and {} alpha powers are computed, not hinted", + alpha_powers.len() + ); +} + +// ============================================================================= +// Degenerate parameter: per-CHUNK vs per-FAMILY accumulation +// ============================================================================= + +/// One sender and TWO receiver chunks of the same family, proved together. +/// +/// A continuation epoch splits each table family into chunks, and `VmAirs::new` +/// builds one AIR per chunk (`lib.rs`: `(0..table_counts.cpu).map(|i| … CPU[i])`), +/// so a family of `k` chunks is `k` entries in the AIR vector and `k` +/// sub-proofs. The closure iterates those entries, so it accumulates PER CHUNK. +/// +/// Every fixture up to here had one sub-proof per family, which makes +/// per-chunk and per-family the same sum — the degenerate case. Two chunks of +/// one family is the smallest shape that tells them apart: the sender's two +/// lookups are answered one per chunk, so dropping either chunk leaves a +/// nonzero remainder. +fn chunked_family() -> (Vec, MultiProof) { + use crate::tables::types::{BusId, alu_op}; + use crate::test_utils::multi_prove_ram; + use stark::constraints::builder::EmptyConstraints; + use stark::lookup::{ + AirWithBuses, AuxiliaryTraceBuildData, BusInteraction, BusValue, Multiplicity, + NullBoundaryConstraintBuilder, Packing, + }; + use stark::trace::TraceTable; + + /// The two lookups, answered by one receiver chunk each. + const LOOKUPS: [(u64, u64); 2] = [(5, 3), (9, 6)]; + const NUM_ROWS: usize = 4; + + type Air = AirWithBuses; + let opts = options(); + + let values = || { + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: 0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 1, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: 2, + packing: Packing::Direct, + }, + ] + }; + let build = |sender: bool, name: &str| { + let interaction = if sender { + BusInteraction::sender(BusId::ByteAlu, Multiplicity::Column(3), values()) + } else { + BusInteraction::receiver(BusId::ByteAlu, Multiplicity::Column(3), values()) + }; + Air::new( + 4, + AuxiliaryTraceBuildData { + interactions: vec![interaction], + }, + &opts, + 1, + EmptyConstraints, + ) + .with_name(name) + }; + + // The two receiver chunks are the SAME construction — one family, two + // instances, exactly as `CPU[0]` and `CPU[1]` are. + let sender = build(true, "SENDER"); + let recv0 = build(false, "RECEIVER[0]"); + let recv1 = build(false, "RECEIVER[1]"); + + let trace_for = |rows: &[(u64, u64)]| { + let mut data = vec![FE::zero(); NUM_ROWS * 4]; + for (r, (x, y)) in rows.iter().enumerate() { + data[r * 4] = FE::from(*x); + data[r * 4 + 1] = FE::from(*y); + data[r * 4 + 2] = FE::from(x & y); + data[r * 4 + 3] = FE::one(); + } + TraceTable::::new_main(data, 4, 1) + }; + let mut sender_trace = trace_for(&LOOKUPS); + let mut recv0_trace = trace_for(&LOOKUPS[..1]); + let mut recv1_trace = trace_for(&LOOKUPS[1..]); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![ + (&sender, &mut sender_trace, &()), + (&recv0, &mut recv0_trace, &()), + (&recv1, &mut recv1_trace, &()), + ]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("the chunked family must prove"); + ( + vec![Box::new(sender), Box::new(recv0), Box::new(recv1)], + proof, + ) +} + +/// ★ The closure accumulates per CHUNK, and that is observable. +/// +/// Both halves of the degenerate-parameter rule. The machine's three-term sum +/// closes a bus production accepts at target zero; and every two-term sum — a +/// per-family reading, which would collapse the two receiver chunks into one +/// contribution — is NONZERO, so the distinction is load-bearing on this +/// fixture rather than merely stated. +/// +/// Without the second half this test would pass against an emitter that folded +/// a family's chunks into a single term, because on every earlier fixture, and +/// on any workload whose families happen to be one chunk each, the two readings +/// agree. +#[test] +fn the_closure_accumulates_per_chunk_not_per_family() { + let (airs, proof) = chunked_family(); + let air_refs: Vec<&dyn AIR> = + airs.iter().map(|a| &**a).collect(); + assert_eq!(air_refs.len(), 3, "one sender and two chunks of one family"); + + assert!( + Verifier::multi_verify( + &air_refs, + &proof, + &mut DefaultTranscript::::new(&[]), + &FEE::zero(), + ), + "production must accept the chunked family at target zero, or the \ + fixture is not a balanced bus" + ); + + let contributions: Vec = (0..proof.proofs.len()) + .map(|i| { + StarkProofView::Owned(&proof.proofs[i]) + .bus_table_contribution() + .expect("every table here has interactions") + }) + .collect(); + assert!( + contributions.iter().all(|c| *c != FEE::zero()), + "each chunk must carry its OWN nonzero contribution — a zero one would \ + make dropping it invisible: {contributions:?}" + ); + + // Half one: the per-chunk sum closes, in the machine. + let shape = LogUpShape { + num_contributing_tables: 3, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let cells: Vec<_> = (0..3u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + let total = emit_bus_closure(&mut b, &shape, &cells, zero); + b.public(total.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("admissible"); + + let words: Vec = contributions.iter().map(ext_word).collect(); + let exec = execute(&program, std::slice::from_ref(&words), &TestPermutation) + .expect("the per-chunk sum must close"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + FEE::zero() + ); + + // Half two: every two-term reading DISAGREES. Dropping chunk 1 or chunk 2 + // is exactly what a per-family accumulator would do. + for dropped in 0..3usize { + let partial: FEE = contributions + .iter() + .enumerate() + .filter(|(i, _)| *i != dropped) + .map(|(_, c)| *c) + .fold(FEE::zero(), |a, c| a + c); + assert_ne!( + partial, + FEE::zero(), + "dropping table {dropped} must break the balance, or the per-chunk \ + reading is not observable on this fixture" + ); + } + // …and the same statement in the MACHINE: a closure compiled for two + // contributing tables — what a per-family emitter would build, one term per + // family — fed a well-formed two-word arena, must not close. + let family_shape = LogUpShape { + num_contributing_tables: 2, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let cells: Vec<_> = (0..2u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + emit_bus_closure(&mut b, &family_shape, &cells, zero); + let per_family = compile(b.finish()); + for dropped in 1..3usize { + let words: Vec = contributions + .iter() + .enumerate() + .filter(|(i, _)| *i != dropped) + .map(|(_, c)| ext_word(c)) + .collect(); + assert!( + execute(&per_family, &[words], &TestPermutation).is_err(), + "a per-family closure that folded chunk {dropped} away must not \ + close the bus" + ); + } + + println!( + "per-chunk accumulation witnessed: 3 chunks close, all 3 two-term \ + readings nonzero, and a per-family closure rejects both chunk drops" + ); +} + +/// ★ `has_trace_interaction()` is SHAPE, and production checks the proof's +/// presence against it in BOTH directions. +/// +/// `verifier.rs:1238` rejects an AIR with interactions whose proof carries no +/// bus public inputs, and `:1244` rejects the converse. So the count of +/// contributing tables is fixed by the AIR set, never read off the proof — +/// which is why [`LogUpShape::num_contributing_tables`] is a program constant. +/// A machine that sized its sum from the arena would let a prover drop a +/// table's contribution from the bus by omitting it. +#[test] +fn the_contributing_table_count_is_shape() { + let (airs, proof) = chunked_family(); + for (i, air) in airs.iter().enumerate() { + let view = StarkProofView::Owned(&proof.proofs[i]); + assert!( + air.has_trace_interaction(), + "table {i} of this fixture declares interactions" + ); + assert_eq!( + air.has_trace_interaction(), + view.has_bus_public_inputs(), + "table {i}: production rejects any disagreement between the AIR's \ + declared interactions and the proof's bus public inputs, in both \ + directions — so the two can never disagree in a proof that verifies" + ); + } + + // The shape is the AIR set's property. A program compiled for three + // contributing tables cannot read a two-table arena: the schema mismatches. + let shape = LogUpShape { + num_contributing_tables: 3, + num_output_bytes: 0, + }; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let cells: Vec<_> = (0..3u32).map(|i| b.hint_word(arena, i).as_ext()).collect(); + let zero = b.ext_const(&FEE::zero()); + emit_bus_closure(&mut b, &shape, &cells, zero); + let program = compile(b.finish()); + assert!( + execute( + &program, + &[vec![ext_word(&FEE::zero()); 2]], + &TestPermutation + ) + .is_err(), + "a short arena must not satisfy a program compiled for three tables" + ); + println!("contributing-table count is shape: a short arena is rejected"); +} + +// ============================================================================= +// Degenerate parameter: a fixed table with NO rows, on a real epoch +// ============================================================================= + +/// How a fixed table's "no rows on the bus" claim is witnessed from its TRACE. +/// +/// Needed because a zero-row table is not the same thing as a blank one. Two +/// padding conventions are in play, and taking either for the general case would +/// have mislabelled the other: +/// +/// - `generate_keccak_rnd_trace` and ECSM's write nothing at all when there are +/// no operations, so their traces are literally zero. +/// - `generate_keccak_trace` pads with `state_ptr[lane] = 8·lane` (and KECCAK_RC +/// is a preprocessed constant table, ECDAS pads likewise). Those traces are +/// NOT zero, yet no row of them is on any bus, because every interaction's +/// multiplicity column is zero. +/// +/// So the second form names the multiplicity columns. They come from each +/// table's own `bus_interactions()`, read there rather than through the AIR: +/// `&dyn AIR` does not expose the interaction list. KECCAK's eight interactions +/// and KECCAK_RND's fourteen are all `Multiplicity::Column(cols::MU)`, KECCAK_RC's +/// single one likewise, and ECDAS's three are `MU` twice plus `NEXT_OP` once. +/// ECSM's include `cols::k_bit(i)` as well, which is why the blank witness — the +/// stronger of the two — is the one used for it. +enum RowWitness { + /// Every main cell is zero: the generator wrote nothing, so there is no row + /// to participate in anything. + Blank, + /// Padding carries canonical values, so the trace is not blank. The witness + /// is that every column any interaction uses as MULTIPLICITY is zero on + /// every row, which gates every LogUp term off. + GatedOff(&'static [usize]), + /// This workload populates the table; no zero-row claim is made. Checked to + /// be non-blank, so a misclassification here does not pass silently. + Populated, +} + +/// ★ MEASURED: a fixed table with no rows carries `Some(zero)`, never `None`. +/// +/// ## Why this had to be measured +/// +/// `FIXED_TABLE_COUNT` forces a sub-proof for all ten fixed tables whatever the +/// workload, so a real epoch always carries tables with no real rows. The +/// closure's [`LogUpShape::num_contributing_tables`] is a program CONSTANT, so +/// if such a table reported `None` the count would be workload-dependent and the +/// constant wrong. The LogUp leg closed with this labelled INFERENCE: production +/// rejects any AIR/proof disagreement (`verifier.rs:1238`), and +/// `has_trace_interaction()` is shape, so `None` would make every real epoch +/// unverifiable — therefore it must be `Some`. True, but an argument, and the +/// experiment is cheap. This is the experiment. +/// +/// Note where the damage would have been: a zero `L` is arithmetically inert, so +/// dropping one would not move the SUM. What `None` would break is the arena +/// SCHEMA — a program compiled for `n` contributions fed `n − 1` words — which is +/// why the answer matters to the count and not to the balance. +/// +/// ## What is measured, and against what +/// +/// One REAL continuation epoch — epoch 0 of the LFM fixture guest, built by +/// `Traces::from_image_and_logs` and proved over the production epoch AIR set +/// (`VmAirs` + the epoch-local L2G table) under the real epoch statement, then +/// ACCEPTED by `Verifier::multi_verify_views` against production's own +/// `compute_expected_commit_bus_balance_view`. The acceptance is load-bearing +/// twice over: it is what makes this "what a verifying epoch proof carries" +/// rather than "what some prover run emitted", and it is what runs +/// `verifier.rs:1238`'s presence check over these very sub-proofs. +/// +/// "No rows" is read off the TRACE, not inferred from the workload, and not read +/// back off the contribution being measured — see [`RowWitness`] for the two +/// forms it takes and why one would not do. `FIXED_TABLE_COUNT` keeps the +/// sub-proof either way: `generate_keccak_trace` pads a zero-operation table to +/// four rows rather than dropping it. +/// +/// ## Which sub-proof is which table +/// +/// Positional, because `VmAirs::new` builds these nine without `.with_name(…)` — +/// `AIR::name()` answers `"unknown"` for every one of them, so there is no name +/// on the proof side to match. The order is `lib.rs`'s own, and `air_refs()` and +/// `air_trace_pairs()` list it identically; that identity is what makes sub-proof +/// `i` this table's proof. It is not taken on trust: each position's sub-proof +/// must report the trace length that position's TRACE built. +/// +/// ## What this test cannot see +/// +/// The row-count cross-check cannot separate two tables of equal height, so +/// swapping (say) KECCAK and ECSM — both four rows here — would relabel two +/// results without failing. It catches the reorderings that change a height, +/// which is every one that could move a populated table into a zero-row slot. +/// +/// Whether a fixed table whose interactions took a CONSTANT multiplicity would +/// answer differently. None does today — every multiplicity in the five zero-row +/// tables is a column, checked by reading their `bus_interactions()` — but such a +/// table would carry a nonzero `L` with no real rows, and this test would report +/// the changed contribution without explaining it. It measures an INTERMEDIATE +/// epoch, so HALT is out of scope, and one workload, so it says nothing about +/// which tables are unused in general — only what a table with no rows carries. +#[test] +fn a_zero_row_fixed_table_carries_some_zero_not_none() { + use crate::tables::trace_builder::{Traces, build_initial_image_paged}; + use crate::tables::{MaxRowsConfig, bitwise, local_to_global, register}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + use math::field::traits::IsPrimeField; + use stark::proof::view::MultiProofView; + use stark::trace::TraceTable; + + let opts = super::proof_fixture::fixture_options(); + let elf_bytes = super::proof_fixture::read_inner_elf(); + let elf = Elf::load(&elf_bytes).expect("the fixture ELF must load"); + let epoch_size = 1usize << super::proof_fixture::FIXTURE_EPOCH_LOG2; + + // ---- epoch 0, built exactly as `prove_continuation` builds it ---- + let mut executor = Executor::new(&elf, vec![]).expect("executor"); + let image = build_initial_image_paged(&elf, &[]); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let logs = executor + .resume_with_limit(epoch_size) + .expect("resume") + .expect("the guest runs at least one epoch") + .to_vec(); + let is_final = executor.pc() == 0; + assert!( + !is_final, + "wanted an INTERMEDIATE epoch (nine fixed tables, no HALT), but the \ + guest finished inside one epoch of {epoch_size} cycles" + ); + + let mut traces = Traces::from_image_and_logs( + &elf, + &image, + ®ister_init, + &logs, + &MaxRowsConfig::default(), + &[], + is_final, + true, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("the epoch trace must build"); + + let label = local_to_global::epoch_label(0); + let mut provenance = + local_to_global::genesis_provenance(image.iter().map(|(a, v)| (a, v as u64))); + let boundary = + local_to_global::epoch_boundary(&mut provenance, label, &traces.touched_memory_cells); + // `prove_epoch`'s first act: the L2G table's range-check lookups must be + // counted into BITWISE, or the epoch's own bus does not close. + bitwise::update_multiplicities( + &mut traces.bitwise, + &local_to_global::collect_bitwise_from_l2g(&boundary), + ); + + // ---- the trace-side census, taken before proving borrows the traces ---- + let all_main_zero = |t: &TraceTable| -> bool { + (0..t.main_table.height).all(|r| t.main_table.get_row(r).iter().all(|v| *v == FE::zero())) + }; + let columns_zero = |t: &TraceTable, cols: &[usize]| -> bool { + (0..t.main_table.height).all(|r| { + let row = t.main_table.get_row(r); + cols.iter().all(|c| row[*c] == FE::zero()) + }) + }; + // `(name, rows, has_no_bus_rows)`. + // + // ONE list, name and trace and witness together, deliberately: a version + // that kept the nine names in a separate constant and zipped them onto the + // traces passed with two names swapped — the swap moved only the label, so + // the row-count cross-check below still compared the right trace against the + // right sub-proof and saw nothing wrong. Merged, a reordering moves the + // TRACE too, which that cross-check does catch. + let census: Vec<(&str, usize, bool)> = { + use crate::tables::{ecdas, keccak, keccak_rc}; + let fixed: [(&str, &TraceTable, RowWitness); 9] = [ + ("BITWISE", &traces.bitwise, RowWitness::Populated), + ("DECODE", &traces.decode, RowWitness::Populated), + ("COMMIT", &traces.commit, RowWitness::Populated), + ( + "KECCAK", + &traces.keccak, + RowWitness::GatedOff(&[keccak::cols::MU]), + ), + ("KECCAK_RND", &traces.keccak_rnd, RowWitness::Blank), + ( + "KECCAK_RC", + &traces.keccak_rc, + RowWitness::GatedOff(&[keccak_rc::cols::MU]), + ), + ("ECSM", &traces.ecsm, RowWitness::Blank), + ( + "ECDAS", + &traces.ecdas, + RowWitness::GatedOff(&[ecdas::cols::MU, ecdas::cols::NEXT_OP]), + ), + ("REGISTER", &traces.register, RowWitness::Populated), + ]; + fixed + .into_iter() + .map(|(name, t, witness)| { + let no_bus_rows = match witness { + RowWitness::Blank => { + assert!( + all_main_zero(t), + "{name} was expected to have NO rows in this epoch \ + (its generator writes nothing when there is no \ + work), but its main trace is not all zero" + ); + true + } + RowWitness::GatedOff(cols) => { + assert!( + columns_zero(t, cols), + "{name} was expected to have no rows on any bus, but \ + one of its multiplicity columns {cols:?} is nonzero" + ); + true + } + RowWitness::Populated => { + assert!( + !all_main_zero(t), + "{name} was classified as populated by this workload \ + but its main trace is entirely zero — the \ + classification, not the measurement, is wrong" + ); + false + } + }; + (name, t.num_rows(), no_bus_rows) + }) + .collect() + }; + + // ---- prove it, over the production epoch AIR set ---- + let reg_fini = register::fini_from_trace(&traces.register); + let table_counts = traces.table_counts(); + let public_output = traces.public_output_bytes.clone(); + let runtime_page_ranges = traces.runtime_page_ranges(); + + let airs = crate::VmAirs::new( + &elf, + &opts, + false, + &[], + &table_counts, + None, + is_final, + None, + None, + Some(( + register::compute_precomputed_commitment_with_fini(&opts, ®ister_init, ®_fini), + register::NUM_PREPROCESSED_COLS_WITH_FINI, + )), + ); + let l2g_air = crate::continuation::l2g_memory_air(&opts, label); + let mut l2g_trace = local_to_global::generate_local_to_global_trace(&boundary); + + // The real epoch statement, so the challenges are the ones a production + // epoch proof is bound to. + let seed = || { + let mut t = DefaultTranscript::::new(&[]); + crate::statement::absorb_statement( + &mut t, + crate::statement::StatementKind::ContinuationEpoch { epoch_label: label }, + &elf_bytes, + &public_output, + &table_counts, + 0, + &runtime_page_ranges, + opts.fri_final_poly_log_degree, + ); + t + }; + + let proof = { + let mut pairs = airs.air_trace_pairs(&mut traces); + pairs.push((&l2g_air, &mut l2g_trace, &())); + crate::test_utils::multi_prove_ram(pairs, &mut seed()).expect("the epoch must prove") + }; + + let refs = { + let mut r = airs.air_refs(); + r.push(&l2g_air); + r + }; + let view = MultiProofView::Owned(&proof); + assert_eq!( + view.len(), + census.len() + table_counts.total() + 1, + "an intermediate epoch is nine fixed tables, the chunked families, and \ + one L2G_MEMORY" + ); + assert_eq!(refs.len(), view.len(), "one AIR per sub-proof"); + + // ---- production must ACCEPT it, or nothing below is about a real proof ---- + let expected = crate::compute_expected_commit_bus_balance_view( + &refs, + view, + &public_output, + register_init[register::X254_INDEX] as u64, + &mut seed(), + ) + .expect("the COMMIT-bus target must exist"); + assert!( + Verifier::multi_verify_views(&refs, view, &mut seed(), &expected), + "production must ACCEPT this epoch proof — the measurement is about what \ + a VERIFYING proof carries, and this is also the run of \ + verifier.rs:1238's presence check" + ); + + // ---- THE MEASUREMENT ---- + println!( + "\nreal continuation epoch (intermediate, {} sub-proofs), fixed tables:\n\ + \x20 {:<11} {:>9} {:>10} {:>5} {:>5} {:>4} contribution", + view.len(), + "table", + "rows", + "proof_len", + "iact", + "bpi", + "zero" + ); + let mut zero_row = Vec::new(); + let mut with_rows = Vec::new(); + for (i, (name, rows, no_rows)) in census.iter().enumerate() { + let sp = view.get(i); + let interacts = refs[i].has_trace_interaction(); + let present = sp.has_bus_public_inputs(); + let contribution = sp.bus_table_contribution(); + assert_eq!( + sp.trace_length(), + *rows, + "position {i} was labelled {name} but proved a trace of {} rows, not \ + the {rows} that table built — the census order no longer matches \ + air_refs()/air_trace_pairs()", + sp.trace_length() + ); + assert_eq!( + interacts, present, + "{name}: production rejects any disagreement between the AIR's \ + declared interactions and the proof's bus public inputs, in both \ + directions (verifier.rs:1238 and :1244)" + ); + println!( + "\x20 {:<11} {:>9} {:>10} {:>5} {:>5} {:>4} {}", + name, + rows, + sp.trace_length(), + interacts, + present, + contribution.as_ref().is_some_and(|c| *c == FEE::zero()), + match &contribution { + None => "None".to_string(), + Some(c) => format!( + "Some({:?})", + c.value() + .iter() + .map(|l| Gl::canonical(l.value())) + .collect::>() + ), + } + ); + if *no_rows { + // THE ANSWER. A zero-row fixed table is still a contributing table. + assert!( + present, + "{name} has no rows on any bus and must STILL carry bus public \ + inputs — a None here would make num_contributing_tables \ + workload-dependent, and the closure's program constant wrong" + ); + assert_eq!( + contribution, + Some(FEE::zero()), + "{name} has no rows on any bus, so every LogUp term is gated to \ + zero and its L must be exactly zero" + ); + zero_row.push(*name); + } else { + with_rows.push((*name, contribution)); + } + } + + // Non-vacuity, both directions: there IS a zero-row fixed table in a real + // epoch, and the observation distinguishes it from a populated one. Without + // the second half, "every zero-row table reports Some(zero)" could hold + // because every table reports Some(zero). + assert!( + !zero_row.is_empty(), + "this epoch has no zero-row fixed table, so it cannot settle the \ + question — pick a guest that leaves one unused" + ); + assert!( + census + .iter() + .enumerate() + .all(|(i, _)| view.get(i).has_bus_public_inputs()), + "every fixed table of an epoch is a contributing table, populated or not" + ); + assert!( + with_rows.iter().any(|(_, c)| *c != Some(FEE::zero())), + "no fixed table carries a NONZERO contribution, so Some(zero) is not a \ + distinguishing observation on this epoch: {with_rows:?}" + ); + println!( + "\x20 ANSWER: Some(zero), not None. {} zero-row fixed tables {:?}, each \ + Some(zero); {} populated, {} of them nonzero.", + zero_row.len(), + zero_row, + with_rows.len(), + with_rows + .iter() + .filter(|(_, c)| *c != Some(FEE::zero())) + .count() + ); + + // ---- the converse, also measured: None would be REJECTED ---- + // The inference this experiment replaces ran the other way — a zero-row + // table cannot report None, because production checks presence against + // has_trace_interaction() before anything else (verifier.rs:1238), so a + // None would make every real epoch unverifiable. That is now a run: strip + // the bus public inputs off a zero-row sub-proof and watch this very proof + // stop verifying. Only the `is_some` direction can be tested on an epoch — + // all 24 sub-proofs declare interactions, so :1244's converse has no + // subject here. + for (i, (name, _, no_rows)) in census.iter().enumerate() { + if !no_rows { + continue; + } + let mut tampered = proof.clone(); + tampered.proofs[i].bus_public_inputs = None; + assert!( + !Verifier::multi_verify_views( + &refs, + MultiProofView::Owned(&tampered), + &mut seed(), + &expected, + ), + "{name} has no rows, but dropping its bus public inputs must still \ + be REJECTED — that rejection is why Some(zero) is forced rather \ + than merely observed" + ); + } + + // ---- and the closure itself, over the REAL epoch's whole table set ---- + // The handoff's other open item: every earlier fixture is two or three + // tables, so the SUM was exercised but its LENGTH was not. + let contributions: Vec = (0..view.len()) + .filter(|i| refs[*i].has_trace_interaction()) + .map(|i| { + view.get(i) + .bus_table_contribution() + .expect("presence was just checked against the AIR") + }) + .collect(); + let shape = LogUpShape { + num_contributing_tables: contributions.len(), + num_output_bytes: public_output.len(), + }; + let (z, alpha) = crate::replay_transcript_phase_a_view(&refs, view, &mut seed()); + + let n_tables = contributions.len() as u32; + let n_bytes = public_output.len() as u32; + let mut b = LfmBuilder::new(); + // Only the cells the gadget reads may be declared: an unread arena word is a + // compile error, and an empty output makes the target a constant that reads + // neither z, alpha, start nor any byte. + let head = if n_bytes == 0 { 0 } else { 3 + n_bytes }; + let arena = b.declare_arena(head + n_tables); + let target = if n_bytes == 0 { + b.ext_const(&FEE::zero()) + } else { + let z_cell = b.hint_word(arena, 0).as_ext(); + let alpha_cell = b.hint_word(arena, 1).as_ext(); + let start_cell = b.hint_felt(arena, 2); + let byte_cells: Vec<_> = (0..n_bytes).map(|i| b.hint_felt(arena, 3 + i)).collect(); + emit_commit_bus_target(&mut b, &shape, z_cell, alpha_cell, start_cell, &byte_cells) + }; + let contrib_cells: Vec<_> = (0..n_tables) + .map(|i| b.hint_word(arena, head + i).as_ext()) + .collect(); + let total = emit_bus_closure(&mut b, &shape, &contrib_cells, target); + b.public(total.as_cell()); + let program = compile(b.finish()); + validate(&program).expect("the epoch closure program is admissible"); + + let mut words: Vec = Vec::new(); + if n_bytes > 0 { + words.push(ext_word(&z)); + words.push(ext_word(&alpha)); + words.push(base_word(FE::from( + register_init[register::X254_INDEX] as u64, + ))); + words.extend(public_output.iter().map(|v| base_word(FE::from(*v as u64)))); + } + words.extend(contributions.iter().map(ext_word)); + let exec = execute(&program, std::slice::from_ref(&words), &TestPermutation) + .expect("a real epoch's LogUp bus must close in the machine"); + assert_eq!( + word_as_ext(&exec.public_words[0].1).expect("ext"), + expected, + "the machine's published total must be production's own expected balance" + ); + + // Falsification: move any one contribution, in any lane. Every zero-row + // table is in here too, so this is also the check that a Some(zero) term is + // a real summand and not a no-op the emitter could drop. + let mut vectors = 0usize; + for table in 0..contributions.len() { + for lane in 0..3usize { + let mut arenas = vec![words.clone()]; + arenas[0][head as usize + table][lane] += FE::one(); + assert!( + execute(&program, &arenas, &TestPermutation).is_err(), + "table {table} lane {lane}: a moved contribution must not close" + ); + vectors += 1; + } + } + println!( + "\x20 the closure also runs on the real epoch: {} contributing tables of \ + {} sub-proofs, {} output bytes, {vectors} single-lane moves rejected\n", + contributions.len(), + view.len(), + public_output.len() + ); +} diff --git a/prover/src/lfm/machine_tests.rs b/prover/src/lfm/machine_tests.rs new file mode 100644 index 000000000..837a7d1e8 --- /dev/null +++ b/prover/src/lfm/machine_tests.rs @@ -0,0 +1,4494 @@ +//! Milestone B end-to-end: the machine proves a trivial program; valid +//! accepts, tampered variants reject, and the registry drift test pins the +//! program's identity (recompute-and-compare, the static-commitments policy). + +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use crate::tables::types::FE; + +use super::executor::LfmExecError; +use super::fixture::{self, bump_lane0, fixture_prove}; +use super::programs::{fri_toy_program, trivial_program, trivial_program_source}; +use super::proof::{LfmProveError, lfm_prove, lfm_verify}; +use super::registry::{LfmProgramKind, LfmRegistryError, build_artifacts, resolve}; +use super::validator::validate; +use super::word::LfmWord; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +#[test] +fn trivial_program_is_admissible() { + let program = trivial_program(); + validate(&program).expect("the registered program must pass admission"); +} + +#[test] +fn trivial_program_proves_and_verifies() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + let ok = lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("registry entry exists"); + assert!(ok, "honest machine proof must verify"); +} + +#[test] +fn tampered_claimed_public_word_rejects() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + + let mut claimed = proved.public_words.clone(); + claimed[0].1[0] = &claimed[0].1[0] + FE::from(1u64); + let ok = lfm_verify(LfmProgramKind::TrivialV0, &proved.proof, &claimed, &opts) + .expect("registry entry exists"); + assert!(!ok, "a tampered claimed public word must reject"); +} + +#[test] +fn different_arena_values_change_the_public_output_not_the_program() { + // Same program identity, different hints: proves and verifies against its + // own (different) public words. + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + + let other: Vec> = vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(7_777 * (i + 1) + j as u64))) + .collect(), + ]; + let a = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove a"); + let b = lfm_prove(&program, &artifacts, &other, &opts).expect("prove b"); + assert_ne!(a.public_words, b.public_words); + assert!( + lfm_verify(LfmProgramKind::TrivialV0, &b.proof, &b.public_words, &opts).expect("entry") + ); + // Cross-claiming rejects: proof b with proof a's public words. + assert!( + !lfm_verify(LfmProgramKind::TrivialV0, &b.proof, &a.public_words, &opts).expect("entry") + ); +} + +#[test] +fn registry_miss_is_a_hard_error() { + let opts = GoldilocksCubicProofOptions::with_blowup(8).expect("options"); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + let err = lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .unwrap_err(); + assert_eq!( + err, + LfmRegistryError::UnknownProgram { + kind: LfmProgramKind::TrivialV0, + blowup_factor: 8 + }, + "no registry entry ⇒ hard error, never a fallback" + ); +} + +/// The registry drift test — the LFM analogue of +/// `static_commitments_tests.rs`. A failure here means the trivial program, +/// a chip layout, the commit pipeline or the digest changed: investigate, +/// never re-bless. +#[test] +fn registry_drift_trivial_v0_blowup2() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = resolve(LfmProgramKind::TrivialV0, 2).expect("TrivialV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +#[test] +fn trivial_program_source_is_deterministic() { + let a = trivial_program_source(); + let b = trivial_program_source(); + assert_eq!(a.num_addrs, b.num_addrs); + assert_eq!(a.instrs.len(), b.instrs.len()); +} + +// ======================= Milestone C: the FRI verifier ======================= + +fn fri_arenas(proof: &fixture::FriToyProof) -> Vec> { + vec![proof.commitments.clone(), proof.openings.clone()] +} + +#[test] +fn fri_toy_program_is_admissible() { + validate(&fri_toy_program()).expect("the FRI verifier program must pass admission"); +} + +/// The Milestone-C headline: the machine verifies a structurally real FRI +/// commitment-opening proof (sponge transcript, Merkle-authenticated +/// openings, α-combination, two unnormalized folds, terminal check) and the +/// resulting machine proof verifies against the registry. +#[test] +fn machine_verifies_fixture_fri_proof_end_to_end() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts(&program, &opts); + let inner = fixture_prove(); + let proved = + lfm_prove(&program, &artifacts, &fri_arenas(&inner), &opts).expect("machine accepts"); + // The attested public output is the inner proof's identity: both roots. + assert_eq!(proved.public_words[0].1, inner.commitments[0]); + assert_eq!(proved.public_words[1].1, inner.commitments[1]); + assert!( + lfm_verify( + LfmProgramKind::FriToyV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("FriToyV0 is registered"), + "the machine proof of FRI verification must verify" + ); +} + +/// Every tamper vector must make the verification program *unprovable* (the +/// executor hits the same failed assert the AIR's division constraint makes +/// unsatisfiable). +#[test] +fn machine_rejects_tampered_fri_proofs() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts(&program, &opts); + let honest = fixture_prove(); + + let expect_reject = |arenas: Vec>, what: &str| match lfm_prove( + &program, &artifacts, &arenas, &opts, + ) { + Err(LfmProveError::Exec(LfmExecError::DivByZero { .. })) => {} + other => panic!( + "{what}: expected a failed in-machine assert, got {:?}", + other.map(|_| "accepted") + ), + }; + + // (a) a tampered opened row value breaks its Merkle path. + let mut t = fri_arenas(&honest); + t[1][0] = bump_lane0(&t[1][0]); + expect_reject(t, "tampered opened row"); + + // (b) a tampered sibling digest breaks the walk. + let mut t = fri_arenas(&honest); + t[1][2] = bump_lane0(&t[1][2]); + expect_reject(t, "tampered sibling"); + + // (c) a tampered main root diverges the transcript: different queries, + // openings no longer match. + let mut t = fri_arenas(&honest); + t[0][0] = bump_lane0(&t[0][0]); + expect_reject(t, "tampered main root"); + + // (d) a tampered terminal coefficient fails the terminal check. + let mut t = fri_arenas(&honest); + t[0][2] = bump_lane0(&t[0][2]); + expect_reject(t, "tampered terminal coefficient"); + + // (e) a tampered L1 opened value fails fold-consistency or its path. + let mut t = fri_arenas(&honest); + t[1][12] = bump_lane0(&t[1][12]); + expect_reject(t, "tampered layer-1 opening"); +} + +#[test] +fn registry_drift_fri_toy_v0_blowup2() { + let opts = options(); + let program = fri_toy_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = resolve(LfmProgramKind::FriToyV0, 2).expect("FriToyV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +/// The kill-risk-3 instrument on the first real verification program. +#[test] +fn fri_toy_cell_counts() { + let program = fri_toy_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "FriToyV0: {} instructions, {} addresses, {} main value cells, {} aux ext elements", + program.instrs.len(), + program.num_addrs, + main, + aux + ); + assert!(main > 0 && aux > 0); +} + +// ===================== R1b: keccak-f[1600] in the machine ===================== + +use super::compiler::LfmProgram; +use super::keccak_adapter; +use super::layout::keccak as klayout; +use super::programs::{keccak_chain_program, keccak_chain_program_source}; +use super::proof::prove_traces; +use super::registry::LfmArtifacts; +use super::trace::{LfmTraces, build_traces}; +use super::validator::LfmViolation; +use crate::lfm::chips::keccak as kchip; +use crate::tables::types::VmTable; +use stark::prover::ProvingError; + +/// A keccak state derived from `seed`, in the machine's word form. +fn keccak_state(seed: u64) -> [u64; 25] { + core::array::from_fn(|i| { + seed.wrapping_mul(i as u64 + 1) + .wrapping_add(0x9E37_79B9_7F4A_7C15) + ^ 0x0123_4567_89AB_CDEF + }) +} + +fn keccak_arenas(seed: u64) -> Vec> { + vec![keccak_adapter::state_to_words(&keccak_state(seed)).to_vec()] +} + +type KeccakChainProof = stark::proof::stark::MultiProof< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + (), +>; + +/// Execute + build traces, let the caller corrupt them, then prove. +fn prove_keccak_chain_with_tamper( + program: &LfmProgram, + artifacts: &LfmArtifacts, + seed: u64, + mutate: impl FnOnce(&mut LfmTraces), +) -> Result<(KeccakChainProof, Vec<(u32, LfmWord)>), ProvingError> { + let opts = options(); + let exec = + super::executor::execute(program, &keccak_arenas(seed), &super::hash::TestPermutation) + .expect("honest execution"); + let mut traces = build_traces(program, &exec.records); + mutate(&mut traces); + let proof = prove_traces(artifacts, &mut traces, &exec.public_words, &opts)?; + Ok((proof, exec.public_words)) +} + +#[test] +fn keccak_chain_program_is_admissible() { + validate(&keccak_chain_program()).expect("the keccak-chain program must pass admission"); +} + +#[test] +fn keccak_chain_source_is_deterministic() { + let a = keccak_chain_program_source(); + let b = keccak_chain_program_source(); + assert_eq!(a.num_addrs, b.num_addrs); + assert_eq!(a.instrs.len(), b.instrs.len()); +} + +/// The R1b headline: the machine proves two *chained* real `keccak-f[1600]` +/// permutations, with the state bound to `LfmMem` words and the permutation +/// itself discharged by the unchanged production `KECCAK_RND` / `KECCAK_RC` / +/// `BITWISE` chips. +#[test] +fn keccak_chain_proves_and_verifies() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &keccak_arenas(7), &opts).expect("machine proves"); + + // Host-side reference: the same two permutations, same word convention. + let once = keccak_adapter::permute(keccak_state(7)); + let twice = keccak_adapter::permute(once); + let once_words = keccak_adapter::state_to_words(&once); + let twice_words = keccak_adapter::state_to_words(&twice); + assert_eq!(proved.public_words[0].1, once_words[0], "first permutation"); + assert_eq!(proved.public_words[1].1, twice_words[0], "second, word 0"); + assert_eq!(proved.public_words[2].1, twice_words[1], "second, word 1"); + + assert!( + lfm_verify( + LfmProgramKind::KeccakChainV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("KeccakChainV0 is registered"), + "the machine proof of two chained keccak permutations must verify" + ); +} + +/// Flipping one output byte — i.e. one quarter of one `u32` half — must break +/// the proof. The byte columns feed both the `Keccak` reply token and, through +/// the `Linear` half recomposition, the `LfmMem` word the next instruction +/// reads, so either bus catches it. +#[test] +fn tampered_keccak_output_half_rejects() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let (proof, public) = prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + let col = kchip::cols::out_byte(3, 2); + let old = t.keccak.main_table.get_row(0)[col]; + t.keccak.main_table.set_fe(0, col, old + FE::from(1u64)); + }) + .expect("the adapter has no constraints, so the prover accepts"); + + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "a flipped output byte must reject" + ); +} + +/// Flipping an input byte likewise rejects: the request token no longer matches +/// the round chip's first receive. +#[test] +fn tampered_keccak_input_half_rejects() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let (proof, public) = prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + let col = kchip::cols::state_byte(11, 5); + let old = t.keccak.main_table.get_row(1)[col]; + t.keccak.main_table.set_fe(1, col, old + FE::from(1u64)); + }) + .expect("locally consistent"); + + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "a flipped input byte must reject" + ); +} + +/// CLOSES THE R1a HAZARD. +/// +/// `keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` +/// exhibits a live forgery against the raw keccak family: given two rows +/// sharing a tag, swapping their output states leaves the `Keccak` bus +/// balanced, so the verifier accepts two permutations neither of which is +/// genuine. Nothing but the tag binds a request token to its reply. +/// +/// Moving the tag into the preprocessed column group closes it, in three legs +/// asserted below: +/// 1. the compiled program's keccak rows carry *distinct* tags; +/// 2. swapping two rows' output states now REJECTS (it accepted in R1a); +/// 3. the prover cannot repair leg 2 by colliding the tags, because they are +/// preprocessed — editing them fails the recommit before a proof exists. +#[test] +fn preprocessed_tags_close_the_output_swap_hazard() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let group = &program.groups.keccak; + assert_eq!(group.real_rows, 2, "the chain program has two keccak rows"); + + // Leg 1: distinct tags, assigned by the compiler as row ordinals. + let tag = |row: usize| { + ( + *group.at(row, klayout::TAG_LO), + *group.at(row, klayout::TAG_HI), + ) + }; + assert_ne!(tag(0), tag(1), "keccak tags must be distinct"); + + // Leg 2: the R1a forgery, replayed — swap the two rows' 200 output bytes. + let (proof, public) = prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + for col in kchip::cols::OUT..kchip::cols::NUM_COLUMNS { + let a = t.keccak.main_table.get_row(0)[col]; + let b = t.keccak.main_table.get_row(1)[col]; + t.keccak.main_table.set_fe(0, col, b); + t.keccak.main_table.set_fe(1, col, a); + } + }) + .expect("locally consistent"); + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "with distinct tags the swapped outputs must no longer balance" + ); + + // Leg 3: colliding the tags is not available to the prover. The TAG columns + // are PREPROCESSED — pinned by the group's precomputed commitment — so a + // prover cannot obtain a VERIFYING proof with the tags rewritten. Copying + // row 0's tag over row 1's makes the trace's leading columns disagree with + // the committed group. + // + // Where the failure lands is precomputed-tree-cache-dependent (the cache is + // keyed by the expected root): on a cache MISS the prover recomputes the + // preprocessed tree from the tampered columns and refuses with + // `PrecomputedCommitmentMismatch`; on a warm cache HIT it commits the pinned + // tree without re-checking (so it does NOT refuse at prove time) and the + // tampered proof is caught at VERIFY instead. Either way the forgery fails — + // which is the obligation this leg exists to pin. + match prove_keccak_chain_with_tamper(&program, &artifacts, 7, |t| { + let lo = t.keccak.main_table.get_row(0)[klayout::TAG_LO]; + let hi = t.keccak.main_table.get_row(0)[klayout::TAG_HI]; + t.keccak.main_table.set_fe(1, klayout::TAG_LO, lo); + t.keccak.main_table.set_fe(1, klayout::TAG_HI, hi); + }) { + Err(err) => assert!( + matches!(err, ProvingError::PrecomputedCommitmentMismatch), + "expected a preprocessed recommit failure, got {err:?}" + ), + Ok((proof, public)) => assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "a tag-rewrite tamper must not yield a verifying proof" + ), + } +} + +/// The registrar's independent gate on the same obligation: even if a future +/// compiler change stopped assigning distinct tags, admission would catch it. +#[test] +fn duplicate_keccak_tags_fail_admission() { + let mut program = keccak_chain_program(); + let lo = *program.groups.keccak.at(0, klayout::TAG_LO); + let hi = *program.groups.keccak.at(0, klayout::TAG_HI); + program.groups.keccak.set(1, klayout::TAG_LO, lo); + program.groups.keccak.set(1, klayout::TAG_HI, hi); + assert_eq!( + validate(&program), + Err(LfmViolation::DuplicateKeccakTag { tag: (1, 0) }), + "duplicate keccak tags must fail admission" + ); +} + +/// A keccak lane is a `u64`, but a felt lane carrying a half must be a `u32`. +/// A hinted word above that bound is caught by the executor — on the AIR side +/// no such value exists, since each half is a fixed combination of four +/// BITWISE-constrained bytes. +#[test] +fn keccak_rejects_non_u32_half() { + let program = keccak_chain_program(); + let mut arenas = keccak_arenas(7); + arenas[0][0][0] = FE::from(1u64 << 32); + assert_eq!( + super::executor::execute(&program, &arenas, &super::hash::TestPermutation).unwrap_err(), + LfmExecError::NotU32Half { addr: 0, lane: 0 } + ); +} + +/// The state is 50 halves in 52 word slots; the two spare slots are pinned to +/// zero as bus tuple constants, so a nonzero one is unprovable. +#[test] +fn keccak_rejects_nonzero_spare_lane() { + let program = keccak_chain_program(); + let mut arenas = keccak_arenas(7); + let last = klayout::NUM_WORDS - 1; + arenas[0][last][2] = FE::from(1u64); + assert_eq!( + super::executor::execute(&program, &arenas, &super::hash::TestPermutation).unwrap_err(), + LfmExecError::KeccakSpareLaneNonZero { + addr: last as u64, + lane: 2 + } + ); +} + +#[test] +fn registry_drift_keccak_chain_v0_blowup2() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = + resolve(LfmProgramKind::KeccakChainV0, 2).expect("KeccakChainV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +/// The kill-risk-3 instrument with the keccak family in the set. +#[test] +fn keccak_chain_cell_counts() { + let program = keccak_chain_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "KeccakChainV0: {} instructions, {} main value cells, {} aux ext elements", + program.instrs.len(), + main, + aux + ); + assert!(main > 0 && aux > 0); +} + +// ==================== R1c: keccak256 over byte streams ==================== + +use super::keccak_host; +use super::programs::{KECCAK_SPONGE_LEN, keccak_sponge_program}; +use super::proof::verify_against; + +/// Reference messages. Together they cover: the empty string (padding only), +/// a short message, the exact rate boundary, one byte either side of it, and +/// two lengths whose final `u32` half mixes message bytes with padding. +fn reference_messages() -> Vec> { + let lens = [0usize, 1, 4, 135, 136, 137, KECCAK_SPONGE_LEN, 272]; + lens.iter() + .map(|&n| { + (0..n) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect() + }) + .collect() +} + +fn sponge_arenas(msg: &[u8]) -> Vec> { + let halves = keccak_host::pack_stream(msg); + keccak_host::assert_high_bytes_zero(&halves, msg.len()); + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +/// The 32-byte digest from the two public words: byte `j` is byte `j % 4` of +/// half `j / 4`, and half `h` is lane `h % 4` of word `h / 4`. +fn digest_bytes(public: &[(u32, LfmWord)]) -> [u8; 32] { + use math::field::traits::IsPrimeField; + let mut out = [0u8; 32]; + for h in 0..8 { + let lane = public[h / 4].1[h % 4]; + let half = crate::tables::types::GoldilocksField::canonical(lane.value()) as u32; + out[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + out +} + +/// Bit-exactness against the production hasher, execute-only (fast): every +/// reference length must reproduce `PlatformKeccak256` byte for byte. +#[test] +fn keccak256_matches_platform_hasher() { + for msg in reference_messages() { + let program = keccak_sponge_program(msg.len()); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("len {}: execution failed: {e:?}", msg.len())); + assert_eq!( + digest_bytes(&exec.public_words), + keccak_host::keccak256(&msg), + "keccak256 mismatch at len {}", + msg.len() + ); + } +} + +#[test] +fn keccak_sponge_program_is_admissible() { + for msg in reference_messages() { + validate(&keccak_sponge_program(msg.len())) + .unwrap_or_else(|e| panic!("len {} must pass admission: {e:?}", msg.len())); + } +} + +/// The R1c headline: the machine PROVES keccak256 of real byte streams and the +/// proofs verify, with the digest matching `PlatformKeccak256` byte for byte. +/// +/// The four lengths cover the shapes that differ structurally: padding-only +/// (empty), a single block whose last half mixes message and padding bytes, +/// a multi-block message crossing the rate boundary, and an exact multiple of +/// the rate — which `pad10*1` grows by a whole extra block. +#[test] +fn keccak_sponge_reference_lengths_prove_and_verify() { + let opts = options(); + for len in [0usize, 135, KECCAK_SPONGE_LEN, 272] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(len); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts) + .unwrap_or_else(|e| panic!("len {len}: prove failed: {e:?}")); + assert_eq!( + digest_bytes(&proved.public_words), + keccak_host::keccak256(&msg), + "len {len}: digest must match the production hasher" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "len {len}: the machine proof of keccak256 must verify" + ); + } +} + +/// The registered length, through the full registry-resolving verify path. +#[test] +fn keccak_sponge_proves_and_verifies() { + let opts = options(); + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!( + digest_bytes(&proved.public_words), + keccak_host::keccak256(&msg) + ); + assert!( + lfm_verify( + LfmProgramKind::KeccakSpongeV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("KeccakSpongeV0 is registered"), + "the registered keccak256 program must verify" + ); +} + +/// Claiming the honest digest for a message whose stream was altered must +/// reject: the absorbed block differs, so the sponge produces a different +/// digest and the claimed public words no longer match the proof. +#[test] +fn tampered_stream_half_rejects() { + let opts = options(); + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + let mut tampered = sponge_arenas(&msg); + tampered[0][3][0] = &tampered[0][3][0] + FE::from(1u64); + let forged = lfm_prove(&program, &artifacts, &tampered, &opts).expect("prove"); + + assert_ne!( + forged.public_words, honest.public_words, + "a changed stream half must change the digest" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "claiming the honest digest for a tampered stream must reject" + ); +} + +/// An absorb row's XOR is pinned by BITWISE lookups, so corrupting the +/// permutation input the family sees — without touching the state read from +/// memory — must reject. +#[test] +fn tampered_absorb_xor_rejects() { + let opts = options(); + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + let mut traces = build_traces(&program, &exec.records); + // Rate byte 5 of the first absorb row: XOR(state, block) no longer holds. + let col = kchip::cols::PERM_IN + 5; + let old = traces.keccak.main_table.get_row(0)[col]; + traces + .keccak + .main_table + .set_fe(0, col, old + FE::from(1u64)); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + ), + "a broken absorb XOR must reject" + ); +} + +#[test] +fn registry_drift_keccak_sponge_v0_blowup2() { + let opts = options(); + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let artifacts = build_artifacts(&program, &opts); + let entry = + resolve(LfmProgramKind::KeccakSpongeV0, 2).expect("KeccakSpongeV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +#[test] +fn keccak_sponge_cell_counts() { + let program = keccak_sponge_program(KECCAK_SPONGE_LEN); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "KeccakSpongeV0 ({} bytes): {} instructions, {} main value cells, {} aux ext elements", + KECCAK_SPONGE_LEN, + program.instrs.len(), + main, + aux + ); + assert!(main > 0 && aux > 0); +} + +/// Isolates the rate-region pass-through constraint +/// `MODE_PERM · (PERM_IN − STATE) = 0`. +/// +/// Absorb rows get `PERM_IN` pinned by the BYTE_ALU[XOR] lookups; permute rows +/// have no lookups, so without this constraint a prover could feed the keccak +/// family a permutation input unrelated to the state it read from memory. Trace +/// tampering alone does not reach that hole — it desynchronises the round chip +/// and the bus catches it first. So this builds the *coordinated* forgery: the +/// last keccak row's `perm_in` is replaced BEFORE trace generation, so the +/// KECCAK_RND rows, the BITWISE multiplicities, the reply token and the output +/// words are all internally consistent with the forged input, and the claimed +/// public words are recomputed to match. Every bus balances. The only thing +/// standing between this and an accepted proof is the constraint. +#[test] +fn permute_row_cannot_substitute_the_permuted_state() { + let opts = options(); + let program = keccak_chain_program(); + let artifacts = build_artifacts(&program, &opts); + let mut exec = + super::executor::execute(&program, &keccak_arenas(7), &super::hash::TestPermutation) + .expect("honest execution"); + + // Forge the second (last) permutation's input, and make everything + // downstream of it consistent. + let last = exec.records.keccak.len() - 1; + let mut forged = exec.records.keccak[last].perm_in; + forged[0] ^= 1; + let output = keccak_adapter::permute(forged); + exec.records.keccak[last].perm_in = forged; + exec.records.keccak[last].output = output; + + // The chain program publics are once[0], twice[0], twice[1]; the last two + // come from this row, so claim the values the forged run actually produces. + let words = keccak_adapter::state_to_words(&output); + exec.public_words[1].1 = words[0]; + exec.public_words[2].1 = words[1]; + exec.records.public[1] = words[0]; + exec.records.public[2] = words[1]; + + let mut traces = build_traces(&program, &exec.records); + let proof = prove_traces(&artifacts, &mut traces, &exec.public_words, &opts) + .expect("the prover has no constraint checks, so it accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + ), + "a permute row whose PERM_IN differs from the state it read must reject" + ); +} + +// ============ R1d groundwork: DefaultTranscript::sample() replay ============ + +/// The machine's reversed digest must equal the production transcript's +/// `sample()` byte for byte. +/// +/// This is a REAL bit-exactness check against `DefaultTranscript`, not a +/// reimplementation: `sample()` — finalize, reverse the 32 bytes, absorb the +/// reversed bytes, return them — is identical before and after #841, so it can +/// be verified even though this worktree predates that change. The buffered +/// candidate machinery that #841 introduced is what is blocked, not this. +#[test] +fn machine_reversed_digest_matches_default_transcript_sample() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + for len in [0usize, 1, 135, KECCAK_SPONGE_LEN] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = super::programs::keccak_sample_program(len); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("len {len}: execution failed: {e:?}")); + + let mut host = DefaultTranscript::::new(&msg); + let expected = host.sample(); + assert_eq!( + digest_bytes(&exec.public_words), + expected, + "len {len}: reversed digest must match DefaultTranscript::sample()" + ); + } +} + +/// The reversed-digest send must actually reverse: the machine's own +/// non-reversed digest and its reversed digest are byte-reverses of each other. +#[test] +fn reversed_digest_is_the_reverse_of_the_digest() { + let msg: Vec = (0..KECCAK_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let plain = super::executor::execute( + &keccak_sponge_program(msg.len()), + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("exec"); + let reversed = super::executor::execute( + &super::programs::keccak_sample_program(msg.len()), + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("exec"); + + let mut want = digest_bytes(&plain.public_words); + want.reverse(); + assert_eq!(digest_bytes(&reversed.public_words), want); +} + +/// PROVES the `sample()` replay, which the execute-only test above does NOT. +/// +/// This distinction bit me: `execute` writes the reversed words from the host +/// mirror (`keccak_adapter::reversed_digest_words`), so an execute-only test +/// passes no matter what the CHIP's reversed-coefficient `Linear` says. The two +/// have to agree, and only a proof checks that — if the bus send recomposes the +/// bytes in any other order, the words it sends differ from the ones the +/// executor wrote to memory and the `LfmMem` bus stops balancing. Neutralising +/// the reversal in the chip leaves the execute-only test green and makes THIS +/// one fail, which is how it should be. +#[test] +fn machine_proves_the_sample_replay() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + let opts = options(); + for len in [0usize, 135, KECCAK_SPONGE_LEN] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = super::programs::keccak_sample_program(len); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts) + .unwrap_or_else(|e| panic!("len {len}: prove failed: {e:?}")); + + let mut host = DefaultTranscript::::new(&msg); + assert_eq!( + digest_bytes(&proved.public_words), + host.sample(), + "len {len}: proved sample() must match DefaultTranscript" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "len {len}: the machine proof of sample() must verify" + ); + } +} + +// ============ R1d: DefaultTranscript model + candidate identity ============ + +/// The host model must track the real post-#841 `DefaultTranscript` exactly, +/// across an interleaving that exercises buffer refill AND absorb invalidation. +#[test] +fn transcript_model_matches_default_transcript() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let mut host = DefaultTranscript::::new(b"seed"); + let mut model = keccak_host::TranscriptModel::new(b"seed"); + + // `sample_u64(2^n)` has threshold 0, so it consumes exactly one candidate + // and returns its low n bits. Compare the model's raw candidate masked the + // same way; the raw 32-byte squeezes are compared exactly further down. + const MASK: u64 = (1u64 << 63) - 1; + // Drain a full squeeze (4 candidates) and force a refill on the 5th. + for i in 0..5 { + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "candidate {i}" + ); + } + // Absorb mid-buffer: both must drop the remaining squeezed bytes. + host.append_bytes(b"abc"); + model.append(b"abc"); + for i in 0..3 { + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "post-absorb {i}" + ); + } + // A raw sample() also invalidates. + assert_eq!(model.sample(), host.sample(), "raw sample"); + for i in 0..2 { + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "post-sample {i}" + ); + } + // Absorbs of several lengths, including one crossing the keccak rate. + for len in [1usize, 135, 136, 200] { + let msg: Vec = (0..len).map(|i| (i as u8).wrapping_mul(7)).collect(); + host.append_bytes(&msg); + model.append(&msg); + assert_eq!( + model.next_u64() & MASK, + host.sample_u64(1 << 63), + "len {len}" + ); + } +} + +/// THE IDENTITY THE EMITTER RESTS ON: the four big-endian candidates carved out +/// of a reversed digest are the ORIGINAL digest's `u64` lanes 3, 2, 1, 0. +/// +/// If this holds, the machine reads candidates straight off the plain digest +/// words — already `u32` halves on the bus — and never reverses anything to +/// sample. The big-endian read and the byte reversal cancel. +#[test] +fn be_candidates_are_plain_state_lanes() { + for len in [0usize, 1, 135, 202] { + let msg: Vec = (0..len).map(|i| (i as u8).wrapping_mul(13)).collect(); + + // The state whose first 32 bytes are the digest. + let digest = keccak_host::keccak256(&msg); + let mut state = [0u64; 25]; + for (lane, chunk) in state[..4].iter_mut().zip(digest.chunks_exact(8)) { + let mut b = [0u8; 8]; + b.copy_from_slice(chunk); + *lane = u64::from_le_bytes(b); + } + + let mut model = keccak_host::TranscriptModel::new(&msg); + for i in 0..4 { + assert_eq!( + model.next_u64(), + keccak_host::candidate_from_state(&state, i), + "len {len}, candidate {i} must be state lane {}", + 3 - i + ); + } + } +} + +// ================= R1d: the TranscriptReplay emitter ================= + +use super::programs::{ + TRANSCRIPT_ABSORB_A, TRANSCRIPT_ABSORB_B, TRANSCRIPT_ARENA_HALVES, TRANSCRIPT_QUERY_BITS, + TRANSCRIPT_SEED, canonicity_guard_program, transcript_replay_program, + transcript_replay_program_source, +}; + +/// Goldilocks: `p = 2^64 − 2^32 + 1`. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +// ---------------------------- oracle scrutiny ---------------------------- + +/// The assumption that lets a BASE-field `DefaultTranscript` be the oracle for a +/// machine script containing an EXTENSION draw: an ext3 element is three +/// consecutive base draws, in coordinate order 0, 1, 2. +/// +/// Read off `Degree3GoldilocksExtensionField::sample_field_element_from`, which +/// is `from_fn(|_| GoldilocksField::sample_field_element_from(&mut next_u64))`. +/// `from_fn` evaluating in index order is the load-bearing part, so it is pinned +/// here against the real thing rather than trusted. +#[test] +fn ext_draw_is_three_base_draws_in_coordinate_order() { + use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let mut ext = DefaultTranscript::::new(TRANSCRIPT_SEED); + let mut base = DefaultTranscript::::new(TRANSCRIPT_SEED); + // Four draws = twelve candidates, so this spans three refills and cannot be + // satisfied by a coincidence inside one squeeze. + for draw in 0..4 { + let e = ext.sample_field_element(); + let coords: [FE; 3] = core::array::from_fn(|_| base.sample_field_element()); + assert_eq!(*e.value(), coords, "ext draw {draw}"); + } +} + +/// The guard's predicate, host-side: a candidate is out of range exactly when +/// `hi = 2^32 − 1 ∧ lo ≠ 0`. +fn machine_accepts(lo: u64, hi: u64) -> bool { + !(hi == 0xFFFF_FFFF && lo != 0) +} + +/// THE DERIVATION, checked against the production sampler rather than against +/// itself: for every candidate, the machine's one-instruction predicate agrees +/// with `GoldilocksField::sample_field_element_from` on whether the FIRST draw +/// is accepted, and on the value when it is. +/// +/// The production sampler is probed by feeding it the candidate under test and +/// then zeros: it took a second draw exactly when it rejected the first. +#[test] +fn canonicity_predicate_matches_production_sampler() { + use math::field::traits::HasDefaultTranscript; + + let production = |candidate: u64| -> Option { + let mut draws = 0usize; + let v = crate::tables::types::GoldilocksField::sample_field_element_from(|| { + draws += 1; + if draws == 1 { candidate } else { 0 } + }); + (draws == 1).then_some(v) + }; + + let mut candidates: Vec = vec![0, 1, 1 << 32, 0xFFFF_FFFF, u64::MAX]; + // Dense coverage of the boundary itself. + for d in 0..40u64 { + candidates.push(P.wrapping_sub(20).wrapping_add(d)); + } + // The whole of the reject region's shape: hi pinned at 2^32 − 1. + for lo in [0u64, 1, 2, 3, 0x7FFF_FFFF, 0xFFFF_FFFE, 0xFFFF_FFFF] { + candidates.push((0xFFFF_FFFFu64 << 32) | lo); + candidates.push((0xFFFF_FFFEu64 << 32) | lo); + } + // A broad deterministic sweep. + let mut x = 0x1234_5678_9abc_def0u64; + for _ in 0..5000 { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + candidates.push(x); + } + + let mut rejects = 0usize; + for c in candidates { + let (lo, hi) = (c & 0xFFFF_FFFF, c >> 32); + let accepted = production(c); + assert_eq!( + machine_accepts(lo, hi), + accepted.is_some(), + "candidate {c:#018x}: guard and production sampler disagree" + ); + match accepted { + Some(v) => { + let recomposed = &(&FE::from(hi) * &FE::from(1u64 << 32)) + &FE::from(lo); + assert_eq!(recomposed, v, "candidate {c:#018x}: value"); + } + None => rejects += 1, + } + } + assert!( + rejects >= 20, + "the sweep must actually exercise the reject branch, saw {rejects}" + ); +} + +// -------------------------- the canonicity guard -------------------------- + +fn guard_arenas(lo: u64, hi: u64) -> Vec> { + vec![vec![ + super::word::base_word(FE::from(lo)), + super::word::base_word(FE::from(hi)), + ]] +} + +/// The machine's guard at the boundary, which the replay itself cannot reach: +/// producing a digest whose candidate is ≥ p by search costs about 2^32 keccaks. +#[test] +fn machine_canonicity_guard_accepts_and_rejects_at_the_boundary() { + let program = canonicity_guard_program(); + validate(&program).expect("the guard harness must pass admission"); + let run = |c: u64| { + super::executor::execute( + &program, + &guard_arenas(c & 0xFFFF_FFFF, c >> 32), + &super::hash::TestPermutation, + ) + }; + + for c in [0u64, 1, 12345, 1 << 32, P - 2, P - 1] { + let exec = run(c).unwrap_or_else(|e| panic!("{c:#018x} is canonical: {e:?}")); + assert_eq!( + exec.public_words[0].1[0], + FE::from(c), + "{c:#018x}: recomposed value" + ); + } + for c in [P, P + 1, P + 12345, u64::MAX] { + match run(c) { + Err(LfmExecError::DivByZero { .. }) => {} + other => panic!( + "{c:#018x} is ≥ p and must fail the guard, got {:?}", + other.map(|_| "accepted") + ), + } + } +} + +/// The guard has to hold against a prover, not just against the executor. +/// +/// Trace tampering cannot show this — changing the arena makes the executor +/// refuse, and changing one trace cell desynchronises the memory bus, which +/// rejects for the wrong reason. So this is the coherent forgery (§ the +/// permute-row precedent): start from candidate `p − 1`, whose guard row is +/// `div(lo = 0, g = 0)`, and forge `lo = 1` — i.e. candidate `p` — in EVERY row +/// that touches that cell, recomputing the published value to what the forged +/// halves really give ((2^32 − 1)·2^32 + 1 = p ≡ 0). The hint's send, both +/// receives, the mul-add's own constraint and the public output are then all +/// internally consistent and every bus balances. The single division constraint +/// `SEL_DIV·(B·OUT − A) = 0`, which now reads `0·1 − 1 ≠ 0`, is the only thing +/// left standing between this and an accepted proof. +#[test] +fn canonicity_guard_rejects_an_out_of_range_candidate_in_the_proof() { + let opts = options(); + let program = canonicity_guard_program(); + let artifacts = build_artifacts(&program, &opts); + let mut exec = super::executor::execute( + &program, + &guard_arenas(0, 0xFFFF_FFFF), + &super::hash::TestPermutation, + ) + .expect("p − 1 is canonical"); + + let one = FE::one(); + exec.records.hint[0] = super::word::base_word(one); + // BALU rows in emission order: sub (g = 2^32 − 1 − hi), div (the guard), + // mul-add (the value). Only the guard's numerator and the value move. + exec.records.balu[1].a = one; + exec.records.balu[2].c = one; + exec.records.balu[2].out = FE::zero(); + exec.records.public[0] = super::word::base_word(FE::zero()); + exec.public_words[0].1 = super::word::base_word(FE::zero()); + + let mut traces = build_traces(&program, &exec.records); + let proof = prove_traces(&artifacts, &mut traces, &exec.public_words, &opts) + .expect("the prover has no constraint checks, so it accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + ), + "a candidate at p must fail the canonicity guard" + ); +} + +// ---------------------------- the replay itself ---------------------------- + +/// The two absorbed blobs. Both lengths are multiples of four, so packing their +/// CONCATENATION into halves gives each blob its own whole halves — which is +/// also the property `append_halves` relies on. +fn transcript_absorbs() -> (Vec, Vec) { + let a = (0..TRANSCRIPT_ABSORB_A) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let b = (0..TRANSCRIPT_ABSORB_B) + .map(|i| (i as u8).wrapping_mul(17).wrapping_add(3)) + .collect(); + (a, b) +} + +fn transcript_arenas() -> Vec> { + let (a, b) = transcript_absorbs(); + let mut bytes = a; + bytes.extend_from_slice(&b); + let halves = keccak_host::pack_stream(&bytes); + assert_eq!(halves.len(), TRANSCRIPT_ARENA_HALVES as usize); + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +struct ReplayExpectation { + f0: FE, + f1: FE, + e: [FE; 3], + q: u64, + f2: FE, + s: [u8; 32], + f3: FE, +} + +/// The oracle: the REAL `DefaultTranscript`, driven through the same script. +/// +/// Instantiated over the base field so that `sample_field_element` is one draw, +/// matching the machine's `sample_felt`; the extension draw in the middle is +/// three consecutive base draws, which +/// `ext_draw_is_three_base_draws_in_coordinate_order` pins against the real ext +/// sampler independently. +fn host_expectation() -> ReplayExpectation { + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let (a, b) = transcript_absorbs(); + let mut h = DefaultTranscript::::new(TRANSCRIPT_SEED); + h.append_bytes(&a); + let f0 = h.sample_field_element(); + let f1 = h.sample_field_element(); + let e: [FE; 3] = core::array::from_fn(|_| h.sample_field_element()); + h.append_bytes(&b); + let q = h.sample_u64(1 << TRANSCRIPT_QUERY_BITS); + let f2 = h.sample_field_element(); + let s = h.sample(); + let f3 = h.sample_field_element(); + ReplayExpectation { + f0, + f1, + e, + q, + f2, + s, + f3, + } +} + +fn check_replay_publics(public: &[(u32, LfmWord)], what: &str) { + let x = host_expectation(); + assert_eq!(public.len(), 8, "{what}: public word count"); + assert_eq!(public[0].1[0], x.f0, "{what}: first base challenge"); + assert_eq!(public[1].1[0], x.f1, "{what}: second base challenge"); + for i in 0..3 { + assert_eq!(public[2].1[i], x.e[i], "{what}: ext coordinate {i}"); + } + assert_eq!(public[3].1[0], FE::from(x.q), "{what}: sample_u64 draw"); + assert_eq!(public[4].1[0], x.f2, "{what}: post-absorb challenge"); + assert_eq!(digest_bytes(&public[5..7]), x.s, "{what}: raw sample()"); + assert_eq!(public[7].1[0], x.f3, "{what}: post-sample challenge"); +} + +#[test] +fn transcript_replay_program_is_admissible() { + validate(&transcript_replay_program()).expect("the replay must pass admission"); +} + +#[test] +fn transcript_replay_source_is_deterministic() { + let a = transcript_replay_program_source(); + let b = transcript_replay_program_source(); + assert_eq!(a.instrs.len(), b.instrs.len()); + assert_eq!(a.num_addrs, b.num_addrs); + assert_eq!(format!("{:?}", a.instrs), format!("{:?}", b.instrs)); +} + +/// Bit-exactness against the real transcript, execute-only (fast). Validates the +/// EMITTER — the consumption schedule, the invalidation rules, the candidate +/// lane mapping — against `DefaultTranscript` itself. +#[test] +fn transcript_replay_matches_default_transcript() { + let exec = super::executor::execute( + &transcript_replay_program(), + &transcript_arenas(), + &super::hash::TestPermutation, + ) + .expect("the replay must execute"); + check_replay_publics(&exec.public_words, "execute"); +} + +/// The R1d headline: the machine PROVES a scripted `DefaultTranscript` +/// interleaving and the proof verifies through the registry, with every sampled +/// value identical to the real transcript's. +/// +/// The proving half is not redundant with the execute-only test above. Per the +/// R1c lesson, `execute` fills the keccak rows from the host mirror, so an +/// execute-only test says nothing about whether the CHIP agrees — and this +/// program leans on the chip's reversed-digest send (the re-absorb), on `Unpack` +/// of keccak output words (the candidates), and on the BALU division that +/// enforces canonicity. +#[test] +fn transcript_replay_proves_and_verifies() { + let opts = options(); + let program = transcript_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &transcript_arenas(), &opts).expect("prove"); + check_replay_publics(&proved.public_words, "prove"); + assert!( + lfm_verify( + LfmProgramKind::TranscriptReplayV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("TranscriptReplayV0 is registered"), + "the registered transcript replay must verify" + ); +} + +/// Flipping one absorbed half must reject: the absorb feeds a squeeze, so every +/// later challenge moves, and claiming the honest ones no longer matches. +/// +/// Both blobs are covered — the first is absorbed before any squeeze, the second +/// invalidates a buffer mid-flight, and they reach the sponge by different +/// paths. +#[test] +fn tampered_transcript_absorb_half_rejects() { + let opts = options(); + let program = transcript_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &transcript_arenas(), &opts).expect("prove"); + check_replay_publics(&honest.public_words, "honest"); + + for (half, what) in [(5u32, "first absorb"), (20, "second absorb")] { + let mut tampered = transcript_arenas(); + tampered[0][half as usize][0] = &tampered[0][half as usize][0] + FE::from(1u64); + let forged = lfm_prove(&program, &artifacts, &tampered, &opts).expect("prove"); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a changed half must change the challenges" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "{what}: claiming the honest challenges for a tampered absorb must reject" + ); + } +} + +#[test] +fn registry_drift_transcript_replay_v0_blowup2() { + let opts = options(); + let program = transcript_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let entry = resolve(LfmProgramKind::TranscriptReplayV0, 2) + .expect("TranscriptReplayV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +/// Pins the emitted SHAPE, which the value tests would only catch indirectly: +/// the script's five squeezes span six rate blocks (segment #3 is 168 bytes and +/// takes two), so the program must hold exactly six keccak rows. An extra or +/// missing squeeze — the classic invalidation-rule bug — moves this number. +#[test] +fn transcript_replay_cell_counts() { + let program = transcript_replay_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "TranscriptReplayV0: {} instructions, {} addresses, {} main value cells, {} aux ext elements", + program.instrs.len(), + program.num_addrs, + main, + aux + ); + assert_eq!( + program.groups.keccak.real_rows, 6, + "five squeezes over six rate blocks" + ); + assert!(main > 0 && aux > 0); +} + +// ------------------------- emitter-contract guards ------------------------- + +#[test] +#[should_panic(expected = "nbits must be in 1..=32")] +fn sample_u64_pow2_rejects_more_than_32_bits() { + use super::transcript_replay::TranscriptReplay; + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + let _ = t.sample_u64_pow2(&mut b, 33); +} + +/// The packing obligation, made unmissable: a constant whose length is not a +/// multiple of four leaves the segment byte-misaligned, and machine data +/// appended after one would straddle a half boundary — which needs the +/// byte-level splice the statement-absorb leg will build, not a silent +/// miscoding here. +#[test] +#[should_panic(expected = "must start on a 4-byte boundary")] +fn machine_data_after_a_misaligned_constant_is_rejected() { + use super::transcript_replay::TranscriptReplay; + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(b"abc"); + let z = b.felt_const(FE::zero()); + t.append_halves(&[z]); +} + +/// Pins the completeness figures `SOUNDNESS.md` §6.3 quotes, so a doc number +/// cannot drift away from the arithmetic behind it. +#[test] +fn zero_rejection_completeness_bound() { + use super::transcript_replay::{ + reject_probability_per_candidate, reject_probability_per_proof, + }; + + // q = (2^32 − 1)/2^64: just under 2^−32, and within a hair of it. + let q = reject_probability_per_candidate(); + assert!(q < 2f64.powi(-32), "q must be strictly below 2^-32"); + assert!(q > 2f64.powi(-32) * (1.0 - 1e-9), "q ≈ 2^-32"); + + // The verified schedule: E = 4 + T·(3 + L_t) extension draws per proof, each + // three base candidates. L_t = 12 with tables at their 2^19 row cap. + let ext_draws = |tables: usize, fold_challenges: usize| 4 + tables * (3 + fold_challenges); + assert_eq!(ext_draws(24, 12), 364, "T = 24 (the structural minimum)"); + assert_eq!(ext_draws(60, 12), 904, "T ≈ 60 (realistic)"); + + let p_min = reject_probability_per_proof(3 * ext_draws(24, 12)); + let p_real = reject_probability_per_proof(3 * ext_draws(60, 12)); + assert!( + (p_min - 2.54e-7).abs() < 0.01e-7, + "the T = 24 bound moved: {p_min:e}" + ); + assert!( + (p_real - 6.31e-7).abs() < 0.01e-7, + "the T = 60 bound moved: {p_real:e}" + ); + assert!( + p_real < 1e-6, + "the headline claim is < 1e-6 at production shapes" + ); + + // Per-table growth is 15 extension draws, NOT one: the per-draw figure is + // ~7e-10 and the per-table figure is 15x that. Conflating them was a real + // error in an earlier draft of §6.3, so both are pinned. + let per_draw = reject_probability_per_proof(3); + let per_table = (p_real - p_min) / 36.0; + assert!((per_draw - 6.98e-10).abs() < 0.01e-10, "per extension draw"); + assert!( + (per_table - 1.048e-8).abs() < 0.01e-8, + "per additional table" + ); + assert!( + (per_table / per_draw - 15.0).abs() < 1e-6, + "a table is 3 + L_t = 15 extension draws" + ); + + // Where it stops being negligible: ~4.3e7 base candidates for 1%, 2^31 for 50%. + assert!(reject_probability_per_proof(43_000_000) > 0.01); + assert!(reject_probability_per_proof(42_000_000) < 0.01); + assert!((reject_probability_per_proof(1 << 31) - 0.5).abs() < 1e-6); +} + +/// Pins `append_digest`'s word-to-halves byte order: a machine-computed keccak +/// digest absorbed into the replay must reach the sponge as the same 32 bytes +/// `DefaultTranscript::append_bytes` sees. +/// +/// This is the absorb path a real verifier uses for every commitment root, and +/// the acceptance script above does not reach it (it absorbs arena halves +/// directly). Reversing the lane order inside `append_word` fails this test and +/// nothing else. +#[test] +fn absorbed_machine_digest_matches_default_transcript() { + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + for len in [0usize, 135, KECCAK_SPONGE_LEN] { + let msg: Vec = (0..len) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect(); + let program = super::programs::transcript_absorb_digest_program(len); + validate(&program).unwrap_or_else(|e| panic!("len {len}: admission: {e:?}")); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("len {len}: execution failed: {e:?}")); + + let mut h = DefaultTranscript::::new(TRANSCRIPT_SEED); + h.append_bytes(&keccak_host::keccak256(&msg)); + assert_eq!( + exec.public_words[0].1[0], + h.sample_field_element(), + "len {len}: challenge after absorbing a machine-computed digest" + ); + } +} + +/// Makes the buffer-position table in `transcript_replay_program_source`'s doc +/// comment executable, so the documented interleaving cannot drift away from the +/// emitted one. +/// +/// The value tests would catch a schedule change too, but only as "the numbers +/// moved". This says which step moved. +#[test] +fn transcript_replay_schedule_matches_the_documented_table() { + use super::transcript_replay::TranscriptReplay; + + let halves_a = TRANSCRIPT_ABSORB_A / keccak_host::BYTES_PER_HALF; + let mut b = super::builder::LfmBuilder::new(); + let arena = b.declare_arena(TRANSCRIPT_ARENA_HALVES); + let halves: Vec<_> = (0..TRANSCRIPT_ARENA_HALVES) + .map(|i| b.hint_felt(arena, i)) + .collect(); + let (absorb_a, absorb_b) = halves.split_at(halves_a); + + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_halves(absorb_a); + assert_eq!( + (t.out_pos(), t.segment_len()), + (32, TRANSCRIPT_SEED.len() + TRANSCRIPT_ABSORB_A), + "after absorb A: buffer empty, segment is seed ‖ A" + ); + + let _ = t.sample_felt(&mut b); + assert_eq!( + (t.out_pos(), t.segment_len()), + (8, 32), + "squeeze #1, then one candidate; the segment becomes the reversed digest" + ); + let _ = t.sample_felt(&mut b); + assert_eq!(t.out_pos(), 16, "second candidate, no squeeze"); + let _ = t.sample_ext(&mut b); + assert_eq!( + t.out_pos(), + 8, + "three more candidates: squeeze #2 lands INSIDE the extension draw" + ); + + t.append_halves(absorb_b); + assert_eq!( + (t.out_pos(), t.segment_len()), + (32, 32 + TRANSCRIPT_ABSORB_B), + "absorb B invalidates a buffer with 24 live bytes; 168 bytes = two blocks" + ); + let _ = t.sample_u64_pow2(&mut b, TRANSCRIPT_QUERY_BITS); + assert_eq!(t.out_pos(), 8, "squeeze #3"); + let _ = t.sample_felt(&mut b); + assert_eq!(t.out_pos(), 16, "no squeeze"); + let _ = t.sample(&mut b); + assert_eq!( + t.out_pos(), + 32, + "raw sample #4 invalidates a buffer with 16 live bytes" + ); + let _ = t.sample_felt(&mut b); + assert_eq!(t.out_pos(), 8, "squeeze #5"); +} + +/// The segment-level packing rule, made executable: consecutive constant appends +/// are ONE byte run, chunked into halves only at the squeeze. +/// +/// `"abc"` then `"de"` must hash as the five-byte string `"abcde"` — two halves +/// — not as two independently packed pieces (which would give `"abc\0de\0\0"`). +/// Per-append packing cannot pass this. +#[test] +fn constant_appends_concatenate_across_append_boundaries() { + use super::transcript_replay::TranscriptReplay; + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + for pieces in [ + vec![&b"abc"[..], &b"de"[..]], + vec![&b"a"[..], &b"b"[..], &b"c"[..], &b"d"[..], &b"e"[..]], + vec![&b""[..], &b"abcde"[..]], + vec![&b"abcde"[..]], + ] { + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(pieces[0]); + for p in &pieces[1..] { + t.append_const_bytes(p); + } + let f = t.sample_felt(&mut b); + b.public(f.as_cell()); + let program = super::compiler::compile(b.finish()); + let exec = + super::executor::execute(&program, &[], &super::hash::TestPermutation).expect("exec"); + + let mut h = DefaultTranscript::::new(b"abcde"); + assert_eq!( + exec.public_words[0].1[0], + h.sample_field_element(), + "{pieces:?} must absorb as the concatenation \"abcde\"" + ); + } +} + +/// The alignment rule is about the SEGMENT's length, not about whether some +/// earlier append happened to be misaligned: a 3-byte constant followed by a +/// 1-byte constant leaves the segment 4-byte aligned, so machine data may follow. +#[test] +fn machine_data_may_follow_constants_that_together_align() { + use super::transcript_replay::TranscriptReplay; + + let mut b = super::builder::LfmBuilder::new(); + let mut t = TranscriptReplay::new(b"abc"); + t.append_const_bytes(b"d"); + let z = b.felt_const(FE::zero()); + t.append_halves(&[z]); + assert_eq!(t.segment_len(), 8, "4 constant bytes plus one machine half"); +} + +/// Cross-check of the emitter's squeeze economics against the verified +/// production draw schedule. +/// +/// Per table the verifier draws β, z_OOD, γ and `L` FRI fold challenges, each +/// preceded by a root absorb that invalidates the buffer — so each extension +/// draw costs one fresh squeeze and uses three of its four candidates. The `Q` +/// query indices are then drawn back to back, costing `⌈Q/4⌉`. +/// +/// Keccak ROWS equal squeezes here because every segment stays inside one +/// 136-byte rate block: 32 reversed-digest bytes plus a 32-byte root is 64. +#[test] +fn squeeze_economics_match_the_verified_draw_schedule() { + use super::transcript_replay::TranscriptReplay; + + const L: usize = 12; // fold challenges: log2(trace) − 7, tables at the 2^19 cap + const Q: usize = 219; // Preset::Blowup2 query count + let ext_draws = 3 + L; // β, z_OOD, γ, then L fold challenges + let halves_per_root = 8; + + let mut b = super::builder::LfmBuilder::new(); + let arena = b.declare_arena(((ext_draws + 1) * halves_per_root) as u32); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + for r in 0..=ext_draws { + let root: Vec<_> = (0..halves_per_root) + .map(|i| b.hint_felt(arena, (r * halves_per_root + i) as u32)) + .collect(); + t.append_halves(&root); + // The last absorb stands for the grinding/final-poly absorb that precedes + // query sampling; it draws nothing. + if r < ext_draws { + let _ = t.sample_ext(&mut b); + } + } + for _ in 0..Q { + let _ = t.sample_u64_pow2(&mut b, 20); + } + let program = super::compiler::compile(b.finish()); + + let expected = ext_draws + Q.div_ceil(4); + assert_eq!( + program.groups.keccak.real_rows, + expected, + "expected {ext_draws} squeezes for the extension draws (one each, the \ + preceding absorb having invalidated the buffer) plus ⌈{Q}/4⌉ = {} for the \ + query draws", + Q.div_ceil(4) + ); + assert_eq!(expected, 70, "15 extension squeezes + 55 query squeezes"); +} + +// ============ R1e slice a: field elements on the wire (big-endian) ============ + +/// Byte patterns that make an endianness or permutation error impossible to +/// miss: every byte of the first value is distinct, and the boundary values pin +/// the canonical range `bit_dec` enforces. +fn be_reference_felts() -> Vec { + vec![ + 0, + 1, + 0x0123_4567_89ab_cdef, + 0xfedc_ba98_7654_3210, + 0xff, + 0xff00_0000, + 1 << 32, + P - 1, + P - 2, + ] +} + +/// A base field element must reach the sponge as the same 8 big-endian bytes +/// `append_field_element` streams. +/// +/// The program publishes the raw 32-byte squeeze rather than a sampled +/// challenge, so a mismatch localises to the absorbed bytes. +#[test] +fn append_felt_matches_default_transcript() { + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let program = super::programs::append_felt_program(); + validate(&program).expect("admission"); + for v in be_reference_felts() { + let arenas = vec![vec![super::word::base_word(FE::from(v))]]; + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("{v:#018x}: execution failed: {e:?}")); + + let mut h = DefaultTranscript::::new(TRANSCRIPT_SEED); + h.append_field_element(&FE::from(v)); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "{v:#018x}: absorbed bytes must match append_field_element" + ); + } +} + +/// The same for a cubic-extension element — 24 bytes, coordinates 0, 1, 2. +/// +/// The three coordinates are deliberately distinct, so a reversed coordinate +/// order (the other byte order this file offers, which belongs to the raw +/// `[FpE; 3]` type) fails rather than coincidentally passing. +#[test] +fn append_ext_matches_default_transcript() { + use crate::tables::types::GoldilocksExtension; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + use math::field::element::FieldElement; + + let program = super::programs::append_ext_program(); + validate(&program).expect("admission"); + for coords in [ + [0u64, 1, 2], + [0x0123_4567_89ab_cdef, 0xfedc_ba98_7654_3210, 0xff], + [P - 1, 0, 1 << 32], + ] { + let arenas = vec![ + coords + .iter() + .map(|&c| super::word::base_word(FE::from(c))) + .collect::>(), + ]; + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("{coords:?}: execution failed: {e:?}")); + + let e = + FieldElement::::new(core::array::from_fn(|i| FE::from(coords[i]))); + let mut h = DefaultTranscript::::new(TRANSCRIPT_SEED); + h.append_field_element(&e); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "{coords:?}: absorbed bytes must match append_field_element" + ); + } +} + +/// The byteswap gadget PROVED, not just executed. +/// +/// `felt_be_halves` is the first thing in this emitter that leans on `LFM_BITDEC` +/// for a value rather than for index bits, and on a 32-term `MulAdd` chain whose +/// weights carry the byte permutation. Execution alone would not catch a +/// chip-vs-executor disagreement in either. +#[test] +fn append_ext_proves_and_verifies() { + let opts = options(); + let program = super::programs::append_ext_program(); + let artifacts = build_artifacts(&program, &opts); + let coords = [0x0123_4567_89ab_cdefu64, 0xfedc_ba98_7654_3210, P - 1]; + let arenas = vec![ + coords + .iter() + .map(|&c| super::word::base_word(FE::from(c))) + .collect::>(), + ]; + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the big-endian absorb must verify" + ); +} + +/// Pins the gadget's cost, which is the reason `append_field_element` was +/// deferred out of R1d: one `BitDec` plus 64 `BALU` rows per felt. +#[test] +fn felt_be_halves_cost() { + let program = super::programs::append_felt_program(); + println!( + "append_felt: {} instructions, bitdec {}, balu {}", + program.instrs.len(), + program.groups.bitdec.real_rows, + program.groups.balu.real_rows + ); + assert_eq!( + program.groups.bitdec.real_rows, 1, + "one decomposition per felt" + ); + assert_eq!( + program.groups.balu.real_rows, 64, + "two accumulators, each 1 Mul + 31 MulAdd over its 32 bits" + ); +} + +// ==================== R1e slice b: the byte-level splice ==================== + +use super::programs::{ + SPLICE_ALT_DIGEST_HALVES, SPLICE_ALT_FIELD_HALVES, SPLICE_ALT_TAG, splice_alternating_program, + splice_dynamic, splice_prefix, splice_program, +}; + +fn splice_arenas(byte_len: usize) -> Vec> { + vec![ + keccak_host::pack_stream(&splice_dynamic(byte_len)) + .into_iter() + .map(super::word::base_word) + .collect(), + ] +} + +/// The splice at every shift, against the REAL transcript. +/// +/// The oracle is `DefaultTranscript` over the concatenated byte string, which is +/// the definition of what the machine must reproduce: append boundaries leave no +/// trace in the digest input, so the whole segment is one byte string and the +/// machine's job is to hash exactly it. +/// +/// Shift 0 is included as the control — it takes the aligned fast path, so if +/// the splice were silently applied there it would show up here. +#[test] +fn splice_matches_default_transcript_at_every_shift() { + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + const DYN_BYTES: usize = 32; + let halves = (DYN_BYTES / keccak_host::BYTES_PER_HALF) as u32; + for prefix_len in [0usize, 1, 2, 3, 4, 5, 6, 7, 29, 30, 31, 32] { + let program = splice_program(prefix_len, halves); + validate(&program).unwrap_or_else(|e| panic!("prefix {prefix_len}: admission: {e:?}")); + let exec = super::executor::execute( + &program, + &splice_arenas(DYN_BYTES), + &super::hash::TestPermutation, + ) + .unwrap_or_else(|e| panic!("prefix {prefix_len}: execution failed: {e:?}")); + + let mut bytes = splice_prefix(prefix_len); + bytes.extend_from_slice(&splice_dynamic(DYN_BYTES)); + let mut h = DefaultTranscript::::new(&bytes); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "prefix {prefix_len} (shift {}): spliced bytes must equal the concatenation", + prefix_len % keccak_host::BYTES_PER_HALF + ); + } +} + +/// The statement's real shape: alternating constant and dynamic runs where a +/// one-byte field moves the shift from 2 to 3 partway through. +#[test] +fn splice_alternating_runs_match_default_transcript() { + use crate::tables::types::GoldilocksField; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + + let d = SPLICE_ALT_DIGEST_HALVES as usize * keccak_host::BYTES_PER_HALF; + let f = SPLICE_ALT_FIELD_HALVES as usize * keccak_host::BYTES_PER_HALF; + let program = splice_alternating_program(); + validate(&program).expect("admission"); + let exec = super::executor::execute( + &program, + &splice_arenas(d + 2 * f), + &super::hash::TestPermutation, + ) + .expect("execution"); + + // The same byte string, built independently in absorb order. + let dynamic = splice_dynamic(d + 2 * f); + let mut bytes = splice_prefix(SPLICE_ALT_TAG); + bytes.extend_from_slice(&dynamic[..d]); + bytes.extend_from_slice(&splice_prefix(8)); + bytes.extend_from_slice(&dynamic[d..d + f]); + bytes.extend_from_slice(&splice_prefix(1)); + bytes.extend_from_slice(&dynamic[d + f..]); + + let mut h = DefaultTranscript::::new(&bytes); + assert_eq!( + digest_bytes(&exec.public_words), + h.sample(), + "alternating const/dynamic runs across a shift change must match" + ); +} + +/// The splice PROVED, not just executed: it leans on `BitDec` plus a weighted +/// sum plus the recomposition assert, and only a proof sees the chips. +#[test] +fn splice_proves_and_verifies() { + let opts = options(); + let program = splice_program(30, 8); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &splice_arenas(32), &opts).expect("prove"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the spliced absorb must verify" + ); +} + +/// A half at or above `2^32` has no four-byte rendering, so the splice must +/// refuse it rather than silently absorb the wrong bytes. +/// +/// `bit_dec` alone bounds its input by `p`, not by `2^32`; the recomposition +/// assert inside `split_half` is what closes the gap, and this is the test that +/// fails if it is removed. +#[test] +fn splice_rejects_a_non_u32_half() { + let program = splice_program(2, 8); + let mut arenas = splice_arenas(32); + arenas[0][0][0] = FE::from(1u64 << 32); + match super::executor::execute(&program, &arenas, &super::hash::TestPermutation) { + Err(LfmExecError::DivByZero { .. }) => {} + other => panic!( + "a half at 2^32 must fail the splice's recomposition assert, got {:?}", + other.map(|_| "accepted") + ), + } +} + +/// Pins the splice's cost, and that the ALIGNED path is still free. +#[test] +fn splice_cost() { + let spliced = splice_program(2, 8); + let aligned = splice_program(4, 8); + println!( + "splice 8 halves @shift2: bitdec {}, balu {} | aligned: bitdec {}, balu {}", + spliced.groups.bitdec.real_rows, + spliced.groups.balu.real_rows, + aligned.groups.bitdec.real_rows, + aligned.groups.balu.real_rows, + ); + assert_eq!( + spliced.groups.bitdec.real_rows, 8, + "one decomposition per spliced half" + ); + assert_eq!( + aligned.groups.bitdec.real_rows, 0, + "the aligned path must emit no splice at all" + ); + assert_eq!( + aligned.groups.balu.real_rows, 0, + "the aligned path must stay instruction-free" + ); +} + +// ========== R1e slices c+d: the epoch statement and Phase A ========== + +use super::programs::{ + STMT_PREPROCESSED, STMT_PUBLIC_OUTPUT_LEN, epoch_statement_shape, statement_replay_program, + stmt_arena_halves, +}; + +/// The per-proof statement values and the Phase-A roots, as bytes. The machine +/// arena and the host oracle are both built from this, so they cannot drift. +struct StatementFixture { + elf_digest: [u8; 32], + public_output: Vec, + epoch_label: u64, + /// `(preprocessed_root, main_root)` per sub-proof, in air order. + roots: Vec<(Option<[u8; 32]>, [u8; 32])>, +} + +fn statement_fixture() -> StatementFixture { + let root = |seed: u8| -> [u8; 32] { + core::array::from_fn(|i| (i as u8).wrapping_mul(seed).wrapping_add(seed)) + }; + StatementFixture { + elf_digest: root(7), + public_output: (0..STMT_PUBLIC_OUTPUT_LEN) + .map(|i| (i as u8).wrapping_mul(19).wrapping_add(5)) + .collect(), + epoch_label: 0x0123_4567_89ab_cdef, + roots: STMT_PREPROCESSED + .iter() + .enumerate() + .map(|(i, &prep)| { + let p = prep.then(|| root(11 + 2 * i as u8)); + (p, root(31 + 2 * i as u8)) + }) + .collect(), + } +} + +/// Each field gets its OWN halves. The arena is a vector of `u32` words, not a +/// byte stream, so concatenating first and packing after would let a field whose +/// length is not a multiple of four shift every field behind it — which is +/// exactly what an unaligned `public_output` does. `pack_stream` zeroes the +/// trailing half's unused high bytes, the property the machine's mask pins. +fn statement_arenas(f: &StatementFixture) -> Vec> { + let mut halves = keccak_host::pack_stream(&f.elf_digest); + halves.extend(keccak_host::pack_stream(&f.public_output)); + halves.extend(keccak_host::pack_stream(&f.epoch_label.to_le_bytes())); + for (prep, main) in &f.roots { + if let Some(p) = prep { + halves.extend(keccak_host::pack_stream(p)); + } + halves.extend(keccak_host::pack_stream(main)); + } + assert_eq!(halves.len(), stmt_arena_halves() as usize); + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +/// The host reference: the REAL `absorb_statement_with_digest`, then Phase A. +/// +/// The statement half of this is production code, not a reimplementation — which +/// matters, because that encoding has ten fields and is exactly where a replay +/// would go wrong. The Phase-A half is a four-line transcription of +/// `crate::replay_transcript_phase_a_view` (`lib.rs`: for each air, the +/// precomputed commitment when `is_preprocessed()`, then +/// `lde_trace_main_merkle_root()`, then `z` and `α`); calling the helper itself +/// would mean synthesising `dyn AIR`s and proof views for three fake tables, +/// which would test the fakes rather than the replay. +type ExtFE = math::field::element::FieldElement; + +fn host_statement_challenges(f: &StatementFixture) -> (ExtFE, ExtFE) { + use crate::statement::{StatementKind, absorb_statement_with_digest}; + use crate::tables::types::GoldilocksExtension; + use crate::{RuntimePageRange, TableCounts}; + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use crypto::fiat_shamir::is_transcript::IsTranscript; + + let shape = epoch_statement_shape(); + let c = shape.table_counts.map(|v| v as usize); + let counts = TableCounts { + cpu: c[0], + lt: c[1], + memw: c[2], + memw_aligned: c[3], + load: c[4], + mul: c[5], + dvrm: c[6], + shift: c[7], + branch: c[8], + memw_register: c[9], + eq: c[10], + bytewise: c[11], + store: c[12], + cpu32: c[13], + }; + let ranges: Vec = shape + .page_ranges + .iter() + .map(|&(base, count)| RuntimePageRange { base, count }) + .collect(); + + let mut t = DefaultTranscript::::new(&[]); + absorb_statement_with_digest( + &mut t, + StatementKind::ContinuationEpoch { + epoch_label: f.epoch_label, + }, + &f.elf_digest, + &f.public_output, + &counts, + shape.num_private_input_pages as usize, + &ranges, + shape.fri_final_poly_log_degree, + ); + for (prep, main) in &f.roots { + if let Some(p) = prep { + t.append_bytes(p); + } + t.append_bytes(main); + } + (t.sample_field_element(), t.sample_field_element()) +} + +fn assert_challenges_match(public: &[(u32, LfmWord)], f: &StatementFixture, what: &str) { + let (z, alpha) = host_statement_challenges(f); + assert_eq!(public.len(), 2, "{what}: z and alpha"); + for (i, (name, want)) in [("z", z), ("alpha", alpha)].iter().enumerate() { + for lane in 0..3 { + assert_eq!( + public[i].1[lane], + want.value()[lane], + "{what}: {name} coordinate {lane}" + ); + } + } +} + +/// Pins where the epoch statement leaves the byte cursor, which decides whether +/// Phase A is spliced and at what shift. +/// +/// CORRECTION to an earlier claim of mine: the statement is NOT unconditionally +/// 3 bytes past a boundary. Its length is `207 + L + 16R`, so the shift Phase A +/// inherits is `(3 + L) mod 4` — it is 3 only when the public output happens to +/// be a multiple of four, and it is ZERO (Phase A entirely unspliced) whenever +/// `L ≡ 1 (mod 4)`. Since `L` is one byte per COMMIT op and therefore workload- +/// determined, the Phase-A splice cost is workload-dependent and free for about +/// one workload in four. +#[test] +fn epoch_statement_cursor_is_three_plus_output_len() { + let shape = epoch_statement_shape(); + let r = shape.page_ranges.len(); + assert_eq!(shape.byte_len(), 207 + STMT_PUBLIC_OUTPUT_LEN + 16 * r); + for l in 0..8usize { + let total = 207 + l + 16 * r; + assert_eq!( + total % keccak_host::BYTES_PER_HALF, + (3 + l) % keccak_host::BYTES_PER_HALF, + "Phase A inherits shift (3 + L) mod 4" + ); + } + // The acceptance shape is chosen to exercise BOTH new paths at once: an + // unaligned public output (so the trailing half is masked) and a nonzero + // inherited shift (so Phase A is spliced). + assert_ne!(STMT_PUBLIC_OUTPUT_LEN % keccak_host::BYTES_PER_HALF, 0); + assert_ne!(shape.byte_len() % keccak_host::BYTES_PER_HALF, 0); +} + +#[test] +fn statement_replay_program_is_admissible() { + validate(&statement_replay_program()).expect("admission"); +} + +/// R1e's acceptance: the machine's `(z, α)` must equal what the REAL statement +/// absorb plus Phase A produce. +#[test] +fn statement_replay_matches_the_host_challenges() { + let f = statement_fixture(); + let exec = super::executor::execute( + &statement_replay_program(), + &statement_arenas(&f), + &super::hash::TestPermutation, + ) + .expect("execution"); + assert_challenges_match(&exec.public_words, &f, "execute"); +} + +/// The same, PROVED and verified through the registry. +#[test] +fn statement_replay_proves_and_verifies() { + let opts = options(); + let f = statement_fixture(); + let program = statement_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &statement_arenas(&f), &opts).expect("prove"); + assert_challenges_match(&proved.public_words, &f, "prove"); + assert!( + lfm_verify( + LfmProgramKind::StatementReplayV0, + &proved.proof, + &proved.public_words, + &opts, + ) + .expect("StatementReplayV0 is registered"), + "the registered statement replay must verify" + ); +} + +/// Both tamper vectors: a flipped Phase-A root half and a flipped statement +/// byte. Each must move the challenges, and claiming the honest ones must reject. +#[test] +fn tampered_statement_or_root_rejects() { + let opts = options(); + let f = statement_fixture(); + let program = statement_replay_program(); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &statement_arenas(&f), &opts).expect("prove"); + + // Half 0 is the ELF digest (statement); half 14 is inside the first + // sub-proof's preprocessed root (Phase A). + for (half, what) in [(0usize, "statement byte"), (14, "Phase-A root half")] { + let mut arenas = statement_arenas(&f); + arenas[0][half][0] = &arenas[0][half][0] + FE::from(1u64); + let forged = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a flip must move z or alpha" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "{what}: claiming the honest challenges must reject" + ); + } +} + +#[test] +fn registry_drift_statement_replay_v0_blowup2() { + let opts = options(); + let artifacts = build_artifacts(&statement_replay_program(), &opts); + let entry = resolve(LfmProgramKind::StatementReplayV0, 2) + .expect("StatementReplayV0@2 must be registered"); + assert_eq!(entry.roots, artifacts.roots, "group roots drifted"); + assert_eq!( + entry.log_heights, artifacts.log_heights, + "group heights drifted" + ); + assert_eq!( + entry.keccak_rnd_chunks, artifacts.keccak_rnd_chunks, + "KECCAK_RND chunk count drifted" + ); + assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted"); + assert_eq!(entry.program_id, artifacts.program_id, "program_id drifted"); +} + +#[test] +fn statement_replay_cell_counts() { + let program = statement_replay_program(); + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!( + "StatementReplayV0: {} instructions, keccak {}, bitdec {}, balu {}, {main} main cells, {aux} aux", + program.instrs.len(), + program.groups.keccak.real_rows, + program.groups.bitdec.real_rows, + program.groups.balu.real_rows, + ); + assert!(main > 0 && aux > 0); +} + +/// The masked trailing half's soundness obligation: bytes PAST the encoded +/// length must not reach the sponge. +/// +/// `public_output` is length-prefixed, so its final arena half has live bytes +/// only up to `len % 4`. The high bytes of that felt are arena data and +/// otherwise unconstrained — without the zero-pin in `Packer::push_masked` a +/// prover could put anything there and change the absorbed byte string while the +/// length prefix said otherwise. Here byte 3 of the trailing half is past the +/// 14-byte length, so the program must refuse to execute rather than absorb it. +#[test] +fn statement_rejects_garbage_past_the_public_output_length() { + let f = statement_fixture(); + let mut arenas = statement_arenas(&f); + // Halves: elf 0..8, public_output 8..12 (14 bytes = 3 whole + 2 live), + // so half 11's top two bytes are past the length. + arenas[0][11][0] = &arenas[0][11][0] + FE::from(1u64 << 24); + match super::executor::execute( + &statement_replay_program(), + &arenas, + &super::hash::TestPermutation, + ) { + Err(LfmExecError::DivByZero { .. }) => {} + other => panic!( + "bytes past the public-output length must be pinned to zero, got {:?}", + other.map(|_| "accepted") + ), + } +} +// ======================= KECCAK_RND chunking ======================= +// +// `KECCAK_RND` costs 24 rows per permutation, so one instance cannot hold the +// ~460k permutations a real proof wrap needs. These tests cover the split: the +// shape it produces, that a multi-chunk program proves and verifies, and the +// two ways the split itself can be wrong (a corrupted chunk, a dropped +// permutation). + +use super::airs::{keccak_rnd_chunk_permutations, keccak_rnd_chunk_rows, num_lfm_airs}; +use super::chunking::KeccakChunking; +use crate::tables::keccak_rnd; + +/// A 3-permutation sponge: `pad10*1` grows 280 bytes to 3 rate blocks, so at +/// two permutations per chunk it splits unevenly (2 + 1) — the partial-final +/// chunk is the case a uniform split would miss. +const CHUNKED_SPONGE_LEN: usize = 280; + +/// Two permutations per chunk. Small enough that the multi-chunk tests prove in +/// seconds instead of the 21,845 permutations the default policy would need. +fn test_chunking() -> KeccakChunking { + KeccakChunking::from_permutations(2) +} + +fn chunked_sponge_program() -> LfmProgram { + keccak_sponge_program(CHUNKED_SPONGE_LEN).with_keccak_chunking(test_chunking()) +} + +fn chunked_sponge_msg() -> Vec { + (0..CHUNKED_SPONGE_LEN) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(7)) + .collect() +} + +/// The permutation-level round operations a program's execution produces — +/// the same list `build_traces` chunks, rebuilt here so the tamper tests can +/// re-chunk it by hand. +fn round_ops_of( + program: &LfmProgram, + arenas: &[Vec], +) -> Vec { + let exec = super::executor::execute(program, arenas, &super::hash::TestPermutation) + .expect("honest execution"); + let ops: Vec<_> = exec + .records + .keccak + .iter() + .enumerate() + .map(|(row, r)| keccak_adapter::KeccakAdapterOperation { + tag: klayout::tag_for_row(row), + input: r.perm_in, + }) + .collect(); + keccak_adapter::round_operations(&ops) +} + +/// Every registered program is single-chunk under the default policy, so the +/// production path is unchanged by this feature — chunking is dormant until a +/// program exceeds 21,845 permutations. +#[test] +fn registered_programs_are_single_chunk() { + for entry in super::registry::LFM_REGISTRY { + assert_eq!( + entry.keccak_rnd_chunks, 1, + "{:?} is registered with a chunk count other than 1", + entry.kind + ); + } +} + +/// Every `HasherKind` there is. Not derived — a new candidate must be added +/// here by hand, which is the point: the two tests below are what say a new +/// hasher gets its own program identity rather than sharing one. +const ALL_HASHERS: [super::hash::HasherKind; 3] = [ + super::hash::HasherKind::Test, + super::hash::HasherKind::Poseidon, + super::hash::HasherKind::Blake3, +]; + +/// ★ Each registered entry's digest is bound to the hasher the entry names, +/// and to no other. +/// +/// The first assertion is the honest control: the stored `program_id` really is +/// what the stored `(roots, heights, chunks, hasher)` derive, so the table is +/// internally consistent. The second is the property — swapping *only* the +/// hasher, with every root held fixed, must move the digest. Held together they +/// say a Test-backed and a Poseidon-backed machine of the same program are two +/// programs, which is what stops a verifier from pairing one hasher's digest +/// with another hasher's AIR set. +#[test] +fn every_registry_entry_binds_its_hasher_into_its_digest() { + for entry in super::registry::LFM_REGISTRY { + assert_eq!( + super::statement::lfm_program_id( + &entry.roots, + &entry.log_heights, + entry.keccak_rnd_chunks, + entry.hasher, + ), + entry.program_id, + "{:?}: the stored digest must be what the stored shape derives", + entry.kind + ); + for other in ALL_HASHERS { + if other == entry.hasher { + continue; + } + assert_ne!( + super::statement::lfm_program_id( + &entry.roots, + &entry.log_heights, + entry.keccak_rnd_chunks, + other, + ), + entry.program_id, + "{:?}: {other:?} must not share {:?}'s program identity", + entry.kind, + entry.hasher + ); + } + } +} + +/// The registry-path consequence: `lfm_verify` builds its AIR set from the +/// entry's hasher, so an honest proof of a registered program verifies, and the +/// same proof against the same entry's roots and digest under any *other* +/// hasher does not. +/// +/// The accept half is not decoration. A verifier that rejected everything would +/// pass the reject half on its own, so the pair is what makes this a binding +/// test rather than a "does it say no" test. +#[test] +fn the_registry_hasher_is_what_verify_builds() { + let opts = options(); + let program = trivial_program(); + let entry = resolve(LfmProgramKind::TrivialV0, opts.blowup_factor).expect("registered"); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts + ) + .expect("registered"), + "the honest proof must verify under the hasher the entry names" + ); + + for other in ALL_HASHERS { + if other == entry.hasher { + continue; + } + assert!( + !verify_against( + &entry.roots, + &entry.program_id, + entry.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + other, + ), + "the entry's own proof must not verify under {other:?}" + ); + } +} + +/// The split's shape: chunk count, per-chunk permutation counts, per-chunk +/// trace heights, AIR count and trace count all agree. +#[test] +fn chunking_splits_the_sponge_into_two_uneven_chunks() { + let program = chunked_sponge_program(); + assert_eq!( + program.groups.keccak.real_rows, 3, + "a {CHUNKED_SPONGE_LEN}-byte message must be 3 rate blocks" + ); + + assert_eq!(keccak_rnd_chunk_permutations(&program), vec![2, 1]); + // 2 permutations = 48 rows → 64; 1 permutation = 24 rows → 32. + assert_eq!(keccak_rnd_chunk_rows(&program), vec![64, 32]); + + let artifacts = build_artifacts(&program, &options()); + assert_eq!(artifacts.keccak_rnd_chunks, 2); + assert_eq!(num_lfm_airs(2), super::NUM_LFM_CHIPS + 1); + + let exec = super::executor::execute( + &program, + &sponge_arenas(&chunked_sponge_msg()), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + let traces = build_traces(&program, &exec.records); + assert_eq!(traces.keccak_rnd.len(), 2, "one KECCAK_RND trace per chunk"); + assert_eq!( + traces + .keccak_rnd + .iter() + .map(|t| t.num_rows()) + .collect::>(), + vec![64, 32], + "chunk traces must match the heights the artifacts predict" + ); +} + +/// ★ The acceptance test: a program needing more than one `KECCAK_RND` chunk +/// proves and verifies end to end, and its digest still matches the production +/// hasher. +#[test] +fn chunked_sponge_proves_and_verifies() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + assert_eq!(artifacts.keccak_rnd_chunks, 2, "this test needs 2 chunks"); + + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + assert_eq!( + digest_bytes(&proved.public_words), + keccak_host::keccak256(&msg), + "a chunked proof must hash the same as the production hasher" + ); + assert_eq!( + stark::proof::view::MultiProofView::Owned(&proved.proof).len(), + num_lfm_airs(2), + "the proof must carry one sub-proof per AIR instance" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "a two-chunk KECCAK_RND proof must verify" + ); +} + +/// Chunking is a prover-side layout choice, not a semantic one: the same +/// message proved at 1 and at 2 chunks yields the same public output. (The +/// program *identity* does differ — the chunk count is bound into the digest — +/// which is exactly why the two need different artifacts.) +#[test] +fn chunking_does_not_change_what_is_proved() { + let opts = options(); + let msg = chunked_sponge_msg(); + + let one = keccak_sponge_program(CHUNKED_SPONGE_LEN); + let one_artifacts = build_artifacts(&one, &opts); + assert_eq!(one_artifacts.keccak_rnd_chunks, 1); + let one_proof = lfm_prove(&one, &one_artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + let two = chunked_sponge_program(); + let two_artifacts = build_artifacts(&two, &opts); + let two_proof = lfm_prove(&two, &two_artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + assert_eq!( + one_proof.public_words, two_proof.public_words, + "chunking must not change the program's output" + ); + assert_eq!( + one_artifacts.roots, two_artifacts.roots, + "chunking must not move any preprocessed root" + ); + assert_ne!( + one_artifacts.program_id, two_artifacts.program_id, + "the chunk count is program shape and must be bound into the digest" + ); + for (artifacts, proof) in [(&one_artifacts, &one_proof), (&two_artifacts, &two_proof)] { + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof.proof, + &proof.public_words, + &opts, + artifacts.hasher, + ), + "both chunkings must verify against their own artifacts" + ); + } +} + +/// ★ Tamper: corrupting a permutation that lives in the *second* chunk must +/// reject. The first chunk is untouched, so this only rejects if chunk 1's +/// rows are really part of the proof's bus balance. +#[test] +fn tampered_second_chunk_permutation_rejects() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + + let mut traces = build_traces(&program, &exec.records); + assert_eq!(traces.keccak_rnd.len(), 2); + // Byte 0 of lane (0,0) on the second chunk's first row: the `Keccak` + // receive token no longer matches the send that fed it. + let col = keccak_rnd::cols::start(0, 0, 0); + let old = traces.keccak_rnd[1].main_table.get_row(0)[col]; + traces.keccak_rnd[1] + .main_table + .set_fe(0, col, old + FE::from(1u64)); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + ), + "a corrupted permutation in the second chunk must reject" + ); +} + +/// ★ Falsifies the split itself: drop the permutation the second chunk holds. +/// The `LFM_KECCAK` chip still sends its request token, so the `Keccak` bus is +/// left with a send that nothing receives. If this ever accepts, chunks are +/// not actually contributing their rows to the balance. +#[test] +fn dropping_the_second_chunks_permutation_rejects() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + + let mut traces = build_traces(&program, &exec.records); + // Same chunk COUNT — so the AIR set and the digest still match — but the + // last chunk is now empty. + traces.keccak_rnd[1] = keccak_rnd::generate_keccak_rnd_trace(&[]); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + ), + "a chunk missing its permutation must reject" + ); +} + +/// The mechanism's foundation, stated positively: which chunk a permutation +/// lands in is free. `KECCAK_RND` has no row-to-row constraints and its rounds +/// are linked by `Keccak` bus tokens rather than row adjacency, so LogUp +/// cannot tell a 2+1 split from a 1+2 one. This is why chunking needs no +/// pairing logic — and if it ever fails, the round chip has grown a +/// cross-row dependency that chunking would silently break. +#[test] +fn permutations_may_be_reassigned_across_chunk_boundaries() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let exec = super::executor::execute( + &program, + &sponge_arenas(&msg), + &super::hash::TestPermutation, + ) + .expect("honest execution"); + + let round_ops = round_ops_of(&program, &sponge_arenas(&msg)); + assert_eq!(round_ops.len(), 3); + + let mut traces = build_traces(&program, &exec.records); + // Canonical split is 2 + 1; re-split as 1 + 2. + traces.keccak_rnd[0] = keccak_rnd::generate_keccak_rnd_trace(&round_ops[..1]); + traces.keccak_rnd[1] = keccak_rnd::generate_keccak_rnd_trace(&round_ops[1..]); + + let proof = + prove_traces(&artifacts, &mut traces, &exec.public_words, &opts).expect("prover accepts"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proof, + &exec.public_words, + &opts, + artifacts.hasher, + ), + "chunk assignment is free — a 1+2 split proves the same statement as 2+1" + ); +} + +/// The verifier builds its AIR set from the supplied chunk count, so a count +/// that disagrees with the proof's shape must be rejected — including zero, +/// which would drop `KECCAK_RND` and its constraints from the set. +/// +/// Two layers enforce this and the test does not distinguish them: the +/// explicit length check in `verify_against`, and the framework's own +/// AIR-count handling. Measured: deleting the explicit check leaves this test +/// green, so it pins the *behaviour*, not that particular guard. The guard +/// stays because it makes the shape contract local and legible, not because +/// this test would catch its removal. +#[test] +fn verify_rejects_a_chunk_count_that_does_not_match_the_proof() { + let opts = options(); + let msg = chunked_sponge_msg(); + let program = chunked_sponge_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &sponge_arenas(&msg), &opts).expect("prove"); + + for wrong in [0usize, 1, 3, 14] { + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + wrong, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "chunk count {wrong} must not verify a 2-chunk proof" + ); + } +} + +/// What chunking costs: `KECCAK_RND` pads each chunk to its own power of two, +/// so the only overhead is padding, and splitting can even reduce it. +#[test] +fn chunking_cell_cost() { + let one = keccak_sponge_program(CHUNKED_SPONGE_LEN); + let two = chunked_sponge_program(); + let perms = one.groups.keccak.real_rows as u64; + + let (main_one, aux_one) = super::airs::lfm_cell_counts(&one); + let (main_two, aux_two) = super::airs::lfm_cell_counts(&two); + println!( + "{CHUNKED_SPONGE_LEN}-byte sponge, {perms} permutations:\n \ + 1 chunk rows {:?} main {main_one} aux {aux_one}\n \ + 2 chunks rows {:?} main {main_two} aux {aux_two}\n \ + delta main {} aux {}", + keccak_rnd_chunk_rows(&one), + keccak_rnd_chunk_rows(&two), + main_two as i64 - main_one as i64, + aux_two as i64 - aux_one as i64, + ); + + // KECCAK_RND rows are the only thing chunking moves; here 128 padded rows + // in one chunk versus 64 + 32 in two. + assert_eq!(keccak_rnd_chunk_rows(&one).iter().sum::(), 128); + assert_eq!(keccak_rnd_chunk_rows(&two).iter().sum::(), 96); + assert!( + main_two < main_one, + "this split lands on tighter power-of-two boundaries, so it is cheaper" + ); +} + +/// At the default policy's geometry chunking does not cost rows, it saves +/// them. A single table must pad to one power of two for the whole program; N +/// chunks each pad to their own, and every full chunk is within 8 rows of its +/// power of two by construction. +#[test] +fn default_policy_beats_a_single_table_at_wrap_scale() { + let c = KeccakChunking::default(); + let per = c.permutations_per_chunk(); + let full_chunk_rows = (per * 24).next_power_of_two(); + assert_eq!(full_chunk_rows, 1 << 19); + assert_eq!( + full_chunk_rows - per * 24, + 8, + "a full chunk wastes 8 rows of 524,288" + ); + + // The proof wrap this feature exists for. + const WRAP_PERMUTATIONS: usize = 460_000; + let chunks = c.chunk_count(WRAP_PERMUTATIONS); + assert_eq!(chunks, 22, "21 full chunks plus a partial one"); + + let chunked_rows: usize = (0..chunks) + .map(|i| { + let perms = WRAP_PERMUTATIONS.saturating_sub(i * per).min(per); + (perms * 24).next_power_of_two().max(4) + }) + .sum(); + let single_table_rows = (WRAP_PERMUTATIONS * 24).next_power_of_two(); + + println!( + "{WRAP_PERMUTATIONS} permutations: {chunks} chunks = {chunked_rows} rows, \ + single table = {single_table_rows} rows ({:.1}% saved)", + 100.0 * (1.0 - chunked_rows as f64 / single_table_rows as f64), + ); + assert!( + chunked_rows < single_table_rows, + "chunking must not cost more rows than one table would" + ); + // 2^24 rows at 1480 columns is also far past what one table can hold. + assert_eq!(single_table_rows, 1 << 24); +} + +// ================= R1f slice b: real continuation-proof bytes ================= + +use super::proof_fixture; + +/// Cache path for the fixture blob. Outside the repository on purpose: a +/// checked-in binary can drift from the encoder silently, so the generation path +/// is what a cold run exercises. +fn fixture_cache() -> std::path::PathBuf { + std::env::temp_dir().join("lfm-r1f-continuation-fixture.bin") +} + +/// R1f(b): the machine's fixture is a REAL two-epoch continuation proof, encoded +/// by the same function that builds the recursion guest's private input. +#[test] +fn continuation_fixture_generates_two_epochs() { + let (blob, num_epochs) = proof_fixture::generate(); + println!( + "R1f fixture: inner={} epoch_log2={} epochs={} blob={} bytes", + proof_fixture::FIXTURE_INNER_ELF, + proof_fixture::FIXTURE_EPOCH_LOG2, + num_epochs, + blob.len() + ); + assert!( + proof_fixture::has_recursion_prefix(&blob), + "the blob must carry the recursion input wire format's magic prefix" + ); + assert!( + num_epochs >= 2, + "a CONTINUATION fixture needs more than one epoch, got {num_epochs} — \ + lower FIXTURE_EPOCH_LOG2" + ); + // Cache it for the slices that consume it. Atomic: this test runs in + // parallel with readers of the same path, and blobs are not reproducible, + // so a torn write would hand someone a truncated proof. + proof_fixture::write_cache(&fixture_cache(), &blob); +} + +/// R1f(a): the arena filler reads a REAL proof's committed roots out of the +/// guest's wire-format blob, in place, exactly as the recursion guest would. +#[test] +fn arena_filler_reads_real_committed_roots() { + use super::proof_arena; + use super::proof_fixture::FixtureArchive; + + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = FixtureArchive::open(&blob); + + let epochs = proof_arena::num_epochs(&archive); + assert_eq!(epochs, 2, "the fixture is a two-epoch continuation"); + + for epoch in 0..epochs { + let tables = proof_arena::epoch_num_tables(&archive, epoch); + let roots = proof_arena::epoch_main_roots(&archive, epoch); + assert_eq!(roots.len(), tables, "one main root per sub-proof"); + assert!(tables > 0, "epoch {epoch} must have sub-proofs"); + // Real commitments, not defaults: an all-zero root would mean the reader + // is looking at the wrong bytes rather than at the proof. + assert!( + roots.iter().all(|r| *r != [0u8; 32]), + "epoch {epoch}: every committed root must be nonzero" + ); + let halves = proof_arena::roots_to_halves(&roots); + assert_eq!(halves.len(), tables * proof_arena::ROOT_HALVES); + println!( + "R1f arena: epoch {epoch} -> {tables} sub-proofs, {} arena halves, output {} bytes", + halves.len(), + proof_arena::epoch_public_output(&archive, epoch).len() + ); + } +} + +/// Verifies the team lead's ruling premise directly against the blob: the +/// SUPPLIED preprocessed roots really are embedded, so replaying Phase A does +/// not need `build_epoch_airs` reachable. +/// +/// Checked here rather than taken on trust, because the whole leg's shape +/// depends on it. +#[test] +fn supplied_preprocessed_roots_are_embedded_in_the_blob() { + use super::proof_fixture::FixtureArchive; + + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = FixtureArchive::open(&blob); + let gi = archive.guest_input(); + + // DECODE: one commitment, directly in the guest input. + assert_ne!( + gi.decode_commitment, [0u8; 32], + "the DECODE root must be embedded and nonzero" + ); + // Per-page genesis roots: (base, commitment) pairs, also directly embedded. + println!( + "R1f supplied roots: decode present, {} page commitments", + gi.page_commitments.len() + ); + for pair in gi.page_commitments.iter() { + assert_ne!(pair.1, [0u8; 32], "page genesis roots must be nonzero"); + } +} + +// ============ R1f (c)+(d): a REAL Merkle opening, in the machine ============ +// +// Everything up to here ran on data this machine produced. This is the first +// leg that authenticates production-committed data: one FRI query's main-trace +// opening from a real two-epoch continuation proof, walked under the production +// keccak Merkle conventions, against that proof's own committed root. +// +// The oracle is the proof's root. Nothing here recomputes an expected answer +// with a local model and compares the machine against itself. + +use super::programs::{MerkleOpeningShape, keccak_merkle_opening_program}; +use super::proof_arena::MainTraceOpening; + +/// Which opening the leg authenticates. +/// +/// Epoch 0's first sub-proof, chosen on measured grounds. Two things make it +/// the right target, and only the first is stable across blobs. +/// +/// **Depth.** It is one of exactly two depth-20 trees in the fixture (the other +/// is epoch 1's table 0); everything else is depth 7 or less, and half the +/// sub-proofs are depth 2. Depth is SHAPE, so it does not move when the blob +/// does — see `fixture_generation_is_not_reproducible`. +/// +/// **A unique leaf index.** Measured on one blob, 24 of the 49 sub-proofs have +/// exactly one index that verifies and 25 have several: a table whose trace is +/// mostly padding commits identical rows, so identical leaves sit under +/// identical subtrees and every index checks out. On one of those, "flip an +/// index bit" is not a tamper at all and the (d) vector would pass while +/// testing nothing. That split is blob-dependent, so it is NOT pinned as a +/// constant — `real_opening_is_a_usable_tamper_target` asserts uniqueness at +/// run time on whatever blob it is handed. +const R1F_EPOCH: usize = 0; +const R1F_TABLE: usize = 0; +const R1F_QUERY: usize = 0; + +/// The pinned shape, asserted against the real proof rather than read from it — +/// program shape is compile-time by construction, so if the fixture ever moves, +/// this must fail loudly rather than quietly recompile to a new program. +const R1F_SHAPE: MerkleOpeningShape = MerkleOpeningShape { + leaf_values: 20, + depth: 20, +}; + +/// The opening and its recovered leaf index, resolved once per test binary. +/// +/// The index costs a `2^depth` sweep (~4 s at depth 20) because `iota` is a +/// transcript challenge and is not in the proof; see +/// [`MainTraceOpening::indices_that_verify`]. Sharing it across the tests that +/// need it keeps that to one sweep. +fn r1f_opening() -> &'static (MainTraceOpening, usize) { + use std::sync::OnceLock; + static CELL: OnceLock<(MainTraceOpening, usize)> = OnceLock::new(); + CELL.get_or_init(|| { + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + let opening = MainTraceOpening::extract(&archive, R1F_EPOCH, R1F_TABLE, R1F_QUERY); + let hits = opening.indices_that_verify(); + assert_eq!( + hits.len(), + 1, + "the authenticated opening must sit at exactly one index, else the \ + index-tamper vector tests nothing; got {hits:?}" + ); + (opening, hits[0]) + }) +} + +fn merkle_arenas(opening: &MainTraceOpening, index: usize) -> Vec> { + vec![ + opening.leaf_arena(), + opening.sibling_arena(), + vec![super::word::base_word(FE::from(index as u64))], + opening.root_arena(), + ] +} + +/// Scrutinises the oracle before anything is built on it: the opening really is +/// what the leg assumes, and PRODUCTION's own path check accepts it. +#[test] +fn real_opening_is_a_usable_tamper_target() { + let (opening, index) = r1f_opening(); + assert_eq!( + opening.depth(), + R1F_SHAPE.depth, + "the fixture's tree depth moved; R1F_SHAPE is program shape and must be updated deliberately" + ); + assert_eq!( + opening.values.len(), + R1F_SHAPE.leaf_values, + "the fixture's column count moved" + ); + assert_eq!(opening.num_columns, R1F_SHAPE.columns()); + assert!( + opening.verifies_at(*index), + "production's own checker must accept the opening we are about to \ + authenticate in the machine" + ); + assert!( + !opening.verifies_at(index ^ 1), + "flipping the low index bit must break production's check" + ); + println!( + "R1f target: epoch {R1F_EPOCH} table {R1F_TABLE} query {R1F_QUERY} — \ + {} columns, row pair = {} values, depth {}, index {index}", + opening.num_columns, + opening.values.len(), + opening.depth() + ); +} + +/// ★ The headline: the machine walks a real opening to a real committed root, +/// PROVED and verified. +/// +/// Two independent things are checked. The published root equals the root the +/// proof committed to — that is the authentication, and its oracle is the proof +/// itself. And the machine proof verifies against those published words — that +/// is what makes it a proof rather than an execution, which per method rule 2 +/// is the only thing that says anything about the chips. +#[test] +fn keccak_merkle_walk_authenticates_a_real_opening() { + let opts = options(); + let (opening, index) = r1f_opening(); + let program = keccak_merkle_opening_program(R1F_SHAPE); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &merkle_arenas(opening, *index), &opts) + .expect("the honest opening must execute and prove"); + + assert_eq!( + digest_bytes(&proved.public_words), + opening.root, + "the walked root must be the root the proof committed to" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the authenticated opening must verify" + ); +} + +/// One tamper vector: corrupted arenas, plus the root those arenas really fold +/// to — which is what lets the same vector be run both incoherently (claiming +/// the real root) and coherently (claiming its own). +struct TamperVector { + what: &'static str, + arenas: Vec>, + root: [u8; 32], +} + +/// ★ (d) Tamper, both ways round, for all three inputs the walk consumes. +/// +/// INCOHERENT: change one input and leave the claimed root alone. The +/// in-machine root assert makes the program unexecutable — the earliest and +/// loudest failure, and the one that shows the assert is load-bearing. +/// +/// COHERENT (method rule 4): change the input AND supply the root that input +/// really folds to, so every value in the run is consistent with every other, +/// nothing asserts, and a proof comes out. The forgery then fails on the one +/// thing it cannot fake — the published root is not the root the proof +/// committed to, so a verifier claiming the real one rejects. +#[test] +fn tampered_merkle_opening_rejects() { + let opts = options(); + let (opening, index) = r1f_opening(); + let program = keccak_merkle_opening_program(R1F_SHAPE); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove(&program, &artifacts, &merkle_arenas(opening, *index), &opts) + .expect("honest prove"); + + let mut vectors: Vec = Vec::new(); + + // 1. A wrong sibling at the leaf level. + { + let mut siblings = opening.siblings.clone(); + siblings[0][0] ^= 1; + let mut arenas = merkle_arenas(opening, *index); + arenas[1] = siblings + .iter() + .flat_map(super::proof_arena::commitment_words) + .collect(); + let root = super::proof_arena::walk_to_root(opening.leaf_hash(), *index, &siblings); + vectors.push(TamperVector { + what: "wrong sibling", + arenas, + root, + }); + } + + // 2. Wrong index bits: the same leaf and the same path, walked in the other + // order at level 0. + { + let bad = index ^ 1; + let arenas = merkle_arenas(opening, bad); + let root = super::proof_arena::walk_to_root(opening.leaf_hash(), bad, &opening.siblings); + vectors.push(TamperVector { + what: "wrong index bits", + arenas, + root, + }); + } + + // 3. A wrong opened value: one field element of the row pair. + { + let mut tampered = MainTraceOpening { + root: opening.root, + values: opening.values.clone(), + num_columns: opening.num_columns, + siblings: opening.siblings.clone(), + }; + tampered.values[0] = &tampered.values[0] + FE::from(1u64); + let mut arenas = merkle_arenas(opening, *index); + arenas[0] = tampered.leaf_arena(); + let root = + super::proof_arena::walk_to_root(tampered.leaf_hash(), *index, &tampered.siblings); + vectors.push(TamperVector { + what: "wrong leaf value", + arenas, + root, + }); + } + + for TamperVector { + what, + arenas, + root: forged_root, + } in vectors + { + assert_ne!( + forged_root, opening.root, + "{what}: the tamper must actually move the root, or the vector is vacuous" + ); + + // Incoherent: still claiming the real root. + let err = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .err() + .unwrap_or_else(|| panic!("{what}: claiming the real root must not execute")); + println!("R1f tamper {what}: incoherent run rejected with {err:?}"); + + // Coherent: claim the root the tampered inputs really reach. + let mut coherent = arenas; + coherent[3] = super::proof_arena::commitment_words(&forged_root).to_vec(); + let proved = lfm_prove(&program, &artifacts, &coherent, &opts) + .unwrap_or_else(|e| panic!("{what}: the coherent forgery must prove: {e:?}")); + assert_eq!( + digest_bytes(&proved.public_words), + forged_root, + "{what}: the coherent forgery must publish its own root" + ); + assert_ne!( + proved.public_words, honest.public_words, + "{what}: the forgery must not publish the honest root" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "{what}: claiming the real committed root for a forged walk must reject" + ); + } +} + +/// Main-trace cells one byteswap costs: one `LFM_BITDEC` row and 64 `LFM_BALU` +/// rows, each at its chip's non-preprocessed width — the same accounting +/// [`super::airs::lfm_cell_counts`] uses. +pub(super) fn byteswap_cells() -> u64 { + use super::chips::{balu, bitdec}; + use super::layout; + let bitdec_w = (bitdec::cols::NUM_COLUMNS - layout::bitdec::PREP_WIDTH) as u64; + let balu_w = (balu::cols::NUM_COLUMNS - layout::balu::PREP_WIDTH) as u64; + bitdec_w + 64 * balu_w +} + +/// Main-trace cells one keccak permutation costs: the `LFM_KECCAK` row that +/// requests it, plus the 24 `KECCAK_RND` rounds that carry it. +pub(super) fn permutation_cells() -> u64 { + use super::chips::keccak; + use super::chunking::KECCAK_RND_ROWS_PER_PERMUTATION as ROUNDS; + use super::layout; + use crate::tables::keccak_rnd; + let keccak_w = (keccak::cols::NUM_COLUMNS - layout::keccak::PREP_WIDTH) as u64; + keccak_w + ROUNDS as u64 * keccak_rnd::cols::NUM_COLUMNS as u64 +} + +/// Pins the widths the cost model above is built on. +/// +/// Not ceremony: the inline `// 52 / 252 / 388 / 588 / 788` comments on +/// `chips::keccak::cols` were stale by 4 (R1d widened `PREP_WIDTH` for the +/// reversed-digest columns and they were not updated), and reading them instead +/// of the constants is what produced a wrong per-permutation figure on the first +/// pass through this measurement. A wrong width silently rescales every cell +/// number in `keccak_merkle_opening_cost`, so the widths get an assertion of +/// their own rather than a comment. +#[test] +fn cost_model_widths_are_what_the_chips_declare() { + use super::chips::{balu, bitdec, keccak}; + use super::layout; + use crate::tables::keccak_rnd; + assert_eq!(keccak::cols::NUM_COLUMNS, 792, "LFM_KECCAK total width"); + assert_eq!(layout::keccak::PREP_WIDTH, 56, "LFM_KECCAK preprocessed"); + assert_eq!(balu::cols::NUM_COLUMNS - layout::balu::PREP_WIDTH, 4); + assert_eq!(bitdec::cols::NUM_COLUMNS - layout::bitdec::PREP_WIDTH, 66); + assert_eq!(keccak_rnd::cols::NUM_COLUMNS, 1480); + assert_eq!(byteswap_cells(), 322, "66 + 64 x 4"); + assert_eq!(permutation_cells(), 36_256, "736 + 24 x 1480"); +} + +/// ★ The leg's headline measurement — and it REFUTES the prediction it was set +/// up to confirm. +/// +/// The R1f handoff predicted that byteswapping the opened values would dominate +/// the leaf, "not the hashing", on the strength of the row counts: a 10-column +/// table pays 20 `LFM_BITDEC` + 1280 `LFM_BALU` rows of byteswapping against +/// only 22 permutations. Those row counts are right. The conclusion drawn from +/// them is wrong, because rows of different chips are not comparable units. +/// +/// A byteswap's rows are narrow — `LFM_BALU` carries 4 non-preprocessed columns +/// — while a permutation expands into 24 `KECCAK_RND` rounds at 1480 columns +/// each. Priced in main-trace cells, the unit the proof actually pays in, the +/// measured figures are **322 cells per byteswap against 36,256 per +/// permutation, a factor of 113**. Hashing then dominates at every width in the +/// fixture: 124× at the 10-column table this leg authenticates, 8.9× at 511 +/// columns, 7.4× at 1480. The crossover this test was written to find does not +/// exist. Both terms are linear in the column count — `2c` byteswaps against +/// `≈16c/136` rate blocks — so the ratio flattens near 6.6× rather than +/// inverting. +/// +/// This is why a byteswap chiplet is NOT the lever it looked like, and the +/// measurement rather than the intuition is what says so. The attribution is +/// MARGINAL (real rows, not padded), so it answers "what does one more column +/// cost" and not "what does this proof cost"; the whole-program figure is +/// printed alongside because the fixed floor — `BITWISE` is 2^20 rows whatever +/// the program does — dwarfs both terms at these sizes. +#[test] +fn keccak_merkle_opening_cost() { + let (opening, _) = r1f_opening(); + println!( + "one byteswap = {} main cells; one permutation = {} main cells ({:.0}x)", + byteswap_cells(), + permutation_cells(), + permutation_cells() as f64 / byteswap_cells() as f64, + ); + println!("shape instrs keccak bitdec balu select lanes"); + let mut shapes = vec![R1F_SHAPE]; + // Two wider tables from the same fixture, to show the scaling rather than + // assert a single point. 511 and 1480 columns are real widths in it. + for columns in [511usize, 1480] { + shapes.push(MerkleOpeningShape { + leaf_values: 2 * columns, + depth: R1F_SHAPE.depth, + }); + } + for shape in &shapes { + let program = keccak_merkle_opening_program(*shape); + println!( + "{:>4} cols d={:<3} {:>7} {:>7} {:>7} {:>7} {:>7} {:>7}", + shape.columns(), + shape.depth, + program.instrs.len(), + program.groups.keccak.real_rows, + program.groups.bitdec.real_rows, + program.groups.balu.real_rows, + program.groups.select.real_rows, + program.groups.lanes.real_rows, + ); + } + + // The same three shapes priced in main-trace cells, which is where the + // prediction inverts. `swap` counts only the byteswapping; `hash` counts + // every permutation (leaf blocks and walk levels alike). + println!("shape swap cells hash cells hash/swap whole program"); + for shape in &shapes { + let program = keccak_merkle_opening_program(*shape); + let swap = shape.leaf_values as u64 * byteswap_cells(); + let hash = program.groups.keccak.real_rows as u64 * permutation_cells(); + let (main, _aux) = super::airs::lfm_cell_counts(&program); + println!( + "{:>4} cols d={:<3} {:>12} {:>12} {:>11.1} {:>15}", + shape.columns(), + shape.depth, + swap, + hash, + hash as f64 / swap as f64, + main, + ); + assert!( + hash > swap, + "{} columns: hashing must dominate — if this ever flips, the \ + byteswap-chiplet argument becomes live and the docs above are stale", + shape.columns() + ); + } + + // Pin the real shape's decomposition, so a regression in either half shows. + let program = keccak_merkle_opening_program(R1F_SHAPE); + let leaf_bytes = 8 * R1F_SHAPE.leaf_values; + let leaf_perms = super::keccak_host::num_blocks(leaf_bytes); + assert_eq!( + program.groups.keccak.real_rows, + leaf_perms + R1F_SHAPE.depth, + "one permutation per rate block of the leaf, plus one per level" + ); + assert_eq!( + program.groups.bitdec.real_rows, + R1F_SHAPE.leaf_values + 1, + "one decomposition per opened value, plus one for the index" + ); + assert_eq!( + program.groups.balu.real_rows, + 64 * R1F_SHAPE.leaf_values + 8 * 2, + "64 rows per byteswap, plus the two root asserts (4 sub + 4 div each)" + ); + assert_eq!( + program.groups.select.real_rows, + 2 * R1F_SHAPE.depth, + "two selects per level: a digest is two words and both swap together" + ); + println!( + "R1f leaf: {} values -> {leaf_bytes} bytes -> {leaf_perms} permutations, \ + against {} bitdec + {} balu rows of byteswapping", + R1F_SHAPE.leaf_values, + R1F_SHAPE.leaf_values, + 64 * R1F_SHAPE.leaf_values, + ); + // The fixed floor, for scale: BITWISE alone is 2^20 rows regardless of what + // the program does, so nothing above is a claim about total proof cost. + let (main, aux) = super::airs::lfm_cell_counts(&program); + println!("R1f whole program: {main} main cells, {aux} aux cells"); + assert_eq!(opening.values.len(), R1F_SHAPE.leaf_values); +} + +/// ⚠ STANDING EVIDENCE that the fixture is not reproducible. +/// +/// Two `generate()` calls on identical inputs — same ELF, same empty input, +/// same epoch size, same options — produce different blobs, and the difference +/// is semantic rather than archive padding: sub-proof roots move, which moves +/// the Fiat-Shamir challenges, which opens different leaves. Measured at ~65k +/// of 587k bytes differing. +/// +/// This is why R1f pins SHAPE (`R1F_SHAPE`) and RECOVERS per-blob values +/// (`r1f_opening`) instead of pinning a query index or a root. A pinned index +/// would pass for exactly as long as the cache survived and then fail on the +/// next cold run, which is the worst possible failure mode for a fixture. +/// +/// `#[ignore]`d because it costs two full continuation proofs (~17 s) and +/// asserts nothing the rest of the suite depends on. Run it with +/// `--ignored` when the question comes up again; it is here so the claim has +/// evidence attached rather than being folklore in a status log. +#[test] +#[ignore] +fn fixture_generation_is_not_reproducible() { + let (a, _) = proof_fixture::generate(); + let (b, _) = proof_fixture::generate(); + let differing = a.iter().zip(b.iter()).filter(|(x, y)| x != y).count(); + println!( + "two fixture generations: {} vs {} bytes, {differing} differing", + a.len(), + b.len() + ); + assert_ne!( + a, b, + "if this ever passes, the prover became reproducible \ + and the no-pinning rule in proof_fixture can be relaxed" + ); + + // The difference reaches the committed data, so it is not archive padding. + let (aa, bb) = ( + super::proof_fixture::FixtureArchive::open(&a), + super::proof_fixture::FixtureArchive::open(&b), + ); + assert_ne!( + super::proof_arena::epoch_main_roots(&aa, 0), + super::proof_arena::epoch_main_roots(&bb, 0), + "the divergence must be semantic — if the roots match, this is padding \ + and the no-pinning rule is too strong" + ); + + // The tree this leg authenticates keeps its SHAPE across runs, which is the + // property R1F_SHAPE relies on, while its opened values do not. + let oa = MainTraceOpening::extract(&aa, 0, 0, 0); + let ob = MainTraceOpening::extract(&bb, 0, 0, 0); + assert_eq!( + oa.depth(), + ob.depth(), + "tree depth is shape and must be stable" + ); + assert_eq!( + oa.num_columns, ob.num_columns, + "column count is shape and must be stable" + ); + println!( + "table 0 across runs: depth {} stable, same root = {}, same opened values = {}", + oa.depth(), + oa.root == ob.root, + oa.values == ob.values + ); +} + +// ============ R1g (ii): the cross-epoch L2G commitment binding ============ +// +// The first obligation of the chaining leg, and the first time the machine +// reads ACROSS structures: each epoch's own committed L2G root against the +// corresponding sub-proof of the one global proof. R1f stayed inside a single +// epoch's own sub-proof. + +use super::programs::l2g_binding_program; + +/// Epochs the binding program is compiled for. SHAPE, not a blob-derived +/// constant: the epoch count follows from the inner ELF and +/// `FIXTURE_EPOCH_LOG2`, not from anything the prover chooses per run. Asserted +/// against the real bundle rather than read from it, so a fixture change is +/// loud — the same discipline `R1F_SHAPE` uses. +const R1G_EPOCHS: usize = 2; + +/// The `i`-th 32-byte root in a program's published words. +fn published_root(public: &[(u32, LfmWord)], i: usize) -> [u8; 32] { + use math::field::traits::IsPrimeField; + let mut out = [0u8; 32]; + for h in 0..8 { + let lane = public[2 * i + h / 4].1[h % 4]; + let half = crate::tables::types::GoldilocksField::canonical(lane.value()) as u32; + out[4 * h..4 * h + 4].copy_from_slice(&half.to_le_bytes()); + } + out +} + +fn l2g_arenas( + epoch: &[stark::config::Commitment], + global: &[stark::config::Commitment], +) -> Vec> { + use super::proof_arena::commitments_to_arena; + vec![commitments_to_arena(epoch), commitments_to_arena(global)] +} + +/// The real bundle's L2G roots, resolved once. +fn r1g_l2g_roots() -> &'static ( + Vec, + Vec, +) { + use std::sync::OnceLock; + static CELL: OnceLock<( + Vec, + Vec, + )> = OnceLock::new(); + CELL.get_or_init(|| { + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + let epoch = super::proof_arena::epoch_l2g_roots(&archive); + let global = super::proof_arena::global_l2g_roots(&archive, epoch.len()); + (epoch, global) + }) +} + +/// Scrutinises the oracle before building on it: production's binding really +/// does hold on the real bundle, and — the part that matters for (d) — the +/// per-epoch roots are DISTINCT, so swapping two of them is a real tamper. +/// +/// Without that second check the position-swap vector would pass while testing +/// nothing, exactly as an index-bit flip would have on a degenerate tree in R1f. +#[test] +fn l2g_binding_holds_on_the_real_bundle() { + let (epoch, global) = r1g_l2g_roots(); + assert_eq!( + epoch.len(), + R1G_EPOCHS, + "the fixture's epoch count moved; R1G_EPOCHS is program shape" + ); + assert_eq!(epoch, global, "production's own L2G binding must hold"); + assert!( + epoch.iter().all(|r| *r != [0u8; 32]), + "every L2G root must be nonzero" + ); + for i in 0..epoch.len() { + for j in (i + 1)..epoch.len() { + assert_ne!( + epoch[i], epoch[j], + "epochs {i} and {j} share an L2G root, so swapping them is not a tamper" + ); + } + } + println!( + "R1g(ii): {} epochs, binding holds, roots pairwise distinct", + epoch.len() + ); +} + +/// ★ The binding, emitted and PROVED against the real bundle. +#[test] +fn l2g_binding_proves_and_verifies() { + let opts = options(); + let (epoch, global) = r1g_l2g_roots(); + let program = l2g_binding_program(R1G_EPOCHS); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &l2g_arenas(epoch, global), &opts) + .expect("the honest binding must execute and prove"); + + for (i, root) in epoch.iter().enumerate().take(R1G_EPOCHS) { + assert_eq!( + published_root(&proved.public_words, i), + *root, + "published root {i} must be epoch {i}'s committed L2G root" + ); + } + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the L2G binding must verify" + ); +} + +/// ★ (d) Tamper. Same two-mode structure as R1f: incoherent trips the assert, +/// coherent satisfies every assert and then fails on the published roots. +/// +/// The SWAP vector is the one that matters — it is what "position-sensitive" +/// means. Epoch `i`'s root must meet global sub-proof `i` and no other, so a +/// bundle whose L2G roots are correct as a SET but wrong in ORDER must reject. +#[test] +fn tampered_l2g_binding_rejects() { + let opts = options(); + let (epoch, global) = r1g_l2g_roots(); + let program = l2g_binding_program(R1G_EPOCHS); + let artifacts = build_artifacts(&program, &opts); + let honest = + lfm_prove(&program, &artifacts, &l2g_arenas(epoch, global), &opts).expect("honest prove"); + + // Incoherent: one side changed, the other left honest — the assert must fire. + // + // The two byte positions are chosen, not arbitrary. A digest spans TWO + // machine words (bytes 0-15 and 16-31) and each needs its own + // `assert_word_eq`; vectors that all land in byte 0 would leave an emitter + // that compares only the first word completely uncaught. Byte 31 covers the + // second word. Falsification F32 confirmed the gap was real before this. + let mut bad_epoch = epoch.clone(); + bad_epoch[0][0] ^= 1; + let mut bad_global = global.clone(); + bad_global[1][31] ^= 1; + let swapped_one_side = { + let mut s = epoch.clone(); + s.swap(0, 1); + s + }; + for (what, arenas) in [ + ("wrong epoch root", l2g_arenas(&bad_epoch, global)), + ("wrong global root", l2g_arenas(epoch, &bad_global)), + ( + "epoch roots swapped on one side", + l2g_arenas(&swapped_one_side, global), + ), + ] { + let err = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .err() + .unwrap_or_else(|| panic!("{what}: must not execute")); + println!("R1g tamper {what}: rejected with {err:?}"); + } + + // Coherent: BOTH sides swapped consistently. Every assert passes — the + // bundle's roots are the right set — but the order is wrong, so the + // published roots are not the ones the real bundle commits to. + let mut swapped = epoch.clone(); + swapped.swap(0, 1); + let proved = lfm_prove(&program, &artifacts, &l2g_arenas(&swapped, &swapped), &opts) + .expect("the coherent swap must prove — every assert is satisfied"); + assert_eq!( + published_root(&proved.public_words, 0), + epoch[1], + "the coherent forgery publishes the swapped order" + ); + assert_ne!( + proved.public_words, honest.public_words, + "a reordered binding must not publish the honest roots" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "claiming the real per-epoch roots for a reordered binding must reject" + ); +} + +// ============ R1g (iii): the attestation's program id ============ + +use super::programs::{ProgramIdShape, program_id_program}; + +/// Arena for a `program_id` fold, each field in its own halves. +fn program_id_arenas( + elf_digest: &[u8; 32], + pc_start: u64, + decode: &stark::config::Commitment, + pages: &[(u64, stark::config::Commitment)], +) -> Vec> { + let mut halves = keccak_host::pack_stream(elf_digest); + halves.extend(keccak_host::pack_stream(&pc_start.to_le_bytes())); + halves.extend(keccak_host::pack_stream(decode)); + for (base, c) in pages { + halves.extend(keccak_host::pack_stream(&base.to_le_bytes())); + halves.extend(keccak_host::pack_stream(c)); + } + vec![halves.into_iter().map(super::word::base_word).collect()] +} + +/// The real fixture's program-id inputs. +fn r1g_program_id_inputs() -> ( + [u8; 32], + u64, + stark::config::Commitment, + Vec<(u64, stark::config::Commitment)>, +) { + use super::proof_arena; + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + let elf_bytes = proof_arena::inner_elf(&archive).to_vec(); + let elf = executor::elf::Elf::load(&elf_bytes).expect("the fixture's inner ELF must load"); + ( + crate::statement::elf_digest(&elf_bytes), + elf.entry_point, + proof_arena::decode_commitment(&archive), + proof_arena::page_commitments(&archive), + ) +} + +/// ★ The fold, PROVED, bit-exact against production's own `program_id_from_digest`. +/// +/// The oracle is the production function, not a local re-implementation. +#[test] +fn program_id_matches_production_on_the_real_fixture() { + let opts = options(); + let (elf_digest, pc_start, decode, pages) = r1g_program_id_inputs(); + assert!( + pages.is_empty(), + "the fibonacci fixture is expected to touch no data pages; if this \ + changes, the shape below must change with it" + ); + let shape = ProgramIdShape { + num_pages: pages.len(), + }; + let program = program_id_program(shape); + let artifacts = build_artifacts(&program, &opts); + let arenas = program_id_arenas(&elf_digest, pc_start, &decode, &pages); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + + let expected = crate::recursion::program_id_from_digest(&elf_digest, pc_start, &decode, &pages); + assert_eq!( + digest_bytes(&proved.public_words), + expected, + "the machine's program id must equal production's" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the program-id fold must verify" + ); + println!( + "R1g(iii): {} pages, {} bytes hashed, tag {} bytes (shift {})", + shape.num_pages, + shape.byte_len(), + crate::recursion::PROGRAM_ID_TAG.len(), + crate::recursion::PROGRAM_ID_TAG.len() % 4, + ); +} + +/// ★ The page loop, exercised. The fixture has ZERO page commitments, so the +/// sorted-page path is present-but-untested on real data — the caveat the team +/// lead flagged for the supplied roots applies here too. This drives it with a +/// synthetic shape against the same production oracle, so "it compiles" is not +/// mistaken for "it is covered". +/// +/// Proved, not just executed: the fold's byte length changes with the page +/// count, which moves every padding position, and only a proof sees the keccak +/// chip agree with the executor about that. +#[test] +fn program_id_folds_pages_in_the_production_layout() { + let opts = options(); + let (elf_digest, pc_start, decode, _) = r1g_program_id_inputs(); + for num_pages in [1usize, 3] { + let pages: Vec<(u64, stark::config::Commitment)> = (0..num_pages) + .map(|i| { + let base = 0x1000u64 * (i as u64 + 1); + let mut c = [0u8; 32]; + for (j, b) in c.iter_mut().enumerate() { + *b = (17 * i + j) as u8; + } + (base, c) + }) + .collect(); + let shape = ProgramIdShape { num_pages }; + let program = program_id_program(shape); + let artifacts = build_artifacts(&program, &opts); + let arenas = program_id_arenas(&elf_digest, pc_start, &decode, &pages); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + let expected = + crate::recursion::program_id_from_digest(&elf_digest, pc_start, &decode, &pages); + assert_eq!( + digest_bytes(&proved.public_words), + expected, + "{num_pages} pages: the machine's fold must match production's" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "{num_pages} pages: the fold must verify" + ); + } +} + +/// ★ (d) Tamper: every folded field must move the id. +/// +/// Coherent by construction — nothing asserts, so each forgery proves cleanly +/// and fails on the published id, which is the whole mechanism: the id IS the +/// claim, and a consumer comparing against its own recompute rejects. +#[test] +fn tampered_program_id_inputs_change_the_id() { + let opts = options(); + let (elf_digest, pc_start, decode, _) = r1g_program_id_inputs(); + let pages = vec![(0x1000u64, [7u8; 32]), (0x2000u64, [9u8; 32])]; + let shape = ProgramIdShape { + num_pages: pages.len(), + }; + let program = program_id_program(shape); + let artifacts = build_artifacts(&program, &opts); + let honest = lfm_prove( + &program, + &artifacts, + &program_id_arenas(&elf_digest, pc_start, &decode, &pages), + &opts, + ) + .expect("honest prove"); + + let mut d2 = elf_digest; + d2[31] ^= 1; + let mut dec2 = decode; + dec2[0] ^= 1; + let mut pages_value = pages.clone(); + pages_value[1].1[31] ^= 1; + let mut pages_base = pages.clone(); + pages_base[0].0 ^= 1; + let mut pages_order = pages.clone(); + pages_order.swap(0, 1); + + for (what, arenas) in [ + ( + "elf digest", + program_id_arenas(&d2, pc_start, &decode, &pages), + ), + ( + "entry point", + program_id_arenas(&elf_digest, pc_start ^ 1, &decode, &pages), + ), + ( + "decode root", + program_id_arenas(&elf_digest, pc_start, &dec2, &pages), + ), + ( + "page commitment", + program_id_arenas(&elf_digest, pc_start, &decode, &pages_value), + ), + ( + "page base", + program_id_arenas(&elf_digest, pc_start, &decode, &pages_base), + ), + ( + "page ORDER", + program_id_arenas(&elf_digest, pc_start, &decode, &pages_order), + ), + ] { + let forged = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a change must move the program id" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "{what}: claiming the honest id must reject" + ); + } +} + +// ============ R1g (i): the REGISTER preprocessed derivation ============ + +use super::programs::{RegisterDerivationShape, lde_probe_program, register_derivation_program}; + +/// The inner proof's blowup for the real target (`lfm-RESUME.md`), and the +/// sweep the leg was asked to measure. +const INNER_BLOWUPS: [usize; 3] = [2, 4, 8]; + +/// Every production `ProofOptions` uses this offset. +const PRODUCTION_COSET_OFFSET: u64 = 3; + +fn derivation_shape(blowup: usize) -> RegisterDerivationShape { + RegisterDerivationShape { + blowup, + coset_offset: PRODUCTION_COSET_OFFSET, + } +} + +/// The inner proof's options at a given blowup — the ones whose REGISTER +/// commitment is being derived, which are NOT the LFM proof's own options. +fn inner_options(blowup: usize) -> ProofOptions { + let opts = GoldilocksCubicProofOptions::with_blowup(blowup as u8).expect("inner options"); + assert_eq!( + opts.coset_offset, PRODUCTION_COSET_OFFSET, + "the shape constant must track production's coset offset" + ); + opts +} + +/// Deterministic pseudo-random `u32`s (splitmix64, high word) — a register file +/// whose rows are pairwise distinct and all nonzero, asserted by +/// [`the_fixture_register_boundary_is_mostly_zeros`] rather than assumed. +fn synthetic_register_file(seed: u64) -> Vec { + let mut state = seed; + (0..crate::tables::register::NUM_REGISTER_ADDRESSES) + .map(|_| { + state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + ((z ^ (z >> 31)) >> 32) as u32 + }) + .collect() +} + +fn register_arenas(init: &[u32], fini: &[u32]) -> Vec> { + let column = |v: &[u32]| { + v.iter() + .map(|&x| super::word::base_word(FE::from(x as u64))) + .collect() + }; + vec![column(init), column(fini)] +} + +/// ★ Oracle scrutiny, before anything is built on it: the row order this leg +/// restates really is production's. +/// +/// `register_word_address_list` is private, so the program assembles the same +/// list from the public per-register helper. This pins the result against the +/// layout the table's own docs state (x0–x31 at 0..63, x254 at 508, x255 at +/// 510–511) — a check with teeth only because it is written from the docs +/// rather than from the assembly under test. +#[test] +fn the_register_offset_column_is_productions_row_order() { + use crate::tables::register::{ + NUM_REGISTER_ADDRESSES, PC_HI_INDEX, PC_LO_INDEX, X254_INDEX, register_word_addresses, + }; + let mut expected: Vec = (0..64).collect(); + expected.extend([508, 510, 511]); + assert_eq!(expected.len(), NUM_REGISTER_ADDRESSES); + + let mut derived = Vec::new(); + for reg in 0..32u8 { + derived.extend(register_word_addresses(reg)); + } + derived.extend(register_word_addresses(254)); + derived.extend(register_word_addresses(255)); + assert_eq!( + derived, expected, + "the REGISTER row order moved; the derivation's OFFSET column is built \ + from this list and every derived root depends on it" + ); + // The positional constants the table exports must agree with it too. + assert_eq!(derived[X254_INDEX], 508); + assert_eq!(derived[PC_LO_INDEX], 510); + assert_eq!(derived[PC_HI_INDEX], 511); +} + +/// ★ The emitted LDE against production's own `interpolate_fft` + +/// `evaluate_polynomial_on_lde_domain`, at sizes and offsets production never +/// takes. +/// +/// The register leg only ever runs the transform at `n = 128`, offset 3. Per +/// `lfm-target-shape.md`'s degenerate-parameter rule, that means the real +/// differential cannot tell a general emitter from one that is accidentally +/// right there, so the synthetic sizes are the only witness. Execute-only: this +/// is pure `LFM_BALU` arithmetic and the proved test below covers the chips. +#[test] +fn the_emitted_lde_matches_productions_transform() { + use math::polynomial::Polynomial; + use stark::prover::evaluate_polynomial_on_lde_domain; + + for (n, blowup, offset) in [ + (2usize, 2usize, 3u64), + (4, 2, 3), + (8, 2, 3), + (8, 4, 3), + (8, 8, 3), + (16, 4, 7), + (32, 2, 1), + (8, 1, 3), + (128, 2, 3), + ] { + let program = lde_probe_program(n, blowup, offset); + validate(&program).expect("admission"); + let source = synthetic_register_file(n as u64 * 31 + offset); + let values: Vec = (0..n) + .map(|i| FE::from(source[i % source.len()] as u64 + 1)) + .collect(); + let arenas = vec![values.iter().copied().map(super::word::base_word).collect()]; + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("n={n} blowup={blowup}: execution failed: {e:?}")); + + let poly = Polynomial::interpolate_fft::(&values) + .expect("interpolate"); + let expected = + evaluate_polynomial_on_lde_domain(&poly, blowup, n, &FE::from(offset)).expect("lde"); + let got: Vec = exec.public_words.iter().map(|(_, w)| w[0]).collect(); + assert_eq!( + got.len(), + n * blowup, + "n={n} blowup={blowup}: the extension must cover the whole domain" + ); + assert_eq!( + got, expected, + "n={n} blowup={blowup} offset={offset}: the emitted LDE must equal production's" + ); + } +} + +/// ★ The whole derivation, against production's own +/// `compute_precomputed_commitment_with_fini`, across the blowup sweep. +/// +/// Two register files per blowup and they do different jobs. The REAL fixture's +/// file is what the target runs on; a SYNTHETIC one where all 67 entries are +/// distinct and nonzero is what makes the test able to fail — a real register +/// file is mostly zeros, so an emitter that dropped or duplicated rows could +/// agree with production on it and disagree everywhere else. +#[test] +fn the_register_derivation_matches_production() { + for blowup in INNER_BLOWUPS { + let shape = derivation_shape(blowup); + let opts = inner_options(blowup); + let program = register_derivation_program(shape); + validate(&program).expect("admission"); + + for (what, init, fini) in register_file_cases() { + let arenas = register_arenas(&init, &fini); + let exec = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .unwrap_or_else(|e| panic!("blowup {blowup} / {what}: execution failed: {e:?}")); + let expected = crate::tables::register::compute_precomputed_commitment_with_fini( + &opts, &init, &fini, + ); + assert_eq!( + digest_bytes(&exec.public_words), + expected, + "blowup {blowup} / {what}: the derived root must equal production's" + ); + } + } +} + +/// The register files the differential runs on: the fixture's real boundary +/// pair, plus synthetics that exercise every row. +fn register_file_cases() -> Vec<(&'static str, Vec, Vec)> { + let (init, fini) = fixture_register_boundary(); + vec![ + ("the fixture's real epoch boundary", init, fini), + ( + "a synthetic file with every row distinct", + synthetic_register_file(1), + synthetic_register_file(2), + ), + ( + "init and fini equal (an epoch that changed nothing)", + synthetic_register_file(3), + synthetic_register_file(3), + ), + ] +} + +/// Epoch 0's real `(register_init, reg_fini)` from the proof fixture — the +/// verifier-derived INIT from the entry point and the epoch's bound FINI. +fn fixture_register_boundary() -> (Vec, Vec) { + use std::sync::OnceLock; + static CELL: OnceLock<(Vec, Vec)> = OnceLock::new(); + CELL.get_or_init(|| { + let blob = proof_fixture::load_or_generate(&fixture_cache()); + let archive = super::proof_fixture::FixtureArchive::open(&blob); + super::proof_arena::register_boundary(&archive, 0) + }) + .clone() +} + +/// ★ Scrutinise the fixture boundary before trusting the differential that +/// runs on it. +/// +/// A register file is mostly zeros and `init` differs from `fini` in only a +/// handful of places, so "the real case passed" is weak evidence on its own. +/// This says exactly how weak, in numbers, which is what justifies the +/// synthetic cases carrying the load in +/// [`the_register_derivation_matches_production`]. +#[test] +fn the_fixture_register_boundary_is_mostly_zeros() { + use crate::tables::register::{NUM_REGISTER_ADDRESSES, PC_LO_INDEX}; + let (init, fini) = fixture_register_boundary(); + assert_eq!(init.len(), NUM_REGISTER_ADDRESSES); + assert_eq!(fini.len(), NUM_REGISTER_ADDRESSES); + + let nonzero = |v: &[u32]| v.iter().filter(|&&x| x != 0).count(); + let differing = init.iter().zip(&fini).filter(|(a, b)| a != b).count(); + println!( + "R1g(i) fixture boundary: init {}/{} nonzero, fini {}/{} nonzero, \ + {differing} rows differ (pc {} -> {})", + nonzero(&init), + NUM_REGISTER_ADDRESSES, + nonzero(&fini), + NUM_REGISTER_ADDRESSES, + init[PC_LO_INDEX], + fini[PC_LO_INDEX], + ); + assert_ne!( + init, fini, + "an epoch that changed no register would be a degenerate case" + ); + assert!( + nonzero(&fini) > 0, + "an all-zero fini would make the differential blind to the FINI column" + ); + // Nonzero is not enough: rows that share a value are still indistinguishable + // to a differential, so the synthetic file has to be pairwise DISTINCT for + // "it exercises every row" to mean anything. + let synthetic = synthetic_register_file(1); + assert_eq!( + nonzero(&synthetic), + NUM_REGISTER_ADDRESSES, + "the synthetic file must exercise every row, which the real one does not" + ); + let distinct: std::collections::HashSet = synthetic.iter().copied().collect(); + assert_eq!( + distinct.len(), + NUM_REGISTER_ADDRESSES, + "the synthetic file's rows must be pairwise distinct, else a dropped or \ + duplicated row could still agree with production" + ); +} + +/// ★ The measurement the leg was asked for: permutations against the predicted +/// 255 / 511 / 1023, and the derivation's share of an epoch verify. +/// +/// The prediction is `2·leaves − 1` with `leaves = 128·blowup / ROWS_PER_LEAF`, +/// i.e. `128·blowup − 1`. A miss is not something to round off: it would mean +/// the leaf grouping, the domain size or the padding is not what the design +/// says, so this asserts rather than prints. +#[test] +fn register_derivation_cost() { + // `lfm-target-shape.md`'s *Scale*: keccak permutations per epoch verify. + let epoch_permutations = |blowup: usize| match blowup { + 2 => Some(1_400_000f64), + 8 => Some(460_000f64), + _ => None, + }; + println!( + "one permutation = {} main cells; one byteswap = {} main cells; \ + KECCAK_RND chunk ceiling {} rows", + permutation_cells(), + byteswap_cells(), + super::chunking::KECCAK_RND_MAX_CHUNK_ROWS, + ); + println!( + "blowup rows leaves perms predicted instrs const balu bitdec keccak \ + main cells % of epoch hashing" + ); + for blowup in INNER_BLOWUPS { + let shape = derivation_shape(blowup); + let predicted = 128 * blowup - 1; + let program = register_derivation_program(shape); + let (main, _aux) = super::airs::lfm_cell_counts(&program); + let share = epoch_permutations(blowup) + .map(|total| format!("{:.4}%", 100.0 * shape.permutations() as f64 / total)) + .unwrap_or_else(|| "-".to_string()); + println!( + "{blowup:>6} {:>6} {:>8} {:>7} {:>10} {:>8} {:>7} {:>7} {:>7} {:>7} {:>12} {:>19}", + shape.lde_rows(), + shape.leaves(), + program.groups.keccak.real_rows, + predicted, + program.instrs.len(), + program.groups.const_.real_rows, + program.groups.balu.real_rows, + program.groups.bitdec.real_rows, + program.groups.keccak.real_rows, + main, + share, + ); + assert_eq!( + shape.permutations(), + predicted, + "blowup {blowup}: the shape's own arithmetic must give the predicted count" + ); + assert_eq!( + program.groups.keccak.real_rows, predicted, + "blowup {blowup}: the EMITTED permutation count must be 2·leaves − 1 \ + ({predicted}); a miss means the tree's shape is not what the design says" + ); + // Every leaf is 48 bytes and every parent 64 — one rate block each, so + // the permutation count is exactly the node count and nothing else. + assert_eq!( + super::keccak_host::num_blocks(8 * 3 * stark::commitment::ROWS_PER_LEAF), + 1, + "a three-column row pair must fit one keccak rate block" + ); + assert_eq!( + super::keccak_host::num_blocks(2 * super::edsl::COMMITMENT_BYTES), + 1, + "a Merkle parent must fit one keccak rate block" + ); + + // Where the arithmetic goes, to the row. The transform is + // `2 · (n/2·log₂n butterflies + blowup · (n scalings + n/2·log₂n + // butterflies))` at two rows per butterfly and one per scaling, over + // the TWO dynamic columns; the swap is 64 rows for each of the leaf's + // six values. Pinning the split is what makes a later change to either + // half visible instead of showing up as one moved total. + let n = shape.num_rows() as u64; + let butterflies = n / 2 * n.trailing_zeros() as u64; + let per_column = 2 * butterflies + blowup as u64 * (n + 2 * butterflies); + let transform = 2 * per_column; + let swap = shape.leaves() as u64 * 6 * 64; + assert_eq!( + program.groups.balu.real_rows as u64, + transform + swap, + "blowup {blowup}: LFM_BALU rows must be {transform} of transform plus \ + {swap} of byte swapping" + ); + assert_eq!( + program.groups.bitdec.real_rows, + shape.leaves() * 6, + "blowup {blowup}: one bit decomposition per leaf value — the leaf \ + gadget is `keccak_leaf_hash` reused, not a second one" + ); + // Chunking is not a constraint at this scale and the leg should say so + // rather than leave the next reader to work it out: the whole tree at + // blowup 8 is 1023 permutations against a ceiling of 2^19 ROWS. + assert_eq!( + super::chunking::KeccakChunking::default().chunk_count(shape.permutations()), + 1, + "blowup {blowup}: the register tree must fit one KECCAK_RND chunk" + ); + assert_eq!( + program.groups.select.real_rows, 0, + "blowup {blowup}: a TREE build knows every child's side at emission \ + time, so it must emit no Select at all; routing it through \ + `keccak_merkle_walk` would put one per parent here" + ); + println!( + " blowup {blowup}: transform {transform} balu rows ({:.1}%), \ + byteswap {swap} ({:.1}%); a Select would cost {} cells against a \ + permutation's {}", + 100.0 * transform as f64 / (transform + swap) as f64, + 100.0 * swap as f64 / (transform + swap) as f64, + select_cells(), + permutation_cells(), + ); + } +} + +/// Main-trace cells one `LFM_SELECT` row costs — the unit the tree build avoids +/// by knowing child order at emission time. +fn select_cells() -> u64 { + use super::chips::select; + use super::layout; + (select::cols::NUM_COLUMNS - layout::select::PREP_WIDTH) as u64 +} + +/// ★ The derivation PROVED, not merely executed — and against the REAL +/// fixture's own options rather than a reconstruction of them. +/// +/// The executor mirrors the keccak the chip also does, so an execute-only +/// differential cannot see the `LFM_KECCAK` adapter, the `KECCAK_RND` chunking +/// or the lane plumbing agree with it. Blowup 2 (255 permutations) is the cheap +/// end of the sweep; the shape's arithmetic is what carries 4 and 8, and +/// `register_derivation_cost` asserts it. +/// +/// `fixture_options()` is `MIN_PROOF_OPTIONS` — blowup 2, coset offset 3, the +/// only two fields that reach the commitment — so this is not merely "the +/// machine agrees with a production function on some inputs". The root proved +/// here IS the preprocessed REGISTER commitment epoch 0 of the fixture's own +/// continuation was built against. +#[test] +fn the_register_derivation_proves_and_verifies() { + let opts = options(); + let inner = proof_fixture::fixture_options(); + let shape = RegisterDerivationShape { + blowup: inner.blowup_factor as usize, + coset_offset: inner.coset_offset, + }; + assert_eq!( + shape, + derivation_shape(2), + "the fixture is proved at blowup 2 / offset 3; if that moves, this test is no longer about the fixture's own commitment" + ); + let program = register_derivation_program(shape); + let artifacts = build_artifacts(&program, &opts); + let (init, fini) = fixture_register_boundary(); + let arenas = register_arenas(&init, &fini); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("prove"); + + assert_eq!( + digest_bytes(&proved.public_words), + crate::tables::register::compute_precomputed_commitment_with_fini(&inner, &init, &fini), + "the proved root must equal the fixture epoch's own REGISTER commitment" + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the derivation must verify" + ); + println!( + "R1g(i) proved: blowup {}, {} permutations, {} KECCAK_RND chunks", + shape.blowup, program.groups.keccak.real_rows, artifacts.keccak_rnd_chunks, + ); +} + +/// ★ Tamper: every register word must move the derived root, and claiming the +/// honest root for a tampered file must reject. +/// +/// Coherent by construction — the program asserts nothing, so each forgery +/// proves cleanly and fails on the PUBLISHED root. That is the mechanism +/// working: the root is the claim, and in the assembled verifier it is what +/// Phase A absorbs, so a `reg_fini` the prover did not honour produces a root +/// the epoch's own proof was not made against. +#[test] +fn tampering_the_register_files_moves_the_derived_root() { + let opts = options(); + let shape = derivation_shape(2); + let program = register_derivation_program(shape); + let artifacts = build_artifacts(&program, &opts); + let (init, fini) = fixture_register_boundary(); + let honest = lfm_prove(&program, &artifacts, ®ister_arenas(&init, &fini), &opts) + .expect("honest prove"); + + use crate::tables::register::{NUM_REGISTER_ADDRESSES, PC_HI_INDEX, X254_INDEX}; + // A zero row, a row the epoch changed, and the two whose values are not + // plain GPR words — the PC high half and the synthetic commit index, which + // are the rows an implementation is most likely to mislay. + let mut cases: Vec<(String, Vec, Vec)> = Vec::new(); + for row in [ + 0usize, + 5, + X254_INDEX, + PC_HI_INDEX, + NUM_REGISTER_ADDRESSES - 1, + ] { + let mut i2 = init.clone(); + i2[row] ^= 1; + cases.push((format!("init row {row}"), i2, fini.clone())); + let mut f2 = fini.clone(); + f2[row] ^= 1; + cases.push((format!("fini row {row}"), init.clone(), f2)); + } + + for (what, i, f) in cases { + let forged = lfm_prove(&program, &artifacts, ®ister_arenas(&i, &f), &opts) + .unwrap_or_else(|e| panic!("{what}: a tampered file must still prove: {e:?}")); + assert_ne!( + forged.public_words, honest.public_words, + "{what}: a changed register must move the derived root" + ); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &forged.proof, + &honest.public_words, + &opts, + artifacts.hasher, + ), + "{what}: claiming the honest root must reject" + ); + } +} + +/// ⚠ Documents a LIVE gap — **asserts it SUCCEEDS**, per the guard-test map's +/// convention. If this ever starts failing, something began constraining the +/// arena; re-derive before relaxing it. +/// +/// Production's `reg_fini` is a `Vec`, so every value it can commit is +/// below 2^32 and the type is the whole enforcement. An LFM arena is untyped +/// field elements, and nothing in the derivation narrows them: the program +/// happily extends and commits a column production could not have built. +/// +/// ## What is and is not claimed +/// +/// This is NOT a hole in the derivation. A root over a non-`u32` column matches +/// no commitment a production epoch proof was made against, so the epoch fails +/// — the same argument that makes the derivation a binding at all. What it IS +/// is a place where the machine's accepted set is WIDER than the RV64 +/// verifier's, and the assembled verifier owes one of two things: an arena +/// range check (67 extra `bit_dec`s per column, cheap at this scale), or the +/// argument that no epoch proof can exist over such a column in the first +/// place. That second argument is PLAUSIBLE — REG-C2 puts FINI on the Memory +/// bus as a value word and the memory side decomposes values into bytes — but +/// it is unverified here, and per the standing rule a deferral's safety +/// argument is itself a claim needing evidence. Stated, not assumed. +#[test] +fn the_derivation_extends_a_non_u32_register_value_demonstrating_hazard() { + let shape = derivation_shape(2); + let program = register_derivation_program(shape); + let (init, fini) = fixture_register_boundary(); + + let honest = register_arenas(&init, &fini); + let mut wide = honest.clone(); + // 2^32 — one past every value a `Vec` can hold. + wide[1][0] = super::word::base_word(FE::from(1u64 << 32)); + + let run = |arenas: &[Vec]| { + digest_bytes( + &super::executor::execute(&program, arenas, &super::hash::TestPermutation) + .expect("the machine must accept any felt in the arena") + .public_words, + ) + }; + assert_ne!( + run(&wide), + run(&honest), + "the out-of-range value must actually reach the commitment, else this \ + test documents nothing" + ); +} + +/// The honest-path control for the admission gate as a whole: every kind in +/// `LFM_REGISTRY` still passes, together, in one place. +/// +/// Each program has its own `*_is_admissible` test; this one exists because a +/// rejection rule is worth having only if it rejects nothing that ships, and +/// the cheapest way for a new check to be wrong is to be right about the one +/// program its author had in mind. It also fails loudly when a seventh kind is +/// registered without being run past the registrar here. +#[test] +fn every_registered_program_passes_admission() { + let programs: [(&str, LfmProgram); 6] = [ + ("TrivialV0", trivial_program()), + ("FriToyV0", fri_toy_program()), + ("KeccakChainV0", keccak_chain_program()), + ("KeccakSpongeV0", keccak_sponge_program(KECCAK_SPONGE_LEN)), + ("TranscriptReplayV0", transcript_replay_program()), + ("StatementReplayV0", statement_replay_program()), + ]; + for (name, program) in &programs { + validate(program) + .unwrap_or_else(|e| panic!("registered program {name} must pass admission: {e:?}")); + } +} + +/// Check 9 against a *registered* program's committed group — the object the +/// AIR actually reads. The instruction list stays untouched, so checks 1–4 see +/// a pristine program and nothing but check 9 stands between this group and +/// the registry. +#[test] +fn negative_hash_multiplicity_in_a_registered_group_fails_admission() { + let mut program = trivial_program(); + validate(&program).expect("honest control: the registered program is admissible"); + + let negative_one = FE::from(P - 1); // p − 1, i.e. −1 on the bus + program + .groups + .hash + .set(0, super::layout::hash::MULT1, negative_one); + assert!( + matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_HASH", + row: 0, + col, + .. + } if col == super::layout::hash::MULT1 + ), + "a negative multiplicity in the committed group must fail admission" + ); +} diff --git a/prover/src/lfm/mod.rs b/prover/src/lfm/mod.rs new file mode 100644 index 000000000..d65d4a47d --- /dev/null +++ b/prover/src/lfm/mod.rs @@ -0,0 +1,113 @@ +//! LFM — the Lambda Field Machine. +//! +//! A fixed, straight-line, field-native recursion machine for verifying +//! Lambda VM STARK proofs: the SP1 v4 mechanism (the program is the machine's +//! preprocessed columns; write-once memory closed by pure LogUp balance; no +//! pc, no branches, no fetch/decode) with on-demand registration instead of +//! exhaustive shape enumeration — our framework has no keygen, so a program +//! is nothing but a vector of supplied preprocessed roots plus a registry +//! entry. +//! +//! Design authority: `others/lfm-design.md` (v0). This module is the +//! software layer (Milestone A): word model, instruction set, eDSL builder, +//! straight-line compiler, executor/witness generator, admission validator, +//! and the hash interface with a placeholder permutation. The chips and prover +//! integration follow (Milestone B); the fixed AIR set is 14 chips, the last +//! three being the production keccak family hosted unchanged (see `airs`). + +pub mod airs; +pub mod blake3; +pub mod blake3_chip; +pub mod blake3_socket; +pub mod builder; +pub mod chips; +pub mod chunking; +pub mod commit; +pub mod compiler; +pub mod constraints; +pub mod deep; +pub mod edsl; +pub mod epoch; +pub mod epoch_verify; +pub mod executor; +pub mod fixture; +pub mod fri; +pub mod hash; +pub mod instr; +pub mod keccak_adapter; +pub mod keccak_host; +pub mod layout; +pub mod lde; +pub mod logup; +pub mod poseidon; +pub mod programs; +pub mod proof; +pub mod proof_arena; +pub mod proof_fixture; +pub mod registry; +pub mod statement; +pub mod statement_replay; +pub mod sub_proof; +pub mod trace; +pub mod transcript_replay; +pub mod validator; +pub mod word; + +pub use airs::{LfmAirs, NUM_LFM_CHIPS, num_lfm_airs}; +pub use builder::{ArenaSchema, LfmBuilder, LfmProgramSource}; +pub use chunking::{KECCAK_RND_MAX_CHUNK_ROWS, KeccakChunking}; +pub use commit::{commit_columns, commit_group}; +pub use compiler::{ColumnGroup, LfmColumnGroups, LfmProgram, compile}; +pub use executor::{LfmExecError, LfmExecution, LfmRecords, execute}; +pub use hash::{HasherKind, LfmHasher, TestPermutation}; +pub use instr::{Addr, ArenaId, BaseOp, ExtOp, HashMode, Instr}; +pub use proof::{LfmProof, LfmProveError, lfm_prove, lfm_verify}; +pub use registry::{ + LFM_REGISTRY, LfmArtifacts, LfmProgramKind, LfmRegistryEntry, LfmRegistryError, + build_artifacts, build_artifacts_with_hasher, resolve, +}; +pub use statement::{LFM_MACHINE_VERSION, lfm_program_id}; +pub use transcript_replay::{Candidate, TranscriptReplay}; +pub use validator::{LfmViolation, validate}; +pub use word::{LfmWord, base_word, ext_word, pack_digest, unpack_digest}; + +#[cfg(test)] +mod blake3_probe; +#[cfg(test)] +mod blake3_socket_kats; +#[cfg(test)] +mod blake3_socket_tests; +#[cfg(test)] +mod constraint_tests; +#[cfg(test)] +mod epoch_tests; +#[cfg(test)] +mod epoch_verify_tests; +#[cfg(test)] +mod framework_probe; +#[cfg(test)] +mod fri_tests; +#[cfg(test)] +mod join_tests; +#[cfg(test)] +mod keccak_probe; +#[cfg(test)] +mod leaf_kats; +#[cfg(test)] +mod leaf_tests; +#[cfg(test)] +mod logup_tests; +#[cfg(test)] +mod machine_tests; +#[cfg(test)] +mod poseidon_chip_tests; +#[cfg(test)] +mod step_size_tests; +#[cfg(test)] +mod tests; +#[cfg(test)] +mod transcript_kats; +#[cfg(test)] +mod transcript_tests; +#[cfg(test)] +mod wrap_tests; diff --git a/prover/src/lfm/poseidon.rs b/prover/src/lfm/poseidon.rs new file mode 100644 index 000000000..fd27aab38 --- /dev/null +++ b/prover/src/lfm/poseidon.rs @@ -0,0 +1,768 @@ +//! Poseidon-original ("Poseidon1") over Goldilocks at width 12 — the hash +//! matrix's first real candidate behind the `LFM_HASH` socket. +//! +//! # Why this hash and not Poseidon2 +//! +//! Poseidon2 is BROKEN (eprint 2026/306 exploits the structure of its linear +//! layers) and must never be built here. Poseidon-original is the same S-box +//! family with a dense MDS that lacks the exploited structure, and it is where +//! the ecosystem moved. +//! +//! # Parameter provenance — READ THIS BEFORE TRUSTING A MEASUREMENT +//! +//! Parameters are taken from the vendored Plonky3 tree, +//! `others/Plonky3/goldilocks/src/poseidon1.rs` @ 4aed8fe4, which documents +//! them as generated by the Grain LFSR of the Poseidon paper (Appendix E) with +//! `field_type=1, alpha=7 (exp_flag=0), n=64, t=12, R_F=8, R_P=22`, via +//! `poseidon/generate_constants.py --field goldilocks --width 12`: +//! +//! - `ALPHA = 7` — the smallest valid exponent, because Goldilocks has +//! `p - 1 = 2^32 · 3 · 5 · 17 · 257 · 65537`, so neither 3 nor 5 is coprime to +//! `p - 1`. **A cube S-box is not a permutation over this field**, which is +//! why the in-tree HADES skeleton in `crypto/crypto/src/hash/poseidon/` can +//! never serve as an oracle: it hardcodes `x^3`. +//! - `R_F = 8` full rounds (4 initial + 4 terminal), `R_P = 22` partial. +//! - The MDS matrix is CIRCULANT with first row `MDS_CIRC_ROW`. +//! +//! ⚠ **Ship-grade parameter selection is a separate cryptographic decision for +//! the ecosystem and is NOT settled by this measurement.** These parameters are +//! adequate to measure the AIR's SHAPE — cells depend on round counts and the +//! S-box degree, not on the numeric values of the constants — and they come from +//! a published generator, but choosing what to ship is not ours. +//! +//! # Oracle +//! +//! [`tests::the_permutation_matches_the_plonky3_known_answer_vector`] pins the +//! whole permutation against Plonky3's own known-answer test +//! (`goldilocks/src/poseidon1.rs::test_poseidon_goldilocks_width_12`), an +//! EXTERNAL vector we did not compute. That is the differential this module +//! rests on. + +use crate::tables::types::FE; + +use super::hash::{HASH_STATE_FELTS, LfmHasher}; +use super::word::LfmWord; + +/// The S-box exponent. See the provenance note: 7 is forced by Goldilocks. +pub const ALPHA: u32 = 7; +/// Full rounds per half; `R_F = 2 * HALF_FULL_ROUNDS = 8`. +pub const HALF_FULL_ROUNDS: usize = 4; +/// Partial rounds, S-box on lane 0 only. +pub const PARTIAL_ROUNDS: usize = 22; +/// Total rounds — the figure the AIR's column count is linear in. +pub const NUM_ROUNDS: usize = 2 * HALF_FULL_ROUNDS + PARTIAL_ROUNDS; + +/// First ROW of the circulant MDS matrix, so `M[i][j] = MDS_CIRC_ROW[(j - i) mod 12]`. +/// +/// From `others/Plonky3/goldilocks/src/mds.rs:92` (`MATRIX_CIRC_MDS_12_SML_ROW`). +pub const MDS_CIRC_ROW: [u64; HASH_STATE_FELTS] = [1, 1, 2, 1, 8, 9, 10, 7, 5, 9, 4, 10]; + +/// Round constants, `[round][lane]`, in the order +/// `[initial_full (4), partial (22), terminal_full (4)]`. +pub const ROUND_CONSTANTS: [[u64; HASH_STATE_FELTS]; NUM_ROUNDS] = [ + [ + 0x13dcf33aba214f46, + 0x30b3b654a1da6d83, + 0x1fc634ada6159b56, + 0x937459964dc03466, + 0xedd2ef2ca7949924, + 0xede9affde0e22f68, + 0x8515b9d6bac9282d, + 0x6b5c07b4e9e900d8, + 0x1ec66368838c8a08, + 0x9042367d80d1fbab, + 0x400283564a3c3799, + 0x4a00be0466bca75e, + ], + [ + 0x7913beee58e3817f, + 0xf545e88532237d90, + 0x22f8cb8736042005, + 0x6f04990e247a2623, + 0xfe22e87ba37c38cd, + 0xd20e32c85ffe2815, + 0x117227674048fe73, + 0x4e9fb7ea98a6b145, + 0xe0866c232b8af08b, + 0x00bbc77916884964, + 0x7031c0fb990d7116, + 0x240a9e87cf35108f, + ], + [ + 0x2e6363a5a12244b3, + 0x5e1c3787d1b5011c, + 0x4132660e2a196e8b, + 0x3a013b648d3d4327, + 0xf79839f49888ea43, + 0xfe85658ebafe1439, + 0xb6889825a14240bd, + 0x578453605541382b, + 0x4508cda8f6b63ce9, + 0x9c3ef35848684c91, + 0x0812bde23c87178c, + 0xfe49638f7f722c14, + ], + [ + 0x8e3f688ce885cbf5, + 0xb8e110acf746a87d, + 0xb4b2e8973a6dabef, + 0x9e714c5da3d462ec, + 0x6438f9033d3d0c15, + 0x24312f7cf1a27199, + 0x23f843bb47acbf71, + 0x9183f11a34be9f01, + 0x839062fbb9d45dbf, + 0x24b56e7e6c2e43fa, + 0xe1683da61c962a72, + 0xa95c63971a19bfa7, + ], + [ + 0x4adf842aa75d4316, + 0xf8fbb871aa4ab4eb, + 0x68e85b6eb2dd6aeb, + 0x07a0b06b2d270380, + 0xd94e0228bd282de4, + 0x8bdd91d3250c5278, + 0x209c68b88bba778f, + 0xb5e18cdab77f3877, + 0xb296a3e808da93fa, + 0x8370ecbda11a327e, + 0x3f9075283775dad8, + 0xb78095bb23c6aa84, + ], + [ + 0x3f36b9fe72ad4e5f, + 0x69bc96780b10b553, + 0x3f1d341f2eb7b881, + 0x4e939e9815838818, + 0xda366b3ae2a31604, + 0xbc89db1e7287d509, + 0x6102f411f9ef5659, + 0x58725c5e7ac1f0ab, + 0x0df5856c798883e7, + 0xf7bb62a8da4c961b, + 0xc68be7c94882a24d, + 0xaf996d5d5cdaedd9, + ], + [ + 0x9717f025e7daf6a5, + 0x6436679e6e7216f4, + 0x8a223d99047af267, + 0xbb512e35a133ba9a, + 0xfbbf44097671aa03, + 0xf04058ebf6811e61, + 0x5cca84703fac7ffb, + 0x9b55c7945de6469f, + 0x8e05bf09808e934f, + 0x2ea900de876307d7, + 0x7748fff2b38dfb89, + 0x6b99a676dd3b5d81, + ], + [ + 0xac4bb7c627cf7c13, + 0xadb6ebe5e9e2f5ba, + 0x2d33378cafa24ae3, + 0x1e5b73807543f8c2, + 0x09208814bfebb10f, + 0x782e64b6bb5b93dd, + 0xadd5a48eac90b50f, + 0xadd4c54c736ea4b1, + 0xd58dbb86ed817fd8, + 0x6d5ed1a533f34ddd, + 0x28686aa3e36b7cb9, + 0x591abd3476689f36, + ], + [ + 0x047d766678f13875, + 0xa2a11112625f5b49, + 0x21fd10a3f8304958, + 0xf9b40711443b0280, + 0xd2697eb8b2bde88e, + 0x3493790b51731b3f, + 0x11caf9dd73764023, + 0x7acfb8f72878164e, + 0x744ec4db23cefc26, + 0x1e00e58f422c6340, + 0x21dd28d906a62dda, + 0xf32a46ab5f465b5f, + ], + [ + 0xbfce13201f3f7e6b, + 0xf30d2e7adb5304e2, + 0xecdf4ee4abad48e9, + 0xf94e82182d395019, + 0x4ee52e3744d887c5, + 0xa1341c7cac0083b2, + 0x2302fb26c30c834a, + 0xaea3c587273bf7d3, + 0xf798e24961823ec7, + 0x962deba3e9a2cd94, + 0xb36ee79485ca4707, + 0xd380199eddd2de52, + ], + [ + 0x70971fc4e6f85305, + 0x8e722f6e5dc32699, + 0xa0883df133052b92, + 0x8f86c6a3eb7d01a4, + 0x763649c8b670bdc5, + 0x830d5c82b808759b, + 0xaa1da8bb91da02e7, + 0x9bc9bf629e211c4d, + 0x0f0a899b10a4dea8, + 0xb883bdcee7c6b356, + 0x78c7101e7496ae1e, + 0x2fd6c5a8bf1e5ca6, + ], + [ + 0xe2a6e06e61fcec9c, + 0xebfce7d5c5b3dbd5, + 0xca2eeca4bb485d85, + 0xc2b875537c42eb69, + 0x6faf849976873328, + 0xfc3fcb6e81ad4cc3, + 0x180dd95503955a28, + 0xd40f19a3c9fe1520, + 0x49d178ddbf7fd96d, + 0x3950bee2e10e0297, + 0x437b90cf295be062, + 0xa5cd126edffad23b, + ], + [ + 0xdf58134c134491c2, + 0x0677eca229d9f7bd, + 0x492200a1f7d83a3c, + 0xafb58c9810a43645, + 0x7659077c5a9c208e, + 0x30b4bc83706995cd, + 0xc98fa77bbbef3a3b, + 0x84a82905750b3109, + 0x72f2a02326aeb69b, + 0x8d27a2a2d73a848a, + 0xaa9e30a80bde4b68, + 0x63abb1415e050474, + ], + [ + 0x1c4bd1e816050a7e, + 0x15d1502e4f469dfd, + 0x53989d594b0c4cd8, + 0x7a1a4c83cb7e377e, + 0x1b52f8a9944e480e, + 0xeb7b03f76a91a79e, + 0x0073a4fc9328c69e, + 0x2c7b16f8620d9de4, + 0x950d052963e46bc4, + 0x8d201ba1a9c89fac, + 0xd3502941bdf35503, + 0x7c6dfcd5af8676fb, + ], + [ + 0xf8a6cd02e92cdb0b, + 0x6e7500f3a5464b22, + 0x07637eabba4bdd20, + 0x88b82717beee0e14, + 0xbaa2b1cd3dd4c79a, + 0xdfecc3aebec4cfa6, + 0x7561087b0cff0166, + 0x538fcac317a703a6, + 0xd7d6c6eeeeeeea19, + 0xd647b1ee441658a0, + 0xdf4442110236c546, + 0x559ef2c6dd73ec15, + ], + [ + 0x4c0f5fc6c0dda3d1, + 0x685010cc3100cea7, + 0x2fb6ba8aa0344440, + 0xb515f0a3ca75f1fb, + 0x886887eaecb87c10, + 0xf03ec3fd710abb04, + 0xd3b4763e17f543ef, + 0x50d9e5716e78083a, + 0x0bce2385cf8d74ff, + 0xaf23032cd5f0e04b, + 0xd366aa112b6159d9, + 0x810a3ad3ac7979db, + ], + [ + 0x0a4a11d794be40a2, + 0xeebf0cf23b668a3f, + 0x600873fb011d761b, + 0x0bfb5591a02ff618, + 0xa16e2a528910af52, + 0xf6553653e2878421, + 0xccbe7c7a601a30c0, + 0xb18b214fe489f5b3, + 0xe21017ab9e153425, + 0x586099ede17af9a6, + 0x385078b514f50647, + 0xc02b3a9afb89883d, + ], + [ + 0x6d3fbd3b4a9f1de6, + 0x4b4d40a41b0f473c, + 0x838f1887b8f31711, + 0x9396895be5c58a41, + 0x6247a479d66fc2e3, + 0x13fe228a98f2d0a2, + 0x5ba5fde765f9481e, + 0xafb89fa62267e117, + 0xfa4dc1bebcaa6333, + 0xdbab590882b87289, + 0xc3b6c08e23ba9301, + 0xd84b5de94a324fb7, + ], + [ + 0x0d0c371c5b35b850, + 0x7964f570e7188038, + 0x5daf18bbd996604c, + 0x6743bc47b9595258, + 0x5528b9362c59bb71, + 0xac45e25b7127b68c, + 0xa2077d7dfbb606b6, + 0xf3faac6faee378af, + 0x0c6388b51545e884, + 0xd27dbb6944917b61, + 0x89bcac584344c104, + 0x856bab802ce7402d, + ], + [ + 0x2cff3000be1fcd0a, + 0x765f2977fa72a917, + 0x1443711329f5f9d5, + 0xd35cd0261af2f951, + 0x2a1bb986084ec281, + 0x2334a54b758f23f2, + 0xa9b8cb612caf706b, + 0xb6ba11c4ab1a1017, + 0xde96b0824b4b46e2, + 0xc59d4272c6d92e2c, + 0x389bb5107611754d, + 0x23647fbc77657372, + ], + [ + 0xd5ef60d6f76a42fa, + 0xebb406bb79ac9819, + 0x55faccc709a2f423, + 0xd9d6ea97490091cd, + 0xef3ce5069647a7e4, + 0xdf31625d3fa78464, + 0x242e60fd68f10f66, + 0x39c966cc815f084d, + 0x20e2e22e02bae3f7, + 0xb38919d3f1173d7c, + 0xf17769f6c77084d9, + 0xcc051d8094cac41f, + ], + [ + 0x942069f5d6eece7e, + 0x8d61d3e6f141c572, + 0xc5cef9d85dd605f4, + 0x938f2ac2bf885997, + 0x23bddbace7c48f6c, + 0xc90a6c5ba98537e4, + 0x0be6ee2cca90f6ae, + 0xa026175394ae0e90, + 0x29fca3e314c77628, + 0x2aa2aa8738ab7b77, + 0xe11bbd31fbb8cac6, + 0xb5bbbef1b78a23af, + ], + [ + 0x8b62a5551e9a9797, + 0x3f91073d4d491c80, + 0x4cfa44976396424a, + 0xf8dcb2dfb3aa1b44, + 0x3849409eba1a95f5, + 0x070845799f234380, + 0x184c0093667da1ba, + 0xbd66aafccd51601e, + 0xee6d14e92155b490, + 0x626f2ec1865bc544, + 0x1bd2854bf6485986, + 0x368b8497472f12ef, + ], + [ + 0x4f88cdcdfb791921, + 0xe2c0acfeda9ae781, + 0x9739bc21773469b3, + 0x00ce3ad64dc4bb8f, + 0xaab85a321ee7a4c8, + 0xd5de825be97004f4, + 0x48d676d3a043b1c6, + 0x9c6180b1ff643097, + 0x34882a89dd590b09, + 0xae7e6b0d249c3b1d, + 0x8c016908a04885a1, + 0x83ebaaebc9ae0721, + ], + [ + 0xab21b42e0f642307, + 0xdb46631f62bb29c1, + 0xef29f0399e09b5d9, + 0x5b52fbb3613b8ba1, + 0x57e129fcc96922e6, + 0xcdeb14c9d9204b3a, + 0x1341ef0da8536e34, + 0xd7e3400f2bacde63, + 0x6911eeb42f70d7e5, + 0xc3a2a910a4679767, + 0x1773cbe4a0f6bb28, + 0xe17b0d53e843eab5, + ], + [ + 0x587fa39990b62800, + 0x0d5d32788135879d, + 0x277f7b31fd3a4cdb, + 0xa435290ee56d7efa, + 0xea6f40be35159925, + 0xcb73377a506171cb, + 0xe43c367ce731d82a, + 0x6eb305031ca10c43, + 0xc019a8c622cc84cb, + 0xd5614f5658c612e6, + 0x7b1ecbe957c3ff98, + 0x60db6ee9651a8478, + ], + [ + 0x9271d450fc9b4117, + 0xcffeea06b6e3aac1, + 0xfa4a44c748d1cd8e, + 0xe64db01ba569b469, + 0xd31005160e4045fe, + 0x39e0fa013e025f79, + 0xe243be574196a956, + 0x205b2a681e3d2642, + 0x79cae5ad93486bab, + 0xfdf567844e32c295, + 0x331679589bfb7189, + 0xaf06ee32297b89c2, + ], + [ + 0xa6bcae311e498491, + 0x9d16f52c96ac8b3e, + 0x48a674b59393fa35, + 0x0f9e65da3fde3796, + 0x1e098310fc84578c, + 0x559ae5fab1ae8dad, + 0x56bd4d624078881d, + 0xfd8bbbf8fbe817b5, + 0x82d30695c44df534, + 0x3ec0a97bc41127c5, + 0x1eb8b64adaa22078, + 0x82c45e418d60c983, + ], + [ + 0xb092280f484d55bf, + 0xcd317c9537697939, + 0xd3be2e352feb79f3, + 0xca6d866539a390e5, + 0xb5efb1a494e55ee6, + 0xfa9013ac89756e9e, + 0xaeb88efd1e981242, + 0x13ee477cdab6e0dc, + 0xce7df902c40da2d3, + 0xf3fbaf0d4e6f5f34, + 0xf96354ada6785f38, + 0x13b5692812406886, + ], + [ + 0xf03cae030a0f4418, + 0x7d3172887aa98e1a, + 0x8a2c2644f2faf7b9, + 0x80d721abee696d00, + 0x27c8b903a4d68267, + 0xaf0b7b12f90291b8, + 0x00acd08cfdff3817, + 0x4659ee496c634328, + 0xf5b25c10730dbff1, + 0xdde3a153297329c2, + 0x50c0b70d6910a44b, + 0x23c7426af725a6a0, + ], +]; + +/// Is round `r` a full round? Rounds run +/// `[initial_full (4), partial (22), terminal_full (4)]`, matching +/// [`ROUND_CONSTANTS`]' row order. +pub const fn is_full_round(r: usize) -> bool { + r < HALF_FULL_ROUNDS || r >= HALF_FULL_ROUNDS + PARTIAL_ROUNDS +} + +/// S-boxed lanes in round `r`: all 12 in a full round, lane 0 only in a partial +/// one. This is the single rule the AIR's column count and the witness both read +/// — the partial rounds' S-box lane is one of the conventions the KAT pins +/// (`tests::the_permutation_matches_the_plonky3_known_answer_vector`). +pub const fn sboxed_lanes(r: usize) -> usize { + if is_full_round(r) { + HASH_STATE_FELTS + } else { + 1 + } +} + +/// One round's recorded intermediates, in the association the degree-3 AIR +/// lowering needs: `x2 = a·a`, `x3 = x2·a`, and the S-box output `(x3)²·a` +/// entering the MDS. `x2`/`x3` carry [`sboxed_lanes`] entries; the rest of the +/// array is unused (and stays zero) on partial rounds. +#[derive(Clone, Copy, Debug)] +pub struct PoseidonRound { + /// `a_i = state_i + rc[r][i]` — the post-constant state. Recorded for + /// cross-checking only; the AIR recomputes it as a degree-1 expression. + pub a: [FE; HASH_STATE_FELTS], + /// `a_i²` for the S-boxed lanes. + pub x2: [FE; HASH_STATE_FELTS], + /// `a_i³` for the S-boxed lanes. + pub x3: [FE; HASH_STATE_FELTS], + /// The post-MDS state — this round's output, next round's input. + pub out: [FE; HASH_STATE_FELTS], +} + +/// Every intermediate the AIR witnesses, one entry per round. +pub type PoseidonWitness = [PoseidonRound; NUM_ROUNDS]; + +/// Records the permutation's intermediates for the trace generator. +/// +/// ⚠ **Written independently of [`PoseidonGoldilocks::permute`] rather than +/// factored out of it, deliberately.** A recording wrapper that `permute` +/// delegated to would make [`tests::the_witness_agrees_with_the_permutation`] a +/// tautology at the moment of the refactor (standing-decisions rule 7). Both +/// paths are pinned to the SAME external KAT instead, so a divergence between +/// them fails a test that does not compare them to each other. +pub fn permutation_witness(state: [FE; HASH_STATE_FELTS]) -> PoseidonWitness { + let zero = [FE::zero(); HASH_STATE_FELTS]; + let mut rounds = [PoseidonRound { + a: zero, + x2: zero, + x3: zero, + out: zero, + }; NUM_ROUNDS]; + let mut s = state; + for (r, round) in rounds.iter_mut().enumerate() { + round.a = core::array::from_fn(|i| &s[i] + FE::from(ROUND_CONSTANTS[r][i])); + let mut mixed = round.a; + for (lane, m) in mixed.iter_mut().enumerate().take(sboxed_lanes(r)) { + let a = &round.a[lane]; + round.x2[lane] = a * a; + round.x3[lane] = &round.x2[lane] * a; + *m = &(&round.x3[lane] * &round.x3[lane]) * a; + } + round.out = PoseidonGoldilocks::mds(&mixed); + s = round.out; + } + rounds +} + +/// Poseidon-original over Goldilocks, width 12 — a real cryptographic hash +/// behind the `LFM_HASH` contract, replacing `TestPermutation`. +pub struct PoseidonGoldilocks; + +impl PoseidonGoldilocks { + /// `x^7` by square-and-multiply: `x^2`, `x^3 = x^2 * x`, `x^7 = (x^3)^2 * x`. + /// + /// Written in exactly the association the AIR's degree-3 lowering uses, so + /// the executor and the chip agree by construction rather than by luck. + fn sbox(x: &FE) -> FE { + let x2 = x * x; + let x3 = &x2 * x; + let x6 = &x3 * &x3; + &x6 * x + } + + /// The circulant MDS product. + fn mds(state: &[FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + core::array::from_fn(|i| { + let mut acc = FE::zero(); + for (j, s) in state.iter().enumerate() { + let c = FE::from(MDS_CIRC_ROW[(j + HASH_STATE_FELTS - i) % HASH_STATE_FELTS]); + acc += c * s; + } + acc + }) + } + + fn add_round_constants(state: &mut [FE; HASH_STATE_FELTS], round: usize) { + for (k, s) in state.iter_mut().enumerate() { + *s += FE::from(ROUND_CONSTANTS[round][k]); + } + } +} + +impl LfmHasher for PoseidonGoldilocks { + fn permute(&self, state: [FE; HASH_STATE_FELTS]) -> [FE; HASH_STATE_FELTS] { + let mut s = state; + let mut round = 0; + for _ in 0..HALF_FULL_ROUNDS { + Self::add_round_constants(&mut s, round); + for lane in s.iter_mut() { + *lane = Self::sbox(lane); + } + s = Self::mds(&s); + round += 1; + } + for _ in 0..PARTIAL_ROUNDS { + Self::add_round_constants(&mut s, round); + s[0] = Self::sbox(&s[0]); + s = Self::mds(&s); + round += 1; + } + for _ in 0..HALF_FULL_ROUNDS { + Self::add_round_constants(&mut s, round); + for lane in s.iter_mut() { + *lane = Self::sbox(lane); + } + s = Self::mds(&s); + round += 1; + } + debug_assert_eq!(round, NUM_ROUNDS, "every round constant row is consumed"); + s + } + + /// The capacity cell for `Compress` mode. + /// + /// ⚠ ZERO capacity, i.e. the plain sponge compression of `[a ‖ b ‖ 0]`. + /// Domain separation is a cryptographic decision of the same class as + /// parameter selection and is deliberately NOT invented here; a shipped + /// design may well want a nonzero, length- or position-dependent IV. + fn compress_iv(&self) -> LfmWord { + [FE::zero(), FE::zero(), FE::zero(), FE::zero()] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Plonky3's own known-answer vector — an EXTERNAL oracle. + /// + /// Source: `others/Plonky3/goldilocks/src/poseidon1.rs`, + /// `test_poseidon_goldilocks_width_12` (input `0..11`). Nothing in this + /// repository produced these twelve numbers, which is the point: they check + /// the constants, the round order, the S-box exponent, the MDS orientation + /// and the partial-round lane all at once. + const PLONKY3_KAT_OUT: [u64; HASH_STATE_FELTS] = [ + 15595088881848875364, + 9564850329150784619, + 13607005230761744521, + 12117102595842533385, + 2814257411756993122, + 11640647689983397089, + 14363867760831937423, + 13323891071259596526, + 11219803511311150468, + 9221595262780869902, + 5898229059046891887, + 18181291031484020550, + ]; + + #[test] + fn the_permutation_matches_the_plonky3_known_answer_vector() { + let input: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(i as u64)); + let got = PoseidonGoldilocks.permute(input); + let want: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(PLONKY3_KAT_OUT[i])); + assert_eq!( + got, want, + "the permutation must match Plonky3's KAT exactly" + ); + } + + /// The S-box exponent must be coprime to `p - 1`, or the S-box is not a + /// permutation. This is the exact trap the in-tree HADES skeleton falls into + /// with its hardcoded `x^3`, so the property is asserted rather than trusted. + #[test] + fn the_sbox_exponent_is_coprime_to_the_group_order() { + // Goldilocks: p = 2^64 - 2^32 + 1, and p - 1 = 2^32 · 3 · 5 · 17 · 257 · 65537. + const P_MINUS_ONE: u128 = (1u128 << 64) - (1u128 << 32); + fn gcd(a: u128, b: u128) -> u128 { + if b == 0 { a } else { gcd(b, a % b) } + } + assert_eq!( + gcd(ALPHA as u128, P_MINUS_ONE), + 1, + "x^{ALPHA} must be a permutation over Goldilocks" + ); + for bad in [3u128, 5] { + assert_ne!( + gcd(bad, P_MINUS_ONE), + 1, + "{bad} divides p-1, so x^{bad} is NOT a permutation — the skeleton's bug" + ); + } + } + + #[test] + fn the_round_constant_table_has_one_row_per_round() { + assert_eq!(ROUND_CONSTANTS.len(), NUM_ROUNDS); + assert_eq!(NUM_ROUNDS, 30, "8 full + 22 partial"); + } + + /// The witness's last round must reproduce the SAME external vector + /// `permute` is pinned to — not `permute`'s output, which would only say the + /// two agree. This is the absolute pin on the recording path. + #[test] + fn the_witness_final_round_matches_the_plonky3_known_answer_vector() { + let input: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(i as u64)); + let w = permutation_witness(input); + let want: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(PLONKY3_KAT_OUT[i])); + assert_eq!( + w[NUM_ROUNDS - 1].out, + want, + "the witness's final post-MDS state must match Plonky3's KAT" + ); + } + + /// A genuine differential: two independently written round loops, neither + /// delegating to the other (rule 7). It runs on inputs the KAT does not + /// cover, so it catches a divergence the single vector would miss. + #[test] + fn the_witness_agrees_with_the_permutation() { + for seed in 0..8u64 { + let input: [FE; HASH_STATE_FELTS] = + core::array::from_fn(|i| FE::from(seed.wrapping_mul(0x9E37_79B9) + i as u64)); + let w = permutation_witness(input); + assert_eq!( + w[NUM_ROUNDS - 1].out, + PoseidonGoldilocks.permute(input), + "witness and permute must agree at seed {seed}" + ); + } + } + + /// The intermediates must be the ones the AIR constrains: `x2 = a²`, + /// `x3 = a³`, and the S-box output `(x3)²·a = a^7` feeding the MDS. Asserted + /// against `sbox` for the S-boxed lanes and against `a` itself elsewhere, so + /// a partial round that quietly S-boxed twelve lanes would fail here. + #[test] + fn the_witness_records_the_degree_three_association() { + let input: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(3 * i as u64 + 1)); + let w = permutation_witness(input); + for (r, round) in w.iter().enumerate() { + let sboxed = sboxed_lanes(r); + assert_eq!(sboxed, if is_full_round(r) { 12 } else { 1 }); + let mixed: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| { + if i < sboxed { + assert_eq!( + round.x2[i], + &round.a[i] * &round.a[i], + "round {r} lane {i} x2" + ); + assert_eq!( + round.x3[i], + &round.x2[i] * &round.a[i], + "round {r} lane {i} x3" + ); + PoseidonGoldilocks::sbox(&round.a[i]) + } else { + assert_eq!(round.x2[i], FE::zero(), "round {r} lane {i} x2 unused"); + assert_eq!(round.x3[i], FE::zero(), "round {r} lane {i} x3 unused"); + round.a[i] + } + }); + assert_eq!( + round.out, + PoseidonGoldilocks::mds(&mixed), + "round {r} output is the MDS of the S-boxed state" + ); + } + } +} diff --git a/prover/src/lfm/poseidon_chip_tests.rs b/prover/src/lfm/poseidon_chip_tests.rs new file mode 100644 index 000000000..45dbf31a7 --- /dev/null +++ b/prover/src/lfm/poseidon_chip_tests.rs @@ -0,0 +1,674 @@ +//! The Poseidon-original `LFM_HASH` chip: its layout, its degree bound, what it +//! accepts, what it rejects, and the prove+verify that turns a predicted cell +//! count into a measured one. +//! +//! ## What pins what +//! +//! The permutation itself is pinned elsewhere, to an EXTERNAL vector: `poseidon:: +//! tests::the_permutation_matches_the_plonky3_known_answer_vector`. Nothing here +//! re-checks the algebra. This module checks the *chip* — that 601 constraints +//! over 612 value columns say exactly what that permutation does, and that they +//! say it inside a real proof. +//! +//! ## What this suite cannot see +//! +//! It does not choose a hash. The parameters are published ones adequate to +//! measure an AIR's SHAPE (cells depend on round counts and S-box degree, not on +//! the constants' values); ship-grade parameter selection and domain separation +//! are cryptographic decisions for the ecosystem, and `compress_iv` being zero +//! here is a deliberate non-choice, not a recommendation. +//! +//! It also says nothing about the machine's DEFAULT hash, which is still +//! `TestPermutation`: every test below constructs the Poseidon configuration +//! explicitly. + +use math::field::element::FieldElement; +use stark::constraints::builder::{ + CaptureBuilder, ConstraintSet, ProverEvalFolder, RootKind, num_base_from_meta, +}; +use stark::frame::Frame; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::table::TableView; +use stark::traits::TransitionEvaluationContext; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use super::airs::lfm_chip_census_with_hasher; +use super::chips::hash::{self, HashConstraints, poseidon_cols as pc}; +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::poseidon::{NUM_ROUNDS, PoseidonGoldilocks, sboxed_lanes}; +use super::programs::trivial_program; +use super::proof::{lfm_prove_with_hasher, verify_against}; +use super::registry::{build_artifacts, build_artifacts_with_hasher}; +use super::trace::fill_poseidon_witness; +use super::word::LfmWord; + +type Gl = GoldilocksField; +type Gl3 = GoldilocksExtension; + +/// §6.3's pinned layout width, as a literal. This is the number wave 8 derived +/// on paper and handed over to be confirmed or falsified; writing it out rather +/// than recomputing it from the layout is the whole point — a closed form taken +/// from the code under test would agree with any layout, including a wrong one. +const PINNED_VALUE_COLUMNS: usize = 612; +/// §6.4's pinned constraint count, same reasoning. +/// §6.4's pinned constraint count, plus the shared unread-input pins. +/// +/// 601 was the figure §6.4 pinned; the +8 are `chips::hash`'s unread-`IN` pins, +/// which every arm emits since the D1 fix (a leaf row's unread cells were free +/// on this arm, and this arm's round 0 reads them). +const PINNED_CONSTRAINTS: usize = 601 + super::chips::hash::NUM_UNREAD_INPUT_PINS; +/// §6.3's pinned base-equivalent cells per permutation: `612 + 3·3`. +const PINNED_CELLS_PER_PERMUTATION: u64 = 621; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +/// A hash row exactly as `trace::build_traces_with_hasher` fills one, for a +/// permutation of `state`. +/// +/// `compress` selects the mode, which is what the capacity columns key off: +/// `MODE_P = 1` copies `IN8..11` into `S8..11`, `MODE_C = 1` forces them to +/// Poseidon's zero IV. The `IN`/`OUT`/mode cells are written the way the +/// executor records them (`executor.rs`, `Instr::Hash`) and the witness columns +/// by the production filler itself, so a row here is the row the prover builds. +fn hash_row(state: [FE; HASH_STATE_FELTS], compress: bool) -> Vec { + let mut row = vec![FE::zero(); pc::NUM_COLUMNS]; + if compress { + // Compress: IN0..7 = a‖b, IN8..11 stay zero, capacity = the zero IV. + row[hash::cols::IN0..hash::cols::IN0 + 8].copy_from_slice(&state[0..8]); + row[pc::MODE_C] = FE::one(); + } else { + row[hash::cols::IN0..hash::cols::IN0 + HASH_STATE_FELTS].copy_from_slice(&state); + row[pc::MODE_P] = FE::one(); + } + for k in 0..4 { + row[hash::cols::S8 + k] = if compress { FE::zero() } else { state[8 + k] }; + } + let permuted = PoseidonGoldilocks.permute(state); + row[hash::cols::OUT0..hash::cols::OUT0 + HASH_STATE_FELTS].copy_from_slice(&permuted); + fill_poseidon_witness(&mut row); + row +} + +/// A permutation-mode row over a deterministic, non-degenerate state. +fn sample_row() -> Vec { + hash_row( + core::array::from_fn(|i| FE::from(0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1))), + false, + ) +} + +/// Every constraint's value on `row`, via the same `ProverEvalFolder` the prover +/// itself folds with. +fn evaluate(row: &[FE]) -> Vec { + let set = HashConstraints::POSEIDON; + let n = ConstraintSet::::meta(&set).len(); + let no_ch: Vec> = vec![]; + let offset = FieldElement::::zero(); + let frame = Frame::::new(vec![TableView::new(vec![row.to_vec()], vec![vec![]])]); + let ctx = + TransitionEvaluationContext::new_prover(frame.as_row_frame(), &no_ch, &no_ch, &offset); + let mut base_out = vec![FE::zero(); n]; + let mut ext_out = vec![FieldElement::::zero(); n]; + let mut folder = ProverEvalFolder::new(&ctx, &mut base_out, &mut ext_out); + set.eval(&mut folder); + folder.assert_all_emitted(); + base_out +} + +fn violations(row: &[FE]) -> Vec { + evaluate(row) + .iter() + .enumerate() + .filter(|(_, v)| **v != FE::zero()) + .map(|(i, _)| i) + .collect() +} + +// ========================================================================= +// The layout — test 0, and the half of §6.3 that is pure arithmetic +// ========================================================================= + +/// The width wave 8 predicted, confirmed against the layout that was built. +/// +/// Both sides are stated independently: the left is the AIR's own width, the +/// right is §6.3's literal. The closed form is spelled out too, because the +/// prediction and the implementation arrange the same 612 columns differently — +/// §6.4 counts a fresh output block for all 30 rounds and no shared `OUT`, the +/// implementation shares `OUT` with the last round. Equal totals across two +/// arrangements is a stronger check than either alone. +#[test] +fn the_poseidon_layout_is_612_value_columns() { + assert_eq!( + pc::NUM_COLUMNS - pc::PREP_WIDTH, + PINNED_VALUE_COLUMNS, + "the built layout must be the width §6.3 pinned" + ); + // §6.4's arrangement: IN(12) + S(4), then 8 full rounds of 36 and 22 + // partial rounds of 14, the last round's output serving as OUT. + assert_eq!(PINNED_VALUE_COLUMNS, 16 + 8 * 36 + 22 * 14); + // The implemented arrangement: the frozen 28-column IN/S/OUT prefix, seven + // full rounds with their own output block, the eighth (last) round without + // one, and 22 partial rounds. + assert_eq!( + PINNED_VALUE_COLUMNS, + 28 + 7 * 36 + 24 + 22 * 14, + "the two arrangements must agree on the total" + ); + // 13: option B1's `MODE_T` took it from 11 to 12 and option C's `MODE_L` + // to 13 — the same number the BLAKE3 arm pins, because the prefix is the + // hasher-independent instruction group. + assert_eq!(pc::PREP_WIDTH, 13, "the preprocessed prefix does not move"); +} + +/// The layout is injective and gapless — no column is written twice, none is +/// left unread. +/// +/// The totals above cannot see an off-by-one inside `block`/`x2`/`x3`/`out`: two +/// blocks could overlap and the width still come to 612. This walks every index +/// the layout hands out and asserts they are exactly `PREP_WIDTH..NUM_COLUMNS`, +/// once each — with the ONE deliberate alias (the final round's output IS `OUT`) +/// asserted as an alias rather than tolerated as a collision. +#[test] +fn the_poseidon_layout_assigns_every_column_exactly_once() { + assert_eq!( + (0..HASH_STATE_FELTS) + .map(|j| pc::out(NUM_ROUNDS - 1, j)) + .collect::>(), + (0..HASH_STATE_FELTS) + .map(|j| hash::cols::OUT0 + j) + .collect::>(), + "the final round's output must BE the frozen OUT columns, not a copy" + ); + + let mut seen = vec![0usize; pc::NUM_COLUMNS]; + let mut claim = |c: usize| seen[c] += 1; + for i in 0..HASH_STATE_FELTS { + claim(hash::cols::IN0 + i); + } + for k in 0..4 { + claim(hash::cols::S8 + k); + } + for j in 0..HASH_STATE_FELTS { + claim(hash::cols::OUT0 + j); + } + for r in 0..NUM_ROUNDS { + for lane in 0..sboxed_lanes(r) { + claim(pc::x2(r, lane)); + claim(pc::x3(r, lane)); + } + if r + 1 < NUM_ROUNDS { + for j in 0..HASH_STATE_FELTS { + claim(pc::out(r, j)); + } + } + } + for (c, &n) in seen.iter().enumerate().skip(pc::PREP_WIDTH) { + assert_eq!( + n, 1, + "value column {c} is claimed {n} times, want exactly 1" + ); + } + for (c, &n) in seen.iter().enumerate().take(pc::PREP_WIDTH) { + assert_eq!(n, 0, "preprocessed column {c} must not be claimed"); + } +} + +/// The `LfmMem` tuple contract is hasher-INDEPENDENT: the same six +/// interactions, the same tuples, reading the same frozen offsets under every +/// configuration. +/// +/// This is what lets a candidate be swapped in without touching `LfmMem`, and it +/// is why the census's `aux_cols` is 3 in both columns of the Test/Poseidon +/// matrix. +/// +/// What is NOT hasher-independent is the interaction list as a whole: a +/// candidate built from byte operations brings its own lookups, and +/// `HasherKind::Blake3` brings over a thousand `BITWISE` ones. That is asserted +/// here as an inequality rather than left implicit, because "the bus contract +/// does not move" is exactly the sentence a BLAKE3 arm makes half-true. +#[test] +fn the_lfm_mem_tuple_contract_does_not_move_with_the_hasher() { + for kind in [HasherKind::Test, HasherKind::Poseidon] { + assert_eq!(hash::bus_interactions(kind).len(), 6, "{kind:?}"); + } + assert!( + hash::bus_interactions(HasherKind::Blake3).len() > 6, + "BLAKE3 must add its BITWISE lookups to the frozen six" + ); + assert_eq!(hash::num_columns(HasherKind::Test), pc::PREP_WIDTH + 28); + assert_eq!(hash::num_columns(HasherKind::Poseidon), pc::NUM_COLUMNS); + // The tuple columns the bus reads are the frozen prefix in every layout. + const { assert!(hash::cols::OUT0 + HASH_STATE_FELTS <= pc::PREP_WIDTH + 28) }; +} + +// ========================================================================= +// Test 1 — the degree bound +// ========================================================================= + +/// `max_degree()` is what sizes the composition polynomial, so an +/// UNDER-declaration is a soundness bug. The S-box is decomposed as +/// `x⁷ = (x³)²·x` over witnessed `x²`/`x³` precisely to hold this at 3; if that +/// decomposition were ever "simplified" to `a⁷`, this test is what fails. +#[test] +fn every_poseidon_constraint_is_degree_three_or_less() { + let set = HashConstraints::POSEIDON; + let meta = ConstraintSet::::meta(&set); + let n = meta.len(); + assert_eq!( + n, PINNED_CONSTRAINTS, + "the built constraint set must be the size §6.4 pinned" + ); + // 4 capacity copies + the mode-sum booleanity + the rounds, plus the + // shared unread-input pins every arm emits (`chips::hash`'s single + // derivation — the D1 fix). §6.4 pinned the pre-pin figure of 601. + assert_eq!( + PINNED_CONSTRAINTS, + 4 + 1 + 8 * 36 + 22 * 14 + super::chips::hash::NUM_UNREAD_INPUT_PINS + ); + for (i, m) in meta.iter().enumerate() { + assert_eq!(m.constraint_idx, i, "meta must be dense and idx-ordered"); + assert_eq!(m.kind, RootKind::Base, "every hash constraint is base"); + } + + let mut cb = CaptureBuilder::::new(); + set.eval(&mut cb); + let (_prog, degrees) = cb.finish(num_base_from_meta(&meta)); + assert_eq!(degrees.len(), n, "one emit per constraint"); + let mut emitted: Vec = degrees.iter().map(|&(idx, _)| idx).collect(); + emitted.sort_unstable(); + assert!( + emitted.iter().enumerate().all(|(i, &idx)| i == idx), + "emitted indices must be exactly 0..{n}" + ); + + let declared = ConstraintSet::::max_degree(&set); + assert_eq!(declared, 3, "the wrap's blowup 2 depends on this staying 3"); + for &(idx, measured) in °rees { + assert!( + measured <= declared, + "constraint {idx}: measured degree {measured} EXCEEDS declared {declared}" + ); + } + // Not merely `<=`: the MDS output constraints really are cubic, so a + // decomposition that quietly dropped to degree 2 would mean the S-box was + // no longer being computed. + assert_eq!( + degrees.iter().map(|&(_, d)| d).max(), + Some(3), + "some constraint must actually reach degree 3" + ); +} + +// ========================================================================= +// Test 2 — satisfaction +// ========================================================================= + +/// A real Poseidon row satisfies all 601 constraints, in both modes. +#[test] +fn a_real_poseidon_row_satisfies_every_constraint() { + for compress in [false, true] { + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(7 * i as u64 + 1)); + let row = hash_row(state, compress); + assert_eq!( + violations(&row), + Vec::::new(), + "an honest row (compress={compress}) must satisfy every constraint" + ); + } +} + +/// The chip agrees with the permutation the KAT pins, at the one place the two +/// meet: the row's `OUT` columns. +/// +/// Satisfaction alone cannot see this — a chip constraining the WRONG +/// permutation would be satisfied by its own consistent witness. What makes it +/// binding is that `OUT` is where the `LfmMem` bus reads the result, so this is +/// the value the rest of the machine consumes. +#[test] +fn the_chip_output_is_the_externally_pinned_permutation() { + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(i as u64)); + let row = hash_row(state, false); + let want = PoseidonGoldilocks.permute(state); + for j in 0..HASH_STATE_FELTS { + assert_eq!( + row[hash::cols::OUT0 + j], + want[j], + "OUT lane {j} must be the permutation's output" + ); + } + assert!(violations(&row).is_empty()); +} + +// ========================================================================= +// Test 3 — rejection (rule 1: break it deliberately, watch the right thing fail) +// ========================================================================= + +/// Perturbing any single witness column fires a constraint. +/// +/// Four columns, one per structural role: an `x²` (the first S-box step), an +/// `x³` (the second), a round output (the MDS), and a capacity cell (the +/// compress-mode copy). Each is checked separately, and each is asserted to fire +/// a constraint that *reads* it, not merely to fire something. +#[test] +fn perturbing_one_column_is_rejected() { + let base = sample_row(); + assert!( + violations(&base).is_empty(), + "the unperturbed row is honest" + ); + + // An x² in a full round (round 0, lane 5): its own defining constraint, and + // the x³ built on top of it, both read it. + let cases: [(&str, usize); 4] = [ + ("x2 (full round 0, lane 5)", pc::x2(0, 5)), + ("x3 (full round 0, lane 5)", pc::x3(0, 5)), + ("out (round 3, lane 7)", pc::out(3, 7)), + ("capacity S9", hash::cols::S8 + 1), + ]; + for (label, col) in cases { + let mut row = base.clone(); + row[col] = &row[col] + FE::one(); + let fired = violations(&row); + assert!( + !fired.is_empty(), + "perturbing {label} (column {col}) must fire at least one constraint" + ); + } +} + +/// A partial round really is partial: lane 0 only. +/// +/// Perturbing a partial round's single S-box witness must fire, and the AIR must +/// not have allocated (or constrained) witness columns for lanes 1..12 there. +/// This is the convention the KAT pins on the permutation side, asserted again +/// on the chip side — a chip that S-boxed twelve lanes in a partial round would +/// be a different hash with the same round constants. +#[test] +fn a_partial_round_s_boxes_only_lane_zero() { + let partial = 4; // rounds 4..26 are the partial ones + assert_eq!(sboxed_lanes(partial), 1); + assert_eq!(sboxed_lanes(0), HASH_STATE_FELTS); + assert_eq!(sboxed_lanes(NUM_ROUNDS - 1), HASH_STATE_FELTS); + + let base = sample_row(); + let mut row = base.clone(); + row[pc::x2(partial, 0)] = &row[pc::x2(partial, 0)] + FE::one(); + assert!( + !violations(&row).is_empty(), + "the partial round's lane-0 S-box must be constrained" + ); + + // Its block holds exactly two S-box columns plus twelve outputs. + assert_eq!(pc::block(partial + 1) - pc::block(partial), 2 + 12); +} + +/// A row whose witness is internally consistent but describes a DIFFERENT +/// permutation input is rejected. +/// +/// This is the coherent-forgery shape (rule 4) rather than a single-cell smudge: +/// every intermediate agrees with every other, the S-box associations hold, the +/// MDS is right. The one thing that does not hold is that round 0 reads `IN`/`S` +/// — so the capacity/input columns are what reject it, which is exactly the +/// binding the bus depends on. +#[test] +fn a_coherent_witness_for_the_wrong_input_is_rejected() { + let honest: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(7 * i as u64 + 1)); + let other: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| FE::from(9 * i as u64 + 5)); + let mut row = hash_row(honest, false); + + // Overwrite the witness with a fully consistent one for `other`, leaving the + // IN/S columns claiming `honest`. + let mut forged = hash_row(other, false); + let witness = pc::block(0)..pc::NUM_COLUMNS; + row[witness.clone()].copy_from_slice(&forged[witness]); + let out = hash::cols::OUT0..hash::cols::OUT0 + HASH_STATE_FELTS; + row[out.clone()].copy_from_slice(&forged[out]); + let fired = violations(&row); + assert!( + !fired.is_empty(), + "a coherent witness for a different input must still be rejected" + ); + + // And the converse sanity check: the forged row is honest ABOUT ITS OWN + // input, so the rejection above is about binding, not about the witness + // being malformed. + fill_poseidon_witness(&mut forged); + assert!(violations(&forged).is_empty()); +} + +// ========================================================================= +// Test 4 — padding +// ========================================================================= + +/// The all-zero padding row satisfies all 601 constraints. +/// +/// This is what the round constant being scaled by the mode sum buys: with +/// `m = 0` every `a` is zero, so `x² = x³ = 0` and `out = MDS·0 = 0`, +/// inductively through all 30 rounds. Without it the padding rows would need a +/// degree-4 `IS_REAL` gate, which would push `max_degree` to 4 and cost the wrap +/// its blowup 2. The trick is load-bearing; this test is what says so. +#[test] +fn the_all_zero_padding_row_satisfies_every_constraint() { + let row = vec![FE::zero(); pc::NUM_COLUMNS]; + assert_eq!( + violations(&row), + Vec::::new(), + "zero-filled padding must satisfy every constraint" + ); +} + +/// The padding row is not vacuously satisfied by a set that accepts anything: +/// the same all-zero row with one mode bit set (a "real" row with no witness) +/// must be rejected. +#[test] +fn a_padding_row_claiming_to_be_real_is_rejected() { + let mut row = vec![FE::zero(); pc::NUM_COLUMNS]; + row[pc::MODE_P] = FE::one(); + assert!( + !violations(&row).is_empty(), + "a real-marked row with an all-zero witness must be rejected" + ); +} + +// ========================================================================= +// Test 5 — prove and verify (rule 2: this is what makes the number a +// measurement rather than a declaration) +// ========================================================================= + +/// The production prover builds this AIR, proves a program through it, and the +/// production verifier accepts. +/// +/// `trivial_program` exercises both hash modes (two `compress`, one `permute`) +/// plus padding rows, so the proof covers every path the chip has. Artifacts are +/// built fresh rather than resolved from `LFM_REGISTRY`: this is a program SHAPE +/// that is deliberately not registered, and `verify_against` is the +/// supplied-roots entry point that exists for exactly that. +#[test] +fn the_poseidon_chip_proves_and_verifies() { + let opts = options(); + let program = trivial_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, HasherKind::Poseidon); + let proved = + lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, HasherKind::Poseidon) + .expect("proving under Poseidon must succeed"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest Poseidon-configured proof must verify" + ); +} + +/// A proof is bound to the hasher it was produced under, in both directions. +/// +/// The hasher is program shape — supplied by the verifier, never read off the +/// proof — so this is the check that a verifier which builds the wrong hash AIR +/// rejects rather than accepting something it did not verify. +#[test] +fn a_proof_does_not_verify_under_the_other_hasher() { + let opts = options(); + let program = trivial_program(); + + for (proved_under, verified_under) in [ + (HasherKind::Poseidon, HasherKind::Test), + (HasherKind::Test, HasherKind::Poseidon), + ] { + let artifacts = build_artifacts_with_hasher(&program, &opts, proved_under); + let proved = lfm_prove_with_hasher(&program, &artifacts, &arenas(), &opts, proved_under) + .expect("prove"); + // The digest stays the proved-under one: this isolates the AIR-set + // mismatch, rather than passing because the statement also moved. + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + verified_under, + ), + "a proof made under {proved_under:?} must not verify under {verified_under:?}" + ); + } +} + +/// ★ **The binding.** No root moves with the hasher — but the program digest +/// must. +/// +/// Both halves matter and they are in one test because the second exists only +/// because of the first. `build_artifacts` commits the preprocessed column +/// groups, and `PREP_WIDTH` is the same in both layouts (13 since `MODE_L`) +/// with the preprocessed group untouched, so every root really is bit-identical +/// across hashers. That is +/// what makes the commitments unable to carry the hasher, and it is why +/// `lfm_program_id` folds the kind's tag in directly: without the tag, a +/// Test-backed and a Poseidon-backed machine of the same program would share +/// one identity, and the only thing left separating them would be a +/// main-trace width coincidence that a third candidate could collide with. +/// +/// Asserted rather than assumed, in both directions: a hash experiment silently +/// reassigning program identities and a hash choice silently *sharing* one are +/// the two failures this pins. +#[test] +fn the_hasher_choice_moves_the_program_digest_and_no_root() { + let opts = options(); + for program in [trivial_program(), super::programs::fri_toy_program()] { + let test = build_artifacts_with_hasher(&program, &opts, HasherKind::Test); + let pos = build_artifacts_with_hasher(&program, &opts, HasherKind::Poseidon); + + assert_eq!( + build_artifacts(&program, &opts).program_id, + test.program_id, + "build_artifacts must be deterministic and default to Test" + ); + assert_eq!( + test.roots, pos.roots, + "no preprocessed root may move with the hasher" + ); + assert_eq!(test.log_heights, pos.log_heights); + assert_eq!(test.keccak_rnd_chunks, pos.keccak_rnd_chunks); + // The roots agree, so this inequality can only come from the tag. + assert_ne!( + test.program_id, pos.program_id, + "two hashers must be two program identities" + ); + assert_eq!(test.hasher, HasherKind::Test); + assert_eq!(pos.hasher, HasherKind::Poseidon); + + // The census's row counts and preprocessed widths are hasher-independent + // too — only LFM_HASH's value width moves. + let test = lfm_chip_census_with_hasher(&program, HasherKind::Test); + let pos = lfm_chip_census_with_hasher(&program, HasherKind::Poseidon); + assert_eq!(test.len(), pos.len()); + for (t, p) in test.iter().zip(pos.iter()) { + assert_eq!(t.name, p.name); + assert_eq!(t.rows, p.rows, "{}: row count must not move", t.name); + assert_eq!( + t.aux_cols, p.aux_cols, + "{}: aux width must not move", + t.name + ); + if t.name != "LFM_HASH" { + assert_eq!( + t.main_cols, p.main_cols, + "{}: only LFM_HASH may change width", + t.name + ); + } + } + } +} + +/// The tag is the mechanism, so pin it directly rather than only through a +/// digest: a reordered enum must not silently re-map an existing kind's tag +/// onto another's, which would give two permutations one program identity. +#[test] +fn the_hasher_tags_are_stable_and_distinct() { + assert_eq!(HasherKind::Test.as_tag(), 0); + assert_eq!(HasherKind::Poseidon.as_tag(), 1); + assert_eq!(HasherKind::default(), HasherKind::Test); +} + +// ========================================================================= +// The measurement — §6.3's pinned prediction, confirmed or falsified +// ========================================================================= + +/// **The number this leg exists for.** +/// +/// Base-equivalent cells per permutation, read off the same census instrument +/// that produced entry 10's keccak column (`main + 3·aux`, one row per +/// permutation) — so the two columns of the matrix are measured by one +/// instrument and are comparable by construction. +/// +/// Both sides are independent: the left comes from the AIR that was built and +/// proved, the right is §6.3's literal 621. A disagreement falsifies wave 8's +/// arithmetic, which is the outcome this test is here to allow. +#[test] +fn the_measured_cells_per_permutation_match_the_pinned_prediction() { + let program = trivial_program(); + let census = lfm_chip_census_with_hasher(&program, HasherKind::Poseidon); + let hash_chip = census + .iter() + .find(|c| c.name == "LFM_HASH") + .expect("LFM_HASH is slot-registered"); + + assert_eq!( + hash_chip.main_cols, PINNED_VALUE_COLUMNS, + "value columns per permutation row" + ); + assert_eq!( + hash_chip.aux_cols, 3, + "six LfmMem interactions ⇒ three aux columns" + ); + let per_permutation = hash_chip.main_cols as u64 + 3 * hash_chip.aux_cols as u64; + assert_eq!( + per_permutation, PINNED_CELLS_PER_PERMUTATION, + "§6.3 pinned 621 base-equivalent cells per permutation" + ); + + // The keccak column, for the ratio the matrix reports. 77,992 is entry 10's + // measured per-permutation figure; it is quoted, not recomputed here. + const KECCAK_CELLS_PER_PERMUTATION: u64 = 77_992; + assert!( + KECCAK_CELLS_PER_PERMUTATION / per_permutation >= 125, + "the algebraic column must be two orders of magnitude cheaper per permutation" + ); +} diff --git a/prover/src/lfm/programs.rs b/prover/src/lfm/programs.rs new file mode 100644 index 000000000..e4e0b869d --- /dev/null +++ b/prover/src/lfm/programs.rs @@ -0,0 +1,1297 @@ +//! Registered LFM programs. +//! +//! Every program here is deterministic — same builder calls, same +//! instructions, same column groups, same digest — which is what lets the +//! registry pin it and the drift tests recompute it on every PR. Arena +//! *values* vary per proof; the program (and its identity) never does. + +use crate::tables::types::{FE, FEE}; + +use super::builder::{Cell, LfmBuilder, LfmProgramSource}; +use super::compiler::{LfmProgram, compile}; + +/// The Milestone-B trivial program: a few hundred instructions exercising +/// every chip — constants, base ALU (incl. the assert lowering), Fp3 ALU, +/// bit decomposition, selects driven by decomposed bits, a chain of hash +/// compressions, hints and public output. +/// +/// ## It contains no `permute`, deliberately +/// +/// It used to end on a raw `b.permute`, which made it unprovable under the +/// machine's real hash — a REGISTERED program whose cryptographic meaning +/// depended on a placeholder permutation, which is the disclosure this whole +/// effort exists to retire. The permutation is now a third `compress`: every +/// registry entry is provable under every hasher, and the swap is marginally +/// cheaper besides. +/// +/// Permute mode did not disappear with it — `Test` and `Poseidon` still +/// implement it, and it still needs coverage or the arms rot. But coverage does +/// not need a registry ENTRY: [`permute_coverage_program_source`] exercises the +/// arms without claiming a program identity. +pub fn trivial_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + + let arena = b.declare_arena(4); + let h: Vec = (0..4).map(|i| b.hint_word(arena, i)).collect(); + + // Base-field leg: s = 16, m = 112, q = m/s = 7; assert q == x. + let x = b.felt_const(FE::from(7u64)); + let y = b.felt_const(FE::from(9u64)); + let s = b.add(x, y); + let m = b.mul(s, x); + let q = b.div(m, s); + b.assert_eq(q, x); + + // Fp3 leg: product, Horner step, base scaling. + let e1 = b.ext_const(&FEE::new([FE::from(1u64), FE::from(2u64), FE::from(3u64)])); + let e2 = b.ext_const(&FEE::new([FE::from(4u64), FE::from(5u64), FE::from(6u64)])); + let p = b.emul(e1, e2); + let pm = b.emul_add(p, e1, e2); + let _pb = b.emul_base(pm, q); + + // Bit-decomposition leg: m = 112 = 0b1110000; bits drive the selects. + let bits = b.bit_dec(m, 8); + let (l, _r) = b.select(bits[4], h[0], h[1]); // bit 4 of 112 = 1 → swap + let (l2, _r2) = b.select(bits[0], l, h[2]); // bit 0 = 0 → pass through + + // Hash leg: three compressions chained through memory. Feeding `d1` back in + // is the point — a socket's own output must be a legal input to the next + // one, which is what a Merkle walk does at every level. + // + // ✓ SWEPT for the leaf-mode migration and deliberately LEFT as `compress`: + // these are the only place in any registered program where raw arena data + // enters a compress, and they form a CHAIN, not a tree. There is no leaf and + // no parent here, so there is no leaf/parent confusion for `MODE_L` to + // separate — what the mode buys elsewhere it would not buy here. The + // consequence to keep in mind is that this program's arena words must be + // `u32`-laned under BLAKE3 (obligation O1), which its tests supply; data + // that cannot be is what `leaf` exists for. + let d0 = b.compress(h[0].as_digest(), h[1].as_digest()); + let d1 = b.compress(d0, l2.as_digest()); + let d2 = b.compress(d1, h[3].as_digest()); + + // Public output: two chained digests and one ALU result. + b.public(d1.as_cell()); + b.public(d2.as_cell()); + b.public(m.as_cell()); + + b.finish() +} + +pub fn trivial_program() -> LfmProgram { + compile(trivial_program_source()) +} + +/// A `permute`-mode fixture — **not a registry entry, and it must not become +/// one.** +/// +/// [`trivial_program_source`] gave up its raw `b.permute` so that every +/// registered program runs under the machine's real hash. Permute mode is still +/// live under `Test` and `Poseidon`, so it still needs a program that exercises +/// the executor arm, the trace filler and the AIR's three-cell tuple contract — +/// this is that program. It is deliberately unregistered: a registry entry is a +/// claim about a program's identity, and this one exists only to keep two +/// hashers' arms honest. +/// +/// It is unprovable under BLAKE3 by design (`MODE_P = 0`), which is itself worth +/// testing. +#[cfg(test)] +pub fn permute_coverage_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let h: Vec = (0..3).map(|i| b.hint_word(arena, i)).collect(); + + // Two permutations chained, so an output cell is also an input cell. + let s0 = b.permute([h[0], h[1], h[2]]); + let s1 = b.permute([s0[2], s0[0], s0[1]]); + + for c in s1 { + b.public(c); + } + b.finish() +} + +#[cfg(test)] +pub fn permute_coverage_program() -> LfmProgram { + compile(permute_coverage_program_source()) +} + +/// Number of arena words the keccak-chain program ingests: one full state. +pub const KECCAK_CHAIN_ARENA_WORDS: u32 = super::layout::keccak::NUM_WORDS as u32; + +/// The R1b keccak program: a hint-fed state pushed through two *chained* +/// `keccak-f[1600]` permutations, the second consuming the first's output words +/// directly out of memory. +/// +/// Chaining is the point. It proves the `u32`-half word convention round-trips: +/// the output words `LFM_KECCAK` writes are immediately legal input words, so +/// the halves it produces are canonical `u32`s and the state's two unused top +/// lanes come back zero — no repacking instruction in between. +pub fn keccak_chain_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + + let arena = b.declare_arena(KECCAK_CHAIN_ARENA_WORDS); + let state: [Cell; 13] = core::array::from_fn(|i| b.hint_word(arena, i as u32)); + + let once = b.keccak_f(state); + let twice = b.keccak_f(once); + + // Expose enough to pin both permutations: the intermediate state's first + // word and the final state's first two. + b.public(once[0]); + b.public(twice[0]); + b.public(twice[1]); + + b.finish() +} + +pub fn keccak_chain_program() -> LfmProgram { + compile(keccak_chain_program_source()) +} + +/// Message length of the registered `KeccakSpongeV0`. +/// +/// 202 bytes is chosen to exercise all three shapes at once: it crosses the +/// 136-byte rate boundary (2 blocks), it is not a multiple of the rate (so the +/// padding is not a whole block), and `202 % 4 == 2` puts the `0x01` pad byte in +/// the same `u32` half as the message's last two bytes — the mixed-half case the +/// emitter handles by adding a padding constant to the stream half. +pub const KECCAK_SPONGE_LEN: usize = 202; + +/// `keccak256` over a hint-supplied byte stream of exactly `len_bytes`, with +/// the 32-byte digest as public output. +/// +/// Length is program shape, not data: a straight-line machine has no loops, so +/// each length compiles to its own program and its own identity. +pub fn keccak_sponge_program_source(len_bytes: usize) -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let digest = super::edsl::keccak256(&mut b, &stream, len_bytes); + b.public(digest[0]); + b.public(digest[1]); + b.finish() +} + +pub fn keccak_sponge_program(len_bytes: usize) -> LfmProgram { + compile(keccak_sponge_program_source(len_bytes)) +} + +/// `DefaultTranscript::sample()` over a hint-supplied stream: keccak256 of the +/// absorbed bytes, then the 32 digest bytes REVERSED — which is both the +/// challenge the transcript returns and the prefix it re-absorbs. +/// +/// This is the R1d groundwork that is independent of the #841 revision: +/// `sample()` itself is unchanged between them. +pub fn keccak_sample_program_source(len_bytes: usize) -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec<_> = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let rev = super::edsl::keccak256_rev(&mut b, &stream, len_bytes); + b.public(rev[0]); + b.public(rev[1]); + b.finish() +} + +pub fn keccak_sample_program(len_bytes: usize) -> LfmProgram { + compile(keccak_sample_program_source(len_bytes)) +} + +// ==================== R1d: the transcript replay ==================== + +/// Seed the registered transcript-replay program starts from — a program +/// constant, exactly as a domain separator would be. 24 bytes, so the segment +/// stays half-aligned for the machine-supplied absorbs that follow. +pub const TRANSCRIPT_SEED: &[u8] = b"lfm-transcript-replay-v0"; + +/// First absorb: 32 bytes, the shape a commitment root arrives in. +pub const TRANSCRIPT_ABSORB_A: usize = 32; + +/// Second absorb: one full keccak rate, chosen so the segment it lands in +/// (32 reversed-digest bytes + 136) needs TWO rate blocks — the multi-block +/// path inside a replay, which no earlier test reaches. +pub const TRANSCRIPT_ABSORB_B: usize = 136; + +/// Arena words the replay program ingests: both absorbs as `u32` halves. +pub const TRANSCRIPT_ARENA_HALVES: u32 = + ((TRANSCRIPT_ABSORB_A + TRANSCRIPT_ABSORB_B) / super::keccak_host::BYTES_PER_HALF) as u32; + +/// Index bits the replay program's `sample_u64` draw asks for. +pub const TRANSCRIPT_QUERY_BITS: usize = 20; + +/// The R1d headline program: a scripted `DefaultTranscript` interleaving, +/// replayed in the machine, with every sampled value published. +/// +/// The script is chosen so the emitter's bookkeeping is load-bearing at every +/// step. Buffer positions, in bytes, as the emitter tracks them: +/// +/// | step | before | after | squeeze | +/// |-----------------------|--------|-------|---------| +/// | `append` A (32 B) | 32 | 32 | — | +/// | `sample_felt` | 32 | 8 | **#1** | +/// | `sample_felt` | 8 | 16 | — | +/// | `sample_ext` (3 draws)| 16 | 8 | **#2** | +/// | `append` B (136 B) | 8 | 32 | — | +/// | `sample_u64_pow2` | 32 | 8 | **#3** | +/// | `sample_felt` | 8 | 16 | — | +/// | `sample()` | 16 | 32 | **#4** | +/// | `sample_felt` | 32 | 8 | **#5** | +/// +/// So it exercises: a refill in the MIDDLE of an extension draw (squeeze #2 +/// lands between coordinates 1 and 2), an absorb that invalidates a buffer with +/// 24 bytes still in it, a raw `sample()` that invalidates with 16 bytes still +/// in it, a two-block segment (squeeze #3), and both draw kinds. Get any of the +/// invalidation rules wrong and the values diverge from the real transcript. +pub fn transcript_replay_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::edsl::bits_to_felt; + use super::transcript_replay::TranscriptReplay; + + let halves_a = TRANSCRIPT_ABSORB_A / super::keccak_host::BYTES_PER_HALF; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(TRANSCRIPT_ARENA_HALVES); + let halves: Vec = (0..TRANSCRIPT_ARENA_HALVES) + .map(|i| b.hint_felt(arena, i)) + .collect(); + let (absorb_a, absorb_b) = halves.split_at(halves_a); + + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_halves(absorb_a); + let f0 = t.sample_felt(&mut b); + let f1 = t.sample_felt(&mut b); + let e = t.sample_ext(&mut b); + t.append_halves(absorb_b); + let q = t.sample_u64_pow2(&mut b, TRANSCRIPT_QUERY_BITS); + let qf = bits_to_felt(&mut b, &q); + let f2 = t.sample_felt(&mut b); + let s = t.sample(&mut b); + let f3 = t.sample_felt(&mut b); + + b.public(f0.as_cell()); + b.public(f1.as_cell()); + b.public(e.as_cell()); + b.public(qf.as_cell()); + b.public(f2.as_cell()); + b.public(s[0]); + b.public(s[1]); + b.public(f3.as_cell()); + b.finish() +} + +pub fn transcript_replay_program() -> LfmProgram { + compile(transcript_replay_program_source()) +} + +/// Absorbs a machine-COMPUTED keccak digest and samples one challenge from it — +/// the shape a commitment root takes in a real verifier, and the only path that +/// exercises `append_digest`'s word-to-halves byte order. +/// +/// Not registered: it exists to pin that byte order against the real transcript, +/// which execution alone establishes (the executor computes the digest FROM the +/// unpacked halves, so a wrong order moves the sampled value). +pub fn transcript_absorb_digest_program_source(len_bytes: usize) -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new(); + let num_halves = super::keccak_host::num_stream_halves(len_bytes) as u32; + let arena = b.declare_arena(num_halves); + let stream: Vec = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let digest = super::edsl::keccak256(&mut b, &stream, len_bytes); + + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_digest(&mut b, &digest); + let f = t.sample_felt(&mut b); + b.public(f.as_cell()); + b.finish() +} + +pub fn transcript_absorb_digest_program(len_bytes: usize) -> LfmProgram { + compile(transcript_absorb_digest_program_source(len_bytes)) +} + +// ============ R1e slice a: field elements on the wire (big-endian) ============ + +/// Absorbs one hint-supplied BASE field element the way `append_field_element` +/// streams it (canonical `u64`, 8 bytes big-endian) and returns the raw squeeze. +/// +/// Publishing `sample()` rather than a sampled challenge is deliberate: the test +/// then compares the 32 squeezed bytes directly, so a failure means the ABSORBED +/// BYTES are wrong and nothing else. Not registered — proved through +/// `verify_against`, like the per-length keccak programs. +pub fn append_felt_program_source() -> LfmProgramSource { + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(1); + let v = b.hint_felt(arena, 0); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_felt(&mut b, v); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn append_felt_program() -> LfmProgram { + compile(append_felt_program_source()) +} + +/// The same for one CUBIC-EXTENSION element: coordinates 0, 1, 2, each 8 bytes +/// big-endian, 24 bytes total. +pub fn append_ext_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(3); + let coords: [Felt; 3] = core::array::from_fn(|i| b.hint_felt(arena, i as u32)); + let mut t = TranscriptReplay::new(TRANSCRIPT_SEED); + t.append_ext(&mut b, coords); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn append_ext_program() -> LfmProgram { + compile(append_ext_program_source()) +} + +// ==================== R1e slice b: the byte-level splice ==================== + +/// Deterministic constant bytes for the splice programs; the tests build the +/// host reference from the same function, so the two cannot drift apart. +pub fn splice_prefix(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(37).wrapping_add(11)) + .collect() +} + +/// Deterministic machine-supplied bytes for the splice programs. +pub fn splice_dynamic(len: usize) -> Vec { + (0..len) + .map(|i| (i as u8).wrapping_mul(53).wrapping_add(29)) + .collect() +} + +/// A constant prefix of `prefix_len` bytes followed by `num_halves` hinted +/// machine halves, then a raw squeeze. The shift under test is +/// `prefix_len % 4`; at 0 it takes the aligned fast path and serves as control. +pub fn splice_program_source(prefix_len: usize, num_halves: u32) -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(num_halves); + let halves: Vec = (0..num_halves).map(|i| b.hint_felt(arena, i)).collect(); + let mut t = TranscriptReplay::new(&splice_prefix(prefix_len)); + t.append_halves_misaligned(&halves); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn splice_program(prefix_len: usize, num_halves: u32) -> LfmProgram { + compile(splice_program_source(prefix_len, num_halves)) +} + +/// Tag length of the alternating splice program — the real +/// `LAMBDAVM_CONTINUATION_EPOCH_V2` is exactly this long. +pub const SPLICE_ALT_TAG: usize = 30; +pub const SPLICE_ALT_DIGEST_HALVES: u32 = 8; +pub const SPLICE_ALT_FIELD_HALVES: u32 = 2; + +/// The continuation-epoch statement's shape in miniature: alternating constant +/// and dynamic runs, with the shift CHANGING mid-stream. +/// +/// The byte offsets are the whole point. A 30-byte tag leaves shift 2; the +/// 32-byte digest and an 8-byte field keep it there; then a ONE-byte field — +/// standing for the real encoding's `fri_final_poly_log_degree` — moves every +/// later dynamic value to shift 3. A splice that handles only a single fixed +/// shift passes the fixed-prefix test above and fails this one. +pub fn splice_alternating_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::transcript_replay::TranscriptReplay; + + let total = SPLICE_ALT_DIGEST_HALVES + 2 * SPLICE_ALT_FIELD_HALVES; + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(total); + let h: Vec = (0..total).map(|i| b.hint_felt(arena, i)).collect(); + let d = SPLICE_ALT_DIGEST_HALVES as usize; + let f = SPLICE_ALT_FIELD_HALVES as usize; + + let mut t = TranscriptReplay::new(&splice_prefix(SPLICE_ALT_TAG)); + t.append_halves_misaligned(&h[..d]); + t.append_const_bytes(&splice_prefix(8)); + t.append_halves_misaligned(&h[d..d + f]); + t.append_const_bytes(&splice_prefix(1)); + t.append_halves_misaligned(&h[d + f..]); + let s = t.sample(&mut b); + b.public(s[0]); + b.public(s[1]); + b.finish() +} + +pub fn splice_alternating_program() -> LfmProgram { + compile(splice_alternating_program_source()) +} + +// ============ R1e slices c+d: the epoch statement and Phase A ============ + +/// Public-output length of the acceptance shape. Deliberately NOT a multiple of +/// four: an epoch's public output is collected one byte per COMMIT op, so the +/// unaligned case is the general one and the acceptance must exercise it. +pub const STMT_PUBLIC_OUTPUT_LEN: usize = 14; + +/// Whether each of the acceptance shape's sub-proofs is preprocessed. Mixed on +/// purpose: the verifier absorbs a preprocessed commitment only for the airs +/// that have one, so a replay that absorbs unconditionally must diverge. +pub const STMT_PREPROCESSED: [bool; 3] = [true, false, true]; + +/// Halves per 32-byte commitment. +const ROOT_HALVES: u32 = 8; + +/// Arena halves the statement-replay program reads. +pub fn stmt_arena_halves() -> u32 { + let vars = ROOT_HALVES + STMT_PUBLIC_OUTPUT_LEN.div_ceil(4) as u32 + 2; + let roots: u32 = STMT_PREPROCESSED + .iter() + .map(|&p| if p { 2 * ROOT_HALVES } else { ROOT_HALVES }) + .sum(); + vars + roots +} + +/// The acceptance shape's shape-static statement fields. +pub fn epoch_statement_shape() -> super::statement_replay::EpochStatementShape { + super::statement_replay::EpochStatementShape { + public_output_len: STMT_PUBLIC_OUTPUT_LEN, + table_counts: [3, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + num_private_input_pages: 2, + fri_final_poly_log_degree: 7, + page_ranges: vec![(0x1000, 4), (0x8000, 1)], + } +} + +/// The R1e headline program: a continuation-epoch statement bound into the +/// transcript, then Phase A over three sub-proofs, publishing the shared LogUp +/// challenges `z` and `α`. +/// +/// This is the first leg of a real verifier the machine runs end to end — +/// everything a `multi_verify` does before the per-table forks. What `z` and `α` +/// feed into (the bus-balance replay, the chaining obligations) is R1f. +pub fn statement_replay_program_source() -> LfmProgramSource { + use super::builder::Felt; + use super::statement_replay::{ + EpochStatementVars, PhaseATable, absorb_epoch_statement, replay_phase_a, + }; + use super::transcript_replay::TranscriptReplay; + + let shape = epoch_statement_shape(); + let total = stmt_arena_halves(); + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(total); + let h: Vec = (0..total).map(|i| b.hint_felt(arena, i)).collect(); + + let out_halves = STMT_PUBLIC_OUTPUT_LEN.div_ceil(4); + let (elf, rest) = h.split_at(ROOT_HALVES as usize); + let (public_output, rest) = rest.split_at(out_halves); + let (epoch_label, mut roots) = rest.split_at(2); + + // The verifier seeds an empty transcript and binds the statement first. + let mut t = TranscriptReplay::new(&[]); + absorb_epoch_statement( + &mut t, + &shape, + &EpochStatementVars { + elf_digest: elf, + public_output, + epoch_label, + }, + ); + + let mut tables = Vec::new(); + for &preprocessed in &STMT_PREPROCESSED { + let prep = if preprocessed { + let (p, r) = roots.split_at(ROOT_HALVES as usize); + roots = r; + Some(p) + } else { + None + }; + let (main, r) = roots.split_at(ROOT_HALVES as usize); + roots = r; + tables.push(PhaseATable { + // This driver supplies every root as arena cells on purpose: it is the + // statement/Phase-A differential, and where a root COMES FROM is the + // assembled verifier's decision (ledger entry 7), not this program's. + preprocessed_root: prep.map(super::statement_replay::PhaseAPreprocessed::Cells), + main_root: main, + }); + } + let (z, alpha) = replay_phase_a(&mut t, &mut b, &tables); + + b.public(z.as_cell()); + b.public(alpha.as_cell()); + b.finish() +} + +pub fn statement_replay_program() -> LfmProgram { + compile(statement_replay_program_source()) +} + +/// A harness for the candidate canonicity guard alone: `(lo, hi)` arrive as +/// hinted halves, the guard runs, the recomposed felt is published. +/// +/// Not a sound construction on its own — nothing here range-checks the hinted +/// halves to `u32`, which the derivation in +/// [`super::transcript_replay::assert_canonical`] assumes. In the replay they +/// come from an `Unpack` of a `LFM_KECCAK` output word and the adapter +/// range-checks them. This program exists so the guard's PREDICATE can be +/// exercised at the `p − 2 / p − 1 / p` boundary, which is unreachable through +/// the replay: finding a message whose digest yields an out-of-range candidate +/// means about 2^32 keccaks. +pub fn canonicity_guard_program_source() -> LfmProgramSource { + use super::transcript_replay::{Candidate, assert_canonical, candidate_to_felt}; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let c = Candidate { + lo: b.hint_felt(arena, 0), + hi: b.hint_felt(arena, 1), + }; + assert_canonical(&mut b, c); + let v = candidate_to_felt(&mut b, c); + b.public(v.as_cell()); + b.finish() +} + +pub fn canonicity_guard_program() -> LfmProgram { + compile(canonicity_guard_program_source()) +} + +/// The Milestone-C verifier program: verifies the fixture FRI +/// commitment-opening proof (`fixture::fixture_prove`) — sponge transcript +/// replay, two Merkle-authenticated opening sets, the α-combination Horner, +/// two unnormalized folds with index-bit-derived domain-point inverses, and +/// the terminal-polynomial check. Straight-line: every loop below unrolls at +/// emission; the shape is a compile-time constant of the program. +pub fn fri_toy_program_source() -> LfmProgramSource { + use super::builder::Cell; + use super::edsl::{self, SpongeVar}; + use super::fixture::{domain_constants, shape}; + + let (omega, offset) = domain_constants(); + let omega_inv = omega.inv().expect("root of unity is invertible"); + let offset_inv = offset.inv().expect("coset offset is invertible"); + // Fold-0 point inverses over q0's bits: x = c·ω^{q0} ⇒ factors ω^{-2^i}. + let invx_factors: Vec = (0..shape::QUERY_BITS) + .map(|i| omega_inv.pow(1u64 << i)) + .collect(); + // Fold-1 over j = q0 mod 8: y = c²·ω^{2j} ⇒ factors ω^{-2·2^i}, scale c⁻². + let invy_factors: Vec = (0..3).map(|i| omega_inv.pow(2u64 << i)).collect(); + let offset2_inv = offset_inv.square(); + // Terminal point y₂ = c⁴·ω^{4j}. + let y2_factors: Vec = (0..3).map(|i| omega.pow(4u64 << i)).collect(); + let offset4 = offset.square().square(); + + let mut b = LfmBuilder::new(); + let commits = b.declare_arena(4); + let opens = b.declare_arena((shape::NUM_QUERIES * shape::WORDS_PER_QUERY) as u32); + + let mut sponge = SpongeVar::new(&mut b); + let main_root = b.hint_word(commits, 0); + sponge.absorb(&mut b, main_root); + let alpha = sponge.squeeze_ext(&mut b); + let zeta0 = sponge.squeeze_ext(&mut b); + let l1_root = b.hint_word(commits, 1); + sponge.absorb(&mut b, l1_root); + let zeta1 = sponge.squeeze_ext(&mut b); + let t0w = b.hint_word(commits, 2); + let t1w = b.hint_word(commits, 3); + // The terminal coefficients are field DATA, not digests, so they enter the + // transcript through the leaf encoding — the same rule the trees follow. + sponge.absorb_felts(&mut b, t0w); + sponge.absorb_felts(&mut b, t1w); + let t0 = t0w.as_ext(); + let t1 = t1w.as_ext(); + + // Hoisted reference lanes for the per-query root comparisons. + let main_root_lanes = b.unpack(main_root); + let l1_root_lanes = b.unpack(l1_root); + + for q in 0..shape::NUM_QUERIES { + let off = (q * shape::WORDS_PER_QUERY) as u32; + let bits = sponge.squeeze_bits(&mut b, shape::QUERY_BITS); // q0 = b0..b3 + let zero_bit = b.bit_const(false); + let one_bit = b.bit_const(true); + let path_a = [bits[1], bits[2], bits[3], zero_bit]; + let path_b = [bits[1], bits[2], bits[3], one_bit]; + + // Main-tree opening A (rows 2·l_A, 2·l_A+1 with l_A = q0 >> 1). + let row_a_even = b.hint_word(opens, off); + let row_a_odd = b.hint_word(opens, off + 1); + let leaf_a = edsl::leaf_hash_pair(&mut b, row_a_even, row_a_odd); + let sibs_a: Vec = (0..4).map(|i| b.hint_word(opens, off + 2 + i)).collect(); + let root_a = edsl::merkle_walk(&mut b, leaf_a, &path_a, &sibs_a); + edsl::assert_word_eq_lanes(&mut b, root_a.as_cell(), &main_root_lanes); + + // Main-tree opening B (leaf l_A + 8, i.e. rows q0+16's pair). + let row_b_even = b.hint_word(opens, off + 6); + let row_b_odd = b.hint_word(opens, off + 7); + let leaf_b = edsl::leaf_hash_pair(&mut b, row_b_even, row_b_odd); + let sibs_b: Vec = (0..4).map(|i| b.hint_word(opens, off + 8 + i)).collect(); + let root_b = edsl::merkle_walk(&mut b, leaf_b, &path_b, &sibs_b); + edsl::assert_word_eq_lanes(&mut b, root_b.as_cell(), &main_root_lanes); + + // Row parity: q0 and q0+16 share bit 0. + let (row_a, _) = b.select(bits[0], row_a_even, row_a_odd); + let (row_b, _) = b.select(bits[0], row_b_even, row_b_odd); + + // g0 at the two points: α-combination of the opened row columns. + let la = b.unpack(row_a); + let lo = edsl::horner_ext( + &mut b, + alpha, + &[ + la[0].as_ext(), + la[1].as_ext(), + la[2].as_ext(), + la[3].as_ext(), + ], + ); + let lb = b.unpack(row_b); + let hi = edsl::horner_ext( + &mut b, + alpha, + &[ + lb[0].as_ext(), + lb[1].as_ext(), + lb[2].as_ext(), + lb[3].as_ext(), + ], + ); + + // Fold 0 → must equal the opened g1[q0]. + let inv_x = edsl::pow_bits(&mut b, &bits, &invx_factors, offset_inv); + let v1 = edsl::fri_fold(&mut b, lo, hi, zeta0, inv_x); + + let l1_lo = b.hint_word(opens, off + 12); + let l1_hi = b.hint_word(opens, off + 13); + let l1_leaf = edsl::leaf_hash_pair(&mut b, l1_lo, l1_hi); + let l1_sibs: Vec = (0..3).map(|i| b.hint_word(opens, off + 14 + i)).collect(); + let l1_path = [bits[0], bits[1], bits[2]]; + let l1_root_c = edsl::merkle_walk(&mut b, l1_leaf, &l1_path, &l1_sibs); + edsl::assert_word_eq_lanes(&mut b, l1_root_c.as_cell(), &l1_root_lanes); + + let (g1_at_q0, _) = b.select(bits[3], l1_lo, l1_hi); + b.assert_eq_ext(v1, g1_at_q0.as_ext()); + + // Fold 1 → must equal the terminal polynomial at y₂. + let inv_y = edsl::pow_bits(&mut b, &bits[0..3], &invy_factors, offset2_inv); + let v2 = edsl::fri_fold(&mut b, l1_lo.as_ext(), l1_hi.as_ext(), zeta1, inv_y); + + let y2 = edsl::pow_bits(&mut b, &bits[0..3], &y2_factors, offset4); + let t1y = b.emul_base(t1, y2); + let t_eval = b.eadd(t0, t1y); + b.assert_eq_ext(v2, t_eval); + } + + b.public(main_root); + b.public(l1_root); + b.finish() +} + +pub fn fri_toy_program() -> LfmProgram { + compile(fri_toy_program_source()) +} + +// ============ R1f: a real Merkle opening under the production hash ============ + +/// Everything about a Merkle-opening program that is compile-time. +/// +/// Both fields are SHAPE, in the sense of `others/lfm-target-shape.md`: they fix +/// how many arena words the program reads, how many byteswaps it emits and how +/// many permutations the walk costs. A program that read them from an arena +/// would be claiming to authenticate a tree whose geometry the prover chose. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MerkleOpeningShape { + /// Field elements in the leaf. A leaf is a row PAIR (`ROWS_PER_LEAF = 2`), + /// so this is `2 × columns`. + pub leaf_values: usize, + /// Tree depth: index bits consumed, siblings read, permutations walked. + pub depth: usize, +} + +impl MerkleOpeningShape { + pub const fn columns(self) -> usize { + self.leaf_values / 2 + } +} + +/// Authenticates one FRI query's main-trace opening against a committed root, +/// under the PRODUCTION keccak Merkle conventions. +/// +/// Four arenas, each field in its own words (the R1e packing rule): +/// +/// 0. the leaf's field elements, one base word each, in hash order +/// (`evaluations ‖ evaluations_sym`); +/// 1. the sibling digests, two `u32`-half words per level, LEAF LEVEL FIRST; +/// 2. the leaf index, one base word; +/// 3. the committed root, two `u32`-half words. +/// +/// The walked root is asserted equal to arena 3 and then PUBLISHED. Both matter +/// and they do different jobs. The assert is the composition-ready shape — in +/// the assembled verifier the expected root arrives exactly like this, as an +/// arena value that Phase A has already bound into the transcript, and +/// `fri_toy_program` compares its roots the same way. Publishing is what makes +/// the result a claim rather than an internal fact: public words are absorbed +/// into the LFM statement, so a verifier that supplies the real committed root +/// as the claimed output is checking the machine reached THAT root and not some +/// other one the prover found convenient. +/// +/// ## What this program does and does not bind +/// +/// It binds the leaf, the path and the low `depth` bits of the index to the +/// root. It does not bind the index to a transcript — `bit_dec` constrains the +/// hinted index to its own decomposition and the walk uses the low `depth` +/// bits, so a prover may add any multiple of `2^depth` without changing +/// anything. That is correct here and unsound alone: in the assembled verifier +/// the bits come from `TranscriptReplay::sample_u64_pow2`, which produces +/// exactly this `Vec` from a squeezed candidate. This program is the +/// authentication half of that pair, built and measured before the sampler is +/// wired to it. +pub fn keccak_merkle_opening_program_source(shape: MerkleOpeningShape) -> LfmProgramSource { + use super::edsl; + + assert!(shape.leaf_values > 0, "a leaf covers at least one column"); + assert!( + shape.leaf_values.is_multiple_of(2), + "a leaf is a row PAIR, so it holds an even number of values" + ); + assert!( + (1..=32).contains(&shape.depth), + "depth must be in 1..=32: below, there is no path; above, the index \ + would outrun a single transcript candidate half" + ); + + let mut b = LfmBuilder::new(); + let leaf_arena = b.declare_arena(shape.leaf_values as u32); + let sibling_arena = b.declare_arena(2 * shape.depth as u32); + let index_arena = b.declare_arena(1); + let root_arena = b.declare_arena(2); + + let values: Vec<_> = (0..shape.leaf_values as u32) + .map(|i| b.hint_felt(leaf_arena, i)) + .collect(); + let leaf = edsl::keccak_leaf_hash(&mut b, &values); + + let index = b.hint_felt(index_arena, 0); + let bits = b.bit_dec(index, shape.depth); + + let siblings: Vec<[Cell; 2]> = (0..shape.depth as u32) + .map(|l| { + [ + b.hint_word(sibling_arena, 2 * l), + b.hint_word(sibling_arena, 2 * l + 1), + ] + }) + .collect(); + + let root = edsl::keccak_merkle_walk(&mut b, leaf, &bits, &siblings); + + let expected = [b.hint_word(root_arena, 0), b.hint_word(root_arena, 1)]; + edsl::assert_word_eq(&mut b, root[0], expected[0]); + edsl::assert_word_eq(&mut b, root[1], expected[1]); + + b.public(root[0]); + b.public(root[1]); + b.finish() +} + +pub fn keccak_merkle_opening_program(shape: MerkleOpeningShape) -> LfmProgram { + compile(keccak_merkle_opening_program_source(shape)) +} + +// ============ R1g(ii): the cross-epoch L2G commitment binding ============ + +/// Ties each epoch's own committed L2G root to the corresponding sub-proof of +/// the global proof — `verify_l2g_commitment_binding_view` (`lib.rs:993`), +/// emitted. +/// +/// Two arenas, each root in its own two words: +/// +/// 0. the per-epoch L2G roots, `EpochProof::l2g_root`, epoch order; +/// 1. the global proof's first `num_epochs` sub-proof main-trace roots. +/// +/// Every pair is asserted equal and the epoch side is published. As in +/// [`keccak_merkle_opening_program_source`], the assert is the relation and the +/// publish is what makes it a claim: the equality alone would be satisfied by +/// any two matching arena values, so the published roots are what a verifier +/// pins against the real bundle. +/// +/// ## What binds each side, and what this slice does not do +/// +/// This program asserts the RELATION. What binds each root to its proof is the +/// composition's job: the epoch root is bound by that epoch's own Phase A +/// absorb, the global root by the global proof's. Until those legs exist, both +/// sides are arena values and a prover could satisfy the equality with two +/// matching lies — which is exactly why the roots are published rather than +/// merely compared. +/// +/// ## Why the epoch count is a constant +/// +/// `num_epochs` is shape: it fixes how many roots are read and how many asserts +/// are emitted. Production's `final_proof.len() >= epoch_l2g_roots.len()` guard +/// has no counterpart here because a program compiled for `n` epochs cannot read +/// an `n+1`-epoch bundle — the arena schema would not match. +pub fn l2g_binding_program_source(num_epochs: usize) -> LfmProgramSource { + use super::edsl; + + assert!(num_epochs > 0, "a continuation has at least one epoch"); + + let words = 2 * num_epochs as u32; + let mut b = LfmBuilder::new(); + let epoch_arena = b.declare_arena(words); + let global_arena = b.declare_arena(words); + + for i in 0..num_epochs as u32 { + let epoch = [ + b.hint_word(epoch_arena, 2 * i), + b.hint_word(epoch_arena, 2 * i + 1), + ]; + let global = [ + b.hint_word(global_arena, 2 * i), + b.hint_word(global_arena, 2 * i + 1), + ]; + edsl::assert_word_eq(&mut b, epoch[0], global[0]); + edsl::assert_word_eq(&mut b, epoch[1], global[1]); + b.public(epoch[0]); + b.public(epoch[1]); + } + b.finish() +} + +pub fn l2g_binding_program(num_epochs: usize) -> LfmProgram { + compile(l2g_binding_program_source(num_epochs)) +} + +// ============ R1g(iii): the attestation's program id ============ + +/// Halves in a `u64` rendered little-endian. +const U64_HALVES: u32 = 2; + +/// Everything about a `program_id` fold that is compile-time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProgramIdShape { + /// Page genesis commitments folded in. SHAPE: it fixes the byte length of + /// the hashed string, hence the block count and every padding position. + pub num_pages: usize, +} + +impl ProgramIdShape { + /// Bytes the fold hashes — `tag ‖ elf_digest ‖ pc_start ‖ decode ‖ n ‖ + /// (base ‖ commitment)*`. + pub fn byte_len(self) -> usize { + use crate::recursion::PROGRAM_ID_TAG; + PROGRAM_ID_TAG.len() + 32 + 8 + 32 + 8 + 40 * self.num_pages + } +} + +/// Emits `recursion::program_id_from_digest` — the fold the recursion guest +/// commits as the first 32 bytes of its attestation. +/// +/// One arena, each field in its own halves (the R1e packing rule): +/// the 32-byte ELF digest, the `u64` entry point, the 32-byte DECODE root, then +/// per page a `u64` base and a 32-byte commitment. +/// +/// ## Why the tag makes this the splice case +/// +/// `PROGRAM_ID_TAG` is 22 bytes, `≡ 2 (mod 4)`, so the ELF digest immediately +/// after it straddles half boundaries and so does everything behind it — the +/// same shape as R1e's 30-byte epoch tag. [`super::transcript_replay::ByteString`] +/// carries the byte-granular packer that handles it; alignment is a property of +/// the cursor, not of the field. +/// +/// ## What this program does NOT establish +/// +/// The attestation is deliberately **not self-enforcing**, and emitting the fold +/// in the machine does not change that. The guest uses SUPPLIED roots verbatim +/// without binding them to the inner ELF; the binding happens outside, when a +/// consumer recomputes the id from an ELF it trusts and compares +/// (`recursion::check_attestation`, an expensive native FFT + Merkle pass done +/// once at top level, never in-VM). A machine-emitted attestation inherits that +/// model unchanged — the same consumer-side compare closes it. Do not read +/// "the machine folded the roots" as "the machine bound the roots". +/// +/// ## Page ordering +/// +/// `program_id_from_digest` SORTS pages by base before folding. This program +/// folds them in supplied order, so the arena filler owes sortedness. That is +/// not a soundness hole: an unsorted fold yields an id that differs from the +/// consumer's recompute, so the proof is rejected there — the prover only +/// breaks their own attestation. It IS a completeness obligation, so it is +/// stated rather than assumed. +pub fn program_id_program_source(shape: ProgramIdShape) -> LfmProgramSource { + use super::builder::Felt; + + let root_halves = ROOT_HALVES; + let per_page = U64_HALVES + root_halves; + let total = root_halves + U64_HALVES + root_halves + per_page * shape.num_pages as u32; + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(total); + let h: Vec = (0..total).map(|i| b.hint_felt(arena, i)).collect(); + + let (elf_digest, rest) = h.split_at(root_halves as usize); + let (pc_start, rest) = rest.split_at(U64_HALVES as usize); + let (decode, mut pages) = rest.split_at(root_halves as usize); + + let page_cells: Vec<(&[Felt], &[Felt])> = (0..shape.num_pages) + .map(|_| { + let (base, r) = pages.split_at(U64_HALVES as usize); + let (commitment, r) = r.split_at(root_halves as usize); + pages = r; + (base, commitment) + }) + .collect(); + + let id = emit_program_id(&mut b, shape, elf_digest, pc_start, decode, &page_cells); + b.public(id[0]); + b.public(id[1]); + b.finish() +} + +/// The `program_id` fold over cells the caller already holds — the form the +/// ASSEMBLED verifier needs. +/// +/// This exists for assembly ledger entry 7's DECODE half. DECODE's preprocessed +/// commitment is a function of the inner ELF, so it can be neither interned (that +/// would make LFM program identity ELF-dependent) nor left unbound. The +/// resolution ruled on 2026-08-04 is the **attestation join**: the same arena cell +/// Phase A absorbs is the cell this fold consumes, so a prover who substitutes a +/// DECODE root changes the published `program_id` and the consumer's own recompute +/// rejects it. That makes DECODE exactly as bound as `elf_digest` and `pc_start` +/// already are — and the join is only real if it is STRUCTURAL, one cell with two +/// consumers, which is why this takes cells rather than an arena. +/// +/// `elf_digest` is the same eight halves the epoch STATEMENT absorbs, so that +/// value's join comes free. +/// +/// ⚠ The join's strength is the consumer-side compare +/// (`recursion::check_attestation`), which has zero production call sites. Folding +/// the roots does not bind them by itself; it makes a substitution DETECTABLE by a +/// consumer who performs the ritual. +pub fn emit_program_id( + b: &mut LfmBuilder, + shape: ProgramIdShape, + elf_digest: &[super::builder::Felt], + pc_start: &[super::builder::Felt], + decode: &[super::builder::Felt], + pages: &[(&[super::builder::Felt], &[super::builder::Felt])], +) -> super::edsl::KeccakDigest { + use super::transcript_replay::ByteString; + use crate::recursion::PROGRAM_ID_TAG; + + assert_eq!( + elf_digest.len(), + ROOT_HALVES as usize, + "the ELF digest is 32 bytes" + ); + assert_eq!( + pc_start.len(), + U64_HALVES as usize, + "the entry point is one u64" + ); + assert_eq!( + decode.len(), + ROOT_HALVES as usize, + "the DECODE commitment is 32 bytes" + ); + assert_eq!( + pages.len(), + shape.num_pages, + "the page count is SHAPE: it fixes the hashed length and every padding \ + position" + ); + + let mut s = ByteString::new(); + s.push_const(PROGRAM_ID_TAG); + s.push_halves(elf_digest); + s.push_halves(pc_start); + s.push_halves(decode); + s.push_const(&(shape.num_pages as u64).to_le_bytes()); + for (base, commitment) in pages { + assert_eq!(base.len(), U64_HALVES as usize, "a page base is one u64"); + assert_eq!( + commitment.len(), + ROOT_HALVES as usize, + "a page commitment is 32 bytes" + ); + s.push_halves(base); + s.push_halves(commitment); + } + assert_eq!(s.len(), shape.byte_len(), "byte accounting must agree"); + + s.keccak256(b) +} + +pub fn program_id_program(shape: ProgramIdShape) -> LfmProgram { + compile(program_id_program_source(shape)) +} + +// ======== R1g(i): the next epoch's REGISTER preprocessed commitment ======== + +/// Everything about a REGISTER-derivation program that is compile-time. +/// +/// Both fields belong to the INNER proof's `ProofOptions`, and both are SHAPE +/// in the sense of `others/lfm-target-shape.md`: they fix the LDE domain, hence +/// every twiddle, every leaf's byte layout and the whole tree's permutation +/// count. A program that read them from an arena would let the prover pick the +/// domain its commitment was computed over. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RegisterDerivationShape { + /// The inner proof's blowup factor. + pub blowup: usize, + /// The inner proof's coset offset (`ProofOptions::coset_offset`). + pub coset_offset: u64, +} + +impl RegisterDerivationShape { + /// Rows in the interpolation domain — `NUM_REGISTER_ADDRESSES` rounded up. + pub fn num_rows(self) -> usize { + crate::tables::register::NUM_REGISTER_ADDRESSES.next_power_of_two() + } + + /// Rows in the LDE domain. + pub fn lde_rows(self) -> usize { + self.num_rows() * self.blowup + } + + /// Merkle leaves — one per row PAIR (`ROWS_PER_LEAF = 2`). + pub fn leaves(self) -> usize { + self.lde_rows() / stark::commitment::ROWS_PER_LEAF + } + + /// Permutations the tree costs: one per leaf plus one per internal node. + /// Leaves are 48 bytes and parents 64, so each is a single rate block. + pub fn permutations(self) -> usize { + 2 * self.leaves() - 1 + } +} + +/// The REGISTER preprocessed columns' word addresses, in row order. +/// +/// Mirrors the private `register::register_word_address_list`, but assembled +/// from the PUBLIC `register_word_addresses` rather than hand-copied, so only +/// the ORDER is restated here. Nothing pins that order locally and nothing +/// needs to: the derived root is compared against production's own +/// `compute_precomputed_commitment_with_fini`, and any disagreement about which +/// address sits in which row moves the root. +fn register_offsets() -> Vec { + use crate::tables::register::{NUM_REGISTER_ADDRESSES, register_word_addresses}; + let mut addrs = Vec::with_capacity(NUM_REGISTER_ADDRESSES); + for reg in 0..32u8 { + addrs.extend(register_word_addresses(reg)); + } + addrs.extend(register_word_addresses(254)); + addrs.extend(register_word_addresses(255)); + assert_eq!( + addrs.len(), + NUM_REGISTER_ADDRESSES, + "the register address list must cover every table row" + ); + addrs +} + +/// Derives the next epoch's REGISTER preprocessed commitment from `reg_fini` — +/// `register::compute_precomputed_commitment_with_fini`, emitted. +/// +/// Two arenas, one base word per register word address (the R1e packing rule): +/// +/// 0. `R_i`, the epoch's INIT register file; +/// 1. `R_{i+1}`, the epoch's `reg_fini`. +/// +/// The derived root is PUBLISHED. There is nothing to assert it against, and +/// that is the mechanism rather than an omission — see below. +/// +/// ## Why this is a derivation and not a comparison +/// +/// The chaining obligation is often written as "check `reg_fini` against the +/// next epoch's supplied REGISTER root". There is no supplied root. +/// `build_epoch_airs` (`continuation.rs:636`) CONSTRUCTS the preprocessed +/// commitment from `register_init` and `reg_fini`, and `VmAirs::new`'s +/// `register_preprocessed` parameter — which every verify caller passes `None` +/// to — must stay unwired: computing the commitment from the values is what +/// ties the values to it. Supply the root instead and `reg_fini` has no +/// remaining role, so a prover could offer a root consistent with a `reg_fini` +/// it never honoured and the cross-epoch chain would go unenforced. +/// +/// ## Three columns, one of them free +/// +/// Production commits OFFSET ‖ INIT ‖ FINI. OFFSET holds the register word +/// addresses, which are fixed, so its LDE is a program CONSTANT — the shape +/// rule applying in the machine's favour for once. Only INIT and FINI carry +/// arena values and only they pay for a transform, which is why the derivation +/// emits two LDEs for three columns. +/// +/// The constant column still costs at leaf-hashing time: its values are +/// byte-swapped into the leaf like any other. That swap is what +/// [`RegisterDerivationShape::permutations`] does NOT count, and the cost test +/// prints both. +/// +/// ## What this program does NOT bind +/// +/// The two arenas are unbound here. In the assembled verifier `R_{i+1}` is the +/// same vector the next epoch reads as its INIT and the published root is what +/// that epoch's Phase A absorbs; until those joins exist a prover may supply +/// any pair and get the honestly-derived root for it. The derivation is +/// correct in isolation and binds nothing in isolation — the same standing +/// caveat as the L2G binding leg. +pub fn register_derivation_program_source(shape: RegisterDerivationShape) -> LfmProgramSource { + use crate::tables::register::NUM_REGISTER_ADDRESSES; + + let supplied = NUM_REGISTER_ADDRESSES as u32; + let mut b = LfmBuilder::new(); + let init_arena = b.declare_arena(supplied); + let fini_arena = b.declare_arena(supplied); + let init: Vec<_> = (0..supplied).map(|r| b.hint_felt(init_arena, r)).collect(); + let fini: Vec<_> = (0..supplied).map(|r| b.hint_felt(fini_arena, r)).collect(); + + let root = emit_register_commitment(&mut b, shape, &init, &fini); + b.public(root[0]); + b.public(root[1]); + b.finish() +} + +/// The REGISTER preprocessed commitment over INIT and FINI cells the caller +/// already holds — [`register_derivation_program_source`] without the arenas. +/// +/// This is the form the ASSEMBLED verifier needs, and the reason it exists is +/// assembly ledger entries 7 and 2, which close together. The spine declares the +/// register boundary vector as one arena and reads `start_index` out of slot 64 +/// for the COMMIT-bus target; passing those very cells here means the root Phase +/// A absorbs is COMPUTED from them. That computation is the binding: production +/// has no arithmetic `start + len` check anywhere, it rebuilds the commitment +/// from the register vectors and rejects unless the absorbed root matches, so +/// `start_index` is tied to the chain exactly when the machine does the same. +/// +/// Hinting the root instead — which the spine did until this existed — leaves +/// `start_index` a free arena word: a prover supplies whatever index makes the +/// COMMIT bus close, and the unrelated hinted root satisfies Phase A. +/// +/// `init` and `fini` are `NUM_REGISTER_ADDRESSES` cells each. Rows past that are +/// the pooled ZERO constant, matching `zeroed_fe_vec`: production writes only the +/// supplied prefix, and making the padding program text rather than arena data is +/// the same discipline the OOD next-row pruning follows. +pub fn emit_register_commitment( + b: &mut LfmBuilder, + shape: RegisterDerivationShape, + init: &[super::builder::Felt], + fini: &[super::builder::Felt], +) -> super::edsl::KeccakDigest { + use super::edsl; + use super::lde::coset_lde; + use crate::tables::register::{NUM_PREPROCESSED_COLS_WITH_FINI, NUM_REGISTER_ADDRESSES}; + use math::fft::bit_reversing::reverse_index; + use stark::commitment::ROWS_PER_LEAF; + + assert_eq!( + NUM_PREPROCESSED_COLS_WITH_FINI, 3, + "the derivation commits OFFSET ‖ INIT ‖ FINI; a fourth preprocessed \ + column changes the leaf layout and the arena schema together" + ); + assert_eq!( + init.len(), + NUM_REGISTER_ADDRESSES, + "one INIT cell per register word address" + ); + assert_eq!( + fini.len(), + NUM_REGISTER_ADDRESSES, + "one FINI cell per register word address" + ); + let num_rows = shape.num_rows(); + let coset_offset = FE::from(shape.coset_offset); + + // Padding rows are zero in all three columns, exactly as `zeroed_fe_vec` + // leaves them: production writes only the first NUM_REGISTER_ADDRESSES. + let zero = b.felt_const(FE::zero()); + let offsets = register_offsets(); + let offset_col: Vec = (0..num_rows) + .map(|r| offsets.get(r).map_or(FE::zero(), |&a| FE::from(a))) + .collect(); + let column = |supplied: &[super::builder::Felt]| { + (0..num_rows) + .map(|r| supplied.get(r).copied().unwrap_or(zero)) + .collect::>() + }; + let init_col = column(init); + let fini_col = column(fini); + + // OFFSET is fixed, so its extension is interned constants rather than an + // emitted transform — and it is taken from PRODUCTION's own transform, not + // from `lde`'s. That is deliberate: the three columns land in one tree, so + // a root that matches production pins the emitter against the very function + // it is emitting, inside the same hash. + let offset_lde: Vec<_> = { + use math::polynomial::Polynomial; + use stark::prover::evaluate_polynomial_on_lde_domain; + let poly = + Polynomial::interpolate_fft::(&offset_col) + .expect("the OFFSET column interpolates"); + evaluate_polynomial_on_lde_domain(&poly, shape.blowup, num_rows, &coset_offset) + .expect("the OFFSET polynomial extends") + .into_iter() + .map(|v| b.felt_const(v)) + .collect() + }; + let init_lde = coset_lde(b, &init_col, shape.blowup, coset_offset); + let fini_lde = coset_lde(b, &fini_col, shape.blowup, coset_offset); + + // Leaf `i` hashes the bit-reversed rows `2i` and `2i+1`, each written + // column by column in big-endian — `keccak_leaves_bit_reversed_grouped`. + let lde_rows = shape.lde_rows(); + let leaves: Vec<_> = (0..shape.leaves()) + .map(|leaf| { + let mut values = Vec::with_capacity(ROWS_PER_LEAF * NUM_PREPROCESSED_COLS_WITH_FINI); + for k in 0..ROWS_PER_LEAF { + let row = reverse_index(ROWS_PER_LEAF * leaf + k, lde_rows as u64); + values.extend([offset_lde[row], init_lde[row], fini_lde[row]]); + } + edsl::keccak_leaf_hash(b, &values) + }) + .collect(); + + edsl::keccak_merkle_tree_root(b, &leaves) +} + +pub fn register_derivation_program(shape: RegisterDerivationShape) -> LfmProgram { + compile(register_derivation_program_source(shape)) +} + +/// A bare [`super::lde::coset_lde`], publishing every extended value. +/// +/// The instrument behind the LDE differential. `register_derivation_program` +/// exercises the transform only at the register shape — `n = 128`, coset offset +/// 3 — and every production REGISTER table has exactly that shape, so a +/// differential over production data cannot distinguish an emitter that is +/// right in general from one that is accidentally right at 128. This drives +/// synthetic sizes and offsets against production's own transform. +pub fn lde_probe_program_source(n: usize, blowup: usize, coset_offset: u64) -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(n as u32); + let values: Vec<_> = (0..n as u32).map(|i| b.hint_felt(arena, i)).collect(); + for v in super::lde::coset_lde(&mut b, &values, blowup, FE::from(coset_offset)) { + b.public(v.as_cell()); + } + b.finish() +} + +pub fn lde_probe_program(n: usize, blowup: usize, coset_offset: u64) -> LfmProgram { + compile(lde_probe_program_source(n, blowup, coset_offset)) +} diff --git a/prover/src/lfm/proof.rs b/prover/src/lfm/proof.rs new file mode 100644 index 000000000..5521a1e24 --- /dev/null +++ b/prover/src/lfm/proof.rs @@ -0,0 +1,324 @@ +//! LFM prove / verify entry points. +//! +//! Prove: execute → traces → statement-bound transcript → the same generic +//! `multi_prove` the RV64 VM uses. Verify: registry-resolve the program's +//! roots (hard error on a miss — no fallback), rebuild the AIR set, replay +//! Phase A on a forked transcript to recover the shared LogUp challenges, +//! compute the expected `LfmPublic` balance from the *claimed* public words +//! (the COMMIT-bus pattern), and run `multi_verify_views`. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use math::field::element::FieldElement; +use math::field::traits::IsPrimeField; +use stark::config::Commitment; +use stark::proof::options::ProofOptions; +use stark::proof::stark::MultiProof; +use stark::proof::view::MultiProofView; +use stark::prover::{IsStarkProver, Prover, ProvingError}; +use stark::residency_mode::ResidencyMode; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{BusId, GoldilocksExtension, GoldilocksField}; + +use super::airs::{LfmAirs, NUM_LFM_CHIPS, num_lfm_airs}; +use super::compiler::LfmProgram; +use super::executor::{LfmExecError, execute}; +use super::hash::HasherKind; +use super::registry::{LfmArtifacts, LfmProgramKind, LfmRegistryError, resolve}; +use super::statement::absorb_lfm_statement; +use super::trace::{LfmTraces, build_traces_with_hasher}; +use super::word::LfmWord; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +pub struct LfmProof { + pub proof: MultiProof, + /// The public output the execution produced, in emission order. + pub public_words: Vec<(u32, LfmWord)>, +} + +#[derive(Debug)] +pub enum LfmProveError { + Exec(LfmExecError), + Prover(ProvingError), +} + +/// Proves under the permutation `artifacts` was built for. +/// +/// The hasher comes from the artifacts rather than from a default, because +/// `artifacts.program_id` is derived from it: taking it from anywhere else +/// would let the statement claim one permutation while the AIRs prove another. +pub fn lfm_prove( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, +) -> Result { + lfm_prove_with_hasher(program, artifacts, arenas, options, artifacts.hasher) +} + +/// [`lfm_prove`] with the `LFM_HASH` permutation named explicitly at the call +/// site instead of read off `artifacts`. +/// +/// The chips bake their hasher's constants into their constraints, so execution +/// must use the same hasher — this function is the single place that holds them +/// together, passing one `hasher` to the executor, the trace filler and the AIR +/// set. Verification needs the same value ([`verify_against`]). +/// +/// # Panics +/// +/// If `hasher` is not the one `artifacts` was built for. The two are not +/// independent: `artifacts.program_id` binds the hasher, so a mismatch would +/// produce a proof whose statement names a permutation the trace does not use — +/// unverifiable everywhere, and confusing at exactly the point (registry +/// regeneration) where it would be introduced. The agreement is a caller bug, +/// not a proof outcome, so it is asserted rather than returned. +pub fn lfm_prove_with_hasher( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, + hasher: HasherKind, +) -> Result { + assert_eq!( + artifacts.hasher, hasher, + "artifacts were built for {:?} but proving was asked for {hasher:?}; \ + program_id binds the hasher, so the two must agree", + artifacts.hasher + ); + lfm_prove_with_residency( + program, + artifacts, + arenas, + options, + hasher, + decide_lfm_residency(), + ) +} + +/// [`lfm_prove_with_hasher`] with the residency mode supplied instead of read +/// from the environment, so a test can prove the same program under both modes +/// in one process without touching global state. +pub(crate) fn lfm_prove_with_residency( + program: &LfmProgram, + artifacts: &LfmArtifacts, + arenas: &[Vec], + options: &ProofOptions, + hasher: HasherKind, + residency: ResidencyMode, +) -> Result { + let exec = execute(program, arenas, &hasher).map_err(LfmProveError::Exec)?; + let mut traces = build_traces_with_hasher(program, &exec.records, hasher); + let proof = prove_traces_with_hasher( + artifacts, + &mut traces, + &exec.public_words, + options, + hasher, + residency, + ) + .map_err(LfmProveError::Prover)?; + + Ok(LfmProof { + proof, + public_words: exec.public_words, + }) +} + +/// Proves an already-built trace set against `artifacts`. +/// +/// Split out of [`lfm_prove`] so callers that need to inspect or corrupt a +/// trace between generation and proving (the tamper tests) share this +/// transcript setup instead of reimplementing it. `lfm_prove` itself goes +/// through [`prove_traces_with_hasher`], so this artifacts-hasher form has only +/// test callers. +#[cfg(test)] +pub(crate) fn prove_traces( + artifacts: &LfmArtifacts, + traces: &mut LfmTraces, + public_words: &[(u32, LfmWord)], + options: &ProofOptions, +) -> Result, ProvingError> { + prove_traces_with_hasher( + artifacts, + traces, + public_words, + options, + artifacts.hasher, + decide_lfm_residency(), + ) +} + +/// [`prove_traces`] against an AIR set built for `hasher`. The traces must have +/// been built with the same one. +/// +/// Storage mode comes from [`crate::auto_storage::decide_lfm`] and residency +/// mode from [`decide_lfm_residency`] rather than parameters: both are resource +/// decisions, invisible to the proof — spilling changes where a trace lives and +/// recompute changes how long an LDE lives, never a byte the transcript absorbs +/// — so threading them through the prove signature would put knobs with no wire +/// meaning in front of every caller. +pub(crate) fn prove_traces_with_hasher( + artifacts: &LfmArtifacts, + traces: &mut LfmTraces, + public_words: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, + residency: ResidencyMode, +) -> Result, ProvingError> { + let airs = LfmAirs::new_with_hasher( + &artifacts.roots, + options, + artifacts.keccak_rnd_chunks, + hasher, + ); + let mut transcript = DefaultTranscript::::new(&[]); + absorb_lfm_statement( + &mut transcript, + &artifacts.program_id, + public_words, + options.fri_final_poly_log_degree, + ); + Prover::multi_prove( + airs.air_trace_pairs(traces), + &mut transcript, + #[cfg(feature = "disk-spill")] + crate::auto_storage::decide_lfm(), + residency, + ) +} + +/// The LFM wrap's [`ResidencyMode`]: `RecomputeLde` when `LAMBDA_VM_RESIDENCY` +/// is set to `recompute`, else `Retain`. +/// +/// An explicit knob for the same reason the storage mode is one: the wrap has +/// no calibrated peak estimate to decide from, and the trade this mode makes — +/// one extra forward NTT per table against dropping the `O(N)` main-LDE +/// retention — is only worth taking when `N` is large. The fixture wrap has one +/// or two `KECCAK_RND` chunks and would just pay the NTT. +/// +/// `RecomputeLde` also releases each table's aux columns once its proof exists, +/// so callers that read the traces after proving must leave this unset. Nothing +/// on the wrap path does. +pub(crate) fn decide_lfm_residency() -> ResidencyMode { + match std::env::var("LAMBDA_VM_RESIDENCY").as_deref() { + Ok("recompute") => { + log::info!("lfm residency_mode: RecomputeLde (LAMBDA_VM_RESIDENCY=recompute)"); + ResidencyMode::RecomputeLde + } + _ => ResidencyMode::Retain, + } +} + +/// `Err` = registry miss (the hard, no-fallback path). `Ok(false)` = invalid +/// proof or claimed-public mismatch. +pub fn lfm_verify( + kind: LfmProgramKind, + proof: &MultiProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, +) -> Result { + let entry = resolve(kind, options.blowup_factor)?; + Ok(verify_against( + &entry.roots, + &entry.program_id, + entry.keccak_rnd_chunks, + proof, + claimed_public, + options, + entry.hasher, + )) +} + +/// Verifies against a supplied root vector, program digest, `KECCAK_RND` chunk +/// count and hasher instead of a registry entry. +/// +/// The registry lookup in [`lfm_verify`] is the soundness argument's first +/// premise and has no off-switch; this is not one. It exists for callers that +/// legitimately hold freshly built artifacts — the registry regeneration path, +/// and tests covering program shapes that are not (and need not be) registered, +/// such as the per-length keccak256 programs. +/// +/// Every piece is supplied for the same reason: it is program shape the +/// verifier must know to build the AIR set, and none of it is ever read off the +/// proof. That includes the hasher — which a caller holding artifacts should +/// pass as `artifacts.hasher`, since the digest it is paired with was derived +/// from exactly that value. There is deliberately no defaulting form: a +/// verifier that silently assumed a permutation would be assuming the one thing +/// the roots cannot tell it. +#[allow(clippy::too_many_arguments)] +pub fn verify_against( + roots: &[Commitment; NUM_LFM_CHIPS], + program_id: &Commitment, + keccak_rnd_chunks: usize, + proof: &MultiProof, + claimed_public: &[(u32, LfmWord)], + options: &ProofOptions, + hasher: HasherKind, +) -> bool { + // A zero chunk count would drop KECCAK_RND — and its constraints — from + // the set entirely. Reject the shape rather than build it. + if keccak_rnd_chunks == 0 { + return false; + } + let view = MultiProofView::Owned(proof); + if view.len() != num_lfm_airs(keccak_rnd_chunks) { + return false; + } + + let airs = LfmAirs::new_with_hasher(roots, options, keccak_rnd_chunks, hasher); + let refs = airs.air_refs(); + + let mut transcript = DefaultTranscript::::new(&[]); + absorb_lfm_statement( + &mut transcript, + program_id, + claimed_public, + options.fri_final_poly_log_degree, + ); + + // Fork the statement-bound state and replay Phase A to recover the shared + // LogUp challenges; the expected balance is the LfmPublic sum recomputed + // from the claimed words (all other LFM buses balance to zero internally). + let mut replay = transcript.clone(); + let (z, alpha) = crate::replay_transcript_phase_a_view(&refs, view, &mut replay); + let Some(expected) = expected_public_balance(claimed_public, &z, &alpha) else { + return false; + }; + + Verifier::multi_verify_views(&refs, view, &mut transcript, &expected) +} + +/// `Σ_i 1/(z − (LfmPublic + index_i·α + Σ_l v_l·α^{2+l}))` — the fingerprint +/// layout matches the `LFM_PUBLIC` sender token `(index, v0..v3)`. +fn expected_public_balance( + words: &[(u32, LfmWord)], + z: &FieldElement, + alpha: &FieldElement, +) -> Option> { + let bus = FieldElement::::from(BusId::LfmPublic as u64); + let mut powers = [FieldElement::::zero(); 5]; + powers[0] = *alpha; + for i in 1..5 { + powers[i] = &powers[i - 1] * alpha; + } + let mut fingerprints: Vec> = words + .iter() + .map(|(index, word)| { + let mut acc = &bus + FieldElement::::from(*index as u64) * &powers[0]; + for (l, lane) in word.iter().enumerate() { + let v = GoldilocksField::canonical(lane.value()); + acc += FieldElement::::from(v) * &powers[1 + l]; + } + z - acc + }) + .collect(); + // A zero fingerprint (a collision with z) is a failure, like COMMIT's. + FieldElement::inplace_batch_inverse(&mut fingerprints).ok()?; + Some( + fingerprints + .iter() + .fold(FieldElement::::zero(), |acc, t| acc + t), + ) +} diff --git a/prover/src/lfm/proof_arena.rs b/prover/src/lfm/proof_arena.rs new file mode 100644 index 000000000..75740383b --- /dev/null +++ b/prover/src/lfm/proof_arena.rs @@ -0,0 +1,346 @@ +//! Host-side arena filler: real proof BYTES → LFM arena words. +//! +//! Input is the guest's wire-format blob, not an in-memory bundle — see +//! [`super::proof_fixture`] for why that fidelity matters. Everything here reads +//! the archived view in place, exactly as the recursion guest does. +//! +//! ## The packing rule this module exists to enforce +//! +//! An arena is a vector of `u32` words, NOT a byte stream. Every field must be +//! packed into its OWN halves; concatenating fields and packing afterwards lets +//! a field whose length is not a multiple of four shift every field behind it. +//! That bug cost real debugging time in R1e and it is silent — the halves count +//! still comes out right, only the values are wrong. + +use crypto::merkle_tree::proof::verify_merkle_path_from_leaf_hash; +use math::field::element::FieldElement; +use stark::config::{BatchedMerkleTreeBackend, Commitment}; + +use crate::tables::types::GoldilocksField; + +use super::keccak_host::pack_stream; +use super::proof_fixture::FixtureArchive; +use super::word::{LfmWord, base_word}; + +type FE = FieldElement; + +/// The Merkle backend the main trace is committed under — the production alias, +/// not a locally chosen equivalent, so a backend change reaches this module. +type MainBackend = BatchedMerkleTreeBackend; + +/// Halves in one 32-byte commitment. +pub const ROOT_HALVES: usize = 8; + +/// The main-trace Merkle roots an epoch's sub-proofs commit to, in air order. +/// +/// These are the roots Phase A absorbs and, more importantly for R1f, the roots +/// a Merkle opening is authenticated AGAINST. They come straight off the proof. +/// +/// NOTE for the Phase-A leg: the verifier also absorbs each air's PREPROCESSED +/// commitment, and that one does NOT live in the proof — it comes from the AIR +/// set (`air.precomputed_commitment()`), which means replaying Phase A over a +/// real proof needs the epoch's AIRs rebuilt, not just its bytes. Out of scope +/// here and flagged rather than papered over. +pub fn epoch_main_roots(archive: &FixtureArchive, epoch: usize) -> Vec { + let bundle = &archive.guest_input().bundle; + assert!( + epoch < bundle.num_epochs(), + "epoch {epoch} out of range ({} epochs)", + bundle.num_epochs() + ); + let proofs = bundle.epoch_proof(epoch); + (0..proofs.len()) + .map(|i| *proofs.get(i).lde_trace_main_merkle_root()) + .collect() +} + +/// Number of sub-proofs (tables) in an epoch. +pub fn epoch_num_tables(archive: &FixtureArchive, epoch: usize) -> usize { + archive.guest_input().bundle.epoch_proof(epoch).len() +} + +pub fn num_epochs(archive: &FixtureArchive) -> usize { + archive.guest_input().bundle.num_epochs() +} + +/// Bytes epoch `epoch` committed — the statement's `public_output` field. +pub fn epoch_public_output(archive: &FixtureArchive, epoch: usize) -> &[u8] { + archive.guest_input().bundle.epoch_public_output(epoch) +} + +/// Packs commitments into arena halves, each root into its OWN eight halves. +pub fn roots_to_halves(roots: &[Commitment]) -> Vec { + let mut out = Vec::with_capacity(roots.len() * ROOT_HALVES); + for root in roots { + let halves = pack_stream(root); + debug_assert_eq!(halves.len(), ROOT_HALVES); + out.extend(halves); + } + out +} + +/// Wraps packed halves as arena words. +pub fn halves_to_arena(halves: Vec) -> Vec { + halves.into_iter().map(base_word).collect() +} + +/// A 32-byte commitment as the two machine words a keccak digest occupies: +/// four `u32` halves per word, half `h` = bytes `4h..4h+4` little-endian. +/// +/// This is NOT [`super::word::pack_digest`]'s layout. That one packs four FULL +/// felts, which is the `LFM_HASH` (Milestone-C) digest; a keccak digest lives on +/// the bus as eight `u32` halves and must be handed to the chip that way. +pub fn commitment_words(c: &Commitment) -> [LfmWord; 2] { + let halves = pack_stream(c); + debug_assert_eq!(halves.len(), ROOT_HALVES); + [ + [halves[0], halves[1], halves[2], halves[3]], + [halves[4], halves[5], halves[6], halves[7]], + ] +} + +// ==================== one query's main-trace opening ==================== + +/// One FRI query's MAIN-trace opening, in the form the machine consumes it. +/// +/// This is the input to [`crate::lfm::edsl::keccak_merkle_walk`] and the thing +/// R1f authenticates: a real row pair from a real continuation-epoch proof, +/// against that proof's own committed root. +/// +/// ## What the verifier does with these fields +/// +/// `Verifier::verify_opening_pair` hashes `evaluations ‖ evaluations_sym` into +/// one leaf and folds it up `merkle_path` at index `iota`. The pair is one leaf +/// because `ROWS_PER_LEAF = 2`: a query opens a value and its symmetric +/// counterpart, which are the two bit-reversed rows `2·iota` and `2·iota+1`, so +/// a single path authenticates both. +pub struct MainTraceOpening { + /// The committed root, read off the proof — the oracle for the whole leg. + pub root: Commitment, + /// `evaluations ‖ evaluations_sym` in hash order: the row pair written + /// column by column, each element rendered big-endian by the leaf hasher. + pub values: Vec, + /// Where `evaluations_sym` starts — i.e. the table's column count. + pub num_columns: usize, + /// Sibling digests, LEAF LEVEL FIRST. That is the order + /// `verify_merkle_path_from_leaf_hash` consumes them in: it walks the vector + /// forwards while shifting the index right, so element 0 pairs with the + /// index's least significant bit. (`Proof`'s doc comment describes the + /// reverse; the code is what this mirrors.) + pub siblings: Vec, +} + +impl MainTraceOpening { + /// Reads query `query` of sub-proof `table` in epoch `epoch`. + pub fn extract( + archive: &FixtureArchive, + epoch: usize, + table: usize, + query: usize, + ) -> MainTraceOpening { + let bundle = &archive.guest_input().bundle; + assert!(epoch < bundle.num_epochs(), "epoch {epoch} out of range"); + let proofs = bundle.epoch_proof(epoch); + assert!(table < proofs.len(), "table {table} out of range"); + let proof = proofs.get(table); + assert!( + query < proof.deep_poly_openings_len(), + "query {query} out of range ({} openings)", + proof.deep_poly_openings_len() + ); + let opening = proof.deep_poly_opening(query).main_trace_polys(); + let evaluations = opening.evaluations(); + let sym = opening.evaluations_sym(); + assert_eq!( + evaluations.len(), + sym.len(), + "a row pair's two rows must have the same width" + ); + MainTraceOpening { + root: *proof.lde_trace_main_merkle_root(), + num_columns: evaluations.len(), + values: evaluations.iter().chain(sym.iter()).cloned().collect(), + siblings: opening.merkle_path().to_vec(), + } + } + + /// Path length = tree depth = the number of index bits the walk consumes. + pub fn depth(&self) -> usize { + self.siblings.len() + } + + /// The leaf hash, computed by the PRODUCTION hasher on the production + /// split — literally the call `verify_opening_pair` makes. + pub fn leaf_hash(&self) -> Commitment { + MainBackend::hash_data_from_slices( + &self.values[..self.num_columns], + &self.values[self.num_columns..], + ) + } + + /// Whether production's own path check accepts this opening at `index`. + pub fn verifies_at(&self, index: usize) -> bool { + verify_merkle_path_from_leaf_hash::( + &self.siblings, + &self.root, + index, + self.leaf_hash(), + ) + } + + /// Every leaf index at which this opening authenticates. + /// + /// ## Why a search, and why that is honest + /// + /// The index is the FRI query challenge `iota`, and it is NOT in the proof — + /// the verifier derives it from the transcript, which needs the epoch's + /// statement and its AIR set, neither of which a byte blob carries (the + /// preprocessed commitments come from `air.precomputed_commitment()`). Since + /// the path, the leaf and the root are all fixed by the proof, the index is + /// nonetheless determined by them, so recovering it by exhaustion asks the + /// proof rather than inventing an answer — and the oracle doing the asking + /// is production's `verify_merkle_path_from_leaf_hash`, not a local model. + /// + /// The result is a LIST because a degenerate tree has several: a table + /// whose trace is mostly padding commits identical rows, so identical + /// leaves sit under identical subtrees and many indices verify. Any opening + /// used for an index-tamper vector must have exactly one — otherwise + /// "flip an index bit" is not a tamper at all. Callers assert that. + /// + /// Costs `2^depth` path walks; fine at the fixture's depths, not a + /// mechanism anything but a fixture should use. + pub fn indices_that_verify(&self) -> Vec { + (0..(1usize << self.depth())) + .filter(|i| self.verifies_at(*i)) + .collect() + } + + /// The leaf's field elements as arena words: one base word each, since the + /// machine byteswaps them itself (they are full felts, not `u32` halves). + pub fn leaf_arena(&self) -> Vec { + self.values.iter().copied().map(base_word).collect() + } + + /// The sibling digests as arena words, two per level, leaf level first. + pub fn sibling_arena(&self) -> Vec { + self.siblings.iter().flat_map(commitment_words).collect() + } + + /// The committed root as arena words. + pub fn root_arena(&self) -> Vec { + commitment_words(&self.root).to_vec() + } +} + +/// Host mirror of the machine's walk, returning the root it reaches. +/// +/// Production's checker returns a bool, so it cannot supply the root a TAMPERED +/// input folds to — which a coherent forgery needs (the forged run must claim a +/// root consistent with its own inputs, or it fails in-machine before the +/// interesting check). Built from the production parent hash, so the only thing +/// local about it is the loop. +pub fn walk_to_root(leaf: Commitment, index: usize, siblings: &[Commitment]) -> Commitment { + use crypto::merkle_tree::traits::IsMerkleTreeBackend; + let mut node = leaf; + let mut index = index; + for sibling in siblings { + node = if index.is_multiple_of(2) { + MainBackend::hash_new_parent(&node, sibling) + } else { + MainBackend::hash_new_parent(sibling, &node) + }; + index >>= 1; + } + node +} + +// ==================== the cross-epoch L2G binding ==================== + +/// Each epoch's own committed L2G table root, in epoch order. +/// +/// The left-hand side of `verify_l2g_commitment_binding_view`: epoch `i`'s +/// `EpochProof::l2g_root`, which that epoch's own proof commits to. +pub fn epoch_l2g_roots(archive: &FixtureArchive) -> Vec { + let bundle = &archive.guest_input().bundle; + (0..bundle.num_epochs()) + .map(|i| bundle.epoch_l2g_root(i)) + .collect() +} + +/// The global proof's first `count` sub-proof main-trace roots — the right-hand +/// side of the same binding. +/// +/// The global proof carries one L2G sub-proof per epoch FIRST, then +/// GLOBAL_MEMORY, so sub-proof `i` is epoch `i`'s L2G table. Production also +/// checks `final_proof.len() >= epoch_l2g_roots.len()`; here that is structural, +/// since a machine program compiled for `n` epochs reads exactly `n` roots and +/// this function panics rather than short-reading. +pub fn global_l2g_roots(archive: &FixtureArchive, count: usize) -> Vec { + let global = archive.guest_input().bundle.global_proof(); + assert!( + global.len() >= count, + "the global proof has {} sub-proofs, need {count}", + global.len() + ); + (0..count) + .map(|i| *global.get(i).lde_trace_main_merkle_root()) + .collect() +} + +/// Commitments as arena words, two per root, in order. +pub fn commitments_to_arena(roots: &[Commitment]) -> Vec { + roots.iter().flat_map(commitment_words).collect() +} + +// ==================== the attestation's program id ==================== + +/// The inner ELF bytes the guest input carries. +pub fn inner_elf(archive: &FixtureArchive) -> &[u8] { + archive.guest_input().inner_elf.as_slice() +} + +/// The supplied DECODE preprocessed root. +pub fn decode_commitment(archive: &FixtureArchive) -> Commitment { + archive.guest_input().decode_commitment +} + +/// The supplied per-page genesis roots, `(base, commitment)`. +/// +/// ⚠ EMPTY for the `fibonacci` fixture — that guest touches no data pages — so +/// any test that only uses the fixture leaves the page path unexercised. Drive +/// it with a synthetic shape rather than treating it as covered. +pub fn page_commitments(archive: &FixtureArchive) -> Vec<(u64, Commitment)> { + archive + .guest_input() + .page_commitments + .iter() + .map(|p| (p.0.to_native(), p.1)) + .collect() +} + +// ============ the cross-epoch REGISTER boundary ============ + +/// Epoch `i`'s `(register_init, reg_fini)` — the pair +/// `register::compute_precomputed_commitment_with_fini` turns into that epoch's +/// preprocessed REGISTER commitment. +/// +/// INIT is the VERIFIER's derivation, never a bundled value: epoch 0's comes +/// from the inner ELF's entry point and every later epoch's is the previous +/// epoch's `reg_fini`. That is the whole point of the chaining obligation, so +/// reading INIT off the proof here would quietly test a different mechanism — +/// the walk below is the same one `verify_continuation_archived` performs. +pub fn register_boundary(archive: &FixtureArchive, epoch: usize) -> (Vec, Vec) { + let bundle = &archive.guest_input().bundle; + assert!( + epoch < bundle.num_epochs(), + "epoch {epoch} out of range ({} epochs)", + bundle.num_epochs() + ); + let elf = executor::elf::Elf::load(inner_elf(archive)).expect("the inner ELF must load"); + let mut init = crate::tables::register::register_init_from_entry_point(elf.entry_point); + for i in 0..epoch { + init = bundle.epoch_reg_fini(i).expect("reg_fini deserializes"); + } + let fini = bundle.epoch_reg_fini(epoch).expect("reg_fini deserializes"); + (init, fini) +} diff --git a/prover/src/lfm/proof_fixture.rs b/prover/src/lfm/proof_fixture.rs new file mode 100644 index 000000000..d256d0ef3 --- /dev/null +++ b/prover/src/lfm/proof_fixture.rs @@ -0,0 +1,186 @@ +//! Real continuation-proof BYTES for the machine to consume. +//! +//! R1f's premise: everything before this point ran on synthetic or +//! self-generated data. This module produces an actual two-epoch continuation +//! proof in exactly the encoding the RV64 recursion guest receives. +//! +//! ## Why bytes, and why THESE bytes +//! +//! The guest never sees a `ContinuationProof`. It gets a blob in private input +//! and reads it zero-copy through rkyv. So a machine-side reader whose input is +//! a byte blob is the direct analogue of the guest's reader, and a disagreement +//! between the two is a meaningful signal; a reader that consumed an in-memory +//! `ContinuationProof` would be exercising a path production does not have. +//! +//! The encoding is therefore NOT invented here. It is +//! [`crate::recursion::encode_continuation_guest_input`] — the same encoder the +//! guest's blob comes from — so the fixture cannot drift from production without +//! the encoder itself changing. +//! +//! ## Why not the existing dump test +//! +//! `tests::recursion_smoke_test::test_dump_recursion_input` produces exactly +//! these bytes, but it is `#[ignore]`d as a diagnostic, is driven by five +//! environment variables, and writes to a fixed `/tmp` path. None of that is +//! usable from a deterministic unit test. This module calls the same two public +//! functions it calls — `prove_continuation` then +//! `encode_continuation_guest_input` — and nothing else, so the ENCODER (the +//! part that must not drift) is shared while the harness around it is not. + +use std::path::{Path, PathBuf}; + +use stark::proof::options::ProofOptions; + +use crate::recursion::MIN_PROOF_OPTIONS; + +/// Inner guest whose execution the fixture proves. `fibonacci` rather than +/// `empty`: the fixture needs enough cycles to actually split into two epochs, +/// and `empty` collapses to a single (monolithic-style) one. +pub const FIXTURE_INNER_ELF: &str = "fibonacci"; + +/// Epoch size, as `log2(cycles)`. +/// +/// Measured, not guessed: this guest runs **15 cycles** — the fixture passes no +/// private input, so `n` reads as 0 and the loop body never executes — which an +/// 8-cycle epoch splits into two and a 16-cycle one does not. A single-epoch +/// fixture would defeat the point, since the whole target is a CONTINUATION, +/// and `continuation_fixture_generates_two_epochs` is the canary for it. +/// +/// ⚠ **The cycle count is a property of the compiled ELF, not of the guest +/// source.** `bench_vs/lambda/fibonacci` has no dependencies, so nothing in this +/// workspace moves it — but the pinned nightly and the sysroot do, and a +/// codegen change of two instructions is enough to cross an epoch boundary at +/// this size. If the canary reports one epoch, re-measure rather than guess: +/// run the ELF to completion under `Executor::resume_with_limit` and count the +/// logs, one per cycle, then set this to a `log2` strictly below the count. +/// +/// Blob sizes for the record: 947,340 bytes at the two epochs this selects, +/// against 309,084 for the single epoch a 16-cycle one collapses to. +pub const FIXTURE_EPOCH_LOG2: u32 = 3; + +/// Proof options the fixture is proved under: the `min` preset, which is the +/// cheapest to generate. It is explicitly NOT a secure parameter set — this +/// fixture exists to exercise byte layout and Merkle structure, not to stand in +/// for a production proof's security. +pub fn fixture_options() -> ProofOptions { + MIN_PROOF_OPTIONS +} + +/// Repository root, derived from this crate's manifest directory. +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("prover/ has a parent") + .to_path_buf() +} + +/// Reads a recursion-suite guest ELF (built by `make compile-recursion-elfs`). +pub fn read_inner_elf() -> Vec { + let path = workspace_root().join(format!( + "executor/program_artifacts/recursion/{FIXTURE_INNER_ELF}.elf" + )); + std::fs::read(&path).unwrap_or_else(|e| { + panic!( + "failed to read {} — run `make compile-recursion-elfs`: {e}", + path.display() + ) + }) +} + +/// Proves the fixture continuation and encodes the guest blob. +/// +/// Returns `(blob, num_epochs)`. The epoch count is read before encoding +/// because the encoder consumes the bundle. +pub fn generate() -> (Vec, usize) { + let elf = read_inner_elf(); + let opts = fixture_options(); + let bundle = crate::continuation::prove_continuation(&elf, &[], FIXTURE_EPOCH_LOG2, &opts) + .expect("fixture continuation must prove"); + let num_epochs = bundle.num_epochs(); + let blob = crate::recursion::encode_continuation_guest_input(bundle, &elf, &opts) + .expect("fixture blob must encode"); + (blob, num_epochs) +} + +/// Loads the cached blob, generating and caching it when absent. +/// +/// Proving is slow enough that regenerating per test is not viable, but a +/// checked-in binary is worse: it can drift from the encoder silently. So the +/// cache lives outside the repository and the GENERATION path is what tests +/// exercise on a cold cache. +/// +/// ## ⚠ The blob is NOT reproducible — measured, and it constrains callers +/// +/// Two `generate()` calls on identical inputs produce blobs that differ in +/// ~65k of 587k bytes, and the difference is SEMANTIC, not archive padding: +/// some sub-proofs commit to different roots, which moves the Fiat-Shamir +/// challenges, which opens different leaves. (`machine_tests:: +/// fixture_generation_is_not_reproducible` is the standing evidence.) +/// +/// So **nothing derived from a specific blob may be pinned as a constant** — +/// not a query index, not a leaf value, not a root. Pin SHAPE (column counts, +/// tree depths), which is stable, and recover per-blob values from the blob. +/// R1f's `R1F_SHAPE` and its recovered leaf index are built that way; a pinned +/// index would have broken on the very next cold cache. +/// +/// The write is atomic (temp file then rename) because the cache is shared by +/// tests that run in parallel and one of them regenerates it: without the +/// rename a reader can observe a half-written blob, and since blobs differ, +/// "it was fine last time" proves nothing. +pub fn load_or_generate(cache: &Path) -> Vec { + if let Ok(bytes) = std::fs::read(cache) { + return bytes; + } + let (blob, _) = generate(); + write_cache(cache, &blob); + blob +} + +/// Publishes `blob` at `cache` atomically, so a concurrent reader sees either +/// the old complete blob or the new one, never a torn prefix. +pub fn write_cache(cache: &Path, blob: &[u8]) { + if let Some(dir) = cache.parent() { + let _ = std::fs::create_dir_all(dir); + } + let staging = cache.with_extension(format!("tmp{}", std::process::id())); + if std::fs::write(&staging, blob).is_ok() { + let _ = std::fs::rename(&staging, cache); + } +} + +/// Checks the blob carries the recursion input's magic prefix — i.e. that it is +/// the guest's wire format and not some other encoding. +pub fn has_recursion_prefix(blob: &[u8]) -> bool { + blob.len() > crate::RECURSION_INPUT_PREFIX_LEN + && blob.starts_with(&crate::RECURSION_INPUT_MAGIC) +} + +/// An opened fixture blob, holding the aligned bytes the archived view borrows +/// from. +/// +/// Mirrors `recursion::verify_continuation_and_attest`'s decode exactly: strip +/// the magic/version prefix, re-align if the host `Vec` is not on rkyv's +/// alignment (guest slices are aligned by construction; host ones carry no such +/// guarantee), then `rkyv::access` with validation. The owning struct exists +/// because the archived view borrows from the aligned buffer. +pub struct FixtureArchive { + aligned: rkyv::util::AlignedVec<{ crate::RECURSION_INPUT_ALIGN }>, +} + +impl FixtureArchive { + pub fn open(blob: &[u8]) -> Self { + let archive_bytes = crate::recursion_archive_bytes(blob) + .expect("fixture blob must carry the recursion magic and version"); + let mut aligned = rkyv::util::AlignedVec::new(); + aligned.extend_from_slice(archive_bytes); + Self { aligned } + } + + /// The validated archived guest input. + pub fn guest_input(&self) -> &crate::recursion::ArchivedContinuationGuestInput { + rkyv::access::( + &self.aligned, + ) + .expect("fixture blob must validate") + } +} diff --git a/prover/src/lfm/registry.rs b/prover/src/lfm/registry.rs new file mode 100644 index 000000000..21cdacae0 --- /dev/null +++ b/prover/src/lfm/registry.rs @@ -0,0 +1,757 @@ +//! `LFM_REGISTRY` — the machine's program registry. +//! +//! A program is nothing but a vector of supplied preprocessed roots plus this +//! table's entry. Maintained exactly like `compute_static_commitments`: +//! regenerated by `cargo run --bin compute_lfm_registry --release`, pinned by +//! drift tests that run on every PR, and governed by the same standing policy +//! — **a drift failure is investigated, never re-blessed to silence the +//! test.** +//! +//! Verify-side resolution returns a hard `Err` on a missing entry. There is +//! **no runtime off-switch**: no environment variable, no feature flag, no +//! fallback that recomputes or skips. The registry check is the soundness +//! argument's first premise (see `SOUNDNESS.md`). + +use stark::config::{Commitment, CommitmentHash}; +use stark::proof::options::ProofOptions; + +use crate::tables::{bitwise, keccak_rc}; + +use super::airs::NUM_LFM_CHIPS; +use super::commit::commit_group; +use super::compiler::LfmProgram; +use super::hash::HasherKind; +use super::statement::lfm_program_id; +use super::trace::range_group; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LfmProgramKind { + /// The Milestone-B trivial program (`programs::trivial_program`). + TrivialV0, + /// The Milestone-C FRI-opening verifier (`programs::fri_toy_program`). + FriToyV0, + /// The R1b two-permutation keccak chain (`programs::keccak_chain_program`). + KeccakChainV0, + /// The R1c keccak256 sponge at `programs::KECCAK_SPONGE_LEN` bytes. + KeccakSpongeV0, + /// The R1d scripted `DefaultTranscript` replay + /// (`programs::transcript_replay_program`). + TranscriptReplayV0, + /// The R1e continuation-epoch statement bind plus Phase A + /// (`programs::statement_replay_program`). + StatementReplayV0, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LfmRegistryError { + UnknownProgram { + kind: LfmProgramKind, + blowup_factor: u8, + }, +} + +pub struct LfmRegistryEntry { + pub kind: LfmProgramKind, + pub blowup_factor: u8, + pub roots: [Commitment; NUM_LFM_CHIPS], + pub log_heights: [u8; NUM_LFM_CHIPS], + /// `KECCAK_RND` instances this program is proved and verified with. + pub keccak_rnd_chunks: usize, + /// The `LFM_HASH` permutation this program is proved and verified under. + /// + /// Program shape, like the roots and the chunk count — never read off the + /// proof. `lfm_verify` builds the AIR set with this value, and it is folded + /// into `program_id`, so the digest and the permutation it names are + /// computed together and cannot be paired up wrongly at verify time. + pub hasher: HasherKind, + pub program_id: Commitment, +} + +/// A program's committed artifacts (what a registry entry pins). +pub struct LfmArtifacts { + pub roots: [Commitment; NUM_LFM_CHIPS], + pub log_heights: [u8; NUM_LFM_CHIPS], + pub keccak_rnd_chunks: usize, + /// The hasher `program_id` was derived under; the prove and verify paths + /// both take it from here rather than defaulting. + pub hasher: HasherKind, + pub program_id: Commitment, +} + +/// Commits every instruction column group (plus the fixed tables) at the given +/// options and derives the program digest. Host-side, seconds — there is no +/// keygen in this framework. +/// +/// What the digest binds, per slot class: +/// +/// - **slots 0–9** — the program-dependent instruction column groups. Root and +/// height both vary with the program; this is the program's identity. +/// - **slot 10 (`LFM_RANGE`)** — a fixed table whose group is +/// program-independent but still committed and bound, so a change to the +/// table or to the commit pipeline moves every program digest. +/// - **slots 12–13 (`KECCAK_RC`, `BITWISE`)** — same treatment as `LFM_RANGE`, +/// except their preprocessed columns are owned by `tables/`, so the roots come +/// from those modules' own `preprocessed_commitment` — which is both what the +/// AIRs are built with and what the prover recommits against. Binding them +/// means a change to either production table moves every LFM program digest; +/// that is deliberate, since those tables are now part of the machine. +/// - **slot 11 (`KECCAK_RND`)** — has no preprocessed columns at all, so there +/// is nothing to commit. Its entry stays the all-zero sentinel at height 0 and +/// binds nothing. Sound because the chip is program-independent in both +/// directions: its constraints are fixed, and its trace height is free (extra +/// rows are padding with `MU = 0`, which emits no bus tokens and satisfies +/// every constraint) — the same freedom the production VM's chips have. +/// What the entry *does* pin for this slot is `keccak_rnd_chunks`: how many +/// instances of it the proof carries. That freedom is the same freedom as the +/// height — a prover who used a different count could not forge anything, +/// only fail to balance the `Keccak` bus (too few chunks) or waste rows (too +/// many) — so pinning it is a shape decision, not a soundness gate. Pinning +/// it here keeps the verifier's AIR set derivable from the registry alone, +/// with nothing about proof shape read off the proof. +/// +/// The `LFM_HASH` permutation is bound too, but not through a root: it selects +/// which chip fills the `LFM_HASH` slot, and every candidate's preprocessed +/// width is the same (the instruction group is hasher-independent), so no root +/// moves with the choice. [`build_artifacts_with_hasher`] folds the kind's tag +/// into the digest instead — measured by +/// `the_blake3_choice_moves_the_program_digest_and_no_root`. +/// +/// That is a statement about the machine's *own* hash, not about the hash these +/// roots are built with. The two are separate axes today and the second one is +/// not chosen here; see [`build_artifacts_with_hasher`]'s guard for what keeps +/// them separate. If the machine's hash ever also selects the commitment scheme +/// the roots are committed under, every root above moves with it and the tag on +/// its own stops being the whole binding. +pub fn build_artifacts(program: &LfmProgram, options: &ProofOptions) -> LfmArtifacts { + build_artifacts_with_hasher(program, options, HasherKind::default()) +} + +/// [`build_artifacts`] for a program proved under an explicitly chosen +/// `LFM_HASH` permutation. +/// +/// The returned artifacts carry `hasher`, and `program_id` is derived from it — +/// so the same program under two hashers is two program identities, and the +/// prove/verify paths that read `LfmArtifacts` cannot pair one hasher's digest +/// with another hasher's AIR set. +/// +/// # What `hasher` does not say +/// +/// `hasher` names the `LFM_HASH` chip the machine runs. It says nothing about +/// the hash the roots below are built with: `commit_group` and the two +/// `preprocessed_commitment` helpers all commit through `stark`'s Merkle layer, +/// which is pinned to [`CommitmentHash::Keccak256`]. So under +/// `HasherKind::Blake3` this returns keccak-built roots inside artifacts that +/// name Blake3 — honest only because the name makes no claim about them. +/// +/// The `match` below is what keeps it honest. It is exhaustive over +/// [`CommitmentHash`], so the change that gives `stark` a second commitment +/// hash cannot compile until someone decides here what the artifacts should say +/// — rather than inheriting a digest that names one hash over roots built with +/// another, which nothing downstream would catch. +pub fn build_artifacts_with_hasher( + program: &LfmProgram, + options: &ProofOptions, + hasher: HasherKind, +) -> LfmArtifacts { + // Exhaustive on purpose — see the doc above. Not a runtime check: today + // every arm of `hasher` is legitimately paired with keccak roots. + // + // `CommitmentHash::Blake3` now exists (P-a Stage 1), and this is the + // decision the doc above says has to be taken here rather than inherited. + // + // The decision: the guard stays pointed at the ALIASES, and the Blake3 arm + // is a hard stop rather than an accepted case. `COMMITMENT_HASH` describes + // the default configuration, and the three helpers below — `commit_group` + // and the two `preprocessed_commitment`s — are hard-wired to the aliases, so + // while the aliases are keccak this function's roots are keccak and the doc + // above is true as written. If the aliases ever move, those roots change + // hash and `program_id`'s meaning changes with them: the digest folds in the + // `hasher` tag but says nothing about the commitment hash, so two builds + // committing under different hashes would give the same program the same + // `program_id`. That has to be decided, not defaulted. + // + // It is not claimed this arm is the FIRST thing to fail when the aliases + // move — `stark::config`'s own `assert_keccak_backend` and the + // `COMMITMENT_HASH`-to-`KeccakStarkHash` pin sit in front of it and were + // observed to fire first. It is the one that fails for THIS crate's reason, + // and it is what makes the decision unskippable once those are dealt with. + // + // What this still does not catch, unchanged: a prover running under an + // explicit `Blake3StarkHash` while the aliases stay keccak. The const is + // global, the configuration is per-type. Closing that means making this + // function generic over `H` and reading `H::COMMITMENT_HASH` — Stage 5 work, + // recorded in PA-PLAN §4.2 and in `stark::config::COMMITMENT_HASH`'s doc. + const _: () = match stark::config::COMMITMENT_HASH { + CommitmentHash::Keccak256 => (), + CommitmentHash::Blake3 => panic!( + "the commitment aliases moved to BLAKE3: decide what LfmArtifacts \ + should say about program_id before letting this build through" + ), + }; + + let range = range_group(); + let groups = [ + &program.groups.const_, + &program.groups.balu, + &program.groups.xalu, + &program.groups.select, + &program.groups.bitdec, + &program.groups.hash, + &program.groups.keccak, + &program.groups.lanes, + &program.groups.hint, + &program.groups.public, + &range, + ]; + let mut roots = [[0u8; 32]; NUM_LFM_CHIPS]; + let mut log_heights = [0u8; NUM_LFM_CHIPS]; + for (i, g) in groups.iter().enumerate() { + roots[i] = commit_group(g, options); + log_heights[i] = g.padded_rows.trailing_zeros() as u8; + } + // Slot 11 (KECCAK_RND) keeps the all-zero sentinel installed above. + roots[12] = keccak_rc::preprocessed_commitment(options); + log_heights[12] = keccak_rc::NUM_ROWS.trailing_zeros() as u8; + roots[13] = bitwise::preprocessed_commitment(options); + log_heights[13] = bitwise::NUM_ROWS.trailing_zeros() as u8; + + let keccak_rnd_chunks = program + .chunking + .chunk_count(program.groups.keccak.real_rows); + + let program_id = lfm_program_id(&roots, &log_heights, keccak_rnd_chunks, hasher); + LfmArtifacts { + roots, + log_heights, + keccak_rnd_chunks, + hasher, + program_id, + } +} + +/// Resolves a registry entry or fails hard. No fallback path exists or may +/// ever be added. +pub fn resolve( + kind: LfmProgramKind, + blowup_factor: u8, +) -> Result<&'static LfmRegistryEntry, LfmRegistryError> { + LFM_REGISTRY + .iter() + .find(|e| e.kind == kind && e.blowup_factor == blowup_factor) + .ok_or(LfmRegistryError::UnknownProgram { + kind, + blowup_factor, + }) +} + +// ========================================================================= +// GENERATED — do not edit by hand. Regenerate with: +// cargo run --bin compute_lfm_registry --release +// and paste the output below. Drift tests recompute and compare on every PR. +// ========================================================================= +pub static LFM_REGISTRY: &[LfmRegistryEntry] = &[ + LfmRegistryEntry { + kind: LfmProgramKind::TrivialV0, + blowup_factor: 2, + roots: [ + [ + 0xb7, 0x0f, 0x25, 0x13, 0xc0, 0xd3, 0x94, 0x78, 0x37, 0xe4, 0x2a, 0x0f, 0xa7, 0x2f, + 0xd4, 0x8a, 0xd6, 0xcc, 0x90, 0x12, 0x55, 0x04, 0x59, 0x10, 0xb6, 0x2a, 0x05, 0xe8, + 0xdb, 0x16, 0x01, 0x43, + ], + [ + 0x16, 0xfb, 0x28, 0xa0, 0xb1, 0x38, 0x73, 0x32, 0x92, 0xa7, 0xaf, 0xed, 0x98, 0xa1, + 0x33, 0xcb, 0x69, 0xcb, 0xf9, 0x79, 0xd4, 0xd1, 0x98, 0x71, 0xe5, 0xe8, 0x07, 0xc6, + 0x36, 0x8e, 0x6a, 0x06, + ], + [ + 0x76, 0x7e, 0x07, 0x30, 0x86, 0x40, 0x3e, 0x03, 0xe4, 0x24, 0x01, 0x62, 0xb0, 0x46, + 0x59, 0x1f, 0x40, 0x5b, 0xc1, 0x5f, 0x5d, 0x75, 0x76, 0xbd, 0x55, 0x4a, 0xe0, 0x5b, + 0x7e, 0xc5, 0x7a, 0xf2, + ], + [ + 0xa0, 0x8d, 0x0e, 0x61, 0xde, 0x60, 0x5c, 0xd8, 0xe0, 0x5f, 0x65, 0x7f, 0x62, 0x28, + 0x71, 0x4c, 0x81, 0x5a, 0x77, 0x6f, 0xe4, 0x00, 0x7e, 0x85, 0x77, 0xac, 0x2d, 0x98, + 0x77, 0x0c, 0x4a, 0x57, + ], + [ + 0x8e, 0x50, 0x10, 0xa4, 0x99, 0xe9, 0x74, 0xd7, 0x58, 0xc5, 0xe2, 0xe2, 0xad, 0xd5, + 0x0c, 0x01, 0xda, 0x15, 0xd6, 0x61, 0xad, 0xdc, 0xef, 0xab, 0x8e, 0xa7, 0xee, 0x32, + 0x10, 0xc9, 0x66, 0x31, + ], + [ + 0x06, 0x2e, 0xdc, 0xb0, 0xc0, 0x4a, 0x48, 0x8a, 0xb2, 0xbb, 0xa9, 0xb9, 0x60, 0x79, + 0x89, 0x09, 0xd3, 0xed, 0x12, 0xbc, 0x7d, 0x3a, 0x43, 0xdf, 0x68, 0xae, 0xc7, 0x27, + 0xa9, 0xae, 0xba, 0x3d, + ], + [ + 0x0a, 0xdf, 0x11, 0xf2, 0x5f, 0x56, 0x8c, 0x8f, 0x5f, 0x21, 0xc9, 0xc6, 0x59, 0xcb, + 0x74, 0x11, 0xf2, 0x19, 0x83, 0x86, 0xe5, 0xe7, 0x01, 0xe7, 0xf2, 0xce, 0x93, 0x50, + 0x2d, 0xdc, 0x42, 0x0e, + ], + [ + 0x16, 0x71, 0x3f, 0x10, 0xf8, 0x4c, 0xd2, 0xbd, 0xf3, 0xa3, 0x59, 0xfa, 0xe9, 0x9e, + 0xa8, 0xe0, 0x12, 0xad, 0x40, 0x1c, 0xc0, 0xfa, 0x3e, 0x8b, 0xaf, 0xcd, 0xe4, 0x96, + 0x74, 0xd3, 0xe3, 0x28, + ], + [ + 0x99, 0x62, 0x5f, 0x63, 0x83, 0xdd, 0xa5, 0x73, 0x56, 0xc9, 0x79, 0xea, 0x7a, 0x28, + 0xa7, 0xf4, 0xe1, 0xb4, 0x0f, 0xe6, 0x42, 0xd2, 0x9a, 0x97, 0xd8, 0x76, 0x93, 0x0f, + 0xe6, 0xf5, 0x54, 0xae, + ], + [ + 0x24, 0xb2, 0xdc, 0x4d, 0x70, 0x05, 0xf2, 0x7e, 0x36, 0x3b, 0x06, 0x91, 0x44, 0x65, + 0x05, 0x68, 0x23, 0x07, 0x82, 0x1a, 0x8e, 0x1d, 0x46, 0x88, 0x3f, 0x16, 0xe6, 0xa7, + 0x9e, 0x3c, 0xbf, 0x2e, + ], + [ + 0x30, 0x30, 0xd0, 0x58, 0x2b, 0xf0, 0x84, 0x5a, 0x38, 0x4b, 0xc6, 0x20, 0x48, 0x1f, + 0x0c, 0x3f, 0x08, 0x61, 0x6c, 0x5c, 0x2e, 0x9d, 0x46, 0xdc, 0xfc, 0x2a, 0x50, 0xb2, + 0xf6, 0x27, 0x05, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, + 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, + 0xa9, 0x2c, 0x85, 0xe3, + ], + [ + 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, + 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, + 0x0c, 0x6d, 0xc4, 0xcf, + ], + ], + log_heights: [3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 16, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + program_id: [ + 0x70, 0x87, 0xe2, 0x83, 0x8d, 0xae, 0x11, 0x71, 0x74, 0x1f, 0x49, 0xa3, 0x2c, 0x47, + 0x00, 0x9a, 0x79, 0x49, 0x4e, 0x82, 0x24, 0x9c, 0x8c, 0xee, 0x8c, 0x9e, 0x86, 0x74, + 0x3b, 0xaf, 0x9f, 0x4b, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::FriToyV0, + blowup_factor: 2, + roots: [ + [ + 0xdd, 0xd5, 0x8c, 0x48, 0xfc, 0xb7, 0x6f, 0x9d, 0x4a, 0x24, 0xa3, 0x9a, 0x13, 0xa1, + 0x21, 0x12, 0x57, 0x4b, 0xf3, 0x2d, 0x21, 0x74, 0xd0, 0x07, 0x25, 0xaa, 0x84, 0x96, + 0x3d, 0x98, 0x76, 0xfa, + ], + [ + 0x75, 0xaa, 0xd2, 0x56, 0x3c, 0x29, 0xb9, 0x60, 0x98, 0x86, 0xa9, 0x97, 0x73, 0xba, + 0x2f, 0x95, 0x85, 0x56, 0x15, 0xdb, 0x2f, 0xca, 0xe7, 0xe4, 0x96, 0xff, 0x64, 0x5b, + 0xb2, 0xa7, 0xb0, 0x33, + ], + [ + 0xd1, 0x20, 0x64, 0xd8, 0x59, 0x78, 0x44, 0x90, 0x12, 0x67, 0x51, 0xee, 0xf7, 0xe3, + 0x54, 0x83, 0x24, 0xd7, 0x42, 0x0a, 0x07, 0x48, 0x45, 0x40, 0x84, 0x4f, 0x75, 0x6f, + 0x83, 0xbf, 0xa2, 0xef, + ], + [ + 0x4f, 0x5b, 0x15, 0x14, 0x8c, 0x2d, 0x8a, 0xc8, 0x0d, 0x89, 0x75, 0xb2, 0x8d, 0x3e, + 0x03, 0x61, 0xee, 0x0d, 0x3a, 0x4f, 0xd8, 0xb1, 0xf8, 0x15, 0x35, 0x0e, 0x11, 0x59, + 0x28, 0x50, 0x70, 0x06, + ], + [ + 0x30, 0xe0, 0x5f, 0x36, 0xbc, 0x30, 0xb2, 0x86, 0x4b, 0x6b, 0x00, 0xcb, 0xb3, 0x86, + 0x4e, 0xd1, 0x42, 0x51, 0xc8, 0x06, 0x24, 0x35, 0xb8, 0x43, 0x98, 0xf3, 0x82, 0xc6, + 0xd1, 0xe7, 0xda, 0xc5, + ], + [ + 0xd6, 0x1f, 0x7b, 0xf1, 0xc9, 0x2c, 0x8c, 0x95, 0xc6, 0x0e, 0x6c, 0x24, 0x7d, 0xdb, + 0x99, 0x9d, 0x99, 0xc8, 0x09, 0x3c, 0x5b, 0x0a, 0xbe, 0xb0, 0x05, 0x63, 0x59, 0x40, + 0xfa, 0x13, 0xa4, 0x9f, + ], + [ + 0x0a, 0xdf, 0x11, 0xf2, 0x5f, 0x56, 0x8c, 0x8f, 0x5f, 0x21, 0xc9, 0xc6, 0x59, 0xcb, + 0x74, 0x11, 0xf2, 0x19, 0x83, 0x86, 0xe5, 0xe7, 0x01, 0xe7, 0xf2, 0xce, 0x93, 0x50, + 0x2d, 0xdc, 0x42, 0x0e, + ], + [ + 0xc5, 0xcb, 0xad, 0xe1, 0x7c, 0xe6, 0x84, 0x0b, 0xbc, 0x0e, 0x94, 0x6a, 0x31, 0xdf, + 0x6e, 0x08, 0x73, 0xb5, 0x3d, 0xfe, 0xb8, 0x88, 0x2e, 0x6c, 0x21, 0xbf, 0xf9, 0xb8, + 0x91, 0x51, 0x41, 0xce, + ], + [ + 0x24, 0x57, 0xd8, 0x67, 0xd5, 0x18, 0x2d, 0xc7, 0x2f, 0x1f, 0x7d, 0x51, 0x2d, 0xb8, + 0x15, 0x3e, 0x87, 0x61, 0x13, 0x9a, 0x6f, 0x25, 0x14, 0xed, 0x95, 0x16, 0x66, 0xcb, + 0xfb, 0x8f, 0x50, 0x20, + ], + [ + 0x44, 0xdf, 0x05, 0xc9, 0xc5, 0x12, 0x1f, 0xb1, 0x41, 0xb2, 0xe1, 0x46, 0x8e, 0xdd, + 0x27, 0x14, 0x50, 0x92, 0xe5, 0x6d, 0xfb, 0x53, 0xc2, 0xc4, 0x53, 0x9f, 0xd3, 0xee, + 0xba, 0xb7, 0x0e, 0xfa, + ], + [ + 0x30, 0x30, 0xd0, 0x58, 0x2b, 0xf0, 0x84, 0x5a, 0x38, 0x4b, 0xc6, 0x20, 0x48, 0x1f, + 0x0c, 0x3f, 0x08, 0x61, 0x6c, 0x5c, 0x2e, 0x9d, 0x46, 0xdc, 0xfc, 0x2a, 0x50, 0xb2, + 0xf6, 0x27, 0x05, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, + 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, + 0xa9, 0x2c, 0x85, 0xe3, + ], + [ + 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, + 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, + 0x0c, 0x6d, 0xc4, 0xcf, + ], + ], + log_heights: [5, 8, 7, 7, 2, 7, 2, 5, 7, 2, 16, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + program_id: [ + 0xb1, 0x40, 0xc0, 0x43, 0xb6, 0xc0, 0x60, 0x87, 0x11, 0x29, 0xc0, 0xd3, 0xb7, 0xb0, + 0x7c, 0x49, 0x30, 0x75, 0x4d, 0x89, 0xe6, 0x91, 0x67, 0xf7, 0x65, 0xe0, 0xe3, 0x8f, + 0xb4, 0x33, 0xd7, 0x99, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::KeccakChainV0, + blowup_factor: 2, + roots: [ + [ + 0x26, 0x6f, 0x52, 0xb1, 0x61, 0x65, 0xe9, 0xe2, 0x25, 0x3b, 0xe1, 0x06, 0xc3, 0x77, + 0x49, 0x31, 0xe0, 0x9b, 0xc1, 0xaf, 0xf9, 0x89, 0x55, 0xa7, 0x48, 0x14, 0x78, 0x8f, + 0xba, 0xba, 0x6b, 0x7f, + ], + [ + 0x3f, 0xdd, 0x51, 0x75, 0x20, 0x94, 0x88, 0x4c, 0xac, 0x75, 0x11, 0x7c, 0x9f, 0xe7, + 0x07, 0x69, 0xd2, 0x54, 0xe8, 0x1e, 0x3d, 0x98, 0x4d, 0x1b, 0x9c, 0x72, 0x53, 0xbf, + 0x99, 0x99, 0xd4, 0xb0, + ], + [ + 0xaf, 0xb2, 0xb2, 0x9d, 0x0c, 0x27, 0x86, 0xc9, 0x1e, 0x64, 0x45, 0xea, 0x78, 0x1e, + 0x7e, 0x22, 0x4c, 0x6c, 0x24, 0xe3, 0x4d, 0x79, 0x11, 0x31, 0xc1, 0x19, 0xcb, 0x10, + 0xdd, 0xcc, 0x2a, 0xbb, + ], + [ + 0x17, 0xd3, 0xb1, 0x28, 0xb5, 0x42, 0xdd, 0xeb, 0x28, 0x11, 0x91, 0x67, 0x34, 0xdf, + 0x4d, 0xa9, 0xbc, 0x03, 0x54, 0x5d, 0xc7, 0x41, 0xcf, 0xce, 0x55, 0x84, 0x8a, 0xd4, + 0x90, 0x56, 0x7a, 0x9d, + ], + [ + 0x68, 0xb4, 0x20, 0x9c, 0xc1, 0x43, 0x22, 0x27, 0xcc, 0x98, 0x54, 0x74, 0x9b, 0x34, + 0xb2, 0x68, 0xe3, 0x76, 0xc9, 0x15, 0xd6, 0xce, 0x61, 0xf7, 0x32, 0xa5, 0x80, 0x3b, + 0x58, 0xeb, 0x0d, 0x65, + ], + [ + 0xf3, 0x46, 0x5a, 0x7c, 0x66, 0x03, 0xa5, 0x66, 0x7c, 0x10, 0x1f, 0xc4, 0x40, 0xc6, + 0x44, 0x83, 0x33, 0x0a, 0x44, 0xd7, 0x29, 0x57, 0x65, 0xc0, 0x93, 0x12, 0x52, 0x60, + 0x62, 0x86, 0x90, 0x7c, + ], + [ + 0x90, 0x28, 0x1b, 0x93, 0x87, 0x82, 0x46, 0x3b, 0x83, 0x25, 0x32, 0x18, 0x66, 0x93, + 0x7f, 0xc5, 0x6e, 0x5f, 0xf0, 0x6d, 0x3e, 0x62, 0xc3, 0xf0, 0x60, 0xc5, 0x9c, 0x86, + 0x10, 0x58, 0x10, 0x5e, + ], + [ + 0x16, 0x71, 0x3f, 0x10, 0xf8, 0x4c, 0xd2, 0xbd, 0xf3, 0xa3, 0x59, 0xfa, 0xe9, 0x9e, + 0xa8, 0xe0, 0x12, 0xad, 0x40, 0x1c, 0xc0, 0xfa, 0x3e, 0x8b, 0xaf, 0xcd, 0xe4, 0x96, + 0x74, 0xd3, 0xe3, 0x28, + ], + [ + 0x7f, 0x12, 0xf0, 0xa9, 0xcc, 0xec, 0xa0, 0x84, 0x75, 0xde, 0xc9, 0xd1, 0x06, 0x29, + 0xb0, 0x41, 0xfd, 0x46, 0x7e, 0x12, 0x9d, 0x27, 0x01, 0x0c, 0x42, 0xf5, 0xd1, 0x97, + 0x46, 0x82, 0xa4, 0xc8, + ], + [ + 0x5d, 0xfa, 0x18, 0x3f, 0xb9, 0x5c, 0x86, 0x90, 0xa5, 0xb7, 0xcd, 0xea, 0xa7, 0x97, + 0x4d, 0x97, 0x60, 0x8e, 0x3b, 0x0e, 0x16, 0xfa, 0x95, 0x5a, 0x6e, 0x6b, 0x5e, 0xa7, + 0x7b, 0x5d, 0x22, 0x2f, + ], + [ + 0x30, 0x30, 0xd0, 0x58, 0x2b, 0xf0, 0x84, 0x5a, 0x38, 0x4b, 0xc6, 0x20, 0x48, 0x1f, + 0x0c, 0x3f, 0x08, 0x61, 0x6c, 0x5c, 0x2e, 0x9d, 0x46, 0xdc, 0xfc, 0x2a, 0x50, 0xb2, + 0xf6, 0x27, 0x05, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, + 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, + 0xa9, 0x2c, 0x85, 0xe3, + ], + [ + 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, + 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, + 0x0c, 0x6d, 0xc4, 0xcf, + ], + ], + log_heights: [2, 2, 2, 2, 2, 2, 2, 2, 4, 2, 16, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + program_id: [ + 0xe8, 0x30, 0xe1, 0xf5, 0xf9, 0xf1, 0xeb, 0xaf, 0x62, 0x33, 0xc1, 0x9a, 0x9a, 0x76, + 0x5e, 0x0d, 0x2d, 0x0f, 0xbd, 0x14, 0x10, 0xe4, 0x59, 0x89, 0x6b, 0x70, 0x51, 0x4d, + 0x95, 0xb6, 0xf4, 0x72, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::KeccakSpongeV0, + blowup_factor: 2, + roots: [ + [ + 0xa9, 0x7b, 0x23, 0x11, 0xd9, 0xf2, 0xc1, 0xc5, 0x7d, 0x53, 0xf3, 0x8e, 0x2e, 0x9e, + 0xf6, 0xc1, 0xdf, 0xb8, 0xfc, 0xfd, 0x59, 0x65, 0x83, 0xe7, 0xf9, 0xf4, 0xde, 0x0b, + 0x38, 0x46, 0x40, 0x5f, + ], + [ + 0xc0, 0x27, 0x98, 0x50, 0xe0, 0x5e, 0x03, 0xc7, 0xe9, 0x2a, 0xe0, 0xa6, 0x59, 0xc7, + 0x3a, 0x7f, 0x86, 0x85, 0x71, 0x37, 0xb3, 0x29, 0xed, 0xaa, 0x3a, 0x38, 0x6a, 0xfc, + 0xb7, 0xb0, 0x22, 0xd9, + ], + [ + 0xaf, 0xb2, 0xb2, 0x9d, 0x0c, 0x27, 0x86, 0xc9, 0x1e, 0x64, 0x45, 0xea, 0x78, 0x1e, + 0x7e, 0x22, 0x4c, 0x6c, 0x24, 0xe3, 0x4d, 0x79, 0x11, 0x31, 0xc1, 0x19, 0xcb, 0x10, + 0xdd, 0xcc, 0x2a, 0xbb, + ], + [ + 0x17, 0xd3, 0xb1, 0x28, 0xb5, 0x42, 0xdd, 0xeb, 0x28, 0x11, 0x91, 0x67, 0x34, 0xdf, + 0x4d, 0xa9, 0xbc, 0x03, 0x54, 0x5d, 0xc7, 0x41, 0xcf, 0xce, 0x55, 0x84, 0x8a, 0xd4, + 0x90, 0x56, 0x7a, 0x9d, + ], + [ + 0x68, 0xb4, 0x20, 0x9c, 0xc1, 0x43, 0x22, 0x27, 0xcc, 0x98, 0x54, 0x74, 0x9b, 0x34, + 0xb2, 0x68, 0xe3, 0x76, 0xc9, 0x15, 0xd6, 0xce, 0x61, 0xf7, 0x32, 0xa5, 0x80, 0x3b, + 0x58, 0xeb, 0x0d, 0x65, + ], + [ + 0xf3, 0x46, 0x5a, 0x7c, 0x66, 0x03, 0xa5, 0x66, 0x7c, 0x10, 0x1f, 0xc4, 0x40, 0xc6, + 0x44, 0x83, 0x33, 0x0a, 0x44, 0xd7, 0x29, 0x57, 0x65, 0xc0, 0x93, 0x12, 0x52, 0x60, + 0x62, 0x86, 0x90, 0x7c, + ], + [ + 0x14, 0xf5, 0xaa, 0x7c, 0x1f, 0xc6, 0xde, 0xbd, 0x4c, 0x17, 0x21, 0x55, 0xff, 0xfc, + 0xa3, 0x12, 0x76, 0x49, 0x55, 0xcf, 0xe4, 0x9c, 0x09, 0xa1, 0x8e, 0xf2, 0x8c, 0x94, + 0x5f, 0x84, 0x2a, 0x06, + ], + [ + 0x77, 0x4a, 0x63, 0xa9, 0xa8, 0x60, 0xde, 0xeb, 0x69, 0x2b, 0x96, 0x6b, 0x5c, 0xbc, + 0xa2, 0x3f, 0x1d, 0x10, 0x8c, 0xf6, 0x11, 0x65, 0x0e, 0x3c, 0x36, 0x2b, 0xdc, 0x3e, + 0x53, 0xbc, 0x6f, 0xb2, + ], + [ + 0xf6, 0xec, 0xc4, 0xd6, 0xc0, 0x18, 0x06, 0x84, 0xf5, 0xc7, 0xaf, 0x24, 0xe1, 0xd1, + 0x19, 0x3b, 0x3d, 0xaa, 0x0f, 0x34, 0x3a, 0x8a, 0x83, 0xd8, 0xa9, 0x43, 0x64, 0x39, + 0xd0, 0x2e, 0x60, 0xe9, + ], + [ + 0xeb, 0x5b, 0x58, 0x3e, 0x83, 0xe5, 0x6d, 0x83, 0x53, 0xe5, 0x2a, 0xca, 0x38, 0x30, + 0x80, 0x91, 0x4e, 0x16, 0x91, 0xc2, 0xf3, 0x05, 0x82, 0x95, 0xd3, 0x63, 0x02, 0xca, + 0xce, 0xcf, 0x3b, 0xfd, + ], + [ + 0x30, 0x30, 0xd0, 0x58, 0x2b, 0xf0, 0x84, 0x5a, 0x38, 0x4b, 0xc6, 0x20, 0x48, 0x1f, + 0x0c, 0x3f, 0x08, 0x61, 0x6c, 0x5c, 0x2e, 0x9d, 0x46, 0xdc, 0xfc, 0x2a, 0x50, 0xb2, + 0xf6, 0x27, 0x05, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, + 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, + 0xa9, 0x2c, 0x85, 0xe3, + ], + [ + 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, + 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, + 0x0c, 0x6d, 0xc4, 0xcf, + ], + ], + log_heights: [2, 2, 2, 2, 2, 2, 2, 5, 6, 2, 16, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + program_id: [ + 0xd4, 0xf9, 0x49, 0x44, 0x58, 0x0b, 0x18, 0xeb, 0x88, 0xd0, 0xe8, 0xe0, 0xc1, 0x1c, + 0x7f, 0x04, 0xdc, 0x69, 0xc3, 0x2a, 0xff, 0x42, 0x89, 0xc9, 0xc7, 0x10, 0x18, 0x1c, + 0x6f, 0x85, 0x40, 0xc0, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::TranscriptReplayV0, + blowup_factor: 2, + roots: [ + [ + 0x70, 0x57, 0xd8, 0x9a, 0x1a, 0xfb, 0xcf, 0x15, 0xdd, 0x21, 0x28, 0xab, 0x41, 0x78, + 0x9c, 0xb1, 0xff, 0x98, 0x26, 0x3b, 0xe3, 0x55, 0x04, 0xa3, 0x87, 0x2a, 0xeb, 0xe7, + 0xa1, 0x5e, 0x35, 0xd9, + ], + [ + 0x7a, 0xcc, 0x68, 0x9d, 0x03, 0xea, 0xfc, 0x86, 0x70, 0x33, 0x88, 0xaf, 0x19, 0xfa, + 0x9d, 0x4f, 0x1a, 0x06, 0x2d, 0x22, 0x61, 0xf2, 0xd7, 0x48, 0x70, 0x7a, 0xd4, 0xa8, + 0xc4, 0x11, 0x34, 0x30, + ], + [ + 0xaf, 0xb2, 0xb2, 0x9d, 0x0c, 0x27, 0x86, 0xc9, 0x1e, 0x64, 0x45, 0xea, 0x78, 0x1e, + 0x7e, 0x22, 0x4c, 0x6c, 0x24, 0xe3, 0x4d, 0x79, 0x11, 0x31, 0xc1, 0x19, 0xcb, 0x10, + 0xdd, 0xcc, 0x2a, 0xbb, + ], + [ + 0x17, 0xd3, 0xb1, 0x28, 0xb5, 0x42, 0xdd, 0xeb, 0x28, 0x11, 0x91, 0x67, 0x34, 0xdf, + 0x4d, 0xa9, 0xbc, 0x03, 0x54, 0x5d, 0xc7, 0x41, 0xcf, 0xce, 0x55, 0x84, 0x8a, 0xd4, + 0x90, 0x56, 0x7a, 0x9d, + ], + [ + 0xd3, 0x70, 0xb3, 0xe6, 0x2b, 0x69, 0x16, 0x22, 0x88, 0x58, 0x1b, 0xf6, 0x5c, 0x7a, + 0xe7, 0xc3, 0xaf, 0xc5, 0xff, 0xa6, 0xcf, 0x49, 0x09, 0x08, 0xa9, 0xb2, 0x63, 0xb9, + 0x62, 0xd6, 0x6e, 0xd1, + ], + [ + 0xf3, 0x46, 0x5a, 0x7c, 0x66, 0x03, 0xa5, 0x66, 0x7c, 0x10, 0x1f, 0xc4, 0x40, 0xc6, + 0x44, 0x83, 0x33, 0x0a, 0x44, 0xd7, 0x29, 0x57, 0x65, 0xc0, 0x93, 0x12, 0x52, 0x60, + 0x62, 0x86, 0x90, 0x7c, + ], + [ + 0xd0, 0x8f, 0x27, 0x5f, 0x01, 0x42, 0xb7, 0x56, 0x00, 0x01, 0x81, 0x5c, 0x39, 0xc8, + 0x4f, 0xee, 0xf0, 0x64, 0xd9, 0xb2, 0xb9, 0xdc, 0x03, 0x5c, 0xb7, 0xba, 0xe2, 0xef, + 0x5d, 0x9d, 0xdb, 0xc0, + ], + [ + 0xf2, 0x8d, 0x95, 0x9b, 0xf8, 0xd6, 0x0c, 0x68, 0x22, 0xee, 0x97, 0xbf, 0x2d, 0x47, + 0x6c, 0x1d, 0x58, 0x6c, 0x6c, 0x93, 0x5a, 0x6a, 0xf0, 0x5a, 0x3e, 0x15, 0xe1, 0x46, + 0x9d, 0xd5, 0x2c, 0xc6, + ], + [ + 0xa2, 0xd5, 0x9c, 0xd2, 0x13, 0x08, 0xb1, 0x0e, 0xd8, 0x34, 0xcf, 0x18, 0xfb, 0x79, + 0xd9, 0x65, 0xaa, 0xa8, 0x1c, 0x8c, 0x07, 0xb4, 0x99, 0xb1, 0x07, 0xaf, 0x88, 0xa2, + 0xc0, 0xbf, 0xab, 0xed, + ], + [ + 0xff, 0xf7, 0xd5, 0x4f, 0x5a, 0xe0, 0x92, 0xa2, 0xde, 0x9d, 0x78, 0x1f, 0xd3, 0x04, + 0x5a, 0xb1, 0x2a, 0x33, 0xa5, 0x13, 0x82, 0x35, 0x68, 0xf0, 0x9d, 0x54, 0xc1, 0x02, + 0x6d, 0xfd, 0x29, 0x82, + ], + [ + 0x30, 0x30, 0xd0, 0x58, 0x2b, 0xf0, 0x84, 0x5a, 0x38, 0x4b, 0xc6, 0x20, 0x48, 0x1f, + 0x0c, 0x3f, 0x08, 0x61, 0x6c, 0x5c, 0x2e, 0x9d, 0x46, 0xdc, 0xfc, 0x2a, 0x50, 0xb2, + 0xf6, 0x27, 0x05, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, + 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, + 0xa9, 0x2c, 0x85, 0xe3, + ], + [ + 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, + 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, + 0x0c, 0x6d, 0xc4, 0xcf, + ], + ], + log_heights: [4, 6, 2, 2, 2, 2, 3, 7, 6, 3, 16, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + program_id: [ + 0x99, 0x82, 0x73, 0xf0, 0x96, 0xab, 0x6b, 0x57, 0xe5, 0x9e, 0x1b, 0x95, 0x3e, 0xef, + 0x76, 0x15, 0x7f, 0x6d, 0x01, 0x1b, 0x6a, 0x3f, 0xa2, 0x07, 0x74, 0x10, 0x66, 0xb5, + 0x14, 0xd9, 0xbe, 0x3a, + ], + }, + LfmRegistryEntry { + kind: LfmProgramKind::StatementReplayV0, + blowup_factor: 2, + roots: [ + [ + 0xb9, 0x07, 0x6c, 0x6f, 0x4b, 0x2e, 0xb6, 0xf9, 0x5b, 0x15, 0x76, 0x06, 0xc5, 0xa7, + 0x40, 0xa0, 0xa9, 0x28, 0x94, 0x75, 0x92, 0x15, 0xc9, 0xa0, 0x21, 0x65, 0xe4, 0xf4, + 0xc1, 0x18, 0xf5, 0x67, + ], + [ + 0xbb, 0x87, 0x72, 0x9f, 0x21, 0x18, 0x76, 0xcc, 0x20, 0xc9, 0xf0, 0xf7, 0x1c, 0x01, + 0x83, 0xec, 0x49, 0x2c, 0x47, 0xd6, 0xae, 0x5c, 0x6e, 0xa2, 0x6b, 0x1c, 0x2c, 0xd4, + 0x3f, 0x9f, 0x0c, 0x77, + ], + [ + 0xaf, 0xb2, 0xb2, 0x9d, 0x0c, 0x27, 0x86, 0xc9, 0x1e, 0x64, 0x45, 0xea, 0x78, 0x1e, + 0x7e, 0x22, 0x4c, 0x6c, 0x24, 0xe3, 0x4d, 0x79, 0x11, 0x31, 0xc1, 0x19, 0xcb, 0x10, + 0xdd, 0xcc, 0x2a, 0xbb, + ], + [ + 0x17, 0xd3, 0xb1, 0x28, 0xb5, 0x42, 0xdd, 0xeb, 0x28, 0x11, 0x91, 0x67, 0x34, 0xdf, + 0x4d, 0xa9, 0xbc, 0x03, 0x54, 0x5d, 0xc7, 0x41, 0xcf, 0xce, 0x55, 0x84, 0x8a, 0xd4, + 0x90, 0x56, 0x7a, 0x9d, + ], + [ + 0x63, 0xc0, 0x5c, 0x80, 0xf4, 0x2a, 0x8a, 0x77, 0xb4, 0xb3, 0x38, 0xbd, 0xc0, 0x2e, + 0x98, 0x84, 0xc4, 0xf0, 0x84, 0x0a, 0x16, 0x83, 0x98, 0x1e, 0xa6, 0x5e, 0xbb, 0x46, + 0x19, 0x4c, 0x42, 0xde, + ], + [ + 0xf3, 0x46, 0x5a, 0x7c, 0x66, 0x03, 0xa5, 0x66, 0x7c, 0x10, 0x1f, 0xc4, 0x40, 0xc6, + 0x44, 0x83, 0x33, 0x0a, 0x44, 0xd7, 0x29, 0x57, 0x65, 0xc0, 0x93, 0x12, 0x52, 0x60, + 0x62, 0x86, 0x90, 0x7c, + ], + [ + 0x80, 0xd2, 0x69, 0x13, 0x3a, 0x9f, 0x8b, 0xf7, 0x71, 0xeb, 0x48, 0x4e, 0xe5, 0x8a, + 0xfd, 0x4d, 0x6a, 0x6c, 0xcb, 0x3a, 0xca, 0xd7, 0x42, 0x29, 0x71, 0xdf, 0xa2, 0x44, + 0x9c, 0xb4, 0xe6, 0x98, + ], + [ + 0x59, 0x7f, 0x5e, 0x01, 0x6e, 0xb7, 0x88, 0x3f, 0x84, 0x16, 0xb1, 0x56, 0x93, 0x29, + 0x09, 0x90, 0x61, 0x65, 0xfc, 0x65, 0xaa, 0x37, 0x9e, 0x20, 0x33, 0x85, 0x97, 0xe9, + 0xcf, 0x8f, 0xd3, 0xf0, + ], + [ + 0xf4, 0x9e, 0xb4, 0x46, 0x04, 0x9f, 0xaa, 0xce, 0x60, 0xf1, 0x8e, 0xde, 0x20, 0xd4, + 0xa9, 0x53, 0xe3, 0xeb, 0xf9, 0xc0, 0x38, 0x5b, 0xb7, 0x7a, 0xc6, 0xf7, 0x1e, 0x2e, + 0x8a, 0x22, 0xdb, 0x22, + ], + [ + 0xf9, 0xe0, 0x5e, 0x52, 0xbe, 0x28, 0xb3, 0xf2, 0x30, 0xfe, 0xfd, 0xa9, 0x50, 0x30, + 0xb1, 0x7b, 0x53, 0x24, 0xfd, 0x8c, 0x94, 0x16, 0x3a, 0x3d, 0x18, 0xae, 0x30, 0x90, + 0xf2, 0x81, 0xa1, 0xe7, + ], + [ + 0x30, 0x30, 0xd0, 0x58, 0x2b, 0xf0, 0x84, 0x5a, 0x38, 0x4b, 0xc6, 0x20, 0x48, 0x1f, + 0x0c, 0x3f, 0x08, 0x61, 0x6c, 0x5c, 0x2e, 0x9d, 0x46, 0xdc, 0xfc, 0x2a, 0x50, 0xb2, + 0xf6, 0x27, 0x05, 0x41, + ], + [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + ], + [ + 0xab, 0x7a, 0xad, 0xf5, 0xbf, 0xa2, 0xd5, 0x5c, 0x29, 0x83, 0x83, 0xe6, 0x2e, 0x47, + 0xa0, 0xa5, 0x22, 0xf9, 0x57, 0x89, 0x5a, 0x5c, 0xbb, 0x1f, 0x34, 0xbc, 0x21, 0x72, + 0xa9, 0x2c, 0x85, 0xe3, + ], + [ + 0xfa, 0x3e, 0xcf, 0x80, 0xfd, 0x95, 0xe5, 0x09, 0x74, 0xd4, 0x55, 0x23, 0xf6, 0x42, + 0xb6, 0x4b, 0x05, 0xc4, 0xf9, 0x66, 0xc2, 0x4d, 0xff, 0xda, 0x31, 0x47, 0xab, 0x7b, + 0x0c, 0x6d, 0xc4, 0xcf, + ], + ], + log_heights: [5, 11, 2, 2, 6, 2, 3, 6, 6, 2, 16, 0, 5, 20], + keccak_rnd_chunks: 1, + hasher: HasherKind::Test, + program_id: [ + 0x78, 0x81, 0x29, 0x77, 0x5d, 0xb2, 0x48, 0xd2, 0xb6, 0x77, 0xe7, 0x94, 0xd6, 0x68, + 0x52, 0x45, 0xfe, 0x00, 0x2f, 0xf2, 0x54, 0x06, 0xff, 0x16, 0xa1, 0x38, 0x04, 0x38, + 0x57, 0x71, 0x6b, 0xae, + ], + }, +]; diff --git a/prover/src/lfm/statement.rs b/prover/src/lfm/statement.rs new file mode 100644 index 000000000..fd91f5458 --- /dev/null +++ b/prover/src/lfm/statement.rs @@ -0,0 +1,96 @@ +//! LFM program identity and statement binding. +//! +//! `lfm_program_id` binds the instruction column groups (roots + heights), +//! the machine version and the preset — it is the digest the registry pins +//! and the consumer's attestation folds. Keccak today; `_V2` rides the +//! ecosystem hash migration (a host/consumer-side artifact). +//! +//! The statement absorb seeds the Fiat–Shamir transcript before +//! `multi_prove` / `multi_verify_views`, exactly like the RV64 VM's +//! `statement.rs`: any divergence in the absorbed bytes changes every derived +//! challenge and verification rejects. + +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; +use digest::Digest; +use math::field::traits::IsPrimeField; +use stark::config::Commitment; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +use super::airs::NUM_LFM_CHIPS; +use super::hash::HasherKind; +use super::word::LfmWord; + +type E = GoldilocksExtension; + +pub const LFM_MACHINE_VERSION: u32 = 1; +/// Single preset in v0; becomes the preset ladder tag later. +pub const LFM_PRESET_TAG: u32 = 0; + +const LFM_PROGRAM_TAG: &[u8] = b"LAMBDAVM_LFM_PROGRAM_V1"; +const LFM_STATEMENT_TAG: &[u8] = b"LAMBDAVM_LFM_STATEMENT_V1"; + +/// The program digest over the frozen chip order. +/// +/// `keccak_rnd_chunks` is bound alongside the roots and heights because it is +/// program shape too: it decides how many `KECCAK_RND` instances the verifier +/// builds. Binding it here is what makes the registry entry — rather than the +/// proof — the authority on that shape. +/// +/// `hasher` is bound for the same reason and is the one piece of program shape +/// the roots cannot carry: `LFM_HASH`'s preprocessed group is its INSTRUCTION +/// group — addresses, mode selectors and multiplicities — which no candidate +/// changes, so every hasher commits the same width (13 since `MODE_L`) and the +/// commitments are hasher-independent by construction (`airs.rs`). Without this +/// tag the only thing separating one permutation's machine from another's would +/// be a main-trace width coincidence, which a third candidate could collide +/// with. The tag is what makes two hashers two programs. +pub fn lfm_program_id( + roots: &[Commitment; NUM_LFM_CHIPS], + log_heights: &[u8; NUM_LFM_CHIPS], + keccak_rnd_chunks: usize, + hasher: HasherKind, +) -> Commitment { + let mut h = Keccak256::new(); + h.update(LFM_PROGRAM_TAG); + h.update(LFM_MACHINE_VERSION.to_le_bytes()); + h.update(LFM_PRESET_TAG.to_le_bytes()); + h.update([hasher.as_tag()]); + for i in 0..NUM_LFM_CHIPS { + h.update([i as u8]); + h.update(roots[i]); + h.update([log_heights[i]]); + } + h.update((keccak_rnd_chunks as u64).to_le_bytes()); + h.finalize().into() +} + +/// Binds the LFM statement: program identity, machine version, the claimed +/// public words and the FRI terminal degree. Exhaustive by construction — +/// extending the statement means extending this function, in one place. +/// +/// Generic over the transcript because the statement bind is hash-agnostic: it +/// only absorbs, so it is the same sequence of `append_bytes` calls whichever +/// sponge the proof runs on. Pinning it to `DefaultTranscript` would have made +/// the machine's own transcript a fork of this function rather than a caller of +/// it, and two copies of a statement encoding is exactly the drift the +/// "exhaustive by construction" note above exists to prevent. +pub fn absorb_lfm_statement( + transcript: &mut impl IsTranscript, + program_id: &Commitment, + public_words: &[(u32, LfmWord)], + fri_final_poly_log_degree: u8, +) { + transcript.append_bytes(LFM_STATEMENT_TAG); + transcript.append_bytes(program_id); + transcript.append_bytes(&LFM_MACHINE_VERSION.to_le_bytes()); + transcript.append_bytes(&(public_words.len() as u64).to_le_bytes()); + for (index, word) in public_words { + transcript.append_bytes(&index.to_le_bytes()); + for lane in word { + transcript.append_bytes(&GoldilocksField::canonical(lane.value()).to_le_bytes()); + } + } + transcript.append_bytes(&[fri_final_poly_log_degree]); +} diff --git a/prover/src/lfm/statement_replay.rs b/prover/src/lfm/statement_replay.rs new file mode 100644 index 000000000..14fb97b7b --- /dev/null +++ b/prover/src/lfm/statement_replay.rs @@ -0,0 +1,190 @@ +//! The continuation-epoch statement and Phase A, replayed in the machine. +//! +//! This is the first leg of a REAL verifier: everything a `multi_verify` does to +//! its transcript before the per-table forks. Two pieces, in order: +//! +//! 1. `absorb_statement(StatementKind::ContinuationEpoch { .. })` — the +//! canonical, domain-separated statement encoding from `crate::statement`; +//! 2. the Phase-A commitment absorbs from `crate::replay_transcript_phase_a_view` +//! — per air an optional preprocessed root then the main trace root — followed +//! by the two shared LogUp challenges `z` and `α`. +//! +//! The target is a continuation EPOCH, not a monolithic proof (see +//! `others/lfm-target-shape.md`), so the tag is `LAMBDAVM_CONTINUATION_EPOCH_V2` +//! and the encoding carries a trailing `epoch_label` the monolithic variant +//! lacks. +//! +//! ## Why this leg is misaligned end to end +//! +//! The tag is 30 bytes, `≡ 2 (mod 4)`, so the ELF digest immediately after it +//! straddles half boundaries; the one-byte `fri_final_poly_log_degree` later +//! moves the cursor again. The whole statement is +//! `207 + public_output_len + 16·page_ranges` bytes, which is `≡ 3 (mod 4)` +//! whenever `public_output_len ≡ 0 (mod 4)` — so **every Phase-A root absorb is +//! spliced at shift 3 too**, at about one `BitDec` and 34 `BALU` rows per half. +//! A single pad byte at the end of the statement encoding would make all of +//! Phase A free; that is a production-encoding change and is not taken here. +//! +//! ## Which fields are program constants +//! +//! Shape-static fields are emitted as constants, not read from an arena, because +//! they DETERMINE the program's shape: the table counts and the page-range list +//! fix how many sub-proofs Phase A absorbs, and `num_private_input_pages` fixes +//! the AIR layout. A program that read them from an arena would be claiming to +//! verify a shape it was not compiled for. Only the genuinely per-proof +//! values — the ELF digest, the public output and the epoch label — come from +//! the arena. + +use crate::statement::CONTINUATION_EPOCH_TAG; + +use super::builder::{Ext, Felt, LfmBuilder}; +use super::keccak_host::BYTES_PER_HALF; +use super::transcript_replay::TranscriptReplay; + +/// `TableCounts` has fourteen split-table families. +pub const NUM_TABLE_COUNTS: usize = 14; + +/// The shape-static half of the statement — emitted as program constants. +#[derive(Debug, Clone)] +pub struct EpochStatementShape { + /// Length of the public output in bytes. Shape-static: it fixes how many + /// arena halves the program reads. + pub public_output_len: usize, + /// The fourteen split-table chunk counts, in `TableCounts` declaration order. + pub table_counts: [u64; NUM_TABLE_COUNTS], + pub num_private_input_pages: u64, + pub fri_final_poly_log_degree: u8, + /// `(base, count)` per runtime page range. + pub page_ranges: Vec<(u64, u64)>, +} + +impl EpochStatementShape { + /// Total bytes the statement absorbs — the emitter's own accounting, so a + /// test can pin the resulting misalignment instead of trusting prose. + pub fn byte_len(&self) -> usize { + CONTINUATION_EPOCH_TAG.len() + + 32 + + 8 + + self.public_output_len + + 8 * NUM_TABLE_COUNTS + + 8 + + 1 + + 8 + + 16 * self.page_ranges.len() + + 8 + } +} + +/// The per-proof half of the statement — arena halves, four bytes each, +/// little-endian, in absorb order. +pub struct EpochStatementVars<'a> { + /// The 32-byte ELF digest: 8 halves. + pub elf_digest: &'a [Felt], + /// `public_output_len / 4` halves. + pub public_output: &'a [Felt], + /// The `u64` epoch label, little-endian: `[low32, high32]`. + pub epoch_label: &'a [Felt], +} + +/// Emits `absorb_statement(ContinuationEpoch)` byte for byte. +/// +/// Every multi-byte field in this encoding is LITTLE-endian (`to_le_bytes`), +/// unlike `append_field_element`'s big-endian rendering — so a `u64` carried as +/// `[low32, high32]` halves needs no byte manipulation at all, and the only cost +/// here is the misalignment splice. +pub fn absorb_epoch_statement( + t: &mut TranscriptReplay, + shape: &EpochStatementShape, + vars: &EpochStatementVars, +) { + assert_eq!( + vars.public_output.len(), + shape.public_output_len.div_ceil(BYTES_PER_HALF), + "public_output halves must match the declared length" + ); + assert_eq!(vars.elf_digest.len(), 8, "the ELF digest is 32 bytes"); + assert_eq!(vars.epoch_label.len(), 2, "the epoch label is one u64"); + + t.append_const_bytes(CONTINUATION_EPOCH_TAG); + t.append_halves_misaligned(vars.elf_digest); + t.append_const_bytes(&(shape.public_output_len as u64).to_le_bytes()); + // Byte-granular on purpose. `public_output` is collected one byte per COMMIT + // operation (`trace_builder`), so an epoch's length is whatever the workload + // produced — nothing aligns it, and the trailing half must be masked rather + // than absorbed whole. + t.append_bytes_misaligned(vars.public_output, shape.public_output_len); + + // One constant run: the counts, the page total, the FRI byte and the range + // list are all shape-static, so they concatenate into a single run and the + // packer chunks them together. + let mut consts = Vec::new(); + for count in shape.table_counts { + consts.extend_from_slice(&count.to_le_bytes()); + } + consts.extend_from_slice(&shape.num_private_input_pages.to_le_bytes()); + consts.push(shape.fri_final_poly_log_degree); + consts.extend_from_slice(&(shape.page_ranges.len() as u64).to_le_bytes()); + for (base, count) in &shape.page_ranges { + consts.extend_from_slice(&base.to_le_bytes()); + consts.extend_from_slice(&count.to_le_bytes()); + } + t.append_const_bytes(&consts); + + // Continuation epochs bind their position last (replay protection). + t.append_halves_misaligned(vars.epoch_label); +} + +/// A preprocessed commitment as Phase A absorbs it — and the distinction is +/// which SOURCE the root has, not how it is encoded. +/// +/// Production reads every one of these from the AIR and never from the proof +/// (`verifier.rs:1187`), so what the machine must reproduce is the root's +/// provenance: a commitment that is a function of the proof options alone is +/// program text and absorbs as literal bytes; one that is a function of per-proof +/// data (an ELF, a register boundary) is cells, and something else in the program +/// owes their binding (assembly ledger entry 7). +pub enum PhaseAPreprocessed<'a> { + /// Program text — 32 literal bytes, absorbed with no arithmetic at all. + Constant(&'a [u8; 32]), + /// Cells: eight `u32` halves, derived in-machine or read from the arena. + Cells(&'a [Felt]), +} + +/// One sub-proof's Phase-A commitments, as arena halves (8 per 32-byte root). +pub struct PhaseATable<'a> { + /// Present exactly when the air is preprocessed — the verifier absorbs the + /// precomputed commitment only then. + pub preprocessed_root: Option>, + pub main_root: &'a [Felt], +} + +/// Replays Phase A: the commitment absorbs, then the two shared LogUp +/// challenges. +/// +/// Mirrors `crate::replay_transcript_phase_a_view` — for each air, the +/// preprocessed commitment when it has one, then the main trace root, and +/// finally `z` and `α` sampled as cubic-extension elements in that order. +/// +/// The absorbs use the misaligned path because the statement leaves the cursor +/// at `≡ 3 (mod 4)`; nothing about a 32-byte root is itself misaligned. +pub fn replay_phase_a( + t: &mut TranscriptReplay, + b: &mut LfmBuilder, + tables: &[PhaseATable], +) -> (Ext, Ext) { + for table in tables { + match &table.preprocessed_root { + Some(PhaseAPreprocessed::Constant(bytes)) => t.append_const_bytes(&bytes[..]), + Some(PhaseAPreprocessed::Cells(prep)) => { + assert_eq!(prep.len(), 8, "a commitment is 32 bytes"); + t.append_halves_misaligned(prep); + } + None => {} + } + assert_eq!(table.main_root.len(), 8, "a commitment is 32 bytes"); + t.append_halves_misaligned(table.main_root); + } + let z = t.sample_ext(b); + let alpha = t.sample_ext(b); + (z, alpha) +} diff --git a/prover/src/lfm/step_size_tests.rs b/prover/src/lfm/step_size_tests.rs new file mode 100644 index 000000000..0ac912f75 --- /dev/null +++ b/prover/src/lfm/step_size_tests.rs @@ -0,0 +1,767 @@ +//! Assembly ledger entries 8 and 9 — the two OOD-grid blindnesses, witnessed. +//! +//! Both entries are members of the phase's degenerate-parameter family: a defect +//! that no test can see because every proof the phase has shares a parameter +//! value. Their shared cause is `num_eval_points = transition_offsets.len() · +//! step_size` (`verifier.rs:179`, `prover.rs:1452`) being 2 for every production +//! AIR, which collapses the OOD grid to one row per block. +//! +//! - **Entry 8, the ABSORB ORDER.** Production absorbs both pruned OOD blocks +//! COLUMN-major (`verifier.rs:1421-1431`). Row-major is indistinguishable while +//! every block is one row tall. Witnessed here by a THREE-offset AIR: the +//! next-row block's height is `num_eval_points − step_size`, so three offsets at +//! `step_size = 1` make it two rows tall. +//! - **Entry 9, the FRAME-STEP VIEW.** `Op::Var{offset}` indexes the constraint +//! frame's evaluation STEP, and a step is `step_size` grid rows. Witnessed here +//! by comparing [`super::epoch_verify::frame_step_view`] against production's +//! own `StarkTableView::into_frame` at `step_size = 2`. +//! +//! ## Two different witnesses, and why one AIR could not carry both +//! +//! The brief asked for ONE synthetic AIR with three transition offsets AND +//! `step_size > 1`, on the grounds that a witness for one entry closes the other +//! only if it exercises both. That is right about the requirement and wrong about +//! the vehicle, for two reasons found by reading and then measured: +//! +//! 1. **Three offsets and `step_size > 1` cannot coexist in a provable AIR.** +//! `AirWithBuses::new` hardcodes `transition_offsets: vec![0, 1]` +//! (`lookup.rs:922`), so three offsets means an `AIR` impl, and the only ones +//! outside `crypto/**`'s example tree are the examples themselves — none of +//! which has `step_size > 1`. Writing one means adding to `crypto/**`, which is +//! on the standing always-stop list. +//! 2. **`step_size > 1` is not provable at all** — a framework ceiling, measured +//! by [`the_prover_cannot_prove_a_step_size_two_air`] rather than argued (in +//! debug the prover panics on the `RowFrame` shape assert; in release, with +//! that assert compiled out, it emits a proof production's own verifier +//! rejects). So entry 9 cannot be closed by a proof of any kind, from any AIR. +//! +//! What closes entry 9 instead is that it does not need a proof. The defect is in +//! how the machine maps a reconstructed grid onto frame steps, and production has +//! its own function for exactly that mapping (`into_frame`), which is a pure +//! function of a grid and a `step_size`. Differentialling against it needs no +//! prover, and it is a stronger oracle than a proof would have been: it is the +//! very code the real verifier runs. +//! +//! So the two entries get two witnesses, each with a production oracle, and +//! neither witness is of the other's defect. What is NOT covered, stated plainly: +//! no test here runs the ASSEMBLED verifier at `step_size > 1`, because nothing +//! can produce such a proof. Entry 9's closure is therefore about the emitter's +//! grid indexing, not about an end-to-end run. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use math::field::traits::IsField; +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; +use stark::examples::fibonacci_multi_column::{ + FibonacciMultiColumnAIR, FibonacciMultiColumnPublicInputs, compute_trace, +}; +use stark::lookup::{AirWithBuses, AuxiliaryTraceBuildData, NullBoundaryConstraintBuilder}; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::view::{MultiProofView, StarkProofView, StarkTableView}; +use stark::table::Table; +use stark::trace::TraceTable; +use stark::traits::AIR; +use stark::verifier::{IsStarkVerifier, Verifier}; + +use crate::tables::types::{FE, FEE, GoldilocksExtension, GoldilocksField}; + +use super::builder::LfmBuilder; +use super::compiler::compile; +use super::epoch::{ + RootCells, TableAbsorbs, TableChallengeShape, emit_table_challenges, fork_table, +}; +use super::executor::execute; +use super::fri::FriShape; +use super::hash::TestPermutation; +use super::transcript_replay::TranscriptReplay; +use super::validator::validate; +use super::word::{base_word, ext_word, word_as_base, word_as_ext}; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("fixture options") +} + +// ============================================================================= +// Entry 9 — the frame-step view, against production's own frame assembly +// ============================================================================= + +/// A `num_eval_points × width` grid of distinct extension values, so a +/// mis-indexed read is caught by VALUE and not merely by shape. +fn distinct_grid(rows: usize, width: usize) -> Vec> { + (0..rows) + .map(|r| { + (0..width) + .map(|c| { + FEE::new([ + FE::from((100 * r + c + 1) as u64), + FE::from((7 * r + 2 * c + 3) as u64), + FE::from((13 * r + 5 * c + 11) as u64), + ]) + }) + .collect() + }) + .collect() +} + +/// What production's constraint interpreter resolves `Op::Var{offset, col}` to, +/// for every offset and column of a grid — via its OWN frame assembly. +/// +/// `into_frame` groups the grid into `step_size`-row steps +/// (`proof/view.rs:269-294`) and the interpreter takes row 0 of the step +/// (`constraint_ir/interp.rs:240-242`, which asserts `row == 0`). Nothing here is +/// our arithmetic: the grid goes in, production decides which value each offset +/// sees. +fn production_frame_values( + grid: &[Vec], + main_width: usize, + step_size: usize, +) -> Vec> { + let width = grid[0].len(); + let flat: Vec = grid.iter().flat_map(|r| r.iter().cloned()).collect(); + let table = Table::new(flat, width); + let frame = StarkTableView::Owned(&table).into_frame(main_width, step_size); + (0..grid.len() / step_size) + .map(|offset| { + let step = frame.get_evaluation_step(offset); + (0..width) + .map(|col| { + if col < main_width { + *step.get_main_evaluation_element(0, col) + } else { + *step.get_aux_evaluation_element(0, col - main_width) + } + }) + .collect() + }) + .collect() +} + +/// ★ ENTRY 9: the machine's frame-step view is production's, at a `step_size` +/// where the two possible answers differ. +/// +/// The oracle is `StarkTableView::into_frame` — the function the real verifier +/// calls on the reconstructed grid (`verifier.rs:320-321`) — so this is not a +/// comparison of two of our own passes. +/// +/// The `step_size = 1` case is included deliberately and it is the point of the +/// entry: there the strided view and the whole grid are the SAME vector, so the +/// test passes for a correct emitter and for the defective one alike. `step_size = +/// 2` separates them, and the negative half below is what shows it. +#[test] +fn the_frame_step_view_matches_productions_own_frame_assembly() { + use super::epoch_verify::frame_step_view; + + let main_width = 3usize; + let width = 4usize; // one aux column, so the aux branch is exercised too + for (offsets, step_size) in [(2usize, 1usize), (3, 1), (2, 2), (3, 2), (2, 4)] { + let rows = offsets * step_size; + let grid = distinct_grid(rows, width); + let expected = production_frame_values(&grid, main_width, step_size); + let got = frame_step_view(&grid, step_size); + assert_eq!( + got.len(), + offsets, + "offsets {offsets}, step_size {step_size}: one view row per frame step" + ); + assert_eq!( + got, expected, + "offsets {offsets}, step_size {step_size}: the machine's frame-step \ + view must be the values production's own frame assembly hands the \ + interpreter" + ); + } + + // ---- ★ the negative half: the wave-5 defect, and the fact that only + // step_size > 1 can see it. + // + // M2 passed the WHOLE grid to the constraint fold. At step_size 1 that is + // literally the same vector, so the mutation was invisible to every test in + // the suite. At step_size 2 it is a different vector, and this is the + // comparison that says so. + for step_size in [1usize, 2] { + let grid = distinct_grid(3 * step_size, width); + let expected = production_frame_values(&grid, main_width, step_size); + let whole_grid = grid.clone(); + if step_size == 1 { + assert_eq!( + whole_grid, expected, + "at step_size 1 the whole grid IS the frame view — this is the \ + blindness the entry records, not a bug" + ); + } else { + assert_ne!( + whole_grid, expected, + "at step_size {step_size} the whole grid must NOT be the frame \ + view, or this witness sees nothing" + ); + // And precisely which rows differ: production sees rows 0, 2, 4. + assert_eq!( + expected, + vec![grid[0].clone(), grid[2].clone(), grid[4].clone()], + "production's frame reads every step_size-th row" + ); + } + } +} + +// ============================================================================= +// The framework ceiling: step_size > 1 is not provable +// ============================================================================= + +/// The parameter the ceiling test uses. +const STEP_SIZE: usize = 2; +const STRIDED_COLS: usize = 3; +const STRIDED_ROWS: usize = 64; + +/// Reads both transition steps, so the AIR would have a non-empty next-row +/// column set if it could be proved. +struct StridedConstraints; + +type StridedAir = AirWithBuses; + +impl ConstraintSet for StridedConstraints { + fn eval>(&self, b: &mut B) { + let here = b.main(0, 0); + let there = b.main(1, 0); + b.emit_base(0, there - here); + } +} + +/// The `step_size = 2` fixture, shared by both halves of the ceiling test. +/// +/// Column 0 — the only column `StridedConstraints` reads — is CONSTANT, so +/// `main(1, 0) − main(0, 0)` is zero under ANY choice of which rows the two +/// transition offsets resolve to. That is what makes the release half below +/// meaningful: the proof it produces is rejected for a structural +/// prover/verifier disagreement, not because some other frame reading would +/// violate the constraint. +fn strided_fixture() -> (StridedAir, TraceTable) { + let air = StridedAir::new( + STRIDED_COLS, + AuxiliaryTraceBuildData { + interactions: vec![], + }, + &options(), + STEP_SIZE, + StridedConstraints, + ) + .with_name("STRIDED"); + + let mut main = Vec::with_capacity(STRIDED_ROWS * STRIDED_COLS); + for r in 0..STRIDED_ROWS as u64 { + main.push(FE::from(7u64)); + main.push(FE::from(1_000 + r)); + main.push(FE::from(2_000 + 3 * r)); + } + let trace = TraceTable::new_main(main, STRIDED_COLS, STEP_SIZE); + + assert_eq!(air.step_size(), STEP_SIZE, "the fixture's step size"); + assert_eq!( + air.context().transition_offsets.len() * air.step_size(), + 4, + "num_eval_points is offsets x step_size, so this AIR WOULD have two-row \ + blocks and a stride of two — the shape both entries want" + ); + + (air, trace) +} + +/// ★ A FRAMEWORK CEILING, measured rather than asserted from reading, and +/// reported as a finding (standing decisions: report ceilings, do not work around +/// them silently). +/// +/// The production prover cannot prove ANY AIR with `step_size > 1`, but it fails +/// in two DIFFERENT ways depending on the build profile, so the ceiling is +/// witnessed twice — once per profile — rather than in a single `should_panic` +/// that only holds in one of them: +/// +/// - **Debug (this body).** The CPU transition evaluator borrows one row per +/// transition offset (`RowFrame::from_lde`, `evaluator.rs:72`) and asserts the +/// single-row shape outright: `debug_assert_eq!(lde_trace.lde_step_size, +/// lde_trace.blowup_factor, "RowFrame requires single-row steps (step_size 1)")` +/// — and `lde_step_size = trace_step_size · blowup_factor`, so the equality IS +/// `step_size == 1`. The prover panics before emitting anything. +/// - **Release (the sibling body below, selected by `cfg(not(debug_assertions))`).** +/// `debug_assert` is compiled out, so the prover runs to completion and returns +/// `Ok(proof)` — and production's own verifier REJECTS that proof. Measured, not +/// read: the ceiling is a completeness failure, not a soundness one, and it is +/// NOT the one assert. Relaxing the assert alone would not lift it. +/// +/// Nothing production-reachable is affected either way: every AIR in the tree +/// reports `step_size = 1` — the VM tables and the LFM chips pass it through +/// their `build_air` helpers, the continuation AIRs pass it to +/// `AirWithBuses::new` directly, and every example AIR's `step_size` impl returns +/// the literal `1` — so this shape exists only in this fixture. +/// +/// What this costs the ledger: entry 9 can have no end-to-end witness, from any +/// AIR, until the ceiling lifts — +/// [`the_frame_step_view_matches_productions_own_frame_assembly`] closes the +/// emitter's half against production's own frame assembly instead. +/// +/// Both halves are self-updating: if the ceiling is ever lifted, the debug half +/// stops panicking and the release half starts verifying, and each fails saying +/// entry 9 became closeable end to end. +#[cfg(debug_assertions)] +#[test] +#[should_panic(expected = "RowFrame requires single-row steps")] +fn the_prover_cannot_prove_a_step_size_two_air() { + use crate::test_utils::multi_prove_ram; + + let (air, mut trace) = strided_fixture(); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let _ = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])); +} + +/// ★ The same ceiling, as release actually reaches it — see the debug body above +/// for the full finding. +/// +/// With the `RowFrame` `debug_assert` compiled out the prover does NOT stop: it +/// emits a proof. What still holds is the claim the test's name makes, one level +/// out — that proof does not round-trip, because production's own verifier +/// rejects it. Asserting the rejection (rather than skipping the test in release) +/// is what keeps the required release CI gate covering this path. +#[cfg(not(debug_assertions))] +#[test] +fn the_prover_cannot_prove_a_step_size_two_air() { + use crate::test_utils::multi_prove_ram; + + let (air, mut trace) = strided_fixture(); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &())]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("with the debug_assert compiled out the prover runs to completion"); + + let refs: Vec<&dyn AIR> = vec![&air]; + assert!( + !Verifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut DefaultTranscript::::new(&[]), + &FEE::zero(), + ), + "production accepted a step_size = 2 proof — the framework ceiling lifted, \ + so entry 9 is now closeable end to end and this test should be replaced by \ + the end-to-end witness" + ); +} + +// ============================================================================= +// Entry 8 — the absorb order, on a real proof with a multi-row OOD block +// ============================================================================= + +type FibAir = FibonacciMultiColumnAIR; +type FibPi = FibonacciMultiColumnPublicInputs; + +/// Columns the three-offset fixture carries. More than one, or a column-major and +/// a row-major absorb of the block coincide. +const FIB_COLS: usize = 3; +const FIB_ROWS: usize = 64; + +fn fib_initial_values() -> Vec<(FE, FE)> { + (0..FIB_COLS as u64) + .map(|c| (FE::from(1 + c), FE::from(3 + 2 * c))) + .collect() +} + +/// A real proof of a real three-offset AIR, produced and accepted by production. +fn fib_proof() -> ( + FibAir, + FibPi, + stark::proof::stark::MultiProof, +) { + use crate::test_utils::multi_prove_ram; + + let opts = options(); + let air = FibAir::with_num_columns(&opts, FIB_COLS); + let initial_values = fib_initial_values(); + let pi = FibPi { + initial_values: initial_values.clone(), + }; + let mut trace = compute_trace::(&initial_values, FIB_ROWS); + + let pairs: Vec<( + &dyn AIR, + _, + _, + )> = vec![(&air, &mut trace, &pi)]; + let proof = multi_prove_ram(pairs, &mut DefaultTranscript::::new(&[])) + .expect("the three-offset fixture must prove"); + (air, pi, proof) +} + +/// The challenges production derives from this proof, and the shape the machine +/// must replay. +struct FibReplay { + shape: TableChallengeShape, + main_root: stark::config::Commitment, + composition_root: stark::config::Commitment, + ood_current: Vec, + ood_next: Vec, + parts: Vec, + fri_roots: Vec, + fri_coeffs: Vec, + nonce: Option, + beta: FEE, + z: FEE, + gamma: FEE, + zetas: Vec, + iotas: Vec, +} + +fn fib_replay( + air: &FibAir, + pi: &FibPi, + proof: &stark::proof::stark::MultiProof, +) -> FibReplay { + use stark::domain::new_verifier_domain; + + let view = StarkProofView::Owned(&proof.proofs[0]); + let opts = air.options(); + let trace_length = view.trace_length(); + let log2_trace_length = trace_length.trailing_zeros(); + let log2_blowup = (opts.blowup_factor as usize).trailing_zeros(); + + // Single-table Phase A, transcribed from `multi_verify_views`: this AIR is not + // preprocessed and has no aux trace, so it is the main root and nothing else. + assert!(!air.is_preprocessed(), "the fixture is not preprocessed"); + assert!(!air.has_aux_trace(), "the fixture has no aux trace"); + let mut transcript = DefaultTranscript::::new(&[]); + transcript.append_bytes(view.lde_trace_main_merkle_root()); + + let domain = new_verifier_domain(air, trace_length); + let layout = Verifier::::ood_layout(air); + let challenges = Verifier::::replay_rounds_after_round_1( + air, + view, + pi, + &domain, + &mut transcript, + Vec::new(), + &layout, + ); + + let nt = challenges.transition_coeffs.len(); + let beta = if nt > 1 { + challenges.transition_coeffs[1] + } else { + challenges.boundary_coeffs[0] + }; + let gamma = challenges.trace_term_coeffs[1][0]; + + let ood_c = view.trace_ood_evaluations(); + let ood_n = view.trace_ood_next_evaluations(); + let shape = TableChallengeShape { + index: 0, + num_tables: 1, + has_aux_root: view.lde_trace_aux_merkle_root().is_some(), + has_contribution: view.bus_table_contribution().is_some(), + log2_trace_length, + log2_blowup, + coset_offset: FE::from(opts.coset_offset), + ood_current_dims: (ood_c.width(), ood_c.height()), + ood_next_dims: (ood_n.width(), ood_n.height()), + num_parts: view.composition_poly_parts_ood_evaluation().len(), + fri: FriShape::from_options(opts, log2_trace_length + log2_blowup), + grinding_factor: opts.grinding_factor, + num_queries: opts.fri_number_of_queries, + }; + + FibReplay { + shape, + main_root: *view.lde_trace_main_merkle_root(), + composition_root: *view.composition_poly_root(), + ood_current: ood_c.row_major_data().to_vec(), + ood_next: ood_n.row_major_data().to_vec(), + parts: view.composition_poly_parts_ood_evaluation().to_vec(), + fri_roots: view.fri_layers_merkle_roots().to_vec(), + fri_coeffs: view.fri_final_poly_coeffs().to_vec(), + nonce: view.nonce(), + beta, + z: challenges.z, + gamma, + zetas: challenges.zetas.clone(), + iotas: challenges.iotas.clone(), + } +} + +/// The machine's replay of `r`'s rounds, publishing every challenge. +fn fib_challenge_program( + r: &FibReplay, +) -> (super::compiler::LfmProgram, Vec>) { + let s = &r.shape; + let mut b = LfmBuilder::new(); + + let a_main = b.declare_arena(2); + let a_composition = b.declare_arena(2); + let a_current = b.declare_arena((s.ood_current_dims.0 * s.ood_current_dims.1) as u32); + let a_next = b.declare_arena((s.ood_next_dims.0 * s.ood_next_dims.1) as u32); + let a_parts = b.declare_arena(s.num_parts as u32); + let a_fri_roots = b.declare_arena(2 * s.fri.num_committed() as u32); + let a_fri_coeffs = b.declare_arena(s.fri.num_terminal_coeffs() as u32); + let a_nonce = (s.grinding_factor > 0).then(|| b.declare_arena(1)); + + let mut t = TranscriptReplay::new(&[]); + let main = RootCells::hint(&mut b, a_main, 0); + t.append_halves(&main.halves()); + + let composition = RootCells::hint(&mut b, a_composition, 0); + let current: Vec<_> = (0..(s.ood_current_dims.0 * s.ood_current_dims.1) as u32) + .map(|i| b.hint_word(a_current, i).as_ext()) + .collect(); + let next: Vec<_> = (0..(s.ood_next_dims.0 * s.ood_next_dims.1) as u32) + .map(|i| b.hint_word(a_next, i).as_ext()) + .collect(); + let parts: Vec<_> = (0..s.num_parts as u32) + .map(|i| b.hint_word(a_parts, i).as_ext()) + .collect(); + let fri_roots: Vec<_> = (0..s.fri.num_committed()) + .map(|i| RootCells::hint(&mut b, a_fri_roots, 2 * i as u32)) + .collect(); + let fri_coeffs: Vec<_> = (0..s.fri.num_terminal_coeffs() as u32) + .map(|i| b.hint_word(a_fri_coeffs, i).as_ext()) + .collect(); + let nonce = a_nonce.map(|id| b.hint_felt(id, 0)); + + let mut fork = fork_table(&t, s.index, s.num_tables); + let ch = emit_table_challenges( + &mut b, + &mut fork, + s, + &TableAbsorbs { + aux_root: None, + contribution: None, + composition_root: &composition, + ood_current: ¤t, + ood_next: &next, + parts: &parts, + fri_roots: &fri_roots, + fri_coeffs: &fri_coeffs, + nonce, + }, + ); + b.public(ch.beta.as_cell()); + b.public(ch.z.as_cell()); + b.public(ch.gamma.as_cell()); + for zeta in &ch.zetas { + b.public(zeta.as_cell()); + } + for bits in &ch.iota_bits { + let felt = super::edsl::bits_to_felt(&mut b, bits); + b.public(felt.as_cell()); + } + + let program = compile(b.finish()); + validate(&program).expect("the three-offset replay must be admissible"); + + let mut arenas = vec![ + super::proof_arena::commitments_to_arena(&[r.main_root]), + super::proof_arena::commitments_to_arena(&[r.composition_root]), + r.ood_current.iter().map(ext_word).collect(), + r.ood_next.iter().map(ext_word).collect(), + r.parts.iter().map(ext_word).collect(), + super::proof_arena::commitments_to_arena(&r.fri_roots), + r.fri_coeffs.iter().map(ext_word).collect(), + ]; + if let Some(n) = r.nonce { + arenas.push(vec![base_word(FE::from(n))]); + } + (program, arenas) +} + +/// ★ ENTRY 8: the machine absorbs a MULTI-ROW OOD block in production's order. +/// +/// The fixture is `FibonacciMultiColumnAIR` with three columns — three transition +/// offsets at `step_size = 1`, so `num_eval_points = 3` and the next-row block is +/// 2 rows × 3 columns. That is the first block in this phase where column-major +/// and row-major absorbs differ. +/// +/// The oracle is production's own `replay_rounds_after_round_1` on a proof the +/// production verifier accepts, so this is the same differential the epoch spine +/// runs — on a shape the epoch cannot produce. +/// +/// The negative half is not optional: without it a green test here would be +/// consistent with the block still being one row tall. So the same proof's block +/// is absorbed ROW-major through production's own transcript, and the challenge +/// that follows must MOVE. +#[test] +fn the_machine_absorbs_a_multi_row_ood_block_in_productions_order() { + let (air, pi, proof) = fib_proof(); + + // Production must accept it, or the blocks below are not a real proof's. + let refs: Vec<&dyn AIR> = vec![&air]; + assert!( + Verifier::multi_verify_views( + &refs, + MultiProofView::Owned(&proof), + &mut DefaultTranscript::::new(&[]), + &FEE::zero(), + ), + "production must accept the three-offset fixture" + ); + + let r = fib_replay(&air, &pi, &proof); + + // ---- ★ the blindness this fixture removes, ASSERTED before it is relied on. + assert_eq!( + air.context().transition_offsets.len(), + 3, + "the fixture must have three transition offsets" + ); + assert_eq!( + air.context().transition_offsets.len() * air.step_size(), + 3, + "num_eval_points" + ); + println!( + " three-offset fixture: offsets {:?}, step_size {}, ood_current {:?}, \ + ood_next {:?}, next_row_cols {:?}, parts {}", + air.context().transition_offsets, + air.step_size(), + r.shape.ood_current_dims, + r.shape.ood_next_dims, + air.trace_ood_next_row_columns(), + r.shape.num_parts, + ); + assert!( + r.shape.ood_next_dims.1 > 1 && r.shape.ood_next_dims.0 > 1, + "the next-row OOD block must be taller than one row AND wider than one \ + column, or a row-major absorb is indistinguishable: got {:?}", + r.shape.ood_next_dims + ); + + // ---- the differential: every challenge, against production's own replay. + let (program, arenas) = fib_challenge_program(&r); + let exec = + execute(&program, &arenas, &TestPermutation).expect("the three-offset replay must execute"); + + let pub_ext = |i: usize| word_as_ext(&exec.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), r.beta, "beta"); + assert_eq!(pub_ext(1), r.z, "z"); + assert_eq!( + pub_ext(2), + r.gamma, + "gamma — the first challenge AFTER the OOD absorb" + ); + let mut cursor = 3usize; + for (k, want) in r.zetas.iter().enumerate() { + assert_eq!(pub_ext(cursor + k), *want, "zeta {k}"); + } + cursor += r.zetas.len(); + for q in 0..r.shape.num_queries { + let got = word_as_base(&exec.public_words[cursor + q].1).expect("an index is a base felt"); + assert_eq!(got, FE::from(r.iotas[q] as u64), "iota {q}"); + } + cursor += r.shape.num_queries; + assert_eq!( + cursor, + exec.public_words.len(), + "every published challenge must be checked" + ); + + // ---- ★ the negative half: INJECT the row-major absorb and watch it fail. + // + // Not a comparison of two of my own orders — a control program that replays + // the same rounds with the blocks absorbed row-major, checked against the SAME + // production challenge the positive half matched. One side is production's. + // + // The control stops at `gamma`, the first challenge drawn after the OOD + // absorb: everything downstream of a wrong `gamma` is wrong for a derived + // reason, and stopping here says the divergence begins exactly at the absorb. + let control_gamma = row_major_control_gamma(&r); + assert_ne!( + control_gamma, r.gamma, + "a row-major absorb of this proof's OOD blocks must move the first \ + challenge drawn after them — if it does not, the fixture is as blind as \ + the epoch and entry 8 stays open" + ); + println!( + " row-major control: gamma moves ({} != production's), so the absorb \ + order is load-bearing on this fixture and the differential above covers it", + control_gamma == r.gamma + ); +} + +/// The DENIED absorb order, emitted: Phase A, round 2, `z`, both OOD blocks +/// ROW-major, the parts, then `γ`. +/// +/// This is the mutation entry 8 says nothing can catch — deliberately built so it +/// CAN be caught, on a fixture whose blocks are more than one row tall. It stops +/// at `γ` because that is the first value the absorb order can move, so a +/// difference here is attributable to the order and to nothing else. +/// +/// It duplicates the round structure of [`super::epoch::emit_table_challenges`] +/// rather than calling it with a flag: a production emitter should not carry a +/// switch for its own denied behaviour, and the duplication is bounded because +/// the control needs nothing past `γ`. +fn row_major_control_gamma(r: &FibReplay) -> FEE { + let s = &r.shape; + let mut b = LfmBuilder::new(); + + let a_main = b.declare_arena(2); + let a_composition = b.declare_arena(2); + let a_current = b.declare_arena((s.ood_current_dims.0 * s.ood_current_dims.1) as u32); + let a_next = b.declare_arena((s.ood_next_dims.0 * s.ood_next_dims.1) as u32); + let a_parts = b.declare_arena(s.num_parts as u32); + + let mut t = TranscriptReplay::new(&[]); + let main = RootCells::hint(&mut b, a_main, 0); + t.append_halves(&main.halves()); + let mut fork = fork_table(&t, s.index, s.num_tables); + + let _beta = fork.sample_ext(&mut b); + let composition = RootCells::hint(&mut b, a_composition, 0); + fork.append_halves(&composition.halves()); + + let _z = super::epoch::emit_z_ood(&mut b, &mut fork, s); + let current: Vec<_> = (0..(s.ood_current_dims.0 * s.ood_current_dims.1) as u32) + .map(|i| b.hint_word(a_current, i).as_ext()) + .collect(); + let next: Vec<_> = (0..(s.ood_next_dims.0 * s.ood_next_dims.1) as u32) + .map(|i| b.hint_word(a_next, i).as_ext()) + .collect(); + let parts: Vec<_> = (0..s.num_parts as u32) + .map(|i| b.hint_word(a_parts, i).as_ext()) + .collect(); + + // ★ THE MUTATION: rows outside, columns inside — production has it the other + // way round (`verifier.rs:1421-1431`). + for (dims, block) in [(s.ood_current_dims, ¤t), (s.ood_next_dims, &next)] { + let (width, height) = dims; + for row in 0..height { + for col in 0..width { + let coords = b.unpack(block[row * width + col].as_cell()); + fork.append_ext(&mut b, [coords[0], coords[1], coords[2]]); + } + } + } + for part in &parts { + let coords = b.unpack(part.as_cell()); + fork.append_ext(&mut b, [coords[0], coords[1], coords[2]]); + } + let gamma = fork.sample_ext(&mut b); + b.public(gamma.as_cell()); + + let program = compile(b.finish()); + validate(&program).expect("the control must be admissible"); + let arenas = vec![ + super::proof_arena::commitments_to_arena(&[r.main_root]), + super::proof_arena::commitments_to_arena(&[r.composition_root]), + r.ood_current.iter().map(ext_word).collect(), + r.ood_next.iter().map(ext_word).collect(), + r.parts.iter().map(ext_word).collect(), + ]; + let exec = execute(&program, &arenas, &TestPermutation).expect("the control must execute"); + word_as_ext(&exec.public_words[0].1).expect("gamma is ext") +} diff --git a/prover/src/lfm/sub_proof.rs b/prover/src/lfm/sub_proof.rs new file mode 100644 index 000000000..0e9b4ffb9 --- /dev/null +++ b/prover/src/lfm/sub_proof.rs @@ -0,0 +1,628 @@ +//! One sub-proof's query verification: DEEP reconstruction over the SAME arena +//! cells the Merkle authentication authenticates. +//! +//! The [constraint](super::constraints) and [DEEP](super::deep) legs consume +//! opened values; the [Merkle walk](super::edsl::keccak_merkle_walk) +//! authenticates them. Built separately the two are each correct and neither +//! says anything about the other — a program could fold one set of values and +//! authenticate a different set, and every test that fed both halves the same +//! data would pass. This module is the join, and it is a join by CONSTRUCTION +//! rather than by convention: [`emit_group_authentication`] takes cells and +//! cannot hint, so the only values it can authenticate are the caller's, and +//! [`emit_query`] hands those same cells to the DEEP fold. +//! +//! # The two consumers disagree about layout, which is the whole difficulty +//! +//! A query opens four committed matrices — precomputed, main, aux, composition +//! — and each is a SEPARATE Merkle tree with its own root and its own path. The +//! leaf of one tree is that matrix's own row pair: +//! +//! ```text +//! leaf(main) = keccak( main[υ] ‖ main[−υ] ) +//! ``` +//! +//! while DEEP walks one POINT across all matrices: +//! +//! ```text +//! DEEP(υ) folds precomputed[υ] ‖ main[υ] ‖ aux[υ] +//! ``` +//! +//! So the authentication groups by matrix and the fold groups by point. The +//! two orders cross, which is exactly the situation that invites two parallel +//! copies of the same values in two arenas — sound only as long as the host +//! filling them agrees with itself, which no in-machine constraint requires. +//! Here `values` is one vector of cells per matrix, in LEAF order, and DEEP +//! indexes into it: column `c` at the regular point is `values[c]`, at the +//! symmetric point `values[num_columns + c]`. +//! +//! # The query point is derived from the index bits, not hinted +//! +//! `DEEP(υ)` is meaningless unless `υ` is the point the authenticated leaf +//! sits at. Production derives both from one challenge `iota` +//! (`query_challenge_to_evaluation_point`); a machine that hinted the point +//! separately would let a prover authenticate a leaf at one index and evaluate +//! DEEP at another. [`emit_query`] decomposes the hinted index ONCE and uses +//! the same bits for the walk and for the point, so the two cannot disagree. +//! +//! `υ = offset · g^{br(2·iota)}` where `br` is the bit reversal over the LDE +//! domain. Reversing `2·iota` maps index bit `i` to weight `2^{depth-1-i}`, so +//! the point is `offset · Π (g^{2^{depth-1-i}})^{b_i}` — one `Select` and one +//! `Mul` per bit against program constants, via [`super::edsl::pow_bits`]. The +//! symmetric point is `−υ`: `br(2·iota+1) = br(2·iota) + L/2` and `g^{L/2} = +//! −1`, so it costs one subtraction rather than a second derivation. + +use math::field::traits::IsFFTField; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::deep::{DeepInvariants, DeepOpening, DeepShape, emit_deep_point}; +use super::edsl::{self, KeccakDigest}; + +/// Rows a Merkle leaf covers — `crypto/stark`'s `ROWS_PER_LEAF`, mirrored here +/// because it fixes program shape: a leaf holds a row PAIR, which is why one +/// path authenticates both of a query's two points. +pub const ROWS_PER_LEAF: usize = 2; + +/// The compile-time shape of one committed matrix of a sub-proof. +/// +/// `is_ext` is the element kind, and it is not cosmetic: a base element is +/// rendered into the leaf as 8 big-endian bytes and an extension element as 24 +/// (components 0, 1, 2, each big-endian — `write_bytes_be` for +/// `FieldElement`). Getting it wrong changes +/// the byte string and therefore the leaf. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GroupShape { + /// Columns at ONE point. A leaf covers `ROWS_PER_LEAF · num_columns`. + pub num_columns: usize, + pub is_ext: bool, +} + +impl GroupShape { + /// Cells one query's opening of this group occupies — both points. + pub fn num_values(&self) -> usize { + ROWS_PER_LEAF * self.num_columns + } + + /// Bytes the leaf hash covers. + pub fn leaf_bytes(&self) -> usize { + self.num_values() * if self.is_ext { 24 } else { 8 } + } +} + +/// One sub-proof's per-query verification shape. +/// +/// Every field is program SHAPE. In particular the group list is: a proof that +/// carried an aux opening where the program expects none would not match the +/// arena schema, which is the straight-line discipline standing in for +/// production's `(Some(root), Some(opening)) | (None, None)` presence check. +#[derive(Clone, Debug)] +pub struct SubProofShape { + /// The DEEP fold's shape — column count, OOD grid, part count. + pub deep: DeepShape, + /// The TRACE matrices in DEEP column order: precomputed, then main, then + /// aux. Absent groups are omitted, exactly as the proof omits them. Their + /// widths must sum to `deep.num_total_cols`. + pub trace_groups: Vec, + /// Merkle depth — `log2(lde_length) − 1`, since a leaf is a row pair. All + /// four trees commit over the same LDE domain, so one depth serves them + /// all and one index addresses them all. + pub merkle_depth: usize, + /// `log2` of the LDE domain — `log2_trace_length + log2(blowup)`. + pub log2_lde_length: u32, + /// The LDE coset offset, `ProofOptions::coset_offset`. + pub coset_offset: FE, +} + +impl SubProofShape { + /// The composition-parts group. Its width is the part count and its + /// elements are extension, both of which are already DEEP shape. + pub fn parts_group(&self) -> GroupShape { + GroupShape { + num_columns: self.deep.num_composition_parts, + is_ext: true, + } + } + + /// Every group a query authenticates: the trace matrices then the parts. + pub fn groups(&self) -> Vec { + let mut all = self.trace_groups.clone(); + all.push(self.parts_group()); + all + } + + /// Arena words one query's openings occupy — every group's values, plus + /// the index and the sibling digests (two words per level per group). + pub fn query_words(&self) -> usize { + 1 + self.opening_words() + } + + /// [`Self::query_words`] WITHOUT the index word. + /// + /// The assembled verifier's stride: its query index is not proof data at + /// all but the transcript's own bits, so the arena carries only the opened + /// values and the paths. An arena that still carried an index would be + /// offering the prover a second one. + pub fn opening_words(&self) -> usize { + let values: usize = self.groups().iter().map(GroupShape::num_values).sum(); + let siblings = 2 * self.merkle_depth * self.groups().len(); + values + siblings + } + + /// Checked invariants of a shape, so a caller cannot assemble one whose + /// groups do not cover the fold. + fn check(&self) { + let width: usize = self.trace_groups.iter().map(|g| g.num_columns).sum(); + assert_eq!( + width, self.deep.num_total_cols, + "the trace groups must cover exactly the DEEP column set" + ); + assert!( + self.merkle_depth + 1 == self.log2_lde_length as usize, + "a leaf is a row pair, so the tree is one level shallower than the \ + LDE domain: depth {} against log2(lde) {}", + self.merkle_depth, + self.log2_lde_length + ); + assert!( + self.merkle_depth >= 1, + "a tree with no levels has no path to walk" + ); + } +} + +/// A committed matrix's root, unpacked once and shared by every query. +/// +/// Hoisting the unpack is what `fri_toy_program` already does per query: the +/// root is a per-sub-proof value and a 219-query proof would otherwise pay +/// 219 redundant `Unpack`s per group. +pub struct GroupCommitment { + /// The root's two words as lanes. + pub root_lanes: [[Felt; 4]; 2], + pub shape: GroupShape, +} + +impl GroupCommitment { + /// Reads a root out of the arena and hoists its unpack. + pub fn hint( + b: &mut LfmBuilder, + arena: super::instr::ArenaId, + base: u32, + shape: GroupShape, + ) -> Self { + let w0 = b.hint_word(arena, base); + let w1 = b.hint_word(arena, base + 1); + GroupCommitment { + root_lanes: [b.unpack(w0), b.unpack(w1)], + shape, + } + } + + /// A commitment over lanes the caller already holds — the assembled + /// verifier's route, where a root reaches this leg as the SAME cells the + /// transcript absorbed rather than as a second hint. + /// + /// A root has two consumers (`epoch::RootCells`' doc comment names them): + /// the Fiat-Shamir absorb and this comparison. Hinting it twice is the + /// two-consumer hazard — a prover would absorb one root and authenticate + /// against another, and no differential over honest data could see it, + /// because the host packs the same bytes into both. This constructor is the + /// join, and it takes lanes rather than words precisely so there is nothing + /// left to hint. + pub fn from_lanes(root_lanes: [[Felt; 4]; 2], shape: GroupShape) -> Self { + GroupCommitment { root_lanes, shape } + } +} + +/// One query's opening of one committed matrix, as CELLS. +/// +/// There is deliberately no constructor that hints: the values are whatever the +/// caller already holds, which is what makes the authentication and the fold +/// share them rather than agree about them. +pub struct GroupOpening { + /// `evaluations ‖ evaluations_sym` in LEAF order — the row pair written + /// column by column, the regular point first. + pub values: Vec, + /// Sibling digests, LEAF LEVEL FIRST — the order + /// `verify_merkle_path_from_leaf_hash` consumes them in. + pub siblings: Vec, +} + +/// The leaf hash of one group's row pair, in the production commitment layout. +/// +/// Base groups go through [`edsl::keccak_leaf_hash`] unchanged. Extension +/// groups render each element as its three components, each big-endian — +/// `write_bytes_be` writes components 0, 1, 2 in that order, so the machine +/// unpacks the word and byteswaps lanes 0, 1, 2. +/// +/// Lane 3 is NOT hashed, which is correct (production hashes three components) +/// and worth stating: an extension cell whose lane 3 is nonzero would hash the +/// same as one whose lane 3 is zero. It cannot arise here because every +/// extension value a query opens is also consumed as an ext operand by the DEEP +/// fold, and an ext read of a word with a nonzero lane 3 is unprovable. A +/// caller that authenticated an extension group WITHOUT folding it would owe +/// that check itself. +pub fn emit_leaf_hash(b: &mut LfmBuilder, shape: GroupShape, values: &[Cell]) -> KeccakDigest { + use super::keccak_host::BYTES_PER_HALF; + use super::transcript_replay::felt_be_halves; + + assert_eq!( + values.len(), + shape.num_values(), + "a leaf covers the whole row pair" + ); + if !shape.is_ext { + let felts: Vec = values.iter().map(|c| Felt(c.addr())).collect(); + return edsl::keccak_leaf_hash(b, &felts); + } + + let mut stream = Vec::with_capacity(6 * values.len()); + for v in values { + let lanes = b.unpack(*v); + for lane in lanes.iter().take(3) { + stream.extend(felt_be_halves(b, *lane)); + } + } + let len_bytes = BYTES_PER_HALF * stream.len(); + debug_assert_eq!(len_bytes, shape.leaf_bytes()); + edsl::keccak256(b, &stream, len_bytes) +} + +/// Authenticate one group's opened values against its committed root. +/// +/// Takes the caller's cells and never hints a value, so what it authenticates +/// is what the caller folds. The assert is the binding; `bits` are shared with +/// every other group of the same query, which is what makes the four trees +/// agree about WHICH leaf they opened. +pub fn emit_group_authentication( + b: &mut LfmBuilder, + commitment: &GroupCommitment, + opening: &GroupOpening, + bits: &[Bit], +) { + assert_eq!( + opening.siblings.len(), + bits.len(), + "one sibling per level, and every group walks the same index" + ); + let leaf = emit_leaf_hash(b, commitment.shape, &opening.values); + let root = edsl::keccak_merkle_walk(b, leaf, bits, &opening.siblings); + edsl::assert_word_eq_lanes(b, root[0], &commitment.root_lanes[0]); + edsl::assert_word_eq_lanes(b, root[1], &commitment.root_lanes[1]); +} + +/// The LDE-domain constants the point derivation multiplies together: +/// `factors[i] = g^{2^{depth-1-i}}`, matching index bit `i`'s weight after the +/// bit reversal. +fn point_factors(log2_lde_length: u32) -> Vec { + let g = ::get_primitive_root_of_unity(log2_lde_length as u64) + .expect("a power-of-two LDE length has a root of unity"); + let depth = log2_lde_length as usize - 1; + (0..depth).map(|i| g.pow(1u64 << (depth - 1 - i))).collect() +} + +/// `(υ, −υ)` from the query index bits, for the LDE domain given by its size and +/// coset offset. Shape-only inputs: the factors are program constants. +/// +/// Keyed on the domain rather than on a [`SubProofShape`] because the FRI leg +/// needs the same derivation and has no trace shape to hand — it holds a +/// [`super::fri::FriShape`], which carries both of these fields. One derivation +/// serves both, which is the point: `join_tests::the_join_premises_hold_on_a_real_proof` +/// checks THIS function against production's +/// `query_challenge_to_evaluation_point` at every index of a real proof, and a +/// second copy would not be covered by that check. +pub fn emit_points_from_bits( + b: &mut LfmBuilder, + log2_lde_length: u32, + coset_offset: FE, + bits: &[Bit], +) -> (Felt, Felt) { + assert_eq!( + bits.len(), + log2_lde_length as usize - 1, + "a leaf is a row pair, so the index is one bit narrower than the domain" + ); + let point = edsl::pow_bits(b, bits, &point_factors(log2_lde_length), coset_offset); + let zero = b.felt_const(FE::zero()); + (point, b.sub(zero, point)) +} + +/// `(υ, −υ)` from the query index bits. +pub fn emit_query_points(b: &mut LfmBuilder, shape: &SubProofShape, bits: &[Bit]) -> (Felt, Felt) { + assert_eq!(bits.len(), shape.merkle_depth); + emit_points_from_bits(b, shape.log2_lde_length, shape.coset_offset, bits) +} + +/// Everything one query of one sub-proof contributes, emitted. +/// +/// Order of business: decompose the index, authenticate every group against +/// its root, derive the two points from the same bits, then fold DEEP at both. +/// Returns `(DEEP(υ), DEEP(−υ))` for the FRI leg to consume. +/// +/// `trace_openings` is parallel to [`SubProofShape::trace_groups`]; the parts +/// opening is separate because DEEP treats it separately. +pub fn emit_query( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + commitments: &[GroupCommitment], + index: Felt, + openings: &[GroupOpening], +) -> (Ext, Ext) { + emit_query_with_bits(b, shape, gamma, inv, commitments, index, openings).deep +} + +/// What one query contributes when the caller needs more than the DEEP pair. +pub struct QueryOutput { + /// `(DEEP(υ), DEEP(−υ))`. + pub deep: (Ext, Ext), + /// The query index decomposed low-to-high — the SAME cells the Merkle walk + /// consumed and the query points were derived from. + /// + /// Handing these out is what lets a later leg join to this one rather than + /// run beside it. FRI reuses the index per layer (leaf position `index >> 1`, + /// partner `index ^ 1`, halving each layer), and a leg that decomposed its + /// own copy would authenticate one index while folding at another — the + /// exact gap this module exists to close, reopened one level up. There is + /// no way to return a DIFFERENT decomposition from here: `bit_dec` is + /// called once and its result feeds the walk, the points and this field. + pub bits: Vec, + /// `υ` — the cell the DEEP fold above evaluated at. + /// + /// Exposed for the same reason as [`Self::bits`], one step further along. + /// FRI needs `υ⁻¹` for its first fold and `υ^(2^total_folds)` for its + /// terminal check; both are functions of this cell, and a leg that + /// re-derived the point from `bits` would pay `merkle_depth` `Select`s and + /// `Mul`s per query for a value it was already holding. Handing the cell + /// over is not just cheaper, it removes the question: there is exactly one + /// `emit_query_points` call in this function and its outputs go to DEEP and + /// to these fields, so no second point EXISTS to disagree. + /// + /// The structural guard is a count, not a comparison — see + /// `fri_tests::the_fri_join_adds_no_second_point_derivation`. + pub point: Felt, + /// `−υ`, likewise. The zero-fold FRI shape checks the terminal polynomial + /// at both points (production's `zetas.is_empty()` branch tests + /// `terminal[2·iota]` AND `terminal[2·iota+1]`). + pub point_sym: Felt, +} + +/// [`emit_query`], additionally returning the index bits — see [`QueryOutput`]. +/// +/// The index arrives as a FELT here, which is the isolation drivers' route: the +/// differential supplies production's own `iota` and the emitter decomposes it. +/// The assembled verifier does not have a felt to supply — its index is +/// `TranscriptReplay::sample_u64_pow2`'s bits — and takes +/// [`emit_query_from_bits`] instead, which is the same emitter minus this one +/// `bit_dec`. +#[allow(clippy::too_many_arguments)] +pub fn emit_query_with_bits( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + commitments: &[GroupCommitment], + index: Felt, + openings: &[GroupOpening], +) -> QueryOutput { + let bits = b.bit_dec(index, shape.merkle_depth); + emit_query_from_bits(b, shape, gamma, inv, commitments, bits, openings) +} + +/// [`emit_query_with_bits`] over an index the caller already holds as BITS. +/// +/// This is the entry point the assembled epoch verifier uses. Production's query +/// index is `sample_u64(lde_length >> 1)`, whose output is `index_bits()` bits by +/// construction (`verifier.rs:138-141`), and the machine's +/// `TranscriptReplay::sample_u64_pow2` produces exactly those bits. Routing them +/// straight in — rather than recomposing a felt and decomposing it again — is +/// what makes the assembled machine's query index in-range by construction and +/// closes ledger entry 5: with no felt in the program, `ι` and `ι + 2^(n−1)` +/// cannot be the same query, because neither is ever a number. +#[allow(clippy::too_many_arguments)] +pub fn emit_query_from_bits( + b: &mut LfmBuilder, + shape: &SubProofShape, + gamma: Ext, + inv: &DeepInvariants, + commitments: &[GroupCommitment], + bits: Vec, + openings: &[GroupOpening], +) -> QueryOutput { + shape.check(); + let groups = shape.groups(); + assert_eq!(commitments.len(), groups.len(), "one commitment per group"); + assert_eq!(openings.len(), groups.len(), "one opening per group"); + for (c, g) in commitments.iter().zip(&groups) { + assert_eq!(c.shape, *g, "commitment shapes must match the sub-proof"); + } + assert_eq!( + bits.len(), + shape.merkle_depth, + "a query index is exactly the tree's depth in bits" + ); + + for (commitment, opening) in commitments.iter().zip(openings) { + emit_group_authentication(b, commitment, opening, &bits); + } + + let (point, point_sym) = emit_query_points(b, shape, &bits); + + // The crossing: the authenticated cells, re-read by POINT instead of by + // matrix. Nothing is hinted here, so `trace` cannot hold anything the walk + // above did not fold into a leaf. + let mut trace = Vec::with_capacity(shape.deep.num_total_cols); + let mut trace_sym = Vec::with_capacity(shape.deep.num_total_cols); + for (opening, g) in openings.iter().zip(&groups).take(shape.trace_groups.len()) { + for c in 0..g.num_columns { + trace.push(opening.values[c].as_ext()); + trace_sym.push(opening.values[g.num_columns + c].as_ext()); + } + } + + let parts_opening = openings.last().expect("the parts group is always present"); + let num_parts = shape.deep.num_composition_parts; + let parts: Vec = (0..num_parts) + .map(|j| parts_opening.values[j].as_ext()) + .collect(); + let parts_sym: Vec = (0..num_parts) + .map(|j| parts_opening.values[num_parts + j].as_ext()) + .collect(); + + let regular = DeepOpening { + point, + trace, + parts, + }; + let symmetric = DeepOpening { + point: point_sym, + trace: trace_sym, + parts: parts_sym, + }; + QueryOutput { + deep: ( + emit_deep_point(b, &shape.deep, gamma, inv, ®ular), + emit_deep_point(b, &shape.deep, gamma, inv, &symmetric), + ), + bits, + point, + point_sym, + } +} + +// ===================== the whole sub-proof ===================== + +/// The arenas one sub-proof's verification reads, in declaration order. +/// +/// Each field is packed into its OWN arena rather than one concatenated +/// stream — the packing rule [`super::proof_arena`] exists to enforce, applied +/// one level up: a query whose group widths shifted would otherwise silently +/// slide every query behind it. +pub struct SubProofArenas { + /// `γ`, then `ζ`. + pub uniforms: super::instr::ArenaId, + /// The reconstructed OOD grid, row-major, `num_eval_points × + /// num_total_cols` — the same values the constraint leg folds. + pub ood: super::instr::ArenaId, + /// The composition parts claimed at `z^P`. + pub parts: super::instr::ArenaId, + /// Two words per group's committed root, in [`SubProofShape::groups`] order. + pub roots: super::instr::ArenaId, + /// Per query, in order: the index, then per group the row-pair values + /// followed by the sibling digests (two words per level). + pub queries: super::instr::ArenaId, +} + +/// Emit a whole sub-proof's query verification: the invariants once, then every +/// query authenticated and folded. +/// +/// Returns `(DEEP(υ), DEEP(−υ))` per query. The invariant hoist is the reason a +/// 219-query proof is affordable, and it is production's own hoist — the OOD +/// row sums and the block scalars do not depend on the query. +pub fn emit_sub_proof( + b: &mut LfmBuilder, + shape: &SubProofShape, + num_queries: usize, +) -> (SubProofArenas, Vec<(Ext, Ext)>) { + let (arenas, out) = emit_sub_proof_with_bits(b, shape, num_queries); + (arenas, out.into_iter().map(|q| q.deep).collect()) +} + +/// [`emit_sub_proof`], additionally returning each query's index bits — see +/// [`QueryOutput`]. The FRI leg folds from these same cells. +pub fn emit_sub_proof_with_bits( + b: &mut LfmBuilder, + shape: &SubProofShape, + num_queries: usize, +) -> (SubProofArenas, Vec) { + use super::deep::emit_deep_invariants; + + shape.check(); + assert!(num_queries > 0, "a proof carries at least one query"); + let groups = shape.groups(); + + let uniforms = b.declare_arena(2); + let ood = b.declare_arena((shape.deep.num_eval_points * shape.deep.num_total_cols) as u32); + let parts = b.declare_arena(shape.deep.num_composition_parts as u32); + let roots = b.declare_arena(2 * groups.len() as u32); + let queries = b.declare_arena((num_queries * shape.query_words()) as u32); + let arenas = SubProofArenas { + uniforms, + ood, + parts, + roots, + queries, + }; + + let gamma = b.hint_word(uniforms, 0).as_ext(); + let zeta = b.hint_word(uniforms, 1).as_ext(); + + let mut next = 0u32; + let ood_steps: Vec> = (0..shape.deep.num_eval_points) + .map(|_| { + (0..shape.deep.num_total_cols) + .map(|_| { + let c = b.hint_word(ood, next).as_ext(); + next += 1; + c + }) + .collect() + }) + .collect(); + let claimed_parts: Vec = (0..shape.deep.num_composition_parts as u32) + .map(|j| b.hint_word(parts, j).as_ext()) + .collect(); + + let commitments: Vec = groups + .iter() + .enumerate() + .map(|(i, g)| GroupCommitment::hint(b, roots, 2 * i as u32, *g)) + .collect(); + + let inv = emit_deep_invariants(b, &shape.deep, gamma, zeta, &ood_steps, &claimed_parts); + + let mut cursor = 0u32; + let mut out = Vec::with_capacity(num_queries); + for _ in 0..num_queries { + let index = b.hint_felt(queries, cursor); + cursor += 1; + let openings: Vec = groups + .iter() + .map(|g| { + let values: Vec = (0..g.num_values()) + .map(|_| { + let c = b.hint_word(queries, cursor); + cursor += 1; + c + }) + .collect(); + let siblings: Vec = (0..shape.merkle_depth) + .map(|_| { + let lo = b.hint_word(queries, cursor); + let hi = b.hint_word(queries, cursor + 1); + cursor += 2; + [lo, hi] + }) + .collect(); + GroupOpening { values, siblings } + }) + .collect(); + out.push(emit_query_with_bits( + b, + shape, + gamma, + &inv, + &commitments, + index, + &openings, + )); + } + assert_eq!( + cursor as usize, + num_queries * shape.query_words(), + "the emitter's cursor must agree with the declared query stride" + ); + + (arenas, out) +} diff --git a/prover/src/lfm/tests.rs b/prover/src/lfm/tests.rs new file mode 100644 index 000000000..08d1d3cfc --- /dev/null +++ b/prover/src/lfm/tests.rs @@ -0,0 +1,647 @@ +//! Milestone A suite: the software layer round-trips — build → compile → +//! validate → execute — plus the negative paths (validator rejections, +//! executor runtime checks, compiler invariant panics). + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, FEE, GoldilocksField}; + +use super::builder::{LfmBuilder, LfmProgramSource}; +use super::compiler::{LfmProgram, compile}; +use super::executor::{LfmExecError, LfmExecution, execute}; +use super::hash::{LfmHasher, TestPermutation}; +use super::instr::{Addr, HashMode, Instr}; +use super::layout; +use super::validator::{LfmViolation, validate}; +use super::word::{LfmWord, base_word}; + +const GOLDILOCKS_P: u64 = 0xFFFF_FFFF_0000_0001; + +fn fe(v: u64) -> FE { + FE::from(v) +} + +fn ext(a: u64, b: u64, c: u64) -> FEE { + FEE::new([fe(a), fe(b), fe(c)]) +} + +fn run(program: &LfmProgram, arenas: &[Vec]) -> LfmExecution { + validate(program).expect("valid program"); + execute(program, arenas, &TestPermutation).expect("execution succeeds") +} + +fn cell(exec: &LfmExecution, addr: Addr) -> LfmWord { + exec.memory[addr.0 as usize].expect("cell written") +} + +fn base_at(exec: &LfmExecution, addr: Addr) -> FE { + super::word::word_as_base(&cell(exec, addr)).expect("base word") +} + +fn ext_at(exec: &LfmExecution, addr: Addr) -> FEE { + super::word::word_as_ext(&cell(exec, addr)).expect("ext word") +} + +// ---- base ALU ---- + +#[test] +fn base_alu_round_trip() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(7)); + let y = b.felt_const(fe(5)); + let s = b.add(x, y); + let d = b.sub(x, y); + let m = b.mul(x, y); + let q = b.div(m, y); + let h = b.mul_add(x, y, s); // 7·5 + 12 = 47 + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(base_at(&exec, s.addr()), fe(12)); + assert_eq!(base_at(&exec, d.addr()), fe(2)); + assert_eq!(base_at(&exec, m.addr()), fe(35)); + assert_eq!(base_at(&exec, q.addr()), fe(7)); + assert_eq!(base_at(&exec, h.addr()), fe(47)); +} + +#[test] +fn div_zero_conventions() { + // 0/0 = 1 (the assert mechanism's accepting case). + let mut b = LfmBuilder::new(); + let z = b.felt_const(FE::zero()); + let q = b.div(z, z); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(base_at(&exec, q.addr()), FE::one()); + + // x/0 with x ≠ 0 errors. + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(3)); + let z = b.felt_const(FE::zero()); + let _ = b.div(x, z); + let program = compile(b.finish()); + validate(&program).expect("structurally valid"); + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::DivByZero { .. })); +} + +#[test] +fn assert_lowering_pass_and_fail() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(6)); + let y = b.felt_const(fe(2)); + let three = b.felt_const(fe(3)); + let q = b.mul(y, three); + b.assert_eq(x, q); + let program = compile(b.finish()); + run(&program, &[]); // passes + + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(6)); + let y = b.felt_const(fe(5)); + b.assert_eq(x, y); + let program = compile(b.finish()); + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::DivByZero { .. })); +} + +// ---- Fp3 ALU ---- + +#[test] +fn ext_alu_matches_field_reference() { + let av = ext(3, 11, 2026); + let bv = ext(9, 1, 77); + let cv = ext(5, 4, 3); + let f = fe(13); + + let mut b = LfmBuilder::new(); + let a = b.ext_const(&av); + let bb = b.ext_const(&bv); + let c = b.ext_const(&cv); + let s = b.eadd(a, bb); + let d = b.esub(a, bb); + let p = b.emul(a, bb); + let q = b.ediv(p, bb); + let ma = b.emul_add(a, bb, c); + let fl = b.felt_const(f); + let mb = b.emul_base(a, fl); + let program = compile(b.finish()); + let exec = run(&program, &[]); + + assert_eq!(ext_at(&exec, s.addr()), &av + &bv); + assert_eq!(ext_at(&exec, d.addr()), &av - &bv); + assert_eq!(ext_at(&exec, p.addr()), &av * &bv); + assert_eq!(ext_at(&exec, q.addr()), av.clone()); + assert_eq!(ext_at(&exec, ma.addr()), &av * &bv + &cv); + let [a0, a1, a2] = *av.value(); + assert_eq!( + ext_at(&exec, mb.addr()), + FEE::new([&a0 * &f, &a1 * &f, &a2 * &f]) + ); +} + +#[test] +fn ext_assert_and_horner() { + // Horner: evaluate 5x² + 3x + 7 at x = (0,1,0) (i.e. w) via mul_add. + let x = ext(0, 1, 0); + let mut b = LfmBuilder::new(); + let xv = b.ext_const(&x); + let c2 = b.ext_const(&ext(5, 0, 0)); + let c1 = b.ext_const(&ext(3, 0, 0)); + let c0 = b.ext_const(&ext(7, 0, 0)); + let acc = b.emul_add(c2, xv, c1); // 5x + 3 + let acc = b.emul_add(acc, xv, c0); // 5x² + 3x + 7 + let expected = &(&(&ext(5, 0, 0) * &x) + &ext(3, 0, 0)) * &x + &ext(7, 0, 0); + let ex = b.ext_const(&expected); + b.assert_eq_ext(acc, ex); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(ext_at(&exec, acc.addr()), expected); +} + +// ---- select / bitdec ---- + +#[test] +fn select_swaps_on_bit() { + let mut b = LfmBuilder::new(); + let l = b.felt_const(fe(100)); + let r = b.felt_const(fe(200)); + let b0 = b.bit_const(false); + let b1 = b.bit_const(true); + let (l0, r0) = b.select(b0, l.as_cell(), r.as_cell()); + let (l1, r1) = b.select(b1, l.as_cell(), r.as_cell()); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(cell(&exec, l0.addr()), base_word(fe(100))); + assert_eq!(cell(&exec, r0.addr()), base_word(fe(200))); + assert_eq!(cell(&exec, l1.addr()), base_word(fe(200))); + assert_eq!(cell(&exec, r1.addr()), base_word(fe(100))); +} + +#[test] +fn non_boolean_select_bit_rejected() { + let mut b = LfmBuilder::new(); + let l = b.felt_const(fe(1)); + let r = b.felt_const(fe(2)); + let two = b.felt_const(fe(2)); + let (_, _) = b.select(super::builder::Bit(two.addr()), l.as_cell(), r.as_cell()); + let program = compile(b.finish()); + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::NonBooleanBit(_))); +} + +#[test] +fn bit_dec_edge_values() { + // Canonical decomposition + the p-specific gadget witnesses at the edges. + for v in [ + 0u64, + 1, + (1 << 32) - 1, + GOLDILOCKS_P - 1, + 0x1234_5678_9ABC_DEF0, + ] { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(v)); + let bits = b.bit_dec(x, 64); + let program = compile(b.finish()); + let exec = run(&program, &[]); + for (i, bit) in bits.iter().enumerate() { + assert_eq!( + base_at(&exec, bit.addr()), + fe((v >> i) & 1), + "bit {i} of {v:#x}" + ); + } + let row = &exec.records.bitdec[0]; + let top = (v >> 32) as u32; + if top == u32::MAX { + assert_eq!(row.z, FE::one(), "z for {v:#x}"); + assert_eq!(row.ginv, FE::zero()); + } else { + assert_eq!(row.z, FE::zero(), "z for {v:#x}"); + let g = fe(0xFFFF_FFFFu64 - top as u64); + assert_eq!(&row.ginv * &g, FE::one(), "ginv·g = 1 for {v:#x}"); + } + } +} + +#[test] +fn bit_dec_partial_width_allocates_only_requested_cells() { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(0b1011)); + let bits = b.bit_dec(x, 4); + let program = compile(b.finish()); + let exec = run(&program, &[]); + assert_eq!(bits.len(), 4); + let vals: Vec = bits.iter().map(|bit| base_at(&exec, bit.addr())).collect(); + assert_eq!(vals, vec![fe(1), fe(1), fe(0), fe(1)]); + // All 64 witness bits still recorded for the constraint columns. + assert_eq!( + exec.records.bitdec[0].bits[4..] + .iter() + .filter(|b| **b == FE::one()) + .count(), + 0 + ); +} + +// ---- hash ---- + +#[test] +fn hash_compress_and_permute_match_reference() { + let hasher = TestPermutation; + let a: LfmWord = core::array::from_fn(|i| fe(10 + i as u64)); + let c: LfmWord = core::array::from_fn(|i| fe(20 + i as u64)); + + let mut b = LfmBuilder::new(); + let da = b.digest_const(a); + let dc = b.digest_const(c); + let d = b.compress(da, dc); + let s0 = b.digest_const(core::array::from_fn(|i| fe(30 + i as u64))); + let s1 = b.digest_const(core::array::from_fn(|i| fe(40 + i as u64))); + let s2 = b.digest_const(core::array::from_fn(|i| fe(50 + i as u64))); + let out = b.permute([s0.as_cell(), s1.as_cell(), s2.as_cell()]); + let program = compile(b.finish()); + let exec = run(&program, &[]); + + assert_eq!(cell(&exec, d.addr()), hasher.compress(&a, &c)); + + let mut state: [FE; 12] = core::array::from_fn(|_| FE::zero()); + for i in 0..4 { + state[i] = fe(30 + i as u64); + state[4 + i] = fe(40 + i as u64); + state[8 + i] = fe(50 + i as u64); + } + let expected = hasher.permute(state); + for (j, o) in out.iter().enumerate() { + let w = cell(&exec, o.addr()); + for l in 0..4 { + assert_eq!(w[l], expected[4 * j + l]); + } + } +} + +// ---- hints / public ---- + +#[test] +fn hint_and_public_round_trip() { + let w0: LfmWord = core::array::from_fn(|i| fe(100 + i as u64)); + let w1: LfmWord = core::array::from_fn(|i| fe(200 + i as u64)); + + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let h0 = b.hint_word(arena, 0); + let h1 = b.hint_word(arena, 1); + b.public(h0); + b.public(h1); + let program = compile(b.finish()); + let exec = run(&program, &[vec![w0, w1]]); + + assert_eq!(cell(&exec, h0.addr()), w0); + assert_eq!(exec.public_words, vec![(0, w0), (1, w1)]); +} + +#[test] +fn arena_out_of_bounds_rejected_by_validator_and_executor() { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(2); + let h = b.hint_word(arena, 5); // past the declared length + b.public(h); + let program = compile(b.finish()); + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::ArenaOutOfBounds { arena: 0, index: 5 } + ); + let err = execute(&program, &[vec![base_word(fe(1)); 2]], &TestPermutation).unwrap_err(); + assert!(matches!(err, LfmExecError::ArenaOutOfBounds { .. })); +} + +// ---- validator negatives (via mutation of a valid program) ---- + +fn small_valid_program() -> LfmProgram { + let mut b = LfmBuilder::new(); + let x = b.felt_const(fe(4)); + let y = b.felt_const(fe(9)); + let s = b.add(x, y); + let p = b.mul(s, x); + b.public(p.as_cell()); + compile(b.finish()) +} + +#[test] +fn validator_rejects_double_write() { + let mut program = small_valid_program(); + // Point the mul's destination at the add's (already-written) cell. + let add_out = program + .instrs + .iter() + .find_map(|i| match i { + Instr::BaseAlu { + op: super::instr::BaseOp::Add, + out, + .. + } => Some(*out), + _ => None, + }) + .unwrap(); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + out, + .. + } = i + { + *out = add_out; + } + } + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::DoubleWrite { addr: add_out.0 } + ); + // The executor independently catches it. + let err = execute(&program, &[], &TestPermutation).unwrap_err(); + assert_eq!(err, LfmExecError::DoubleWrite(add_out.0)); +} + +#[test] +fn validator_rejects_cycle() { + let mut program = small_valid_program(); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + out, + a, + .. + } = i + { + *a = *out; // a := f(a) — balances for any value; must die here + } + } + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::CyclicRead { .. } + )); +} + +#[test] +fn validator_rejects_wrong_mult() { + let mut program = small_valid_program(); + for i in &mut program.instrs { + if let Instr::Const { mult, .. } = i { + *mult += 1; + break; + } + } + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::MultMismatch { .. } + )); +} + +#[test] +fn validator_rejects_non_one_hot_selector() { + let mut program = small_valid_program(); + // Turn a second selector on in the first real BALU row. + program.groups.balu.set(0, layout::balu::SEL_SUB, FE::one()); + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::NonOneHotSelector { + chip: "LFM_BALU", + row: 0 + } + ); +} + +#[test] +fn validator_rejects_dirty_padding() { + let mut program = small_valid_program(); + let row = program.groups.balu.real_rows; // first padding row + program.groups.balu.set(row, layout::balu::MULT, fe(1)); + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::DirtyPadding { + chip: "LFM_BALU", + row + } + ); +} + +/// Check 9 at the level it operates: a multiplicity column of the COMMITTED +/// group, with the instruction list left honest. +/// +/// `p − 1` is what `−1` looks like as a canonical field element, and a +/// negative send is the one stray multiplicity the LogUp count argument cannot +/// catch: it cancels an honest write instead of adding an unmatched token. +#[test] +fn validator_rejects_negative_multiplicity() { + let mut program = small_valid_program(); + // Honest-path control: the program is otherwise admissible, so the + // rejection below is about the multiplicity and nothing else. + validate(&program).expect("the untampered program must pass admission"); + + program + .groups + .const_ + .set(0, layout::const_::MULT, fe(GOLDILOCKS_P - 1)); + assert!( + matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_CONST", + row: 0, + col: layout::const_::MULT, + .. + } + ), + "a field-negative multiplicity must fail admission" + ); +} + +/// The bound is a bound, not merely a sign test: a multiplicity far above any +/// read count the program can emit is rejected even though it is positive. +#[test] +fn validator_rejects_oversized_multiplicity() { + let mut program = small_valid_program(); + validate(&program).expect("the untampered program must pass admission"); + + program.groups.balu.set(0, layout::balu::MULT, fe(1 << 40)); + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_BALU", + row: 0, + .. + } + )); +} + +/// The forgery this pair of checks exists for: a `Compress` row whose two +/// *spare* output slots carry a `−1` / `+1` pair aimed at one address. +/// +/// `Instr::writes()` hides those slots for `Compress`, so checks 1 and 4 never +/// look at them — yet `emit_column_groups` copies them into the committed +/// group and `chips::hash` sends all three slots gated only by their own +/// `MULT`, with no mode factor. The negative send cancels the victim cell's +/// honest write and the positive one re-supplies it with the row's own +/// permutation output, so the token count is preserved exactly and the reader +/// observes a word nobody committed. This was executed end to end against the +/// real prover and verifier, and accepted, before these checks existed. +#[test] +fn validator_rejects_compress_ghost_slot_forgery() { + let konst = |out: u64, v: u64| Instr::Const { + out: Addr(out), + value: [fe(v), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }; + let source = LfmProgramSource { + instrs: vec![ + konst(0, 1), + konst(1, 2), + konst(2, 3), // the victim: a program constant the ghost pair replaces + Instr::Hash { + mode: HashMode::Compress, + ins: [Addr(0), Addr(1), Addr(0)], + outs: [Addr(3), Addr(2), Addr(2)], + mults: [0, GOLDILOCKS_P - 1, 1], + }, + Instr::Public { + addr: Addr(2), + index: 0, + }, + ], + num_addrs: 4, + read_counts: vec![1, 1, 1, 0], + arena_schema: Default::default(), + public_len: 1, + }; + let mut program = compile(source); + + // Leg 1: the point check on `instr.rs`'s placeholder convention. + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::CompressSlotNotPlaceholder { instr: 3 } + ); + + // Leg 2: and check 9 denies it on its own. Repair the INSTRUCTION list — + // the object checks 1–4 read — and leave the COMMITTED group hostile, + // which is precisely the divergence that made the forgery admissible. + let Instr::Hash { outs, mults, .. } = &mut program.instrs[3] else { + panic!("instruction 3 is the hash row"); + }; + *outs = [Addr(3), Addr(0), Addr(0)]; + *mults = [0, 0, 0]; + assert!(matches!( + validate(&program).unwrap_err(), + LfmViolation::MultOutOfRange { + chip: "LFM_HASH", + row: 0, + col: layout::hash::MULT1, + .. + } + )); +} + +#[test] +fn validator_rejects_read_of_unwritten() { + let mut program = small_valid_program(); + let bogus = Addr(program.num_addrs - 1); // allocated range, but rewire below + // Extend the address space by one and point an operand at the unwritten slot. + program.num_addrs += 1; + let unwritten = Addr(program.num_addrs - 1); + for i in &mut program.instrs { + if let Instr::BaseAlu { + op: super::instr::BaseOp::Mul, + b, + .. + } = i + { + *b = unwritten; + } + } + let _ = bogus; + assert_eq!( + validate(&program).unwrap_err(), + LfmViolation::ReadOfUnwritten { addr: unwritten.0 } + ); +} + +// ---- compiler invariant panics (tripwires behind the validator) ---- + +#[test] +fn compiler_panics_on_double_assignment() { + let source = LfmProgramSource { + instrs: vec![ + Instr::Const { + out: Addr(0), + value: [FE::zero(), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }, + Instr::Const { + out: Addr(0), + value: [FE::one(), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }, + ], + num_addrs: 1, + read_counts: vec![0], + arena_schema: Default::default(), + public_len: 0, + }; + let result = std::panic::catch_unwind(|| compile(source)); + assert!(result.is_err()); +} + +#[test] +fn compiler_panics_on_undrained_read_counts() { + let mut read_counts = vec![0u64; 8]; + read_counts[7] = 1; // a read of an address nothing writes + let source = LfmProgramSource { + instrs: vec![Instr::Const { + out: Addr(0), + value: [FE::zero(), FE::zero(), FE::zero(), FE::zero()], + mult: 0, + }], + num_addrs: 8, + read_counts, + arena_schema: Default::default(), + public_len: 0, + }; + let result = std::panic::catch_unwind(|| compile(source)); + assert!(result.is_err()); +} + +// ---- misc structural ---- + +#[test] +fn const_pool_interns_and_counts_reads() { + let mut b = LfmBuilder::new(); + let x1 = b.felt_const(fe(42)); + let x2 = b.felt_const(fe(42)); // same cell + assert_eq!(x1.addr(), x2.addr()); + let s = b.add(x1, x2); // two reads of the shared cell + b.public(s.as_cell()); + let program = compile(b.finish()); + let mult = program + .instrs + .iter() + .find_map(|i| match i { + Instr::Const { out, mult, .. } if *out == x1.addr() => Some(*mult), + _ => None, + }) + .unwrap(); + assert_eq!(mult, 2); + run(&program, &[]); +} + +#[test] +fn canonical_helper_sanity() { + // p ≡ 0: the canonical map the executor's BitDec relies on. + assert_eq!(GoldilocksField::canonical(fe(GOLDILOCKS_P).value()), 0); + assert_eq!(GoldilocksField::canonical(fe(5).value()), 5); +} + +#[test] +fn digest_packing_round_trips() { + let w: LfmWord = core::array::from_fn(|i| fe(0xDEAD_0000 + i as u64)); + let packed = super::word::pack_digest(&w); + assert_eq!(super::word::unpack_digest(&packed), w); +} diff --git a/prover/src/lfm/trace.rs b/prover/src/lfm/trace.rs new file mode 100644 index 000000000..7cb8063b5 --- /dev/null +++ b/prover/src/lfm/trace.rs @@ -0,0 +1,299 @@ +//! LFM trace generation: instruction column group (preprocessed, leading) +//! plus value columns from the executor's records. Aux/LogUp columns are +//! entirely framework-built; heights equal each group's padded height, so the +//! prover's preprocessed-subset recommit matches the registry root exactly. + +use stark::trace::TraceTable; + +use crate::tables::types::{FE, GoldilocksExtension, GoldilocksField}; + +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; + +use super::blake3_socket; +use super::chips::{balu, bitdec, const_, hash, hint, keccak, lanes, public, select, xalu}; +use super::compiler::{ColumnGroup, LfmProgram}; +use super::executor::LfmRecords; +use super::hash::{HASH_STATE_FELTS, HasherKind, LfmHasher}; +use super::instr::{HashMode, Instr}; +use super::keccak_adapter::{self, KeccakAdapterOperation}; +use super::layout; + +type F = GoldilocksField; +type E = GoldilocksExtension; + +pub struct LfmTraces { + pub const_: TraceTable, + pub balu: TraceTable, + pub xalu: TraceTable, + pub select: TraceTable, + pub bitdec: TraceTable, + pub hash: TraceTable, + pub keccak: TraceTable, + pub lanes: TraceTable, + pub hint: TraceTable, + pub public: TraceTable, + pub range: TraceTable, + /// The three production keccak-family tables, proved unchanged. They carry + /// no LFM instruction column group: `KECCAK_RND` has no preprocessed + /// columns at all, and the other two have fixed, program-independent ones. + /// + /// `KECCAK_RND` is one trace per chunk (see [`super::chunking`]); the + /// other two stay single shared instances whose multiplicities count the + /// lookups from *every* chunk. + pub keccak_rnd: Vec>, + pub keccak_rc: TraceTable, + pub bitwise: TraceTable, +} + +/// The `LFM_RANGE` fixed table's column group (program-independent). +pub fn range_group() -> ColumnGroup { + ColumnGroup { + width: layout::range::PREP_WIDTH, + real_rows: layout::range::NUM_ROWS, + padded_rows: layout::range::NUM_ROWS, + data: (0..layout::range::NUM_ROWS as u64).map(FE::from).collect(), + } +} + +/// Builds one chip's trace: copy the (already padded) group into the leading +/// columns, then let `fill` write the value columns of each real row. +fn chip_trace( + group: &ColumnGroup, + num_columns: usize, + mut fill: impl FnMut(usize, &mut [FE]), +) -> TraceTable { + let rows = group.padded_rows; + let mut data = vec![FE::zero(); rows * num_columns]; + for row in 0..rows { + data[row * num_columns..row * num_columns + group.width] + .copy_from_slice(&group.data[row * group.width..(row + 1) * group.width]); + } + for row in 0..group.real_rows { + fill(row, &mut data[row * num_columns..(row + 1) * num_columns]); + } + TraceTable::new_main(data, num_columns, 1) +} + +/// Writes the Poseidon round witness into a hash row whose `IN`/`S`/`OUT` +/// columns are already filled. +/// +/// The permutation input is read back out of the row's own `IN`/`S` columns — +/// the exact cells round 0's constraints read — rather than from the executor +/// record, so the witness cannot describe a different input than the one the +/// AIR constrains. `permutation_witness` supplies the intermediates in the +/// association the degree-3 lowering needs (`x² = a·a`, `x³ = x²·a`, +/// `a⁷ = (x³)²·a`); any other association is the same field element and a +/// different trace, and the constraints would reject it. +pub(super) fn fill_poseidon_witness(out: &mut [FE]) { + use super::chips::hash::poseidon_cols as pc; + use super::poseidon::{NUM_ROUNDS, permutation_witness, sboxed_lanes}; + + let state: [FE; HASH_STATE_FELTS] = core::array::from_fn(|i| { + if i < 8 { + out[hash::cols::IN0 + i] + } else { + out[hash::cols::S8 + (i - 8)] + } + }); + let witness = permutation_witness(state); + for (r, round) in witness.iter().enumerate() { + for lane in 0..sboxed_lanes(r) { + out[pc::x2(r, lane)] = round.x2[lane]; + out[pc::x3(r, lane)] = round.x3[lane]; + } + for (j, v) in round.out.iter().enumerate() { + out[pc::out(r, j)] = *v; + } + } + debug_assert_eq!( + &out[hash::cols::OUT0..hash::cols::OUT0 + HASH_STATE_FELTS], + witness[NUM_ROUNDS - 1].out.as_slice(), + "the final round's output is the OUT columns the executor already wrote" + ); +} + +pub fn build_traces(program: &LfmProgram, records: &LfmRecords) -> LfmTraces { + build_traces_with_hasher(program, records, HasherKind::default()) +} + +/// [`build_traces`] for a proof under `hasher`. +/// +/// `hasher` must be the one the executor ran (`proof::lfm_prove_with_hasher` +/// passes the same value to both) and the one the AIR set was built with: the +/// hash chip's width and witness columns are its layout's, and the constraints +/// bake its round constants. +pub fn build_traces_with_hasher( + program: &LfmProgram, + records: &LfmRecords, + hasher: HasherKind, +) -> LfmTraces { + let g = &program.groups; + + let hash_modes: Vec = program + .instrs + .iter() + .filter_map(|i| match i { + Instr::Hash { mode, .. } => Some(*mode), + _ => None, + }) + .collect(); + let iv = hasher.compress_iv(); + + // The keccak family's traces are driven by the executor's records; the tag + // is the row ordinal, exactly as the compiler emitted it into the + // preprocessed group (one rule, `layout::keccak::tag_for_row`, two callers). + // The family sees `perm_in` — post-XOR on absorb rows — not the state as + // read from memory. + let keccak_ops: Vec = records + .keccak + .iter() + .enumerate() + .map(|(row, r)| KeccakAdapterOperation { + tag: layout::keccak::tag_for_row(row), + input: r.perm_in, + }) + .collect(); + + // The round operations split across `KECCAK_RND` chunks; the chip has no + // row-to-row constraints, so a chunk is just a slice of the permutations + // (see `chunking`). The chunk *count* is program shape, so this uses the + // program's pinned policy rather than anything derived here. + let round_ops = keccak_adapter::round_operations(&keccak_ops); + let keccak_rnd_traces: Vec<_> = program + .chunking + .split(&round_ops) + .into_iter() + .map(keccak_rnd::generate_keccak_rnd_trace) + .collect(); + + // KECCAK_RC and BITWISE are single shared tables: their multiplicities are + // totals over the whole proof, so they are fed the complete operation list + // regardless of how the round rows were chunked. + let mut keccak_rc_trace = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut keccak_rc_trace, keccak_ops.len()); + + let mut histogram = bitwise::BitwiseHistogram::new(); + histogram.add_ops(&keccak_adapter::bitwise_ops_for(&keccak_ops)); + // Absorb rows additionally send one BYTE_ALU[XOR] lookup per rate byte. + histogram.add_ops(&keccak_adapter::absorb_bitwise_ops(&records.keccak)); + // Under BLAKE3 the hash chip is a BITWISE consumer too — over a thousand + // lookups per compression. Every other hasher sends none, so this is the + // one place the shared table's multiplicities depend on the hash choice. + if hasher == HasherKind::Blake3 { + let rows: Vec<([u32; blake3_socket::cols::NUM_LANES], u32)> = records + .hash + .iter() + .zip(&hash_modes) + .map(|(r, mode)| { + let cell = + |k: usize| -> super::word::LfmWord { core::array::from_fn(|i| r.ins[k + i]) }; + // The message lanes, read the way this row's MODE reads them: + // digest cells throughout, or an accumulator cell followed by + // four felts split into halves. A leaf row sends lookups over + // its halves, so the histogram has to split the row exactly the + // way the witness filler does — which is why both come through + // `lanes_from_cells` instead of each carrying its own split. + let lanes = blake3_socket::lanes_from_cells( + *mode == HashMode::Leaf, + &[cell(0), cell(4), cell(8)], + ); + ( + lanes, + // The row's DOMAIN, not a fixed tag: the lookups a row sends + // are values downstream of the tag word, so a transcript row and a + // compress row over the same cells send different bytes. + blake3_socket::tag_for_mode(*mode) + .expect("BLAKE3 admits no permute row (its AIR pins MODE_P = 0)"), + ) + }) + .collect(); + histogram.add_ops(&blake3_socket::bitwise_ops_for(&rows)); + } + let mut bitwise_trace = bitwise::generate_bitwise_trace(); + histogram.fill_multiplicities(&mut bitwise_trace); + + LfmTraces { + const_: chip_trace(&g.const_, const_::cols::NUM_COLUMNS, |_, _| {}), + balu: chip_trace(&g.balu, balu::cols::NUM_COLUMNS, |row, out| { + let r = &records.balu[row]; + out[balu::cols::A] = r.a; + out[balu::cols::B] = r.b; + out[balu::cols::C] = r.c; + out[balu::cols::OUT] = r.out; + }), + xalu: chip_trace(&g.xalu, xalu::cols::NUM_COLUMNS, |row, out| { + let r = &records.xalu[row]; + out[xalu::cols::A0..xalu::cols::A0 + 3].copy_from_slice(&r.a); + out[xalu::cols::B0..xalu::cols::B0 + 3].copy_from_slice(&r.b); + out[xalu::cols::C0..xalu::cols::C0 + 3].copy_from_slice(&r.c); + out[xalu::cols::OUT0..xalu::cols::OUT0 + 3].copy_from_slice(&r.out); + }), + select: chip_trace(&g.select, select::cols::NUM_COLUMNS, |row, out| { + let r = &records.select[row]; + out[select::cols::BIT] = r.bit; + out[select::cols::INL0..select::cols::INL0 + 4].copy_from_slice(&r.in_l); + out[select::cols::INR0..select::cols::INR0 + 4].copy_from_slice(&r.in_r); + out[select::cols::OUTL0..select::cols::OUTL0 + 4].copy_from_slice(&r.out_l); + out[select::cols::OUTR0..select::cols::OUTR0 + 4].copy_from_slice(&r.out_r); + }), + bitdec: chip_trace(&g.bitdec, bitdec::cols::NUM_COLUMNS, |row, out| { + let r = &records.bitdec[row]; + out[bitdec::cols::BITS0..bitdec::cols::BITS0 + 64].copy_from_slice(&r.bits); + out[bitdec::cols::Z] = r.z; + out[bitdec::cols::GINV] = r.ginv; + }), + hash: chip_trace(&g.hash, hash::num_columns(hasher), |row, out| { + let r = &records.hash[row]; + out[hash::cols::IN0..hash::cols::IN0 + 12].copy_from_slice(&r.ins); + for k in 0..4 { + // S_i = MODE_P·IN_i + (MODE_C + MODE_T + MODE_L)·IV_i, + // materialized. Every mode but the permutation takes the IV. + out[hash::cols::S8 + k] = if hash_modes[row] == HashMode::Permute { + r.ins[8 + k] + } else { + iv[k] + }; + } + out[hash::cols::OUT0..hash::cols::OUT0 + 12].copy_from_slice(&r.outs); + match hasher { + HasherKind::Test => {} + HasherKind::Poseidon => fill_poseidon_witness(out), + // The domain is read off the row's own mode columns, which + // `chip_trace` populated before calling this — the same + // discipline `fill_poseidon_witness` follows for its input. + HasherKind::Blake3 => blake3_socket::fill_socket_witness(out), + } + }), + keccak: chip_trace(&g.keccak, keccak::cols::NUM_COLUMNS, |row, out| { + let r = &records.keccak[row]; + for lane in 0..25 { + for b in 0..8 { + let byte = |v: u64| FE::from(u64::from((v >> (8 * b)) as u8)); + out[keccak::cols::state_byte(lane, b)] = byte(r.state[lane]); + out[keccak::cols::perm_in_byte(lane, b)] = byte(r.perm_in[lane]); + out[keccak::cols::out_byte(lane, b)] = byte(r.output[lane]); + } + } + for (k, &v) in r.block.iter().enumerate() { + out[keccak::cols::BLOCK + k] = FE::from(u64::from(v)); + } + }), + lanes: chip_trace(&g.lanes, lanes::cols::NUM_COLUMNS, |row, out| { + out[lanes::cols::V0..lanes::cols::V0 + 4].copy_from_slice(&records.lanes[row]); + }), + hint: chip_trace(&g.hint, hint::cols::NUM_COLUMNS, |row, out| { + out[hint::cols::V0..hint::cols::V0 + 4].copy_from_slice(&records.hint[row]); + }), + public: chip_trace(&g.public, public::cols::NUM_COLUMNS, |row, out| { + out[public::cols::V0..public::cols::V0 + 4].copy_from_slice(&records.public[row]); + }), + range: chip_trace( + &range_group(), + super::chips::range::cols::NUM_COLUMNS, + |_, _| {}, + ), + keccak_rnd: keccak_rnd_traces, + keccak_rc: keccak_rc_trace, + bitwise: bitwise_trace, + } +} diff --git a/prover/src/lfm/transcript_kats.rs b/prover/src/lfm/transcript_kats.rs new file mode 100644 index 000000000..c78758c1a --- /dev/null +++ b/prover/src/lfm/transcript_kats.rs @@ -0,0 +1,164 @@ +//! Transcript KATs for the LFM compress-chain Fiat–Shamir transcript, at 6 and +//! 7 rounds. +//! +//! GENERATED — do not hand-edit. The INPUTS come from +//! `thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.json`, which +//! the oracle produced from a Python reference written **before any Rust +//! existed**. That ordering is the point: these vectors are a specification the +//! implementation is checked against, not a recording of what the +//! implementation happened to do. +//! +//! ⚠ **The results were re-pinned when the socket widened to twelve lanes**, by +//! `leaf-spec/rate4_kat_gen.py` out of the same oracle. All 12 moved and no +//! input did: `block_len` is `v[14]` and cannot be made mode-dependent, so the +//! transcript domain re-blesses alongside the leaf domain that needed the width +//! (COMMIT.md §1.4.4 H9). +//! +//! Framing (transcript spec §1.2 at the COMMIT.md §1.2 width): identical to the +//! Merkle socket in every respect except the tag word, which is `"LFMT"` +//! instead of `"LFMC"`. So h = BLAKE3_IV, m[0..4] = state, m[4..8] = operand, +//! m[8..12] = 0 — the third input cell, which the unread-`IN` pins force to +//! zero — m[12] = "LFMT", m[13..16] = 0, t = 0, block_len = 52, flags = 0x0B, +//! digest = out[0..4]. At 7 rounds a step is still literally +//! `blake3::hash(state ‖ operand ‖ 0^16 ‖ "LFMT")[..16]`. + +/// One transcript step: state, operand, and the resulting state at each round +/// count. +pub struct StepVector { + pub name: &'static str, + pub state: [u32; 4], + pub operand: [u32; 4], + /// Result at 6 rounds (the A6R variant; no library computes it). + pub result_6: [u32; 4], + /// Result at 7 rounds — `blake3::hash(state ‖ operand ‖ "LFMT")[..16]`. + pub result_7: [u32; 4], +} + +pub const STEP_VECTORS: [StepVector; 6] = [ + StepVector { + name: "zero_state_zero_operand", + state: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + operand: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + result_6: [0x58A784C6, 0xCA20122A, 0x574D1385, 0x4C7F61AC], + result_7: [0xBB5DF0AD, 0xBB660FC6, 0x401C1FAD, 0x651C297C], + }, + StepVector { + name: "zero_state_main_root", + state: [0x00000000, 0x00000000, 0x00000000, 0x00000000], + operand: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + result_6: [0xBFD9E2ED, 0x726EDE27, 0x91805DE1, 0xC11F0DA8], + result_7: [0x503FDDF4, 0x48633531, 0x8EEA401C, 0x213213C8], + }, + StepVector { + name: "ramp_state_ramp_operand", + state: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + operand: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + result_6: [0x00A8B31B, 0x0C48A09A, 0x1D06A9A8, 0x6C27BD61], + result_7: [0x29D95598, 0x69E4FD73, 0x243BFCE9, 0x14598F96], + }, + StepVector { + name: "max_state", + state: [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], + operand: [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE], + result_6: [0xA710A43E, 0x62E96839, 0xE00D7CA2, 0x1E054FEF], + result_7: [0xBA98C5EF, 0xAFDC8C3E, 0xFA425A12, 0xF35C1B47], + }, + StepVector { + name: "squeeze_operand_0", + state: [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + operand: [0x305A5153, 0x00000000, 0x00000000, 0x00000000], + result_6: [0xE1F1E0DF, 0x9B1491E4, 0x26F46CE4, 0x644BA9F0], + result_7: [0x2EBCFDA8, 0x2F7C4E72, 0xAE841641, 0x6751FE80], + }, + StepVector { + name: "squeeze_operand_255", + state: [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + operand: [0x305A5153, 0x000000FF, 0x00000000, 0x00000000], + result_6: [0xDD47DC57, 0x9AC95714, 0xE774A0DA, 0xD4703C0B], + result_7: [0x33396F61, 0x832BA04F, 0x2BB788AB, 0xE9FE006B], + }, +]; + +/// The END-TO-END vector: a `FriToyV0`-preamble-shaped transcript, op by op. +/// +/// The operation sequence is fixed and lives in the test that replays it — +/// `absorb(main_root), squeeze, squeeze, absorb(l1_root), squeeze, +/// absorb_felts(t0w), absorb_felts(t1w), 4× squeeze_bits`, ✓ VERIFIED against +/// `programs::fri_toy_program_source`. What is pinned here is the STATE after +/// every recorded op, so a divergence is located at the step it happened rather +/// than at the end. +/// +/// The last two absorbs are `absorb_felts`, not `absorb2`: the terminal +/// coefficients are field DATA, so each is leaf-hashed under `"LFML"` and the +/// DIGEST is absorbed. The transcript's step count is the same either way, which +/// is why this vector had to be re-pointed deliberately when the program moved +/// rather than caught by a red test. +pub struct EndToEndVector { + /// State after each recorded op, in order. + pub states: [[u32; 4]; 11], + /// The three ext challenges (lanes 0–2 of a squeezed cell). + pub alpha: [u32; 3], + pub zeta0: [u32; 3], + pub zeta1: [u32; 3], + /// `QUERY_BITS` index bits per query, low-to-high. + pub query_bits: [[u8; 4]; 4], +} + +/// The transcript's inputs — the four cells the preamble absorbs. +pub const MAIN_ROOT: [u32; 4] = [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10]; + +pub const L1_ROOT: [u32; 4] = [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20]; + +pub const T0W: [u32; 4] = [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE]; + +pub const T1W: [u32; 4] = [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD]; + +/// Compressions the whole preamble costs — the oracle's cost claim. +/// +/// **13, not 11.** Eleven are TRANSCRIPT steps (5 absorbs, 6 squeezes); the +/// other two are the LEAF rows `absorb_felts` adds, one per data cell. Counting +/// them here is the oracle's convention and it is the one that closes the +/// `FriToyV0` total: 4 queries × 20 + 13 = 93. +pub const FRI_TOY_COMPRESSIONS: usize = 13; + +/// The end-to-end vector at 7 rounds (the default build). +pub const FRI_TOY_7: EndToEndVector = EndToEndVector { + states: [ + [0x503FDDF4, 0x48633531, 0x8EEA401C, 0x213213C8], + [0x02FEA9A6, 0xE6BF885C, 0xF174E65F, 0x4EC9AB10], + [0xF91F56DE, 0x62F37956, 0xE67D5421, 0xD82727D0], + [0x7A31B840, 0xAD7F2625, 0xE27D1C56, 0xCDB0E9A7], + [0xA6790B7B, 0x00695D49, 0xA663DC33, 0x2E849F0C], + [0xFC40465E, 0x0092B147, 0x2FA48645, 0x9755608B], + [0x6BD6F0B0, 0x634CF1C6, 0x3CBD9D2D, 0x349F278B], + [0xD86E1D3F, 0xDD1CFBC3, 0x1C8E8F14, 0x22D35494], + [0x5B449138, 0x3435B7D5, 0x7CFE4C06, 0x1C022FCF], + [0xE8D9848C, 0x0429B6F7, 0xDD5CBA1A, 0xBD465F16], + [0xD1E3472D, 0xD945386A, 0xC746A9B3, 0x8FD73C31], + ], + alpha: [0x503FDDF4, 0x48633531, 0x8EEA401C], + zeta0: [0x02FEA9A6, 0xE6BF885C, 0xF174E65F], + zeta1: [0x7A31B840, 0xAD7F2625, 0xE27D1C56], + query_bits: [[0, 0, 0, 0], [1, 1, 1, 1], [0, 0, 0, 1], [0, 0, 1, 1]], +}; + +/// The end-to-end vector at 6 rounds (`--features blake3-6round`). +pub const FRI_TOY_6: EndToEndVector = EndToEndVector { + states: [ + [0xBFD9E2ED, 0x726EDE27, 0x91805DE1, 0xC11F0DA8], + [0x9DDE5E52, 0xD7018142, 0x978528FF, 0xA01782B7], + [0x189BAD31, 0x364E3C30, 0x6B4D5516, 0x78D7FE7B], + [0x8E3E80BB, 0xFCA1AF96, 0xA59E0F41, 0x18C9AB19], + [0x4DF9BD75, 0x2E131E7D, 0x2DEB348E, 0x62BC30F3], + [0x202DB868, 0x8FF72AC4, 0x452D536A, 0x78DECD7B], + [0xA3D1E95E, 0x8C68ECF6, 0x91D7DF5E, 0x28D8CBB2], + [0xFA4222A2, 0x18DA7862, 0x3BBDA144, 0xAB453013], + [0x525CE059, 0x0E9B9EAB, 0xFB57633B, 0xE43490A7], + [0x261309AA, 0x84C675B7, 0xD5BBF0EB, 0x4AF40850], + [0x4D62F602, 0x2CC5D660, 0x99C44AF7, 0xEB502D8A], + ], + alpha: [0xBFD9E2ED, 0x726EDE27, 0x91805DE1], + zeta0: [0x9DDE5E52, 0xD7018142, 0x978528FF], + zeta1: [0x8E3E80BB, 0xFCA1AF96, 0xA59E0F41], + query_bits: [[0, 1, 1, 1], [0, 1, 0, 0], [1, 0, 0, 1], [0, 1, 0, 1]], +}; diff --git a/prover/src/lfm/transcript_replay.rs b/prover/src/lfm/transcript_replay.rs new file mode 100644 index 000000000..f22efc14a --- /dev/null +++ b/prover/src/lfm/transcript_replay.rs @@ -0,0 +1,879 @@ +//! `TranscriptReplay` — the production `DefaultTranscript` replayed inside the +//! machine. +//! +//! This is an eDSL library, not a chip: it is ordinary Rust that tracks the +//! transcript's state AT EMIT TIME and emits the instructions that reproduce +//! the transcript's VALUES at run time. The split matters. Which squeeze a +//! challenge comes from, where a refill lands, which absorb invalidates the +//! output buffer — all of that is decided by the emitter and baked into the +//! program's shape. Only the field arithmetic and the keccak rows are machine +//! work. +//! +//! The mirror it must match is `crypto::fiat_shamir::default_transcript` +//! (post-#841): a keccak sponge with a Plonky3-style duplex output buffer. +//! [`super::keccak_host::TranscriptModel`] is the host model of the same state +//! machine and is checked against the real thing in `machine_tests`; this type +//! tracks the identical `segment` / `out_pos` pair. +//! +//! ## What makes the replay cheap +//! +//! Two identities do all the work: +//! +//! 1. **The reversal cancels.** `sample()` returns the digest byte-REVERSED, and +//! candidates are read big-endian out of those reversed bytes. The two +//! reversals cancel exactly: candidate `i` is the PLAIN digest's `u64` lane +//! `3 − i` (see [`super::keccak_host::candidate_from_state`]). So sampling +//! never reverses anything — it reads `u32` halves straight off the keccak +//! state words, two `Unpack`s per squeeze. The reversed digest is emitted +//! only for the RE-ABSORB (and for a raw [`TranscriptReplay::sample`], whose +//! return value *is* those bytes). +//! +//! 2. **Canonicity is one instruction.** `p = (2^32 − 1)·2^32 + 1`, so a +//! candidate `hi·2^32 + lo` is out of range exactly when +//! `hi = 2^32 − 1 ∧ lo ≠ 0` — and `div` is constrained as `OUT·B = A`, which +//! is provable with `B = 0` only when `A = 0`. See [`assert_canonical`]. +//! +//! A third property is about the emitter rather than the machine: the segment is +//! packed into `u32` halves per SEGMENT, never per append, which is what makes +//! constants of arbitrary length safe anywhere in the stream. The argument is in +//! [`TranscriptReplay::append_const_bytes`] and should be read before touching +//! the append path. +//! +//! ## The zero-rejection restriction +//! +//! A straight-line program has one shape. The production sampler rejects +//! out-of-range candidates and draws again, so the number of candidates a draw +//! consumes — and therefore every later draw's buffer position — is +//! DATA-DEPENDENT. A machine with no branches cannot follow that. The emitted +//! program therefore encodes the no-rejection schedule and is unprovable for the +//! (vanishingly rare) transcript that rejects. See `SOUNDNESS.md` §6.3 for the +//! completeness bound, for why this costs COMPLETENESS only — a rejecting +//! transcript yields no proof, never a wrong one — and for why supporting one +//! rejection is NOT an emitter parameter but a change to the production +//! sampler. + +use crate::tables::types::FE; + +use super::builder::{Bit, Cell, Ext, Felt, LfmBuilder}; +use super::edsl; +use super::keccak_host::{BYTES_PER_HALF, SQUEEZE_LEN}; +use super::layout::keccak::DIGEST_WORDS; + +/// `u32` halves in one 32-byte squeeze. +const SQUEEZE_HALVES: usize = SQUEEZE_LEN / BYTES_PER_HALF; + +/// Bytes in one 64-bit candidate. +const CANDIDATE_BYTES: usize = 8; + +/// Candidates one squeeze yields. +const CANDIDATES_PER_SQUEEZE: usize = SQUEEZE_LEN / CANDIDATE_BYTES; + +/// `2^32 − 1` — the only `hi` half that can put a candidate at or above `p`. +const HI_MAX: u64 = 0xFFFF_FFFF; + +/// A piece of the pending segment, held UNPACKED until the squeeze. +/// +/// Packing is per SEGMENT, never per append — see +/// [`TranscriptReplay::append_const_bytes`] for why that distinction is the +/// whole design. +#[derive(Clone)] +enum SegPiece { + /// Compile-time bytes. Consecutive runs of these are concatenated before + /// being chunked into halves, so a constant of any length may sit anywhere. + Const(Vec), + /// Machine-computed `u32` halves, four bytes each little-endian. Opaque + /// felts, so they must land on a 4-byte boundary of the segment. + Halves(Vec), + /// A machine half carrying only its low `n` bytes (`n` in `1..4`) — the + /// trailing piece of a byte string whose length is not a multiple of four. + /// The packer masks it and pins the unused high bytes to zero. + Partial(Felt, usize), +} + +/// A 64-bit candidate as the two `u32` halves the machine actually holds: +/// `value = hi·2^32 + lo`. +#[derive(Debug, Clone, Copy)] +pub struct Candidate { + pub lo: Felt, + pub hi: Felt, +} + +/// The squeeze currently backing the output buffer. +/// +/// Held as the PLAIN digest's two words with their lane unpacks memoized: +/// candidates 0 and 1 live in word 1 and candidates 2 and 3 in word 0, so a +/// draw that consumes one or two candidates emits a single `Unpack`. +#[derive(Clone)] +struct SqueezeBuf { + words: [Cell; DIGEST_WORDS], + lanes: [Option<[Felt; 4]>; DIGEST_WORDS], +} + +/// Emit-time replay of `DefaultTranscript`. +/// +/// `Clone` is production's `transcript.clone()` — the per-table fork +/// (`verifier.rs:1263`), and the only reason this type is cloneable. It is a +/// pure emitter-state copy: the pending segment is a list of program constants +/// and CELL handles, so a fork shares the cells its prefix already produced and +/// emits nothing. That is exactly the production semantics — the shared prefix +/// is hashed once and each fork diverges only past its domain separator. +#[derive(Clone)] +pub struct TranscriptReplay { + /// The pending segment — the hasher's unfinalized input — as unpacked + /// pieces. Packed into halves at squeeze time, not at append time. + segment: Vec, + /// The segment's length in BYTES: what drives keccak's length-dependent + /// padding, and what decides where every half boundary falls. + segment_len: usize, + buf: Option, + /// Bytes already handed out of the buffer; `SQUEEZE_LEN` means "empty, the + /// next candidate forces a squeeze". + out_pos: usize, +} + +impl TranscriptReplay { + /// `DefaultTranscript::new(seed)` — an empty sponge with `seed` absorbed. + /// + /// The seed is a program constant, interned by the builder. Anything the + /// machine COMPUTES is absorbed with [`TranscriptReplay::append_halves`]. + pub fn new(seed: &[u8]) -> Self { + let mut t = Self { + segment: Vec::new(), + segment_len: 0, + buf: None, + out_pos: SQUEEZE_LEN, + }; + t.append_const_bytes(seed); + t + } + + /// Absorb machine-computed data: `4 · halves.len()` bytes, four per half, + /// little-endian. + /// + /// Whole halves only, and there is no length parameter. A half is four + /// consecutive bytes of the SEGMENT, and these felts are opaque to the + /// emitter — it cannot split or shift their bytes — so machine data must + /// land on a 4-byte boundary. [`TranscriptReplay::assert_appendable`] is the + /// loud check. + /// + /// This is not a real restriction for the FRI-verifier scope: every + /// production rendering is a multiple of four bytes. A commitment root is + /// 32, a Goldilocks felt streams as 8, a cubic-extension felt as 24. + pub fn append_halves(&mut self, halves: &[Felt]) { + self.assert_appendable(); + self.segment.push(SegPiece::Halves(halves.to_vec())); + self.segment_len += BYTES_PER_HALF * halves.len(); + // Absorbing invalidates the buffer: a later challenge must depend on + // this input, so bytes squeezed before it are dropped. + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + /// Absorb one machine word — its four lanes as four halves, 16 bytes. + /// + /// The word must be a `u32`-half word (a keccak state/digest word, which is + /// where transcript-bound data comes from). Feeding one whose lanes are full + /// felts is not a silent miscoding: the halves end up as lanes of a keccak + /// input word, and the adapter refuses anything at or above `2^32` with + /// `LfmExecError::NotU32Half` — and would be unprovable regardless, since the + /// bus range-checks them. + pub fn append_word(&mut self, b: &mut LfmBuilder, w: Cell) { + let lanes = b.unpack(w); + self.append_halves(&lanes); + } + + /// Absorb a 32-byte keccak digest carried as two machine words — the shape a + /// commitment root arrives in. + pub fn append_digest(&mut self, b: &mut LfmBuilder, words: &[Cell; DIGEST_WORDS]) { + for w in words { + self.append_word(b, *w); + } + } + + /// Absorb one base field element the way production streams it: the + /// canonical `u64` in BIG-endian byte order, 8 bytes. + /// + /// `FieldElement::stream_bytes` is + /// `sink(&self.canonical_u64().to_be_bytes())`, so the endianness flip is + /// real work for this machine — see [`felt_be_halves`] for the gadget and + /// its cost. + pub fn append_felt(&mut self, b: &mut LfmBuilder, v: Felt) { + let halves = felt_be_halves(b, v); + self.append_halves(&halves); + } + + /// Absorb one cubic-extension element: coordinates 0, 1, 2, each as its own + /// 8 big-endian bytes — 24 bytes in total. + /// + /// Coordinate order is FORWARD and was verified against the source, because + /// the file offers both orders and picking the wrong one is invisible until + /// a challenge diverges: `FieldElement`'s + /// `write_bytes_be` (which `stream_bytes` calls) writes components 0, 1, 2, + /// while the REVERSED 2, 1, 0 order belongs to the raw `[FpE; 3]` array + /// type. Different types, no contradiction — but do not "fix" this to match + /// the other impl. + pub fn append_ext(&mut self, b: &mut LfmBuilder, coords: [Felt; 3]) { + for c in coords { + self.append_felt(b, c); + } + } + + /// Absorb a compile-time constant byte string of ANY length, anywhere in the + /// segment. No alignment requirement, no builder — the bytes are stored + /// unpacked and interned at the squeeze. + /// + /// ## Why packing is per SEGMENT, not per append + /// + /// Append boundaries are not machine-visible. Between two finalize points + /// the production hasher sees one concatenated byte stream; `append_bytes` + /// boundaries leave no trace in the digest input. Every length here is + /// compile-time. So the emitter's correct unit of packing is the segment, + /// and it packs by concatenating consecutive constant runs and only THEN + /// chunking into halves. + /// + /// That is what makes "a partial half in the middle of a segment that the + /// next append must continue into" impossible rather than merely rejected: + /// when the emitter packs, it already holds every later constant in the + /// segment. Appending `b"abc"` then `b"de"` yields the five-byte run + /// `abcde`, chunked as two halves — it is not two independently packed + /// pieces. **Do not reintroduce per-append packing.** + /// + /// Segment prefixes are safe by construction too: every segment after the + /// first begins with the 32-byte reversed digest, a multiple of four. + /// + /// ## The one case that remains, deliberately unbuilt + /// + /// A constant of length ≢ 0 (mod 4) followed by MACHINE data — a 27-byte + /// domain tag ahead of a root word, say — leaves the dynamic value straddling + /// a half boundary, and re-aligning opaque felts by 1–3 bytes needs a + /// byte-level splice (BitDec-32 per affected half, or a byte-table route). + /// [`TranscriptReplay::append_halves`] rejects it loudly. It arises only in + /// the statement-absorb leg, at a volume of a few dozen halves per proof, and + /// never in FRI or Merkle traffic; when that leg is built it gets a + /// `splice_misaligned(constant_prefix_len, dynamic_halves)` helper. That is + /// an extension point, not a redesign. + pub fn append_const_bytes(&mut self, bytes: &[u8]) { + self.segment.push(SegPiece::Const(bytes.to_vec())); + self.segment_len += bytes.len(); + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + fn assert_appendable(&self) { + assert_eq!( + self.segment_len % BYTES_PER_HALF, + 0, + "machine-computed data must start on a 4-byte boundary of the segment, \ + but {} bytes are already absorbed: a constant of length not a multiple \ + of four leaves the dynamic value straddling a half, which needs the \ + byte-level splice that only the statement-absorb leg will build", + self.segment_len + ); + } + + /// Absorb machine-computed data that does NOT start on a 4-byte boundary. + /// + /// Same bytes as [`TranscriptReplay::append_halves`], but it permits the + /// misalignment that method rejects, and pays for it: each half then + /// straddles two output halves and has to be split byte-wise (see + /// [`split_half`] for the gadget and its cost). Use the aligned method + /// wherever the encoding allows — this one exists for the statement leg, + /// where a 30-byte domain tag and a 1-byte `fri` field between fixed-width + /// fields make misalignment unavoidable. + /// + /// The splice itself happens in [`TranscriptReplay::pack_segment`], not + /// here, because only the packer knows the byte cursor. + pub fn append_halves_misaligned(&mut self, halves: &[Felt]) { + self.segment.push(SegPiece::Halves(halves.to_vec())); + self.segment_len += BYTES_PER_HALF * halves.len(); + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + /// Absorb `byte_len` machine-computed bytes carried in + /// `ceil(byte_len / 4)` halves — the general case, where the byte string's + /// length need not be a multiple of four. + /// + /// The trailing half is masked to its live bytes and its unused high bytes + /// are pinned to zero (see [`Packer::push_masked`]). That matters for any + /// length-prefixed field: `public_output` in the epoch statement is + /// collected one byte per COMMIT operation, so its length is whatever the + /// workload produced and is not aligned in general. + pub fn append_bytes_misaligned(&mut self, halves: &[Felt], byte_len: usize) { + assert_eq!( + halves.len(), + byte_len.div_ceil(BYTES_PER_HALF), + "byte_len must match the supplied halves" + ); + let full = byte_len / BYTES_PER_HALF; + let rem = byte_len % BYTES_PER_HALF; + if full > 0 { + self.segment.push(SegPiece::Halves(halves[..full].to_vec())); + } + if rem > 0 { + self.segment.push(SegPiece::Partial(halves[full], rem)); + } + self.segment_len += byte_len; + self.out_pos = SQUEEZE_LEN; + self.buf = None; + } + + /// Packs the segment into `u32` halves, walking it at BYTE granularity. + /// + /// Constant bytes accumulate host-side; a machine half drops straight in + /// when the cursor is 4-byte aligned — the path every aligned program takes, + /// which must stay instruction-free — and is split when it is not. The + /// packer is the only place that knows the cursor, which is why the splice + /// lives here rather than at the append. + fn pack_segment(&self, b: &mut LfmBuilder) -> Vec { + pack_pieces(&self.segment, b) + } + + /// `DefaultTranscript::sample()` — finalize, reverse the 32 digest bytes, + /// re-absorb them, return them. + /// + /// The returned bytes and the re-absorbed bytes are the SAME 32 bytes; one + /// keccak row produces both. Also invalidates the output buffer, exactly as + /// production does. + pub fn sample(&mut self, b: &mut LfmBuilder) -> [Cell; DIGEST_WORDS] { + let (_plain, rev) = self.squeeze(b); + self.buf = None; + self.out_pos = SQUEEZE_LEN; + rev + } + + /// One squeeze: emits the keccak row over the current segment, sets the + /// segment to the reversed digest, and hands back both digests — the plain + /// one because candidates are read off it, the reversed one because it is + /// what `sample()` returns. + fn squeeze(&mut self, b: &mut LfmBuilder) -> ([Cell; DIGEST_WORDS], [Cell; DIGEST_WORDS]) { + let packed = self.pack_segment(b); + let (plain, rev) = edsl::keccak256_with_rev(b, &packed, self.segment_len); + // The transcript absorbs the reversed bytes into a freshly reset hasher, + // so they are the WHOLE of the next segment, not a suffix of this one. + let mut halves = Vec::with_capacity(SQUEEZE_HALVES); + for w in rev { + halves.extend_from_slice(&b.unpack(w)); + } + self.segment = vec![SegPiece::Halves(halves)]; + self.segment_len = SQUEEZE_LEN; + (plain, rev) + } + + /// Refill the output buffer with one squeeze, as `next_sample_u64` does. + fn refill(&mut self, b: &mut LfmBuilder) { + let (plain, _rev) = self.squeeze(b); + self.buf = Some(SqueezeBuf { + words: plain, + lanes: [None; DIGEST_WORDS], + }); + self.out_pos = 0; + } + + /// The next 64-bit candidate, refilling when fewer than 8 bytes remain. + /// + /// Returns the candidate as its two `u32` halves rather than a felt: a + /// candidate is a 64-BIT integer and values in `[p, 2^64)` are not + /// felt-representable, so it cannot be one cell until it has been range- + /// checked. Consumers either check it ([`TranscriptReplay::sample_felt`]) or + /// use only the low half ([`TranscriptReplay::sample_u64_pow2`]). + pub fn next_candidate(&mut self, b: &mut LfmBuilder) -> Candidate { + if self.out_pos + CANDIDATE_BYTES > SQUEEZE_LEN { + self.refill(b); + } + debug_assert_eq!( + self.out_pos % CANDIDATE_BYTES, + 0, + "candidates are the buffer's only consumer, so out_pos moves in 8s" + ); + let i = self.out_pos / CANDIDATE_BYTES; + debug_assert!(i < CANDIDATES_PER_SQUEEZE); + // Candidate i is the plain digest's u64 lane 3 − i (the reversal + // cancellation), and lane j is halves 2j (low) and 2j + 1 (high). + let lo = self.half(b, 2 * (CANDIDATES_PER_SQUEEZE - 1 - i)); + let hi = self.half(b, 2 * (CANDIDATES_PER_SQUEEZE - 1 - i) + 1); + self.out_pos += CANDIDATE_BYTES; + Candidate { lo, hi } + } + + /// Half `h` of the buffered digest: lane `h % 4` of word `h / 4`, unpacking + /// that word on first use. + fn half(&mut self, b: &mut LfmBuilder, h: usize) -> Felt { + let buf = self + .buf + .as_mut() + .expect("next_candidate refills before reading"); + let (w, l) = (h / 4, h % 4); + let lanes = match buf.lanes[w] { + Some(lanes) => lanes, + None => { + let lanes = b.unpack(buf.words[w]); + buf.lanes[w] = Some(lanes); + lanes + } + }; + lanes[l] + } + + /// One base-field challenge: `GoldilocksField::sample_field_element_from` + /// with the rejection branch replaced by a constraint (see the module docs). + pub fn sample_felt(&mut self, b: &mut LfmBuilder) -> Felt { + let c = self.next_candidate(b); + assert_canonical(b, c); + candidate_to_felt(b, c) + } + + /// One cubic-extension challenge: three independent base draws in + /// coordinate order 0, 1, 2 — which is what + /// `Degree3GoldilocksExtensionField::sample_field_element_from` does + /// (`core::array::from_fn` evaluates in index order). + /// + /// This is the production shape: the STARK verifier's challenges are + /// extension elements, so an ext draw is where the completeness bound is + /// paid three times over. + pub fn sample_ext(&mut self, b: &mut LfmBuilder) -> Ext { + let a0 = self.sample_felt(b); + let a1 = self.sample_felt(b); + let a2 = self.sample_felt(b); + b.pack_ext(a0, a1, a2) + } + + /// `sample_u64(1 << nbits)` — the low `nbits` bits of one candidate, as bits + /// low-to-high. + /// + /// No canonicity guard and no rejection, because production has none here: + /// `threshold = upper_bound.wrapping_neg() % upper_bound` is 0 at every + /// power of two, so the loop in `sample_u64` accepts its first candidate + /// unconditionally and returns `candidate % 2^nbits`. An out-of-range + /// candidate is perfectly legal for this draw — which is why `u64` draws + /// contribute NOTHING to the completeness bound. + /// + /// `nbits ≤ 32` keeps the answer inside the candidate's low half. The bound + /// is real rather than defensive: FRI query indices are bounded by the LDE + /// domain, which is ≤ 2^25 here. + pub fn sample_u64_pow2(&mut self, b: &mut LfmBuilder, nbits: usize) -> Vec { + assert!( + (1..=32).contains(&nbits), + "sample_u64_pow2: nbits must be in 1..=32, got {nbits} — above 32 the \ + answer would span both halves of the candidate" + ); + let c = self.next_candidate(b); + b.bit_dec(c.lo, nbits) + } + + /// `DefaultTranscript::state()` — the sponge's digest RIGHT NOW, without + /// advancing it (production finalizes a CLONE of the hasher, + /// `default_transcript.rs:128-130`). + /// + /// Only grinding needs this: the seed it hashes is the state before the + /// nonce is absorbed, and the nonce is then absorbed into the live + /// transcript. Neither the segment nor the output buffer moves here, so a + /// later `sample` still hashes `segment ‖ nonce` exactly as production does. + /// + /// The segment is packed twice as a result (once here, once at that later + /// squeeze). Packing is free for an aligned segment — which this one is, + /// every caller reaching grinding through a `sample` — and a re-emitted + /// splice would only be redundant work, never a different value. + pub fn state(&mut self, b: &mut LfmBuilder) -> [Cell; DIGEST_WORDS] { + let packed = self.pack_segment(b); + edsl::keccak256(b, &packed, self.segment_len) + } + + /// Emit-time buffer position, for tests that pin the consumption schedule. + pub fn out_pos(&self) -> usize { + self.out_pos + } + + /// Emit-time segment length in bytes, for the same reason. + pub fn segment_len(&self) -> usize { + self.segment_len + } +} + +/// Constrains a candidate to be a canonical field element, i.e. `< p`. +/// +/// `p = 2^64 − 2^32 + 1`, so `p − 1 = (2^32 − 1)·2^32` and +/// `p = (2^32 − 1)·2^32 + 1`. For `candidate = hi·2^32 + lo` with both halves +/// below `2^32`: +/// +/// - `hi < 2^32 − 1` ⇒ `candidate ≤ (2^32 − 2)·2^32 + (2^32 − 1)` +/// `= (2^32 − 1)·2^32 − 1 < p`, always in range; +/// - `hi = 2^32 − 1` ⇒ `candidate = (p − 1) + lo`, in range iff `lo = 0`. +/// +/// So `candidate ≥ p ⟺ hi = 2^32 − 1 ∧ lo ≠ 0`. (The `LFM_BITDEC` chip proves +/// canonicity of a 64-bit decomposition with the same predicate over the top and +/// bottom 32 bits — see `chips::bitdec`.) +/// +/// The guard is then a single division. `g = (2^32 − 1) − hi` is zero exactly +/// when `hi = 2^32 − 1`, and `LFM_BALU` constrains division as +/// `SEL_DIV·(B·OUT − A) = 0`: with `B = 0` that reads `−A = 0`, forcing `A = 0` +/// and leaving `OUT` free. So `div(lo, g)` is provable iff `g ≠ 0 ∨ lo = 0` — +/// the exact negation of the reject condition, in one instruction with nothing +/// hinted and nothing to verify. It is the same assert-via-division mechanism +/// `LfmBuilder::assert_eq` is built from. +/// +/// Both halves must be canonical `u32`s for the derivation to hold. They are: +/// they come from `Unpack` of a `LFM_KECCAK` output word, whose halves the +/// keccak adapter range-checks (`keccak_rejects_non_u32_half`). +pub fn assert_canonical(b: &mut LfmBuilder, c: Candidate) { + let hi_max = b.felt_const(FE::from(HI_MAX)); + let g = b.sub(hi_max, c.hi); + let _ = b.div(c.lo, g); +} + +/// `hi·2^32 + lo` as a field element. +/// +/// Only equal to the candidate's INTEGER value once [`assert_canonical`] has +/// pinned that value below `p`; without the guard this silently wraps (a +/// candidate of `p` becomes `0`). +pub fn candidate_to_felt(b: &mut LfmBuilder, c: Candidate) -> Felt { + let two32 = b.felt_const(FE::from(1u64 << 32)); + b.mul_add(c.hi, two32, c.lo) +} + +// =============================== the packer =============================== + +/// The half currently under construction. +enum Partial { + /// Its bytes so far, all compile-time. Always fewer than four. + Const(Vec), + /// A machine value occupying the LOW `filled` bytes of the half, with + /// `filled` in `1..4`. Its unfilled high bytes are zero, so completing it is + /// an addition rather than an or. + Mixed(Felt, usize), +} + +/// Emits a segment's `u32` halves from a byte-granular walk of its pieces. +struct Packer { + out: Vec, + partial: Partial, +} + +/// The little-endian value of up to four bytes. +fn le_value(bytes: &[u8]) -> u64 { + bytes + .iter() + .enumerate() + .fold(0u64, |acc, (i, &v)| acc | (u64::from(v) << (8 * i))) +} + +impl Packer { + fn filled(&self) -> usize { + match &self.partial { + Partial::Const(v) => v.len(), + Partial::Mixed(_, f) => *f, + } + } + + fn push_const(&mut self, b: &mut LfmBuilder, bytes: &[u8]) { + for &byte in bytes { + match core::mem::replace(&mut self.partial, Partial::Const(Vec::new())) { + Partial::Const(mut v) => { + v.push(byte); + if v.len() == BYTES_PER_HALF { + let c = b.felt_const(FE::from(le_value(&v))); + self.out.push(c); + v.clear(); + } + self.partial = Partial::Const(v); + } + Partial::Mixed(m, filled) => { + // The byte lands above what is already there, and the high + // bytes are zero, so `add` is exactly an or. + let w = b.felt_const(FE::from(u64::from(byte) << (8 * filled))); + let m = b.add(m, w); + if filled + 1 == BYTES_PER_HALF { + self.out.push(m); + self.partial = Partial::Const(Vec::new()); + } else { + self.partial = Partial::Mixed(m, filled + 1); + } + } + } + } + } + + /// Merges `v` into the half under construction at byte offset `filled`. + /// The destination's higher bytes are zero, so `mul_add` is exactly an or. + fn merge(&mut self, b: &mut LfmBuilder, v: Felt, filled: usize) -> Felt { + let shift = b.felt_const(FE::from(1u64 << (8 * filled))); + match core::mem::replace(&mut self.partial, Partial::Const(Vec::new())) { + Partial::Const(c) => { + let base = b.felt_const(FE::from(le_value(&c))); + b.mul_add(v, shift, base) + } + Partial::Mixed(m, _) => b.mul_add(v, shift, m), + } + } + + /// Places a machine value carrying `nbytes` live bytes at the cursor. + /// + /// One routine covers both a whole half (`nbytes == 4`) and the masked tail + /// of an odd-length byte string, because they differ only in width. + fn push_partial(&mut self, b: &mut LfmBuilder, v: Felt, nbytes: usize) { + debug_assert!((1..=BYTES_PER_HALF).contains(&nbytes)); + let filled = self.filled(); + if filled == 0 { + // Nothing to merge with: `v` already sits in the low bytes of a + // fresh half and its high bytes are zero. No instructions — the path + // every aligned program takes, which must stay free or every + // registered digest moves. + if nbytes == BYTES_PER_HALF { + self.out.push(v); + } else { + self.partial = Partial::Mixed(v, nbytes); + } + return; + } + let room = BYTES_PER_HALF - filled; + if nbytes < room { + let merged = self.merge(b, v, filled); + self.partial = Partial::Mixed(merged, filled + nbytes); + } else if nbytes == room { + let merged = self.merge(b, v, filled); + self.out.push(merged); + self.partial = Partial::Const(Vec::new()); + } else { + // Crosses the boundary: the low `room` bytes finish this half and + // the rest opens the next. + let (lo, hi) = split_half(b, v, room); + let merged = self.merge(b, lo, filled); + self.out.push(merged); + self.partial = Partial::Mixed(hi, nbytes - room); + } + } + + fn push_half(&mut self, b: &mut LfmBuilder, d: Felt) { + self.push_partial(b, d, BYTES_PER_HALF); + } + + /// Masks a trailing half to its `nbytes` live bytes and PINS the rest to + /// zero, then places it. + /// + /// The zero-pin is a soundness obligation, not tidiness: the high bytes of + /// an arena-supplied felt are otherwise unconstrained, and without it a + /// prover could put arbitrary content there. Those bytes are past the + /// encoding's length prefix, so they would change the absorbed byte string + /// while the length said otherwise. + fn push_masked(&mut self, b: &mut LfmBuilder, v: Felt, nbytes: usize) { + let (lo, hi) = split_half(b, v, nbytes); + let zero = b.felt_const(FE::zero()); + b.assert_eq(hi, zero); + self.push_partial(b, lo, nbytes); + } + + fn finish(mut self, b: &mut LfmBuilder) -> Vec { + match self.partial { + // A trailing partial half's unused high bytes are zero either way, + // which is the property `edsl::keccak256` needs to merge the padding + // constant with an `add`. + Partial::Const(v) if v.is_empty() => {} + Partial::Const(v) => { + let c = b.felt_const(FE::from(le_value(&v))); + self.out.push(c); + } + Partial::Mixed(m, _) => self.out.push(m), + } + self.out + } +} + +/// Splits a `u32` half into its low `k` bytes and its high `4 − k` bytes. +/// +/// This is the byte-level splice the misaligned statement encoding needs. A byte +/// split is not field arithmetic, so it goes through the canonical bit +/// decomposition and two weighted sums over disjoint bit ranges. +/// +/// The recomposition assert is load-bearing, not a belt: `bit_dec` bounds its +/// input by `p`, not by `2^32`, and a "half" at or above `2^32` has no four-byte +/// rendering at all. Pinning `d = lo + hi·2^(8k)` forces `d < 2^32` and the +/// split's correctness in the same constraint. +/// +/// Cost: one `LFM_BITDEC` row and ~33 `LFM_BALU` rows per spliced half. It only +/// ever runs on the statement leg — a few dozen halves per proof — and never in +/// FRI or Merkle traffic. +pub fn split_half(b: &mut LfmBuilder, d: Felt, k: usize) -> (Felt, Felt) { + assert!( + (1..BYTES_PER_HALF).contains(&k), + "split_half: k must be in 1..4, got {k}" + ); + let bits = b.bit_dec(d, 8 * BYTES_PER_HALF); + let lo = edsl::bits_to_felt(b, &bits[..8 * k]); + let hi = edsl::bits_to_felt(b, &bits[8 * k..]); + let shift = b.felt_const(FE::from(1u64 << (8 * k))); + let recomposed = b.mul_add(hi, shift, lo); + b.assert_eq(d, recomposed); + (lo, hi) +} + +/// The two `u32` halves of a base felt's 8-byte BIG-endian rendering — what +/// `append_field_element` puts on the wire, expressed in the machine's +/// little-endian half convention. +/// +/// ## The derivation +/// +/// Write `v = hi·2^32 + lo`. Big-endian, `v`'s bytes are `hi`'s four bytes +/// most-significant-first, then `lo`'s. Half `h` of the segment is the LE `u32` +/// of segment bytes `4h..4h+4`, so +/// +/// - half 0 = `byteswap32(hi)` — the HIGH word leads in big-endian order, +/// - half 1 = `byteswap32(lo)`. +/// +/// A byte swap is not field arithmetic, so it goes through the canonical bit +/// decomposition: bit `j` of byte `k` must land at bit `j` of byte `3 − k`, +/// which is just a different constant weight per bit. Each half is therefore one +/// 32-term linear form, and the whole byte permutation lives in the weights +/// rather than in any emitted instruction. +/// +/// ## Cost +/// +/// One `LFM_BITDEC` row plus 64 `LFM_BALU` rows (per half: a `Mul` to open the +/// accumulator, then 31 `MulAdd`s), and the 32 weight constants are interned +/// once and shared by both halves — they are the powers `2^0..2^31`, since +/// `j + 8(3 − k)` runs over `0..32` bijectively. +/// +/// `bit_dec` also enforces canonicity (`< p`), which is exactly right: production +/// renders `canonical_u64()`. +/// +/// Note for callers re-absorbing a value the transcript just produced: a +/// challenge from [`TranscriptReplay::sample_felt`] arrives as a recomposed +/// `Felt` and is decomposed again here. That round trip is one redundant +/// `BitDec`; carrying the halves through would avoid it, and is worth doing only +/// if a profile says so. +pub fn felt_be_halves(b: &mut LfmBuilder, v: Felt) -> [Felt; 2] { + let bits = b.bit_dec(v, 64); + core::array::from_fn(|h| { + // Half 0 carries the value's HIGH 32 bits: they lead in big-endian order. + let first = if h == 0 { 32 } else { 0 }; + let mut acc: Option = None; + for k in 0..4 { + for j in 0..8 { + let weight = b.felt_const(FE::from(1u64 << (j + 8 * (3 - k)))); + let bit = bits[first + 8 * k + j].as_felt(); + acc = Some(match acc { + None => b.mul(bit, weight), + Some(a) => b.mul_add(bit, weight, a), + }); + } + } + acc.expect("32 bits per half") + }) +} + +/// Per-candidate probability that the production sampler rejects: there are +/// `2^64 − p = 2^32 − 1` out-of-range values among the `2^64` a candidate can +/// take. +/// +/// Only `sample_field_element` draws are exposed — `sample_u64` at a power-of-two +/// bound never rejects. +pub fn reject_probability_per_candidate() -> f64 { + ((1u64 << 32) - 1) as f64 / 2f64.powi(64) +} + +/// Upper bound on the probability that a transcript with `base_draws` base-field +/// challenge draws rejects at least once — i.e. that the emitted zero-rejection +/// program cannot prove it. +/// +/// A cubic-extension challenge is THREE base draws, so pass `3 · ext_draws`. +/// The union bound is what makes this an upper bound; the exact value is +/// `1 − (1 − q)^n`, indistinguishable at these magnitudes. +pub fn reject_probability_per_proof(base_draws: usize) -> f64 { + base_draws as f64 * reject_probability_per_candidate() +} + +/// Packs a piece list into `u32` halves, walking it at BYTE granularity. +/// +/// Constant bytes accumulate host-side; a machine half drops straight in when +/// the cursor is 4-byte aligned — the path every aligned program takes, which +/// must stay instruction-free — and is split when it is not. The packer is the +/// only place that knows the cursor, which is why the splice lives here rather +/// than at the append. +fn pack_pieces(pieces: &[SegPiece], b: &mut LfmBuilder) -> Vec { + let mut p = Packer { + out: Vec::new(), + partial: Partial::Const(Vec::new()), + }; + for piece in pieces { + match piece { + SegPiece::Const(bytes) => p.push_const(b, bytes), + SegPiece::Halves(halves) => { + for h in halves { + p.push_half(b, *h); + } + } + SegPiece::Partial(v, nbytes) => p.push_masked(b, *v, *nbytes), + } + } + p.finish(b) +} + +/// A structured byte string of compile-time constants and machine values, +/// hashed directly rather than absorbed into a transcript. +/// +/// Same byte-granular packer the transcript's segments use — one implementation, +/// so the splice semantics cannot drift between the two callers. This exists for +/// folds like the recursion attestation's `program_id`, which is a plain +/// `keccak256` over `tag ‖ fields`, not a Fiat-Shamir absorb. +/// +/// The alignment lesson from R1e applies unchanged and is why this is not just +/// "concatenate then hash": alignment is a property of the CURSOR, not of the +/// field. `PROGRAM_ID_TAG` is 22 bytes, so every machine value after it lands +/// mid-half and must be spliced. +#[derive(Default)] +pub struct ByteString { + pieces: Vec, + len: usize, +} + +impl ByteString { + pub fn new() -> Self { + Self::default() + } + + /// Append compile-time constant bytes, any length, any alignment. + pub fn push_const(&mut self, bytes: &[u8]) { + self.pieces.push(SegPiece::Const(bytes.to_vec())); + self.len += bytes.len(); + } + + /// Append `4 · halves.len()` machine-computed bytes at any alignment. + pub fn push_halves(&mut self, halves: &[Felt]) { + self.pieces.push(SegPiece::Halves(halves.to_vec())); + self.len += BYTES_PER_HALF * halves.len(); + } + + /// Append `byte_len` machine-computed bytes carried in + /// `ceil(byte_len / 4)` halves, masking the trailing partial half. + pub fn push_bytes(&mut self, halves: &[Felt], byte_len: usize) { + assert_eq!( + halves.len(), + byte_len.div_ceil(BYTES_PER_HALF), + "byte_len must match the supplied halves" + ); + let full = byte_len / BYTES_PER_HALF; + let rem = byte_len % BYTES_PER_HALF; + if full > 0 { + self.pieces.push(SegPiece::Halves(halves[..full].to_vec())); + } + if rem > 0 { + self.pieces.push(SegPiece::Partial(halves[full], rem)); + } + self.len += byte_len; + } + + /// Bytes the string will hash — its own accounting, so a test can pin the + /// resulting alignment rather than trust prose. + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// `keccak256` over the assembled bytes. + pub fn keccak256(&self, b: &mut LfmBuilder) -> [Cell; DIGEST_WORDS] { + let packed = pack_pieces(&self.pieces, b); + edsl::keccak256(b, &packed, self.len) + } +} diff --git a/prover/src/lfm/transcript_tests.rs b/prover/src/lfm/transcript_tests.rs new file mode 100644 index 000000000..56d6d7814 --- /dev/null +++ b/prover/src/lfm/transcript_tests.rs @@ -0,0 +1,754 @@ +//! The compress-chain Fiat–Shamir transcript (option B1): its vectors, its +//! domain separation, its cost, and the machine that computes it. +//! +//! ## What pins what +//! +//! Four layers, deliberately different evidence: +//! +//! 1. **The step function** is `blake3::hash(state ‖ operand ‖ "LFMT")` +//! truncated — asserted against the *crate*, not against an oracle, so the +//! external anchor the 7-round decision was bought for is inherited rather +//! than claimed. +//! 2. **The vectors** are [`super::transcript_kats`], rendered from a Python +//! reference the oracle wrote before any of this Rust existed. Both round +//! counts, per-op and end-to-end. +//! 3. **The host chain** (`fixture::HostSponge`) reproduces them, which is what +//! "bit-exact mirror" has to mean to be checkable. +//! 4. **The machine** (`edsl::SpongeVar` through `LFM_HASH`) reproduces the same +//! challenges *inside a proof the production verifier accepts* — the layer +//! that would catch a host and chip that agree with the spec separately and +//! with each other not at all. +//! +//! Every rejection test here is paired with an honest-path assertion. A test +//! that only checks "the bad thing is rejected" passes just as well when +//! everything is rejected, which is the failure mode a soundness fix has. + +use crate::tables::types::{FE, FEE, GoldilocksField}; +use math::field::traits::IsPrimeField; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use super::blake3_socket::{ + SOCKET_ROUNDS, TAG_LFMT, leaf_digest_rounds, socket_digest_rounds, transcript_digest, + transcript_digest_rounds, word_of, +}; +use super::builder::{Cell, LfmBuilder, LfmProgramSource}; +use super::compiler::{LfmProgram, compile}; +use super::edsl::{SQUEEZE_MARK, SpongeVar}; +use super::executor::execute; +use super::fixture::HostSponge; +use super::hash::HasherKind; +use super::instr::{HashMode, Instr}; +use super::proof::{lfm_prove_with_hasher, verify_against}; +use super::registry::build_artifacts_with_hasher; +use super::transcript_kats::{ + EndToEndVector, FRI_TOY_6, FRI_TOY_7, FRI_TOY_COMPRESSIONS, L1_ROOT, MAIN_ROOT, STEP_VECTORS, + T0W, T1W, +}; +use super::word::LfmWord; + +const KIND: HasherKind = HasherKind::Blake3; + +/// Query shape of the `FriToyV0` preamble the end-to-end vector is shaped like +/// (✓ VERIFIED `fixture::shape`). +const NUM_QUERIES: usize = 4; +const QUERY_BITS: usize = 4; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// The end-to-end vector for the round count this build compiled. +fn compiled_vector() -> &'static EndToEndVector { + if SOCKET_ROUNDS == 7 { + &FRI_TOY_7 + } else { + &FRI_TOY_6 + } +} + +fn lanes(w: &LfmWord) -> [u32; 4] { + core::array::from_fn(|i| { + u32::try_from(GoldilocksField::canonical(w[i].value())).expect("a u32 lane") + }) +} + +// ========================================================================= +// K1 — the step function, and its external anchor +// ========================================================================= + +/// The per-op vectors, at BOTH round counts in one run. +#[test] +fn every_step_vector_reproduces_at_both_round_counts() { + for v in STEP_VECTORS.iter() { + assert_eq!( + transcript_digest_rounds(&v.state, &v.operand, 6), + v.result_6, + "6-round step vector {}", + v.name + ); + assert_eq!( + transcript_digest_rounds(&v.state, &v.operand, 7), + v.result_7, + "7-round step vector {}", + v.name + ); + } +} + +/// The compiled-in entry point agrees with the vector for its round count — so +/// the knob and the table cannot drift apart. +#[test] +fn the_compiled_step_matches_its_round_counts_vectors() { + for v in STEP_VECTORS.iter() { + let expected = if SOCKET_ROUNDS == 7 { + v.result_7 + } else { + v.result_6 + }; + assert_eq!( + transcript_digest(&v.state, &v.operand), + expected, + "vector {}", + v.name + ); + } +} + +/// ★ **The external anchor, direct.** At 7 rounds a transcript step is +/// literally `blake3::hash(state ‖ operand ‖ 0^16 ‖ "LFMT")` truncated to 16 +/// bytes. +/// +/// The message is re-derived from the byte-level framing rather than from +/// `socket_message`, so the word-level and byte-level forms are two statements +/// that can disagree. This is the property option B was chosen for: the +/// transcript inherits the compress socket's anchor because the tag is the only +/// thing that moved — and it kept it through the leaf RATE's widening, because +/// 52 bytes is still one block. +#[test] +fn seven_rounds_is_blake3_of_the_transcript_message() { + for v in STEP_VECTORS.iter() { + let mut msg = Vec::with_capacity(52); + for lane in v.state.iter().chain(v.operand.iter()) { + msg.extend_from_slice(&lane.to_le_bytes()); + } + // The third input cell, pinned to zero on every row that does not read + // it — a transcript step reads two. + msg.extend_from_slice(&[0u8; 16]); + msg.extend_from_slice(b"LFMT"); + assert_eq!(msg.len(), 52, "a transcript step is one 52-byte block"); + + let full = blake3::hash(&msg); + let want: [u32; 4] = core::array::from_fn(|i| { + u32::from_le_bytes(full.as_bytes()[4 * i..4 * i + 4].try_into().unwrap()) + }); + assert_eq!( + transcript_digest_rounds(&v.state, &v.operand, 7), + want, + "7-round step {} must be blake3::hash of its message", + v.name + ); + assert_eq!(want, v.result_7, "the table itself agrees with the crate"); + } +} + +/// The tag word is the ASCII, little-endian — the one place a byte order slip +/// would silently redefine the domain. +#[test] +fn the_transcript_tag_is_lfmt_little_endian() { + assert_eq!(TAG_LFMT, u32::from_le_bytes(*b"LFMT")); + assert_eq!(TAG_LFMT.to_le_bytes(), *b"LFMT"); + assert_eq!(SQUEEZE_MARK.to_le_bytes(), *b"SQZ0"); +} + +// ========================================================================= +// K3 — domain separation, in both directions +// ========================================================================= + +/// A transcript step and a Merkle parent over the SAME two cells are different +/// digests. Without this the chain would be replayable as a tree and vice +/// versa, and the `MODE_T` column would be buying nothing. +#[test] +fn a_transcript_step_is_not_a_merkle_parent() { + for v in STEP_VECTORS.iter() { + for rounds in [6, 7] { + assert_ne!( + transcript_digest_rounds(&v.state, &v.operand, rounds), + socket_digest_rounds(&v.state, &v.operand, rounds), + "vector {} at {rounds} rounds: the tag is not separating the domains", + v.name + ); + } + } +} + +/// The honest-path control for the test above: the two ARE the same function +/// apart from the tag, so a bug that made them differ for some other reason +/// would show up here. +#[test] +fn the_two_domains_differ_only_in_the_tag() { + for v in STEP_VECTORS.iter() { + use super::blake3_socket::{TAG_LFMC, socket_digest_rounds_tagged}; + assert_eq!( + socket_digest_rounds_tagged(&v.state, &v.operand, 7, TAG_LFMT), + transcript_digest_rounds(&v.state, &v.operand, 7) + ); + assert_eq!( + socket_digest_rounds_tagged(&v.state, &v.operand, 7, TAG_LFMC), + socket_digest_rounds(&v.state, &v.operand, 7) + ); + } +} + +// ========================================================================= +// K2 — the end-to-end vector, host side +// ========================================================================= + +/// Replays the `FriToyV0` preamble against the reference step function at an +/// explicit round count, so BOTH vectors are checkable from one build. +/// +/// ✓ VERIFIED sequence, `programs::fri_toy_program_source`: absorb(main_root), +/// squeeze_ext, squeeze_ext, absorb(l1_root), squeeze_ext, **absorb_felts(t0w), +/// absorb_felts(t1w)**, then `NUM_QUERIES` × squeeze_bits. +/// +/// The last two are `absorb_felts`, not `absorb2`: the terminal coefficients are +/// field DATA, so they are leaf-hashed and the DIGEST is absorbed. The step +/// count is the same either way, which is exactly why this had to be re-pointed +/// deliberately rather than caught by a red test. +fn replay_reference(rounds: usize) -> (Vec<[u32; 4]>, Vec<[u32; 4]>, usize) { + let mut state = [0u32; 4]; + let mut squeeze_index = 0u32; + let mut compressions = 0usize; + let mut states = Vec::new(); + let mut outputs = Vec::new(); + + let absorb = |state: &mut [u32; 4], c: &[u32; 4], compressions: &mut usize| { + *state = transcript_digest_rounds(state, c, rounds); + *compressions += 1; + }; + let squeeze = |state: &mut [u32; 4], i: &mut u32, compressions: &mut usize| -> [u32; 4] { + let out = *state; + let sq = [SQUEEZE_MARK, *i, 0, 0]; + *state = transcript_digest_rounds(state, &sq, rounds); + *i += 1; + *compressions += 1; + out + }; + + absorb(&mut state, &MAIN_ROOT, &mut compressions); + states.push(state); + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + absorb(&mut state, &L1_ROOT, &mut compressions); + states.push(state); + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + // DATA, so each goes through the leaf encoding before it is absorbed. + for cell in [&T0W, &T1W] { + let felts: LfmWord = core::array::from_fn(|i| FE::from(u64::from(cell[i]))); + // From the chain start, exactly as `absorb_felts` does: one leaf row per + // data cell, absorbing four felts and chaining in the same compression. + let d = leaf_digest_rounds(&super::fixture::leaf_chain_start(), &felts, rounds) + .expect("the KAT inputs are canonical"); + absorb(&mut state, &d, &mut compressions); + states.push(state); + } + for _ in 0..NUM_QUERIES { + outputs.push(squeeze(&mut state, &mut squeeze_index, &mut compressions)); + states.push(state); + } + (states, outputs, compressions) +} + +fn check_end_to_end(rounds: usize, want: &EndToEndVector) { + let (states, outputs, compressions) = replay_reference(rounds); + assert_eq!(states.len(), want.states.len()); + for (i, (got, expected)) in states.iter().zip(want.states.iter()).enumerate() { + assert_eq!(got, expected, "state after op {i} at {rounds} rounds"); + } + // Challenges are read off the PRE-advance outputs, not off the states. + let ext = |o: &[u32; 4]| [o[0], o[1], o[2]]; + assert_eq!(ext(&outputs[0]), want.alpha, "alpha at {rounds} rounds"); + assert_eq!(ext(&outputs[1]), want.zeta0, "zeta0 at {rounds} rounds"); + assert_eq!(ext(&outputs[2]), want.zeta1, "zeta1 at {rounds} rounds"); + for (q, bits) in want.query_bits.iter().enumerate() { + let lane0 = outputs[3 + q][0]; + let got: [u8; QUERY_BITS] = core::array::from_fn(|k| ((lane0 >> k) & 1) as u8); + assert_eq!(&got, bits, "query {q} bits at {rounds} rounds"); + } + // The reference replay counts TRANSCRIPT steps; the oracle's constant counts + // the leaf rows too, so the two differ by exactly the two data absorbs. + assert_eq!( + compressions + 2, + FRI_TOY_COMPRESSIONS, + "the preamble's compression count is a cost claim, not an accident" + ); +} + +/// K2 at 7 rounds — the default build. +#[test] +fn the_end_to_end_vector_reproduces_at_seven_rounds() { + check_end_to_end(7, &FRI_TOY_7); +} + +/// K2 at 6 rounds — the `blake3-6round` variant, pinned unconditionally. +#[test] +fn the_end_to_end_vector_reproduces_at_six_rounds() { + check_end_to_end(6, &FRI_TOY_6); +} + +/// ★ The HOST CHAIN — `fixture::HostSponge`, the thing the fixture prover and +/// every host-side replay use — reproduces the vector op for op. +/// +/// This is the mirror property stated as something checkable. It runs at the +/// compiled-in round count, which is why the two `check_end_to_end` tests above +/// carry the other one. +#[test] +fn the_host_sponge_reproduces_the_end_to_end_vector() { + let want = compiled_vector(); + let mut sponge = HostSponge::with_hasher(KIND); + let cell = |w: &[u32; 4]| word_of(w); + let mut states = Vec::new(); + + sponge.absorb(&cell(&MAIN_ROOT)); + states.push(sponge.state()); + let alpha = sponge.squeeze_ext(); + states.push(sponge.state()); + let zeta0 = sponge.squeeze_ext(); + states.push(sponge.state()); + sponge.absorb(&cell(&L1_ROOT)); + states.push(sponge.state()); + let zeta1 = sponge.squeeze_ext(); + states.push(sponge.state()); + sponge.absorb_felts(&felts_of(&T0W)); + states.push(sponge.state()); + sponge.absorb_felts(&felts_of(&T1W)); + states.push(sponge.state()); + let mut queries = Vec::new(); + for _ in 0..NUM_QUERIES { + queries.push(sponge.squeeze_index(QUERY_BITS)); + states.push(sponge.state()); + } + + for (i, (got, expected)) in states.iter().zip(want.states.iter()).enumerate() { + assert_eq!(lanes(got), *expected, "host state after op {i}"); + } + + let ext_lanes = |e: &FEE| -> [u32; 3] { + let v = e.value(); + core::array::from_fn(|i| { + u32::try_from(GoldilocksField::canonical(v[i].value())).expect("a u32 lane") + }) + }; + assert_eq!(ext_lanes(&alpha), want.alpha); + assert_eq!(ext_lanes(&zeta0), want.zeta0); + assert_eq!(ext_lanes(&zeta1), want.zeta1); + for (q, bits) in want.query_bits.iter().enumerate() { + let index: u64 = bits + .iter() + .enumerate() + .map(|(k, &b)| u64::from(b) << k) + .sum(); + assert_eq!(queries[q], index, "host query {q}"); + } +} + +// ========================================================================= +// K4/K5 — the counter and the ordering are load-bearing +// ========================================================================= + +/// Without the counter every advance uses ONE fixed operand, so a run of +/// squeezes iterates one fixed map — precisely the structure the FSE-2014 +/// T-sponge attacks exploit. The vectors must notice. +#[test] +fn the_squeeze_counter_is_load_bearing() { + let rounds = 7; + let start = transcript_digest_rounds(&[0; 4], &MAIN_ROOT, rounds); + + let mut with_counter = Vec::new(); + let mut s = start; + for i in 0..4u32 { + with_counter.push(s); + s = transcript_digest_rounds(&s, &[SQUEEZE_MARK, i, 0, 0], rounds); + } + + let mut without = Vec::new(); + let mut s = start; + let fixed = [SQUEEZE_MARK, 0, 0, 0]; + for _ in 0..4 { + without.push(s); + s = transcript_digest_rounds(&s, &fixed, rounds); + } + + // Honest-path half: squeeze 0 uses counter 0, so the two MUST agree there. + // Without this the test would pass for a chain that simply produced noise. + assert_eq!( + with_counter[0], without[0], + "the first squeeze is the same either way — counter 0 is counter 0" + ); + assert_ne!( + with_counter[1..], + without[1..], + "counter-free squeezes must diverge: they iterate one fixed map" + ); +} + +/// Absorbing two cells in the other order is a different transcript. Cheap to +/// state, and the property every Fiat–Shamir argument silently assumes. +#[test] +fn absorb_order_is_load_bearing() { + let mut a = HostSponge::with_hasher(KIND); + a.absorb(&word_of(&MAIN_ROOT)); + a.absorb(&word_of(&L1_ROOT)); + + let mut b = HostSponge::with_hasher(KIND); + b.absorb(&word_of(&L1_ROOT)); + b.absorb(&word_of(&MAIN_ROOT)); + + assert_ne!(a.state(), b.state()); + + // Honest-path control: the same order gives the same state. + let mut c = HostSponge::with_hasher(KIND); + c.absorb(&word_of(&MAIN_ROOT)); + c.absorb(&word_of(&L1_ROOT)); + assert_eq!(a.state(), c.state()); +} + +// ========================================================================= +// K6 + the machine — the emitted program +// ========================================================================= + +/// A program shaped exactly like `FriToyV0`'s preamble, with the absorbed cells +/// as arena words so a test can feed the vector's inputs in. +/// +/// Its public output is every challenge the preamble derives, so a proof of it +/// carries the transcript's answers where a verifier can check them. +fn preamble_program_source() -> LfmProgramSource { + let mut b = LfmBuilder::new(); + let arena = b.declare_arena(4); + let h: Vec = (0..4).map(|i| b.hint_word(arena, i)).collect(); + + let mut sponge = SpongeVar::new(&mut b); + sponge.absorb(&mut b, h[0]); + let alpha = sponge.squeeze_ext(&mut b); + let zeta0 = sponge.squeeze_ext(&mut b); + sponge.absorb(&mut b, h[1]); + let zeta1 = sponge.squeeze_ext(&mut b); + // The last two arena cells stand for the terminal coefficients — DATA — so + // the program absorbs them the way `FriToyV0` does. + sponge.absorb_felts(&mut b, h[2]); + sponge.absorb_felts(&mut b, h[3]); + + b.public(alpha.as_cell()); + b.public(zeta0.as_cell()); + b.public(zeta1.as_cell()); + for _ in 0..NUM_QUERIES { + let bits = sponge.squeeze_bits(&mut b, QUERY_BITS); + let index = super::edsl::bits_to_felt(&mut b, &bits); + b.public(index.as_cell()); + } + b.finish() +} + +fn preamble_program() -> LfmProgram { + compile(preamble_program_source()) +} + +fn preamble_arena() -> Vec> { + vec![vec![ + word_of(&MAIN_ROOT), + word_of(&L1_ROOT), + felts_of(&T0W), + felts_of(&T1W), + ]] +} + +/// The KAT's `u32` inputs read as FIELD ELEMENTS — what the leaf encoding +/// consumes. `word_of` reads the same values as digest lanes; both are the same +/// four numbers, and which reading applies is the mode's business. +fn felts_of(lanes: &[u32; 4]) -> LfmWord { + core::array::from_fn(|i| FE::from(u64::from(lanes[i]))) +} + +/// K6 — the preamble costs exactly the compressions the spec priced it at, and +/// every one of them is a TRANSCRIPT row rather than a Merkle one. +#[test] +fn the_preamble_costs_eleven_transcript_steps() { + let program = preamble_program(); + let modes: Vec = program + .instrs + .iter() + .filter_map(|i| match i { + Instr::Hash { mode, .. } => Some(*mode), + _ => None, + }) + .collect(); + let steps = modes.iter().filter(|m| **m == HashMode::Transcript).count(); + let leaves = modes.iter().filter(|m| **m == HashMode::Leaf).count(); + assert_eq!(steps, 11, "the transcript itself is 11 steps"); + assert_eq!(leaves, 2, "one leaf row per data cell absorbed"); + // The oracle's `FRI_TOY_COMPRESSIONS` counts BOTH kinds — it is the + // preamble's total socket cost, which is the number that closes `FriToyV0` + // at 93 (4 queries × 20 + 13). + assert_eq!( + steps + leaves, + FRI_TOY_COMPRESSIONS, + "the preamble costs {FRI_TOY_COMPRESSIONS} compressions in total" + ); + assert_eq!( + steps + leaves, + modes.len(), + "a transcript preamble emits transcript steps and leaf rows, nothing else" + ); +} + +/// ★ **The machine computes the specified transcript.** The emitted program, +/// executed under BLAKE3, produces the vector's challenges. +/// +/// This is the layer the host tests cannot reach: `SpongeVar` and `HostSponge` +/// are separate code, and this is where they are made to answer the same +/// question. +#[test] +fn the_machine_reproduces_the_end_to_end_vector() { + let want = compiled_vector(); + let program = preamble_program(); + let exec = execute(&program, &preamble_arena(), &KIND).expect("the preamble executes"); + + let public: Vec = exec.public_words.iter().map(|(_, w)| *w).collect(); + assert_eq!(public.len(), 3 + NUM_QUERIES); + + let ext3 = |w: &LfmWord| -> [u32; 3] { + let l = lanes(w); + [l[0], l[1], l[2]] + }; + assert_eq!(ext3(&public[0]), want.alpha, "alpha"); + assert_eq!(ext3(&public[1]), want.zeta0, "zeta0"); + assert_eq!(ext3(&public[2]), want.zeta1, "zeta1"); + for (q, bits) in want.query_bits.iter().enumerate() { + let index: u64 = bits + .iter() + .enumerate() + .map(|(k, &b)| u64::from(b) << k) + .sum(); + assert_eq!( + GoldilocksField::canonical(public[3 + q][0].value()), + index, + "query {q}" + ); + } +} + +/// The same program, PROVED under BLAKE3 and accepted by the production +/// verifier — the transcript is not merely computed, it is constrained. +#[test] +fn the_transcript_proves_and_verifies_under_blake3() { + let opts = options(); + let program = preamble_program(); + let artifacts = build_artifacts_with_hasher(&program, &opts, KIND); + let proved = lfm_prove_with_hasher(&program, &artifacts, &preamble_arena(), &opts, KIND) + .expect("a transcript program must prove under BLAKE3"); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest BLAKE3 transcript proof must verify" + ); + + // The public challenges are checked against the SPEC's vector, not against + // the executor — so the proof's outputs answer to the specification. + let want = compiled_vector(); + let alpha = lanes(&proved.public_words[0].1); + assert_eq!([alpha[0], alpha[1], alpha[2]], want.alpha); +} + +/// The same program under every hasher: B1 changed the transcript for ALL of +/// them, so all of them must still prove and verify. +#[test] +fn the_transcript_proves_and_verifies_under_every_hasher() { + let opts = options(); + let program = preamble_program(); + for kind in [HasherKind::Test, HasherKind::Poseidon, HasherKind::Blake3] { + let artifacts = build_artifacts_with_hasher(&program, &opts, kind); + let proved = lfm_prove_with_hasher(&program, &artifacts, &preamble_arena(), &opts, kind) + .unwrap_or_else(|e| panic!("prove under {kind:?}: {e:?}")); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "an honest transcript proof must verify under {kind:?}" + ); + } +} + +/// The machine's chain and the host's chain agree under every hasher — the +/// property `fixture_prove` depends on and the one a rewrite of either side +/// would break. +#[test] +fn the_machine_and_the_host_chain_agree_under_every_hasher() { + let program = preamble_program(); + for kind in [HasherKind::Test, HasherKind::Poseidon, HasherKind::Blake3] { + // BLAKE3 needs u32 lanes (O1); the vector's inputs are u32 either way, + // so one arena serves all three hashers. + let exec = execute(&program, &preamble_arena(), &kind).expect("executes"); + let public: Vec = exec.public_words.iter().map(|(_, w)| *w).collect(); + + let mut sponge = HostSponge::with_hasher(kind); + sponge.absorb(&word_of(&MAIN_ROOT)); + let alpha = sponge.squeeze_ext(); + let zeta0 = sponge.squeeze_ext(); + sponge.absorb(&word_of(&L1_ROOT)); + let zeta1 = sponge.squeeze_ext(); + sponge.absorb_felts(&felts_of(&T0W)); + sponge.absorb_felts(&felts_of(&T1W)); + + for (i, want) in [alpha, zeta0, zeta1].iter().enumerate() { + let v = want.value(); + for l in 0..3 { + assert_eq!(public[i][l], v[l], "{kind:?} challenge {i} lane {l}"); + } + } + for q in 0..NUM_QUERIES { + let index = sponge.squeeze_index(QUERY_BITS); + assert_eq!( + public[3 + q][0], + FE::from(index), + "{kind:?} query {q} index" + ); + } + } +} + +// ========================================================================= +// The cost claims the decision was made on +// ========================================================================= + +/// Hash rows in `program`, split by mode. +fn hash_row_modes(program: &LfmProgram) -> (usize, usize, usize) { + let mut compress = 0; + let mut transcript = 0; + let mut leaf = 0; + for i in &program.instrs { + if let Instr::Hash { mode, .. } = i { + match mode { + HashMode::Compress => compress += 1, + HashMode::Transcript => transcript += 1, + HashMode::Leaf => leaf += 1, + HashMode::Permute => panic!("no registered program may contain a permute"), + } + } + } + (compress, transcript, leaf) +} + +/// ★ The ratified cost claims, measured on the emitted programs. +/// +/// `leaf-spec/LEAF.md` §5 prices `TrivialV0` at **16,551** cell-equiv at 7 +/// rounds, which reproduces exactly. It prices `FriToyV0` at **502,047**, and +/// the built machine costs **513,081** — see the ⚠ below. Both are +/// `rows × cells_per_compression`, so this asserts the row counts and the +/// per-row price separately: a product that came out right for two wrong +/// reasons is the failure mode. +/// +/// ⚠ **The spec's `FriToyV0` figure rests on a premise that does not hold.** +/// §5 has "transcript unchanged at 11", but two of the four cells `FriToyV0` +/// absorbs are the terminal polynomial's COEFFICIENTS — arbitrary field +/// elements, not digests — so absorbing them raw hands the socket lanes that +/// are not `u32` and the row is unprovable. They now enter through the leaf +/// encoding (`SpongeVar::absorb_felts`), which adds **two `LFML` rows**: 93 +/// rows rather than 91. The transcript's own step count is unchanged at 11, so +/// the spec's sentence is right about the transcript and wrong about the total. +/// +/// ⚠ Both numbers MOVED with the leaf mode, and `TrivialV0`'s moved even though +/// its row count did not: the canonicity witness columns are part of the AIR, so +/// they exist on every compress row, leaf or not. Option B priced the same two +/// programs at 369,103 and 16,527 against a 5,509-cell row; the row went to +/// 5,517 and `FriToyV0` to 91 rows from 67, because each of its three data +/// leaves became two `LFML` rows and a parent. +/// +/// ★★ **And both moved again with the leaf RATE, in opposite directions — which +/// is the whole trade, priced.** The row grew by 16 columns / 28 census cells +/// (5,517 → 5,545) to carry four more lanes, and `FriToyV0` LOST twelve rows +/// (93 → 81): a data leaf is now one two-row `LFML` chain instead of two `LFML` +/// rows plus an `LFMC` fold, so each of the three leaves per query drops its +/// parent. −12.9% on the program against +0.5% on the row. +/// +/// ⚠ Do not read −12.9% as the tower's number. `FriToyV0` is Merkle-walk-heavy +/// at a toy width — 11 of its 17 per-query hashes are path steps the RATE does +/// not touch. What the RATE halves is leaf ABSORPTION, which is ~70% of a +/// recursion tower node's bill (COMMIT.md §1.4.1) and a rounding error here. +/// +/// The per-compression price is `blake3_socket_tests`' own census formula +/// (`main + 3·⌈interactions/2⌉`), 5,545 at 7 rounds and 4,777 at 6. +#[test] +fn the_programs_cost_what_the_leaf_spec_priced_them_at() { + const CELLS_PER_COMPRESSION_7R: usize = 5_545; + const CELLS_PER_COMPRESSION_6R: usize = 4_777; + let price = if SOCKET_ROUNDS == 7 { + CELLS_PER_COMPRESSION_7R + } else { + CELLS_PER_COMPRESSION_6R + }; + + // The price, from the census rather than from a literal. + let census = super::airs::lfm_chip_census_with_hasher( + &super::programs::trivial_program(), + HasherKind::Blake3, + ); + let hash_chip = census + .iter() + .find(|c| c.name == "LFM_HASH") + .expect("the census names the hash chip"); + assert_eq!( + hash_chip.main_cols + 3 * hash_chip.aux_cols, + price, + "the per-compression price must be the census's, not a literal" + ); + + // TrivialV0: three compressions, no transcript, no leaves. Its row COUNT + // is unchanged by the leaf mode and its PRICE is not — see the doc above. + let (c, t, l) = hash_row_modes(&super::programs::trivial_program()); + assert_eq!((c, t, l), (3, 0, 0)); + if SOCKET_ROUNDS == 7 { + assert_eq!((c + t + l) * price, 16_635, "TrivialV0 at 7 rounds"); + } + + // FriToyV0, per query: three data leaves at two chained `LFML` rows each + // (6) and 11 Merkle-walk steps — 11 `LFMC` and 6 `LFML`, i.e. 17. Times 4 + // queries, plus the preamble's 13. The three `LFMC` folds that used to + // combine each leaf's two halves are gone: the chain does that work inside + // the rows it was already paying for. + let (c, t, l) = hash_row_modes(&super::programs::fri_toy_program()); + assert_eq!((c, t, l), (44, 11, 26)); + assert_eq!( + t + 2, + FRI_TOY_COMPRESSIONS, + "the preamble's share: 11 transcript steps plus its 2 leaf rows" + ); + assert_eq!( + c + t + l, + 4 * 17 + FRI_TOY_COMPRESSIONS, + "the decomposition: 4 queries × 17 + the preamble's 13" + ); + assert_eq!(c + t + l, 81, "was 93 before the leaf RATE"); + // ★ The LEAF ROW count is unchanged — 26 either way — which is the point: + // the same felts are absorbed by the same number of `LFML` rows, and what + // disappeared is the FOLD. A change that had merely moved work from the + // parents into more leaf rows would show up right here. + assert_eq!(l, 26, "the RATE removes folds, it does not add leaf rows"); + if SOCKET_ROUNDS == 7 { + assert_eq!((c + t + l) * price, 449_145, "FriToyV0 at 7 rounds"); + } +} diff --git a/prover/src/lfm/validator.rs b/prover/src/lfm/validator.rs new file mode 100644 index 000000000..24acba49f --- /dev/null +++ b/prover/src/lfm/validator.rs @@ -0,0 +1,546 @@ +//! The registry-admission validator — release-mode, always on. +//! +//! A program digest enters the `LFM_REGISTRY` only after this passes. The +//! AIR checks per-op algebra and bus balance; the *registrar* vouches for the +//! structural well-formedness below, and this validator is what makes that +//! vouching real (the reference machine checks less, and only in dev builds). +//! Together: uniqueness + acyclicity + bounded multiplicities + balance ⇒ +//! every read observes the unique written value. +//! +//! "Bounded multiplicities" is load-bearing and easy to lose: balance is a +//! *field* identity, so a send gated by `p − 1` subtracts a token, and a +//! subtract-then-re-add pair over one address keeps the count while changing +//! the value. Every check below that reads `program.instrs` is therefore +//! backed by one that reads the committed `program.groups` directly — the +//! groups are what the AIR sees, and the two are joined by nothing but the +//! emitter. +//! +//! The compiler's invariant panics are tripwires; this validator is the gate; +//! the registry is the record. There is no off-switch, and there must never +//! be one. + +use std::collections::{HashMap, HashSet}; + +use math::field::traits::IsPrimeField; + +use crate::tables::types::{FE, GoldilocksField}; + +use super::compiler::{ColumnGroup, LfmColumnGroups, LfmProgram}; +use super::instr::{Addr, Instr}; +use super::layout; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LfmViolation { + /// Check 1 — write-once uniqueness. + DoubleWrite { addr: u64 }, + /// Check 2 — every read has a writer. + ReadOfUnwritten { addr: u64 }, + /// Check 2 (range) — an address outside the allocated space. + AddressOutOfRange { addr: u64 }, + /// Check 3 — acyclicity: an operand not strictly below its destination. + CyclicRead { instr: usize, addr: u64 }, + /// Check 4 — a write's `mult` differs from the emitted read count. + MultMismatch { + addr: u64, + expected: u64, + found: u64, + }, + /// Check 4 — a `Compress` row whose spare output slots are not the + /// documented placeholders. + /// + /// `Instr::writes()` reports only `outs[0]` for `Compress` (`instr.rs`), + /// so checks 1 and 4 never see slots 1–2 — but `emit_column_groups` copies + /// them into the committed group unconditionally and `chips::hash` sends + /// all three slots. `instr.rs`'s field conventions call them "`Addr(0)` + /// placeholders with `mults` fixed to 0"; this is what makes that a rule + /// rather than a comment. + CompressSlotNotPlaceholder { instr: usize }, + /// Check 5 — opcode selectors not one-hot / flags not boolean on a real row. + NonOneHotSelector { chip: &'static str, row: usize }, + /// Check 6 — nonzero data beyond the program length. + DirtyPadding { chip: &'static str, row: usize }, + /// Check 7 — a `Hint` outside the declared arena schema. + ArenaOutOfBounds { arena: u32, index: u32 }, + /// Check 8 — two `LFM_KECCAK` rows carry the same tag. + /// + /// The tag is the only thing binding a permutation's request token to its + /// reply token: with a duplicate, a prover can swap the two rows' output + /// states and the `Keccak` bus still balances. This is not theoretical — + /// `keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` + /// exhibits the forgery against the raw family. Tags are preprocessed + /// program data, so this check is what makes them trustworthy. + DuplicateKeccakTag { tag: (u64, u64) }, + /// Check 8 — a tag half at or above `2^32`, so it cannot equal the + /// `DWordWL` timestamp any `KECCAK_RND` row carries. + MalformedKeccakTag { row: usize }, + /// Check 9 — a multiplicity column outside the legal range. + /// + /// A multiplicity is a static read count: a small non-negative integer. + /// The field is not ordered, so nothing about `Multiplicity::Column` stops + /// a preprocessed mult from holding `p − 1`, which the bus reads as `−1` — + /// a **negative** send. That is the one shape the LogUp count argument + /// cannot see: a negative ghost send cancels an honest write and a + /// positive one replaces it at the same address, leaving the token count + /// exactly balanced while the cell's value is prover-chosen. Surplus + /// *positive* sends are already denied by the count (they have no matching + /// receive); this check is what denies the negative half. + MultOutOfRange { + chip: &'static str, + row: usize, + col: usize, + mult: u64, + bound: u64, + }, + /// Cross-check — group shape does not match the instruction partition. + GroupShapeMismatch { chip: &'static str }, +} + +pub fn validate(program: &LfmProgram) -> Result<(), LfmViolation> { + check_writes_and_reads(program)?; + check_multiplicities(program)?; + check_arenas(program)?; + check_groups(program)?; + check_keccak_tags(&program.groups.keccak)?; + Ok(()) +} + +/// Check 8: `LFM_KECCAK` tags are well-formed and pairwise distinct. +/// +/// Padding rows are skipped: their `IS_REAL` is zero, so they emit no bus +/// tokens and their all-zero tag binds nothing. +fn check_keccak_tags(group: &ColumnGroup) -> Result<(), LfmViolation> { + let mut seen = HashSet::new(); + for row in 0..group.real_rows { + let lo = GoldilocksField::canonical(group.at(row, layout::keccak::TAG_LO).value()); + let hi = GoldilocksField::canonical(group.at(row, layout::keccak::TAG_HI).value()); + if lo >= 1u64 << 32 || hi >= 1u64 << 32 { + return Err(LfmViolation::MalformedKeccakTag { row }); + } + if !seen.insert((lo, hi)) { + return Err(LfmViolation::DuplicateKeccakTag { tag: (lo, hi) }); + } + } + Ok(()) +} + +/// Checks 1–3: uniqueness, read-has-writer, acyclicity. +fn check_writes_and_reads(program: &LfmProgram) -> Result<(), LfmViolation> { + let n = program.num_addrs as usize; + let mut written = vec![false; n]; + for instr in &program.instrs { + for Addr(w) in instr.writes() { + let slot = written + .get_mut(w as usize) + .ok_or(LfmViolation::AddressOutOfRange { addr: w })?; + if *slot { + return Err(LfmViolation::DoubleWrite { addr: w }); + } + *slot = true; + } + } + for (idx, instr) in program.instrs.iter().enumerate() { + let min_write = instr.writes().iter().map(|a| a.0).min(); + for Addr(r) in instr.reads() { + if !*written + .get(r as usize) + .ok_or(LfmViolation::AddressOutOfRange { addr: r })? + { + return Err(LfmViolation::ReadOfUnwritten { addr: r }); + } + if let Some(w) = min_write + && r >= w + { + return Err(LfmViolation::CyclicRead { + instr: idx, + addr: r, + }); + } + } + } + Ok(()) +} + +/// Check 4: every write's `mult` equals an independent recount of its reads. +fn check_multiplicities(program: &LfmProgram) -> Result<(), LfmViolation> { + let mut counts: HashMap = HashMap::new(); + for instr in &program.instrs { + for r in instr.reads() { + *counts.entry(r).or_insert(0) += 1; + } + } + let check = |addr: Addr, found: u64| -> Result<(), LfmViolation> { + let expected = counts.get(&addr).copied().unwrap_or(0); + if expected != found { + return Err(LfmViolation::MultMismatch { + addr: addr.0, + expected, + found, + }); + } + Ok(()) + }; + for (idx, instr) in program.instrs.iter().enumerate() { + match instr { + Instr::Const { out, mult, .. } + | Instr::BaseAlu { out, mult, .. } + | Instr::ExtAlu { out, mult, .. } + | Instr::Hint { out, mult, .. } + | Instr::Pack { out, mult, .. } => check(*out, *mult)?, + Instr::Unpack { outs, mults, .. } => { + for i in 0..4 { + check(outs[i], mults[i])?; + } + } + Instr::Select { + out_l, + out_r, + mult_l, + mult_r, + .. + } => { + check(*out_l, *mult_l)?; + check(*out_r, *mult_r)?; + } + Instr::BitDec { bits, .. } => { + for (addr, mult) in bits { + check(*addr, *mult)?; + } + } + Instr::Hash { + mode, + ins, + outs, + mults, + } => { + let num_outs = mode.num_output_cells(); + for i in 0..num_outs { + check(outs[i], mults[i])?; + } + // A one-output row's spare slots are outside `writes()` and so + // outside checks 1 and 4 — but they are inside the committed + // group and inside the bus. Pin them to the placeholders + // `instr.rs` documents, so "slot 0 only" is a checked property + // of the program and not a convention the emitter happens to + // follow. + if mode.num_output_cells() == 1 + && (mults[1] != 0 || mults[2] != 0 || outs[1] != Addr(0) || outs[2] != Addr(0)) + { + return Err(LfmViolation::CompressSlotNotPlaceholder { instr: idx }); + } + // The same for the INPUT slots a mode does not read. A leaf row + // reads one cell, so its second and third slots reach the bus as + // addresses nothing receives; pinning them keeps "reads exactly + // `num_input_cells`" a checked property too. + if ins[mode.num_input_cells()..].iter().any(|a| *a != Addr(0)) { + return Err(LfmViolation::CompressSlotNotPlaceholder { instr: idx }); + } + } + Instr::KeccakF(k) => { + for i in 0..layout::keccak::NUM_WORDS { + check(k.outs[i], k.mults[i])?; + } + if let Some(rev) = &k.rev { + for i in 0..layout::keccak::DIGEST_WORDS { + check(rev.outs[i], rev.mults[i])?; + } + } + } + Instr::Public { .. } => {} + } + } + Ok(()) +} + +/// Check 7: arena discipline. +fn check_arenas(program: &LfmProgram) -> Result<(), LfmViolation> { + let lens = &program.arena_schema.lens; + for instr in &program.instrs { + if let Instr::Hint { arena, index, .. } = instr { + let ok = lens.get(*arena as usize).is_some_and(|&len| *index < len); + if !ok { + return Err(LfmViolation::ArenaOutOfBounds { + arena: *arena, + index: *index, + }); + } + } + } + Ok(()) +} + +fn is_bool(v: &FE) -> bool { + *v == FE::zero() || *v == FE::one() +} + +/// Checks 5–6 on the emitted column groups: selector one-hot-ness on real +/// rows, all-zero padding beyond the program length — plus the shape +/// cross-check against the instruction partition. +fn check_groups(program: &LfmProgram) -> Result<(), LfmViolation> { + let g = &program.groups; + + let chip_real = |chip: &'static str, group: &ColumnGroup, count: usize| { + if group.real_rows == count { + Ok(()) + } else { + Err(LfmViolation::GroupShapeMismatch { chip }) + } + }; + let counts = partition_counts(&program.instrs); + chip_real("LFM_CONST", &g.const_, counts.const_)?; + chip_real("LFM_BALU", &g.balu, counts.balu)?; + chip_real("LFM_XALU", &g.xalu, counts.xalu)?; + chip_real("LFM_SELECT", &g.select, counts.select)?; + chip_real("LFM_BITDEC", &g.bitdec, counts.bitdec)?; + chip_real("LFM_HASH", &g.hash, counts.hash)?; + chip_real("LFM_KECCAK", &g.keccak, counts.keccak)?; + chip_real("LFM_LANES", &g.lanes, counts.lanes)?; + chip_real("LFM_HINT", &g.hint, counts.hint)?; + chip_real("LFM_PUBLIC", &g.public, counts.public)?; + + // Selector one-hot / is_real flags on real rows. + one_hot( + &g.balu, + "LFM_BALU", + layout::balu::SEL_ADD, + layout::balu::NUM_SELECTORS, + )?; + one_hot( + &g.xalu, + "LFM_XALU", + layout::xalu::SEL_ADD, + layout::xalu::NUM_SELECTORS, + )?; + one_hot( + &g.hash, + "LFM_HASH", + layout::hash::MODE_C, + layout::hash::NUM_SELECTORS, + )?; + one_hot(&g.lanes, "LFM_LANES", layout::lanes::MODE_PACK, 2)?; + one_hot(&g.keccak, "LFM_KECCAK", layout::keccak::MODE_PERM, 2)?; + flag_is_one(&g.select, "LFM_SELECT", layout::select::IS_REAL)?; + flag_is_one(&g.bitdec, "LFM_BITDEC", layout::bitdec::IS_REAL)?; + flag_is_one(&g.public, "LFM_PUBLIC", layout::public::IS_REAL)?; + + // Padding: everything beyond the real rows is zero. + for (chip, group) in [ + ("LFM_CONST", &g.const_), + ("LFM_BALU", &g.balu), + ("LFM_XALU", &g.xalu), + ("LFM_SELECT", &g.select), + ("LFM_BITDEC", &g.bitdec), + ("LFM_HASH", &g.hash), + ("LFM_KECCAK", &g.keccak), + ("LFM_LANES", &g.lanes), + ("LFM_HINT", &g.hint), + ("LFM_PUBLIC", &g.public), + ] { + for row in group.real_rows..group.padded_rows { + for col in 0..group.width { + if *group.at(row, col) != FE::zero() { + return Err(LfmViolation::DirtyPadding { chip, row }); + } + } + } + } + + check_mult_ranges(g)?; + Ok(()) +} + +/// The hard ceiling on any multiplicity, independent of the per-chip +/// accounting below and of how that accounting might drift: `2^32`, the same +/// canonical-range shape check 8 uses on the keccak tags. Under it no +/// multiplicity can reach the half of the field the bus reads as negative, and +/// no sum of them over a program of any buildable size can wrap the modulus — +/// which is what the LogUp argument needs in order to mean what it says. +const MULT_HARD_CAP: u64 = 1 << 32; + +/// The largest value a multiplicity may legally take. +/// +/// A multiplicity is the number of times its cell is read, so it cannot exceed +/// the number of reads the *whole program* emits — which is bounded by the +/// committed row counts times the per-chip count of `LfmMem` receivers. Those +/// counts mirror `chips::*::bus_interactions`; over-counting only widens an +/// upper bound (safe), under-counting would reject honest programs, so where +/// two receivers are mutually exclusive the count rounds up. +fn mult_bound(g: &LfmColumnGroups) -> u64 { + let receives_per_row: [(&ColumnGroup, u64); 10] = [ + (&g.const_, 0), // reads nothing + (&g.balu, 3), // A, B, C + (&g.xalu, 3), // A, B, C + (&g.select, 3), // BIT, IN_L, IN_R + (&g.bitdec, 1), // IN + (&g.hash, 3), // IN0, IN1, IN2 + ( + &g.keccak, + (layout::keccak::NUM_WORDS + layout::keccak::BLOCK_WORDS) as u64, + ), // state + rate block + (&g.lanes, 5), // the word (Unpack) or the four lanes (Pack) + (&g.hint, 0), // reads nothing + (&g.public, 1), // the published cell + ]; + let reads = receives_per_row.iter().fold(0u64, |acc, (group, per_row)| { + acc.saturating_add((group.real_rows as u64).saturating_mul(*per_row)) + }); + reads.min(MULT_HARD_CAP - 1) +} + +/// Every preprocessed column that gates an `LfmMem` **send** — i.e. every +/// write multiplicity — per chip, mirroring `chips::*::bus_interactions`. +/// +/// `LFM_PUBLIC` contributes none: it only receives, gated by `IS_REAL`, which +/// check 5 pins to 1. Every other receive gate is likewise a selector already +/// pinned to `{0,1}` on real rows by check 5 and to 0 on padding rows by +/// check 6 — the send gates listed here are the only unbounded ones. +fn mult_columns(g: &LfmColumnGroups) -> Vec<(&'static str, &ColumnGroup, Vec)> { + use layout::{balu, bitdec, const_, hash, hint, keccak, lanes, select, xalu}; + vec![ + ("LFM_CONST", &g.const_, vec![const_::MULT]), + ("LFM_BALU", &g.balu, vec![balu::MULT]), + ("LFM_XALU", &g.xalu, vec![xalu::MULT]), + ( + "LFM_SELECT", + &g.select, + vec![select::MULT_L, select::MULT_R], + ), + ( + "LFM_BITDEC", + &g.bitdec, + (0..bitdec::NUM_BITS).map(bitdec::bit_mult).collect(), + ), + ( + "LFM_HASH", + &g.hash, + vec![hash::MULT0, hash::MULT1, hash::MULT2], + ), + ( + "LFM_KECCAK", + &g.keccak, + (0..keccak::NUM_WORDS) + .map(keccak::mult) + .chain((0..keccak::DIGEST_WORDS).map(keccak::rev_mult)) + .collect(), + ), + ( + "LFM_LANES", + &g.lanes, + (0..4) + .map(|i| lanes::LANE_MULT0 + i) + .chain([lanes::WORD_MULT]) + .collect(), + ), + ("LFM_HINT", &g.hint, vec![hint::MULT]), + ] +} + +/// Check 9: every multiplicity column of every group holds a canonically +/// small non-negative integer. +/// +/// This is the one check on this list that is deliberately +/// **instrs-independent**. Checks 1–4 reach the committed columns only through +/// `program.instrs`, so any slot the `Instr` accessors do not expose is +/// invisible to them while still being emitted onto the bus, and the two +/// objects are joined by nothing but the emitter. Reading the committed +/// columns directly is what makes this hold whatever the instruction list +/// claims — and a field-negative multiplicity, the shape the LogUp count +/// argument cannot see (see [`LfmViolation::MultOutOfRange`]), is a canonical +/// value near `p` and dies here. +/// +/// Padding rows are skipped: check 6 already pins them to all-zero. +fn check_mult_ranges(g: &LfmColumnGroups) -> Result<(), LfmViolation> { + let bound = mult_bound(g); + for (chip, group, cols) in mult_columns(g) { + for row in 0..group.real_rows { + for &col in &cols { + let mult = GoldilocksField::canonical(group.at(row, col).value()); + if mult > bound { + return Err(LfmViolation::MultOutOfRange { + chip, + row, + col, + mult, + bound, + }); + } + } + } + } + Ok(()) +} + +fn one_hot( + group: &ColumnGroup, + chip: &'static str, + first_sel: usize, + num_sels: usize, +) -> Result<(), LfmViolation> { + for row in 0..group.real_rows { + let mut ones = 0usize; + for s in 0..num_sels { + let v = group.at(row, first_sel + s); + if !is_bool(v) { + return Err(LfmViolation::NonOneHotSelector { chip, row }); + } + if *v == FE::one() { + ones += 1; + } + } + if ones != 1 { + return Err(LfmViolation::NonOneHotSelector { chip, row }); + } + } + Ok(()) +} + +fn flag_is_one(group: &ColumnGroup, chip: &'static str, col: usize) -> Result<(), LfmViolation> { + for row in 0..group.real_rows { + if *group.at(row, col) != FE::one() { + return Err(LfmViolation::NonOneHotSelector { chip, row }); + } + } + Ok(()) +} + +struct PartitionCounts { + const_: usize, + balu: usize, + xalu: usize, + select: usize, + bitdec: usize, + hash: usize, + keccak: usize, + lanes: usize, + hint: usize, + public: usize, +} + +fn partition_counts(instrs: &[Instr]) -> PartitionCounts { + let mut c = PartitionCounts { + const_: 0, + balu: 0, + xalu: 0, + select: 0, + bitdec: 0, + hash: 0, + keccak: 0, + lanes: 0, + hint: 0, + public: 0, + }; + for i in instrs { + match i { + Instr::Const { .. } => c.const_ += 1, + Instr::BaseAlu { .. } => c.balu += 1, + Instr::ExtAlu { .. } => c.xalu += 1, + Instr::Select { .. } => c.select += 1, + Instr::BitDec { .. } => c.bitdec += 1, + Instr::Hash { .. } => c.hash += 1, + Instr::KeccakF(_) => c.keccak += 1, + Instr::Pack { .. } | Instr::Unpack { .. } => c.lanes += 1, + Instr::Hint { .. } => c.hint += 1, + Instr::Public { .. } => c.public += 1, + } + } + c +} diff --git a/prover/src/lfm/word.rs b/prover/src/lfm/word.rs new file mode 100644 index 000000000..4d66f5908 --- /dev/null +++ b/prover/src/lfm/word.rs @@ -0,0 +1,61 @@ +//! The LFM machine word: `[F; 4]`, four Goldilocks elements. +//! +//! The word is digest-aligned, not extension-aligned: a Goldilocks-native hash +//! at a 128-bit target uses a 4-felt digest and a 12-felt state, so a digest is +//! exactly one cell, the sponge rate two cells and the state three cells. Base +//! values occupy lane 0 with lanes 1–3 zero; extension values (Fp3) occupy +//! lanes 0–2 with lane 3 zero. The zero lanes are enforced on the bus as +//! constant tuple entries, never as trace columns, so a base value cannot +//! smuggle a phantom extension element. + +use crate::tables::types::{FE, FEE, GoldilocksField}; +use math::field::traits::IsPrimeField; + +/// One machine word / memory cell: four Goldilocks elements. +pub type LfmWord = [FE; 4]; + +/// Number of felt lanes in a word. +pub const WORD_LANES: usize = 4; + +/// A base field value embedded as a word: `(v, 0, 0, 0)`. +pub fn base_word(v: FE) -> LfmWord { + [v, FE::zero(), FE::zero(), FE::zero()] +} + +/// An Fp3 extension value embedded as a word: `(a0, a1, a2, 0)`. +pub fn ext_word(e: &FEE) -> LfmWord { + let [a0, a1, a2] = *e.value(); + [a0, a1, a2, FE::zero()] +} + +/// Reads a word as a base value. `None` unless lanes 1–3 are zero — mirrors +/// the bus-level rule that a base receive carries constant zero high lanes. +pub fn word_as_base(w: &LfmWord) -> Option { + (w[1] == FE::zero() && w[2] == FE::zero() && w[3] == FE::zero()).then(|| w[0]) +} + +/// Reads a word as an Fp3 value. `None` unless lane 3 is zero. +pub fn word_as_ext(w: &LfmWord) -> Option { + (w[3] == FE::zero()).then(|| FEE::new([w[0], w[1], w[2]])) +} + +/// Packs a digest word into the 32-byte commitment format: four canonical +/// u64 lanes, little-endian, in lane order. Exact: 4 × 8 bytes. +pub fn pack_digest(w: &LfmWord) -> [u8; 32] { + let mut out = [0u8; 32]; + for (lane, chunk) in w.iter().zip(out.chunks_exact_mut(8)) { + chunk.copy_from_slice(&GoldilocksField::canonical(lane.value()).to_le_bytes()); + } + out +} + +/// Inverse of [`pack_digest`]. Lanes are reduced mod p on the way in. +pub fn unpack_digest(bytes: &[u8; 32]) -> LfmWord { + let mut lanes = [FE::zero(), FE::zero(), FE::zero(), FE::zero()]; + for (lane, chunk) in lanes.iter_mut().zip(bytes.chunks_exact(8)) { + let mut raw = [0u8; 8]; + raw.copy_from_slice(chunk); + *lane = FE::from(u64::from_le_bytes(raw)); + } + lanes +} diff --git a/prover/src/lfm/wrap_tests.rs b/prover/src/lfm/wrap_tests.rs new file mode 100644 index 000000000..8ef68ca33 --- /dev/null +++ b/prover/src/lfm/wrap_tests.rs @@ -0,0 +1,1071 @@ +//! ★ THE WRAP — the assembled epoch verifier PROVED, not just executed. +//! +//! [`super::epoch_verify_tests`] runs the whole epoch verifier under +//! [`super::executor::execute`]: every check is an assert inside the program, so +//! reaching the end of the execution is the verification passing. What that says +//! nothing about is the CHIPS (standing-decisions method rule 2 — where the +//! executor mirrors a computation the chip also does, only a prove+verify test +//! sees the chip). This module is the wrap: the same program, the same real +//! epoch, through [`lfm_prove`] and [`verify_against`]. +//! +//! ## What the numbers here are, and are not +//! +//! Every cost figure names the epoch's trace-length profile (assembly ledger +//! entry 10). Two different epoch shapes are involved and they must never be +//! conflated: +//! +//! - the INNER epoch's shape — the proof being verified: its per-table trace +//! lengths, its blowup and its query count. This is what makes the emitted +//! verifier program big or small. +//! - the WRAP's own [`ProofOptions`] — blowup 2, the same options every other +//! LFM prove test uses. It fixes what proving the verifier costs, not what the +//! verifier does. +//! +//! ## What this module cannot see +//! +//! The hash. Every permutation here is `TestPermutation` inside the LFM chips +//! plus the production keccak family hosted for `keccak256`; the point of +//! measuring cells at all is to have the first column of a matrix whose other +//! columns (blake, Poseidon) do not exist yet. It also cannot see prove time or +//! peak memory as a property of the machine — those are measured around the +//! process, by the harness that runs it, and are reported as observations of one +//! box rather than as machine invariants. + +use std::time::Instant; + +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; + +use super::airs::{LfmChipCells, lfm_cell_counts, lfm_chip_census}; +use super::compiler::LfmProgram; +use super::epoch_tests::EpochInputs; +use super::executor::execute; +use super::hash::TestPermutation; +use super::instr::Instr; +use super::proof::{LfmProveError, lfm_prove, lfm_prove_with_residency, verify_against}; +use super::registry::build_artifacts; + +use crate::tables::types::FE; + +/// The WRAP proof's own options: blowup 2, the framework's 128-bit query count. +/// +/// Deliberately the same `prove_options()` every leg suite proved under +/// (`join_tests`, `fri_tests`, `constraint_tests`), so a wrap cost is comparable +/// with a leg cost. The inner epoch's options are a different thing entirely and +/// are named per measurement. +fn wrap_options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid") +} + +/// Instructions of each kind, for the shape line every measurement prints. +fn instruction_mix(program: &LfmProgram) -> String { + let count = |f: fn(&Instr) -> bool| program.instrs.iter().filter(|i| f(i)).count(); + format!( + "const {} / base-alu {} / ext-alu {} / select {} / bitdec {} / hash {} / \ + keccak {} / hint {} / pack {} / unpack {} / public {}", + count(|i| matches!(i, Instr::Const { .. })), + count(|i| matches!(i, Instr::BaseAlu { .. })), + count(|i| matches!(i, Instr::ExtAlu { .. })), + count(|i| matches!(i, Instr::Select { .. })), + count(|i| matches!(i, Instr::BitDec { .. })), + count(|i| matches!(i, Instr::Hash { .. })), + count(|i| matches!(i, Instr::KeccakF(_))), + count(|i| matches!(i, Instr::Hint { .. })), + count(|i| matches!(i, Instr::Pack { .. })), + count(|i| matches!(i, Instr::Unpack { .. })), + count(|i| matches!(i, Instr::Public { .. })), + ) +} + +/// Keccak permutations a program requests. +pub(super) fn permutations(program: &LfmProgram) -> usize { + program + .instrs + .iter() + .filter(|i| matches!(i, Instr::KeccakF(_))) + .count() +} + +/// Arena words a program declares. +pub(super) fn arena_words(program: &LfmProgram) -> usize { + program.arena_schema.lens.iter().map(|l| *l as usize).sum() +} + +/// ★ The registry-entry shape record: what the machine proves for one program. +/// +/// Prints the chip census — one line per SUB-PROOF, since `KECCAK_RND`'s chunks +/// are separate AIRs at separate heights — and the totals the hash matrix wants. +/// Returns `(main_cells, aux_cells)` so a caller can assert on them. +pub(super) fn report_census(label: &str, program: &LfmProgram) -> (u64, u64) { + let census = lfm_chip_census(program); + let (main, aux) = lfm_cell_counts(program); + // The census is `lfm_cell_counts`' own decomposition, so summing it is not an + // independent check of the total — it is the same arithmetic. What IS + // independent is that the sub-proof COUNT the census implies must equal the + // AIR count the verifier builds from the program's chunk policy. + assert_eq!( + census.len(), + super::airs::num_lfm_airs( + program + .chunking + .chunk_count(program.groups.keccak.real_rows) + ), + "the census must have one entry per sub-proof the AIR set builds" + ); + println!("\n★ CHIP CENSUS — {label}"); + println!( + " {:>12} {:>10} {:>6} {:>6} {:>16} {:>14}", + "chip", "rows", "main", "aux", "main cells", "aux cells" + ); + for c in &census { + println!( + " {:>12} {:>10} {:>6} {:>6} {:>16} {:>14}", + c.name, + c.rows, + c.main_cols, + c.aux_cols, + c.main_cells(), + c.aux_cells() + ); + } + println!( + " {:>12} {:>10} {:>6} {:>6} {:>16} {:>14}", + "TOTAL", + census.iter().map(|c| c.rows).sum::(), + "", + "", + main, + aux + ); + println!( + " cells per verify = {main} main + {aux} aux ext = {} base-field equivalents \ + (an ext element is 3 base felts)", + main + 3 * aux + ); + + // ★ THE HASH SHARE — what the matrix is about. + // + // The whole reason to count cells is to price the hash, so the census says + // outright how much of the machine IS the hash. `LFM_KECCAK` (the adapter row + // that requests a permutation) and `KECCAK_RND` (its 24 rounds) are the + // permutation itself; `KECCAK_RC` and `BITWISE` are the lookup tables it reads, + // and they are reported separately because they are FIXED-height — a different + // hash would delete the first pair and shrink but not necessarily remove the + // second. + let share = |names: &[&str]| -> (u64, u64) { + census + .iter() + .filter(|c| names.contains(&c.name)) + .fold((0u64, 0u64), |(m, a), c| { + (m + c.main_cells(), a + c.aux_cells()) + }) + }; + let (perm_main, perm_aux) = share(&["LFM_KECCAK", "KECCAK_RND"]); + let (tab_main, tab_aux) = share(&["KECCAK_RC", "BITWISE"]); + let total = (main + 3 * aux) as f64; + println!( + " keccak permutation chips (LFM_KECCAK + KECCAK_RND): {perm_main} main + \ + {perm_aux} aux = {:.1}% of cells\n \ + its lookup tables (KECCAK_RC + BITWISE, fixed height): {tab_main} main + \ + {tab_aux} aux = {:.1}%\n \ + everything else (the verifier's own arithmetic): {:.1}%", + 100.0 * (perm_main + 3 * perm_aux) as f64 / total, + 100.0 * (tab_main + 3 * tab_aux) as f64 / total, + 100.0 * (main + 3 * aux - perm_main - 3 * perm_aux - tab_main - 3 * tab_aux) as f64 / total, + ); + println!(" instruction mix: {}", instruction_mix(program)); + (main, aux) +} + +/// The three headline shape numbers, printed with the epoch profile that fixes +/// them (ledger entry 10). +pub(super) fn report_program(label: &str, profile: &str, program: &LfmProgram) { + println!( + "\n★ {label}\n epoch trace lengths (log2): {profile}\n \ + {} instructions / {} keccak permutations / {} arena words / {} chunks", + program.instrs.len(), + permutations(program), + arena_words(program), + program + .chunking + .chunk_count(program.groups.keccak.real_rows), + ); +} + +/// The INNER epoch's own committed trace cells — `(main, aux ext)` — summed over +/// its sub-proofs. +/// +/// The denominator of the recursion ratio, and the only honest one available from +/// shapes alone: `rows x main_width` and `rows x aux_width` per sub-proof, which is +/// the same accounting [`lfm_chip_census`] applies to the machine (value columns +/// plus aux, one ext element per aux column per row). +/// +/// What it CANNOT see, on both sides equally: preprocessed columns, the +/// composition polynomial's own commitment, the LDE, and the Merkle trees. So the +/// ratio it feeds is "trace cells to verify one epoch's trace cells", not "total +/// prover work", and it is quoted that way. +fn inner_epoch_cells(e: &super::epoch_tests::RealEpoch) -> (u64, u64) { + e.legs + .iter() + .map(|l| { + let rows = 1u64 << l.verify.sub.deep.log2_trace_length; + let aux_width = l.verify.sub.deep.num_total_cols - l.verify.main_width; + (rows * l.verify.main_width as u64, rows * aux_width as u64) + }) + .fold((0, 0), |(m, a), (dm, da)| (m + dm, a + da)) +} + +/// Prints the recursion ratio: machine cells per verify against the verified +/// epoch's own cells. The kill-risk-3 question, asked of a real epoch at last. +fn report_ratio(e: &super::epoch_tests::RealEpoch, main: u64, aux: u64) { + let (inner_main, inner_aux) = inner_epoch_cells(e); + let inner = inner_main + 3 * inner_aux; + let outer = main + 3 * aux; + println!( + " the epoch VERIFIED carries {inner_main} main + {inner_aux} aux ext = {inner} \ + base-field-equivalent trace cells\n \ + so verifying it costs {:.1}x its own trace cells (trace-to-trace; neither \ + side counts preprocessed columns, LDEs or trees)", + outer as f64 / inner as f64, + ); +} + +/// The epoch's trace-length profile as ledger entry 10 wants it printed. +pub(super) fn epoch_profile(e: &super::epoch_tests::RealEpoch) -> String { + let mut lengths: Vec = e + .legs + .iter() + .map(|l| l.verify.sub.deep.log2_trace_length) + .collect(); + lengths.sort_unstable(); + let mut runs: Vec = Vec::new(); + for len in &lengths { + let n = lengths.iter().filter(|l| *l == len).count(); + let entry = if n > 1 { + format!("{len} x{n}") + } else { + format!("{len}") + }; + if !runs.contains(&entry) { + runs.push(entry); + } + } + format!("[{}]", runs.join(", ")) +} + +/// ★ SLICE 0 — the wrap on the min-preset fixture epoch: prove, verify, tamper. +/// +/// `#[ignore]`d, and the reason is the cost: the assembled verifier is ~2.25M +/// instructions, so the LFM traces are an order of magnitude past anything else +/// in this suite and the run is minutes of CPU and tens of gigabytes. It is the +/// wrap run's own harness, not a test the suite can afford on every PR. +/// +/// Run with: +/// `cargo test --release -p lambda-vm-prover --lib lfm::wrap_tests::the_wrap_proves_and_verifies -- --ignored --nocapture` +#[test] +#[ignore] +fn the_wrap_proves_and_verifies() { + wrap_run(super::proof_fixture::fixture_options()); +} + +/// ★ SLICE 0's GPU-dispatch census (`thoughts/shared/gpu-recursion/EXPLORATION.md`, +/// Stage 0). The min-preset wrap proved once, with the process-global GPU call +/// counters reset right before `lfm_prove` — after the inner epoch is built, +/// because building it proves an RV64 continuation whose own GPU traffic (the +/// VM's preprocessed tables clear the size gate even for a 16-cycle epoch) would +/// otherwise pollute the machine's numbers. Prints every counter rather than +/// asserting floors: this is the falsification harness for the GPU map, and the +/// predictions are the document's to state, not the test's to freeze. Needs +/// `--test-threads=1` (the counters are process-global) and, like the rest of +/// the cuda suite, `--ignored` so the no-GPU CI path keeps skipping it. +#[cfg(feature = "cuda")] +#[test] +#[ignore] +fn the_wrap_reports_gpu_counters() { + use stark::gpu_lde as g; + + let e = super::epoch_tests::real_epoch_with(super::proof_fixture::fixture_options()); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let opts = wrap_options(); + let artifacts = build_artifacts(&program, &opts); + println!(" chip log-heights: {:?}", artifacts.log_heights); + + g::reset_all_gpu_call_counters(); + let t = Instant::now(); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("the wrap must prove"); + let prove_secs = t.elapsed().as_secs_f64(); + println!( + "\n★ GPU DISPATCH COUNTERS (min-preset wrap, lfm_prove only, {prove_secs:.1}s):\n \ + lde {} / leaf_hash {} / merkle_tree {} / extend_halves {} / logup {}\n \ + composition {} / comp_poly_tree {} / parts_lde {} / bary {} / deep {}\n \ + batch_invert {} / fri {} / opening_gather {} / device_only {}", + g::gpu_lde_calls(), + g::gpu_leaf_hash_calls(), + g::gpu_merkle_tree_calls(), + g::gpu_extend_halves_calls(), + g::gpu_logup_calls(), + g::gpu_composition_calls(), + g::gpu_comp_poly_tree_calls(), + g::gpu_parts_lde_calls(), + g::gpu_bary_calls(), + g::gpu_deep_calls(), + g::gpu_batch_invert_calls(), + g::gpu_fri_calls(), + g::gpu_opening_gather_calls(), + g::gpu_device_only_calls(), + ); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the wrap proof must verify" + ); +} + +/// ★ SLICE 1 (local rung) — the wrap at the inner proof's BLOWUP-8 GEOMETRY. +/// +/// The standing decision is that the inner proof is at blowup 8, and blowup is +/// not a rescaling of blowup 2: the LDE is four times deeper, so every Merkle walk +/// climbs two more levels, the FRI chain commits more layers, and the terminal +/// polynomial is reached from further away. None of that is exercised by slice 0. +/// +/// The QUERY count is the one thing reduced, and reduced for a stated reason: at +/// the real 73 queries the wrap's own trace does not fit in any box we have (see +/// [`the_wrap_census_at_blowup_8`], which measures the program and prints what +/// proving it would need). ONE query is what a 36 GiB local box holds, and it is +/// enough to make every blowup-8 structure real — the deeper walk, the longer fold +/// chain, the terminal polynomial reached from further away — since what falls out +/// at one query is only the REPETITION of that structure. An honest partial: the +/// GEOMETRY is proved, the query COUNT is not, and the two are separable because +/// per-query cost is a closed form over the shapes that +/// [`the_wrap_census_at_blowup_8`] asserts the emitted program against. +/// `LFM_WRAP_QUERIES` raises the inner query count above the 1 this asserts at, +/// which is how the residency ladder walks the wrap up until a box refuses it. +/// Unset — every CI and local run — it is exactly the one-query test described +/// above. +#[test] +#[ignore] +fn the_wrap_proves_at_blowup_8_geometry() { + let queries = match std::env::var("LFM_WRAP_QUERIES") { + Ok(v) => v.parse().expect("LFM_WRAP_QUERIES must be an integer"), + Err(_) => 1, + }; + wrap_run(inner_blowup_8_with_queries(queries)); +} + +/// The inner proof's blowup-8 options with the query count overridden. +/// +/// NOT a security parameter set at anything below 73 queries, and never used as +/// one: the query count is what this reduces and every measurement taken under it +/// says so in its label. +fn inner_blowup_8_with_queries(queries: usize) -> ProofOptions { + let mut o = crate::recursion::Preset::Blowup8.options(); + o.fri_number_of_queries = queries; + o +} + +/// The wrap, end to end, under supplied INNER proof options, over whatever epoch +/// [`EpochInputs::from_env`] names — the fibonacci fixture unless a measurement +/// run overrode it. +fn wrap_run(inner: ProofOptions) { + wrap_run_from(inner, EpochInputs::from_env()); +} + +/// [`wrap_run`] over an explicitly supplied epoch: build it, emit the verifier, +/// prove it, verify it, and run the three falsifications. +fn wrap_run_from(inner: ProofOptions, inputs: EpochInputs) { + let t_epoch = Instant::now(); + let e = super::epoch_tests::real_epoch_from(inner.clone(), inputs); + let profile = epoch_profile(&e); + println!( + "inner epoch: {} sub-proofs, blowup {}, {} quer{} per table, grinding {} — built in {:.1}s", + e.legs.len(), + 1 << e.tables[0].shape.log2_blowup, + e.legs[0].verify.num_queries, + if e.legs[0].verify.num_queries == 1 { + "y" + } else { + "ies" + }, + e.tables[0].shape.grinding_factor, + t_epoch.elapsed().as_secs_f64() + ); + + // The GEOMETRY the blowup fixes, stated per run: what the walks climb and what + // the FRI chain folds. This is what separates a blowup-8 run from a blowup-2 one + // at the same query count, so it is printed rather than left to the label. + let big = e + .legs + .iter() + .max_by_key(|l| l.verify.sub.deep.log2_trace_length) + .expect("the epoch has sub-proofs"); + println!( + " geometry: widest sub-proof 2^{} trace -> 2^{} LDE, {} Merkle levels per group, \ + {} committed FRI layers ({} across the epoch); widest leaf {} bytes", + big.verify.sub.deep.log2_trace_length, + big.verify.sub.log2_lde_length, + big.verify.sub.merkle_depth, + big.verify.fri.num_committed(), + e.legs + .iter() + .map(|l| l.verify.fri.num_committed()) + .sum::(), + e.legs + .iter() + .flat_map(|l| l.verify.sub.groups()) + .map(|g| g.leaf_bytes()) + .max() + .expect("the epoch has groups"), + ); + + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + report_program("THE WRAPPED PROGRAM", &profile, &program); + let (main, aux) = report_census( + &format!("assembled epoch verifier, epoch {profile}"), + &program, + ); + + // ---- the spine/legs split, and the legs' permutations against a CLOSED FORM. + // + // Both halves matter and for different reasons. The split is what makes two + // runs at different query counts comparable at all: the SPINE also grows with + // the query count (it samples an index per query, and every sample is + // transcript work), so "permutations per query" taken from the total is wrong + // and taken from the difference is right. The closed form is the absolute + // check rule 7's refinement demands — `query_permutations` is arithmetic over + // byte widths and tree depths, not a second pass of this emitter, so a leg + // that quietly stopped hashing a group fails here rather than printing a + // smaller number. + let spine = super::epoch_tests::epoch_program(&e, false); + let leg_perms = permutations(&program) - permutations(&spine); + let predicted: usize = e + .legs + .iter() + .map(|l| super::epoch_verify::query_permutations(&l.verify)) + .sum(); + assert_eq!( + leg_perms, predicted, + "the emitted leg permutations must equal the closed form over the shapes" + ); + let queries = e.legs[0].verify.num_queries; + println!( + " spine {} instr / {} perms / {} words legs {} / {} / {} \ + per query: {:.1} perms ({} queries, closed form checked)", + spine.instrs.len(), + permutations(&spine), + arena_words(&spine), + program.instrs.len() - spine.instrs.len(), + leg_perms, + arena_words(&program) - arena_words(&spine), + leg_perms as f64 / queries as f64, + queries, + ); + + report_ratio(&e, main, aux); + + let opts = wrap_options(); + let artifacts = build_artifacts(&program, &opts); + println!( + " wrap options: blowup {}, {} queries, grinding {}\n chip log-heights: {:?}", + opts.blowup_factor, opts.fri_number_of_queries, opts.grinding_factor, artifacts.log_heights + ); + + // ---- PROVE. + let t = Instant::now(); + let proved = lfm_prove(&program, &artifacts, &arenas, &opts).expect("the wrap must prove"); + let prove_secs = t.elapsed().as_secs_f64(); + + let size = rkyv::to_bytes::(&proved.proof) + .expect("the wrap proof must serialize") + .len(); + + // ---- VERIFY. + let t = Instant::now(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the wrap proof must verify" + ); + let verify_secs = t.elapsed().as_secs_f64(); + println!( + "\n★ WRAP PROVED AND VERIFIED (inner epoch {profile}, blowup {}, {} quer{})\n \ + prove {prove_secs:.1}s / verify {verify_secs:.2}s / proof {size} bytes / \ + {} published words / {} sub-proofs\n \ + cells {main} main + {aux} aux ext; the projection for this run was \ + {:.1} GiB of peak RSS — compare against what the harness measured around \ + the process", + inner.blowup_factor, + inner.fri_number_of_queries, + if inner.fri_number_of_queries == 1 { + "y" + } else { + "ies" + }, + proved.public_words.len(), + proved.proof.proofs.len(), + projected_peak_bytes(main, aux) / (1u64 << 30) as f64, + ); + + // ---- the published words are the ones the execution produced, so the + // spine's differential still holds of the PROVED run and not only of an + // execution. Checked by value against the epoch's own oracles. + let pub_ext = + |i: usize| super::word::word_as_ext(&proved.public_words[i].1).expect("an ext challenge"); + assert_eq!(pub_ext(0), e.z_alpha.0, "the proved run publishes z"); + assert_eq!(pub_ext(1), e.z_alpha.1, "the proved run publishes alpha"); + assert_eq!( + super::word::word_as_ext(&proved.public_words[proved.public_words.len() - 1].1) + .expect("the bus total is ext"), + e.expected_bus_balance, + "the proved run reaches production's own COMMIT-bus target" + ); + + // ---- FALSIFICATION 1: a tampered inner proof makes the wrap UNBUILDABLE. + // + // Not "unverifiable": every check is an assert inside a straight-line + // program, so a false statement has no execution at all — there is no branch + // to take and no error path to return, and `lfm_prove` fails in `execute` + // before a trace exists. That is the designed behaviour of the machine, and + // it is why the positive result above ("it proved") is the verification. + let ix = super::epoch_verify_tests::arena_index(&e, 0); + let mut tampered = arenas.clone(); + tampered[ix.openings][0][0] += FE::one(); + match lfm_prove(&program, &artifacts, &tampered, &opts) { + Err(LfmProveError::Exec(err)) => { + println!(" TAMPERED opened value 0 of table 0: the wrap is UNBUILDABLE ({err:?})") + } + Err(LfmProveError::Prover(err)) => { + panic!("a tampered inner proof must fail in execution, not in the prover: {err:?}") + } + Ok(_) => panic!("a tampered opened value must not produce a wrap proof"), + } + + // ---- FALSIFICATION 2: the honest proof against a MOVED claimed statement. + // + // The other half of the pair: the wrap proof is bound to the public words it + // published (`absorb_lfm_statement`), so a verifier handed the real proof and + // a different claim must reject. This is the path that rejects rather than + // failing to build, and both must exist — a machine where only the first + // existed would prove nothing about what the proof says. + let mut moved = proved.public_words.clone(); + moved[0].1[0] += FE::one(); + assert!( + !verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &moved, + &opts, + artifacts.hasher, + ), + "a moved claimed public word must make the wrap proof UNVERIFIABLE" + ); + println!(" MOVED claimed public word 0: the wrap proof is UNVERIFIABLE"); + + // ---- FALSIFICATION 3: the same proof against another program's identity. + // + // The registry premise. `verify_against` takes the roots and the digest, and + // a proof of THIS program must not verify as a proof of a different one. + let mut other = artifacts.program_id; + other[0] ^= 1; + assert!( + !verify_against( + &artifacts.roots, + &other, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "a moved program digest must make the wrap proof UNVERIFIABLE" + ); + println!(" MOVED program digest: the wrap proof is UNVERIFIABLE"); +} + +/// ★ THE RESIDENCY ORACLE — `ResidencyMode::RecomputeLde` produces the same wrap +/// commitments as `Retain`. +/// +/// [`crate::tests::residency_mode_tests`] in the stark crate pins the mechanism +/// on a three-table toy, byte for byte. This pins it on the workload the mode +/// exists for: the wrap has preprocessed tables (whose main LDE carries the +/// precomputed columns the split trees were built from), `KECCAK_RND` chunks +/// (the family whose retention the mode drops), and the real transcript. +/// +/// ## Why this compares commitments and not proof bytes +/// +/// Proving is **not** reproducible run to run, and it is worth being exact +/// about why, because the fixture-blob note in +/// [`super::proof_fixture`] attributes it to sub-proofs committing to different +/// roots — which is not what happens here. Measured on this test: two runs +/// build byte-identical LFM traces, produce identical roots at every stage, and +/// still serialize to different proof bytes. The cause is the grinding search, +/// `grinding::generate_nonce`, which under the `parallel` feature is +/// `into_par_iter().find_any(..)` — *any* valid nonce, so which one comes back +/// depends on thread scheduling. The nonce is absorbed before the query indices +/// are sampled, so a different nonce opens different leaves. Both proofs are +/// valid; grinding is a proof of work and any witness satisfies it. +/// +/// So everything the nonce cannot reach is compared, which is everything the +/// recomputed LDE feeds: the main, aux, precomputed, composition and FRI-layer +/// roots, the out-of-domain evaluations, the final polynomial, and the bus +/// public inputs. A recomputed LDE that disagreed with the tree Round 1 +/// committed would move the composition root and the OOD evaluations — the two +/// values in that set derived from the LDE rather than from the trace. +/// +/// The run is its own control: `Retain` is proved twice, and the first +/// comparison is `Retain` against `Retain`. If that one ever fails, the +/// nondeterminism has reached the commitments and this oracle — not the +/// residency mode — is what needs fixing. +/// +/// `#[ignore]`d for the same reason as [`the_wrap_proves_and_verifies`], three +/// times over: it proves the wrap three times. +/// +/// Run with: +/// `cargo test --release -p lambda-vm-prover --lib lfm::wrap_tests::the_wrap_commitments_match_across_residency_modes -- --ignored --nocapture` +#[test] +#[ignore] +fn the_wrap_commitments_match_across_residency_modes() { + use stark::residency_mode::ResidencyMode; + + let e = super::epoch_tests::real_epoch_with(super::proof_fixture::fixture_options()); + let program = super::epoch_tests::epoch_program(&e, true); + let arenas = super::epoch_tests::epoch_arena_words(&e, true); + let opts = wrap_options(); + let artifacts = build_artifacts(&program, &opts); + + let prove_under = |residency: ResidencyMode| { + let t = Instant::now(); + let proved = lfm_prove_with_residency( + &program, + &artifacts, + &arenas, + &opts, + artifacts.hasher, + residency, + ) + .expect("the wrap must prove"); + let secs = t.elapsed().as_secs_f64(); + assert!( + verify_against( + &artifacts.roots, + &artifacts.program_id, + artifacts.keccak_rnd_chunks, + &proved.proof, + &proved.public_words, + &opts, + artifacts.hasher, + ), + "the wrap proof must verify under {residency:?}" + ); + println!(" {residency:?}: proved in {secs:.1}s, verified"); + proved + }; + + // Everything the grinding nonce cannot reach. + let commitments = |p: &super::proof::LfmProof| { + p.proof + .proofs + .iter() + .map(|q| { + ( + q.trace_length, + q.lde_trace_main_merkle_root, + q.lde_trace_aux_merkle_root, + q.lde_trace_precomputed_merkle_root, + q.composition_poly_root, + q.composition_poly_parts_ood_evaluation.clone(), + q.trace_ood_evaluations.row_major_data().to_vec(), + q.trace_ood_next_evaluations.row_major_data().to_vec(), + q.fri_layers_merkle_roots.clone(), + q.fri_final_poly_coeffs.clone(), + q.bus_public_inputs.as_ref().map(|b| b.table_contribution), + ) + }) + .collect::>() + }; + + let retained = prove_under(ResidencyMode::Retain); + let control = prove_under(ResidencyMode::Retain); + assert!( + commitments(&retained) == commitments(&control), + "CONTROL FAILED: two Retain runs disagree on their commitments, so this \ + oracle cannot say anything about the residency mode" + ); + println!(" control: two Retain runs agree on every commitment"); + + let recomputed = prove_under(ResidencyMode::RecomputeLde); + assert!( + commitments(&retained) == commitments(&recomputed), + "the wrap commitments moved between residency modes" + ); + assert_eq!( + retained.public_words, recomputed.public_words, + "the published words moved between residency modes" + ); + + // Not asserted — recorded. The nonces are expected to differ; printing them + // keeps the reason this test compares commitments visible in its own output + // rather than only in its doc comment. + let nonces = |p: &super::proof::LfmProof| { + p.proof + .proofs + .iter() + .filter_map(|q| q.nonce) + .collect::>() + }; + println!( + " grinding nonces equal across the two Retain runs: {} (expected false under `parallel`)", + nonces(&retained) == nonces(&control) + ); + println!("\n★ RESIDENCY ORACLE: the wrap commits identically under Retain and RecomputeLde"); +} + +/// ★ GATE B — a REAL Ethereum-block epoch, wrapped. +/// +/// Everything else in this module wraps the 16-cycle fibonacci fixture, which +/// exercises every structure but at a size no production workload has. This +/// wraps one epoch of a real mainnet block at a SECURE inner preset +/// (blowup 4 / 110 queries, grinding as the preset sets it): one real block +/// epoch proof, compressed into one LFM proof. +/// +/// The guest and the block input are multi-megabyte binaries that cannot be +/// checked in, so the test requires them by path and says so rather than +/// quietly proving the fixture and reporting it as a block: +/// +/// ```text +/// LFM_CENSUS_ELF=/path/to/ethrex.elf \ +/// LFM_CENSUS_INPUT=/path/to/ethrex_mainnet_25368371.bin \ +/// LFM_CENSUS_EPOCH_LOG2=16 \ +/// cargo test --release -p lambda-vm-prover --lib \ +/// lfm::wrap_tests::the_real_block_epoch_wraps -- --ignored --nocapture +/// ``` +/// +/// ## The two knobs, and which one actually binds +/// +/// `LFM_CENSUS_EPOCH_LOG2` sets the epoch size and `LFM_WRAP_QUERIES` overrides +/// the inner query count (default: the preset's 110, which is the secure one — +/// anything lower is NOT a security parameter set and every number taken under +/// it carries the count, exactly as +/// [`the_wrap_proves_at_blowup_8_geometry`] does). +/// +/// Measured on a 60 GiB box: the epoch size is the *weak* knob and the query +/// count is the strong one. What decides whether a run fits is the number of +/// `KECCAK_RND` chunks the wrap's own trace needs, and that is +/// `(spine + per_query x queries) / 21,845` permutations. Per-query cost is +/// dominated by leaf absorption, which is set by table WIDTH and so barely +/// moves with epoch size — shrinking the epoch does not meaningfully shrink the +/// chunk count, and shrinking the query count does, linearly. +#[test] +#[ignore] +fn the_real_block_epoch_wraps() { + for var in ["LFM_CENSUS_ELF", "LFM_CENSUS_INPUT"] { + assert!( + std::env::var(var).is_ok(), + "{var} must name a file: this test wraps a REAL block epoch, and \ + without it the harness would build the fibonacci fixture and report \ + it under this test's name" + ); + } + let inputs = EpochInputs::from_env(); + let mut inner = crate::recursion::Preset::Blowup4.options(); + if let Ok(v) = std::env::var("LFM_WRAP_QUERIES") { + inner.fri_number_of_queries = v.parse().expect("LFM_WRAP_QUERIES must be an integer"); + } + println!( + "★ REAL-BLOCK WRAP: guest {}, {} bytes of private input, 2^{} cycles/epoch, \ + inner blowup {} / {} queries{}", + inputs.label, + inputs.private_input.len(), + inputs.epoch_log2, + inner.blowup_factor, + inner.fri_number_of_queries, + if inner.fri_number_of_queries < 110 { + " (REDUCED — not a security parameter set)" + } else { + " (the secure preset)" + }, + ); + wrap_run_from(inner, inputs); +} + +/// The census and shape of the assembled verifier WITHOUT proving it — the cheap +/// half of the wrap run, so the numbers exist even where the prove does not fit. +/// +/// Also the spine/legs split, since the census is what says which chips the legs +/// actually cost: at the min preset the verifier is ~50/50 Fiat-Shamir and +/// verification by instruction count, and this is where that becomes a per-chip +/// statement. +#[test] +#[ignore] +fn the_wrap_census() { + let e = super::epoch_tests::real_epoch(); + let profile = epoch_profile(&e); + let program = super::epoch_tests::epoch_program(&e, true); + let spine = super::epoch_tests::epoch_program(&e, false); + + report_program("ASSEMBLED (spine + legs)", &profile, &program); + report_program("SPINE ALONE (no legs)", &profile, &spine); + let (main, aux) = report_census(&format!("assembled, epoch {profile}"), &program); + let (spine_main, spine_aux) = report_census(&format!("spine alone, epoch {profile}"), &spine); + println!( + "\n legs' marginal cells: {} main + {} aux (assembled {} / {} against spine {} / {})", + main - spine_main, + aux - spine_aux, + main, + aux, + spine_main, + spine_aux + ); + + // The fixed-machine floor: what a program of NO instructions still pays for + // the 14 chips. The number every cells-per-verify figure sits on top of. + let empty = super::compiler::compile(super::builder::LfmBuilder::new().finish()); + let (floor_main, floor_aux) = lfm_cell_counts(&empty); + println!( + " fixed-machine floor (an empty program): {floor_main} main + {floor_aux} aux — \ + {:.1}% of the assembled verifier's main cells", + 100.0 * floor_main as f64 / main as f64 + ); + assert!( + main > floor_main, + "the verifier must cost more than the floor" + ); +} + +/// Peak prover memory the census implies, in bytes, from a MEASURED coefficient. +/// +/// The measured point is slice 0: 481,327,124 base-field-equivalent cells peaked +/// at 16,228,499,456 bytes of RSS (15.1 GiB), i.e. 33.7 bytes per cell — a trace word, its +/// blowup-2 LDE, and the Merkle/quotient working set on top. Stated as a +/// coefficient rather than derived from first principles because the derivation +/// would be a guess about the prover's allocation pattern and this is an +/// observation of it. What it CANNOT see: whether the coefficient holds at ten +/// times the size (allocator behaviour, and the fact that a bigger program is +/// bigger in different chips), so it is a projection and is labelled as one +/// wherever it is printed. +const MEASURED_BYTES_PER_CELL: f64 = 16_228_499_456.0 / 481_327_124.0; + +fn projected_peak_bytes(main: u64, aux: u64) -> f64 { + (main + 3 * aux) as f64 * MEASURED_BYTES_PER_CELL +} + +/// ★ SLICE 1 — the PRODUCTION-SHAPED census: the inner epoch at blowup 8 with its +/// real 73-query count, which is the standing decision for the inner proof. +/// +/// This is the cells-per-verify number the hash matrix wants, and it is a +/// MEASUREMENT of the emitted program rather than a projection from a per-leg +/// cost: the same emitter, the same real epoch, the same 24 sub-proofs, with only +/// the inner proof's options moved. Whether the resulting program can be PROVED is +/// a separate question and the test answers it with the projection above rather +/// than by pretending to have run it. +#[test] +#[ignore] +fn the_wrap_census_at_blowup_8() { + let inner = crate::recursion::Preset::Blowup8.options(); + let t = Instant::now(); + let e = super::epoch_tests::real_epoch_with(inner.clone()); + let profile = epoch_profile(&e); + println!( + "inner epoch: {} sub-proofs, blowup {}, {} queries per table, grinding {}, \ + fri final poly log degree {} — proved and accepted in {:.1}s", + e.legs.len(), + inner.blowup_factor, + inner.fri_number_of_queries, + inner.grinding_factor, + inner.fri_final_poly_log_degree, + t.elapsed().as_secs_f64() + ); + + let t = Instant::now(); + let program = super::epoch_tests::epoch_program(&e, true); + println!( + " emitted the assembled verifier in {:.1}s", + t.elapsed().as_secs_f64() + ); + report_program("ASSEMBLED VERIFIER @ inner blowup 8", &profile, &program); + let (main, aux) = report_census( + &format!("assembled, epoch {profile}, inner blowup 8"), + &program, + ); + report_ratio(&e, main, aux); + + // ---- MEASURED against the phase's pinned predictions, number by number. + let openings: usize = e + .legs + .iter() + .map(|l| { + l.verify.num_queries + * (super::epoch_verify::leaf_permutations(&l.verify.sub) + + l.verify.sub.groups().len() * l.verify.sub.merkle_depth) + }) + .sum(); + let fri: usize = e + .legs + .iter() + .map(|l| l.verify.num_queries * l.verify.fri.permutations_per_query()) + .sum(); + let spine = super::epoch_tests::epoch_program(&e, false); + println!( + "\n MEASURED vs PREDICTED (epoch {profile}, inner blowup 8, {} queries):\n\ + \x20 openings {openings:>9} [predicted 100,959 — wave 6's projection of THIS epoch]\n\ + \x20 FRI {fri:>9} [pinned 14,454 per 2^20 sub-proof at blowup 8]\n\ + \x20 legs total {:>9} = emitted assembled - spine\n\ + \x20 epoch bill {:>9} [design target ~460,000 for a PRODUCTION-sized epoch]", + e.legs[0].verify.num_queries, + permutations(&program) - permutations(&spine), + permutations(&program), + ); + assert_eq!( + permutations(&program) - permutations(&spine), + openings + fri, + "the emitted leg permutations must be the closed form over the shapes" + ); + + // ---- can it be proved? The projection, with its coefficient named. + let bytes = projected_peak_bytes(main, aux); + println!( + "\n PROVING THIS: {} main + {} aux ext = {} base-field-equivalent cells\n\ + \x20 projected peak RSS {:.1} GiB at the measured {:.1} bytes/cell \ + (slice 0's 15.1 GiB / 481.3M cells)\n\ + \x20 the measurement box has 124 GiB, so this is {:.1}x what fits", + main, + aux, + main + 3 * aux, + bytes / (1 << 30) as f64, + MEASURED_BYTES_PER_CELL, + bytes / (124.0 * (1u64 << 30) as f64), + ); +} + +/// Falsification of the census instrument itself: it must agree with what the +/// PROVER actually builds and with what the VERIFIER's AIR set declares. +/// +/// A census computed from the program alone would report the same numbers under a +/// broken trace builder, which is the "measures nothing" failure the method rules +/// name. Two independent oracles, both of which the census is not derived from: +/// +/// - the real [`super::trace::LfmTraces`] — the tables `multi_prove` receives — +/// for the heights; +/// - the AIR set [`super::airs::LfmAirs`] builds, for the NAMES and the widths. +/// The names matter more than they look: the census maps `per_chip` array slots +/// onto `LFM_CHIP_NAMES` across the `KECCAK_RND` slot, and nothing about a +/// height or a width can see that mapping being off by one. `air_refs` is the +/// frozen order's own definition, so comparing against it is what catches it. +#[test] +fn the_census_agrees_with_the_traces_the_prover_builds() { + // The two-permutation keccak chain: small, and it exercises every chip class + // the census names except `LFM_PUBLIC`'s value columns. + let program = super::programs::keccak_chain_program(); + let state: [u64; 25] = + core::array::from_fn(|i| 0x9E37_79B9_7F4A_7C15u64.wrapping_mul(i as u64 + 1)); + let arenas = vec![super::keccak_adapter::state_to_words(&state).to_vec()]; + let exec = execute(&program, &arenas, &TestPermutation).expect("the chain program runs"); + let traces = super::trace::build_traces(&program, &exec.records); + let census = lfm_chip_census(&program); + + // The frozen AIR order, as the census emits it and `air_trace_pairs` proves + // it. Built from the trace set so a chip whose height the census got from the + // wrong group shows up here. + let dims = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| (t.num_rows(), t.num_main_columns); + let mut built: Vec<(usize, usize)> = vec![ + dims(&traces.const_), + dims(&traces.balu), + dims(&traces.xalu), + dims(&traces.select), + dims(&traces.bitdec), + dims(&traces.hash), + dims(&traces.keccak), + dims(&traces.lanes), + dims(&traces.hint), + dims(&traces.public), + dims(&traces.range), + ]; + built.extend(traces.keccak_rnd.iter().map(dims)); + built.push(dims(&traces.keccak_rc)); + built.push(dims(&traces.bitwise)); + + assert_eq!( + census.len(), + built.len(), + "the census must have one entry per trace the prover proves" + ); + for (c, (rows, width)) in census.iter().zip(&built) { + assert_eq!( + c.rows, *rows as u64, + "{}: the census height must be the trace's own", + c.name + ); + // The census counts VALUE columns, so the trace's full width less the + // preprocessed prefix must be what it reports. + assert!( + c.main_cols <= *width, + "{}: the census cannot count more value columns than the trace has", + c.name + ); + } + + // ---- the AIR set: the names and the widths, in the frozen order. + let opts = wrap_options(); + let artifacts = build_artifacts(&program, &opts); + let airs = super::airs::LfmAirs::new(&artifacts.roots, &opts, artifacts.keccak_rnd_chunks); + let refs = airs.air_refs(); + assert_eq!( + census.len(), + refs.len(), + "the census must have one entry per AIR the verifier builds" + ); + for (c, air) in census.iter().zip(&refs) { + assert_eq!( + c.name, + air.name(), + "the census and the AIR set disagree about the frozen chip order" + ); + let (main_width, aux_width) = air.trace_layout(); + let prep = if air.is_preprocessed() { + air.num_precomputed_columns() + } else { + 0 + }; + assert_eq!( + c.main_cols, + main_width - prep, + "{}: the census must count the AIR's value columns", + c.name + ); + assert_eq!( + c.aux_cols, aux_width, + "{}: the census must count the AIR's aux columns", + c.name + ); + } + + let cells_of = |c: &[LfmChipCells], name: &str| -> u64 { + c.iter() + .filter(|e| e.name == name) + .map(|e| e.main_cells()) + .sum() + }; + assert!( + cells_of(&census, "KECCAK_RND") > 0, + "the chain program hashes, so KECCAK_RND must carry rows" + ); +} diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 79ef4c715..47a77849e 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -18,6 +18,7 @@ pub mod continuation; mod debug_report; #[cfg(feature = "instruments")] pub mod instruments; +pub mod lfm; mod paged_mem; pub use stark::profile_markers; pub mod recursion; @@ -51,12 +52,12 @@ use crate::tables::trace_builder::Traces; use crate::tables::trace_builder::count_table_lengths; use crate::tables::types::BusId; use crate::test_utils::{ - E, F, VmAir, create_bitwise_air, create_branch_air, create_bytewise_air, create_commit_air, - create_cpu_air, create_cpu32_air, create_decode_air, create_dvrm_air, create_ecdas_air, - create_ecsm_air, create_eq_air, create_halt_air, create_hint_air, create_keccak_air, - create_keccak_rc_air, create_keccak_rnd_air, create_load_air, create_lt_air, create_memw_air, - create_memw_aligned_air, create_memw_register_air, create_mul_air, create_page_air, - create_register_air, create_shift_air, create_store_air, + E, F, VmAir, create_bitwise_air, create_blake3_air, create_branch_air, create_bytewise_air, + create_commit_air, create_cpu_air, create_cpu32_air, create_decode_air, create_dvrm_air, + create_ecdas_air, create_ecsm_air, create_eq_air, create_halt_air, create_hint_air, + create_keccak_air, create_keccak_rc_air, create_keccak_rnd_air, create_load_air, create_lt_air, + create_memw_air, create_memw_aligned_air, create_memw_register_air, create_mul_air, + create_page_air, create_register_air, create_shift_air, create_store_air, }; // Re-exported for downstream hosts and verifier guests (e.g. the in-VM @@ -82,8 +83,13 @@ pub struct RuntimePageRange { /// Number of tables that always contribute exactly one sub-proof, regardless /// of `TableCounts`: bitwise, decode, halt, commit, keccak, keccak_rnd, -/// keccak_rc, register, ecsm, ecdas, hint. -pub const FIXED_TABLE_COUNT: usize = 11; +/// keccak_rc, blake3, register, ecsm, ecdas, hint. +/// +/// ⚠ Every always-on table costs every proof a near-empty AIR even when the +/// workload never touches it (the EC-campaign lesson, PR #871). BLAKE3 adds +/// one (min 4 rows × ~3.2k cols); its real-workload cost must be ABBA-checked +/// before this merges. +pub const FIXED_TABLE_COUNT: usize = 12; /// Number of chunks for each split table. /// The verifier needs this to reconstruct matching AIRs. @@ -520,6 +526,7 @@ pub(crate) struct VmAirs { pub keccak: VmAir, pub keccak_rnd: VmAir, pub keccak_rc: VmAir, + pub blake3: VmAir, pub ecsm: VmAir, pub ecdas: VmAir, pub hint: VmAir, @@ -546,6 +553,7 @@ impl VmAirs { (self.keccak.as_ref(), &mut traces.keccak, &()), (self.keccak_rnd.as_ref(), &mut traces.keccak_rnd, &()), (self.keccak_rc.as_ref(), &mut traces.keccak_rc, &()), + (self.blake3.as_ref(), &mut traces.blake3, &()), (self.ecsm.as_ref(), &mut traces.ecsm, &()), (self.ecdas.as_ref(), &mut traces.ecdas, &()), (self.hint.as_ref(), &mut traces.hint, &()), @@ -621,6 +629,7 @@ impl VmAirs { self.keccak.as_ref(), self.keccak_rnd.as_ref(), self.keccak_rc.as_ref(), + self.blake3.as_ref(), self.ecsm.as_ref(), self.ecdas.as_ref(), self.hint.as_ref(), @@ -789,6 +798,7 @@ impl VmAirs { let commit: VmAir = Box::new(create_commit_air(proof_options)); let keccak: VmAir = Box::new(create_keccak_air(proof_options)); let keccak_rnd: VmAir = Box::new(create_keccak_rnd_air(proof_options)); + let blake3: VmAir = Box::new(create_blake3_air(proof_options)); let keccak_rc: VmAir = Box::new(create_keccak_rc_air(proof_options).with_preprocessed( tables::keccak_rc::preprocessed_commitment(proof_options), tables::keccak_rc::NUM_PRECOMPUTED_COLS, @@ -914,6 +924,7 @@ impl VmAirs { keccak, keccak_rnd, keccak_rc, + blake3, ecsm, ecdas, hint, @@ -985,10 +996,14 @@ pub(crate) fn compute_commit_bus_offset( /// Replay the prover's Phase A (main trace commitments) to recover the shared /// LogUp challenges (z, alpha), over a proof view (owned or archived-in-place) /// — no `MultiProof` deserialization required either way. +/// +/// Generic over the transcript for the same reason as `absorb_lfm_statement`: +/// the replay is `append_bytes` plus `sample_field_element`, both on +/// `IsTranscript`, so it is the same replay under any sponge. pub(crate) fn replay_transcript_phase_a_view<'p>( airs: &[&dyn AIR], proofs: impl ProofViewSource<'p, F, E, ()>, - transcript: &mut DefaultTranscript, + transcript: &mut impl IsTranscript, ) -> (FieldElement, FieldElement) { for (air, proof) in airs.iter().zip(proofs.view_iter()) { if air.is_preprocessed() { @@ -1222,6 +1237,7 @@ pub fn prove_with_options_and_inputs( &mut transcript, #[cfg(feature = "disk-spill")] storage_mode, + stark::residency_mode::ResidencyMode::Retain, ) .map_err(|e| Error::Prover(format!("{e:?}")))?; #[cfg(feature = "instruments")] diff --git a/prover/src/recursion.rs b/prover/src/recursion.rs index efca722c9..d929f6f49 100644 --- a/prover/src/recursion.rs +++ b/prover/src/recursion.rs @@ -194,7 +194,12 @@ pub fn encode_continuation_guest_input( } /// Domain tag for [`program_id`]. -const PROGRAM_ID_TAG: &[u8] = b"LAMBDAVM_PROGRAM_ID_V1"; +/// Domain tag for the attestation's program id. +/// +/// `pub(crate)` so the LFM emitter binds the same literal instead of +/// duplicating it — the precedent `CONTINUATION_EPOCH_TAG` set in R1e. 22 bytes, +/// so it is `≡ 2 (mod 4)` and every machine value folded after it is spliced. +pub(crate) const PROGRAM_ID_TAG: &[u8] = b"LAMBDAVM_PROGRAM_ID_V1"; /// [`program_id`] from a precomputed ELF digest and entry point — the guest /// path, sharing one full-ELF Keccak pass with the verify-side statement diff --git a/prover/src/statement.rs b/prover/src/statement.rs index 81c18baa5..cf5ad7403 100644 --- a/prover/src/statement.rs +++ b/prover/src/statement.rs @@ -155,7 +155,10 @@ pub(crate) fn absorb_statement_with_digest( /// Continuation domain tags. Distinct from the monolithic `DOMAIN_TAG` so a /// monolithic proof and a continuation proof can never share a transcript prefix. -const CONTINUATION_EPOCH_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_EPOCH_V2"; +/// `pub(crate)` so the LFM statement replay emits the identical tag instead of +/// duplicating the literal: a second copy would drift silently on a version +/// bump, and the tag existing at all depends on both sides agreeing on it. +pub(crate) const CONTINUATION_EPOCH_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_EPOCH_V2"; const CONTINUATION_GLOBAL_TAG: &[u8] = b"LAMBDAVM_CONTINUATION_GLOBAL_V2"; /// Statement bound into the cross-epoch **global** proof's transcript before diff --git a/prover/src/tables/blake3.rs b/prover/src/tables/blake3.rs new file mode 100644 index 000000000..9b76fde7f --- /dev/null +++ b/prover/src/tables/blake3.rs @@ -0,0 +1,1476 @@ +//! BLAKE3 6-round compression accelerator chip (syscall variant). +//! +//! One row per compression call, fully unrolled (Layout B of +//! `thoughts/blake3/blake3-chip/DESIGN.md`): all 6 rounds × 8 G-functions are +//! laid out in SSA form across the row, so the message schedule is a +//! compile-time permutation of the 16 committed message words and there is no +//! state/message handoff between rows. +//! +//! I/O follows the KECCAK core idiom (`keccak.rs`): an `Ecall` receiver binds +//! (timestamp, syscall#), a `Memw` register read binds the x10 state pointer, +//! and per-dword `Memw` ops read the 112 input bytes / write the 64 output +//! bytes. The 176-byte state region layout is documented on +//! [`executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER`]. +//! +//! ## The single-dataflow rule +//! +//! The compression dataflow is written ONCE, in [`run_flow`], and interpreted +//! twice: [`WireFlow`] (columns — drives the constraints and bus senders) and +//! [`ValueFlow`] (u32 witness — drives trace filling and the BITWISE +//! multiplicity collection in `trace_builder.rs`). The two cannot diverge on +//! wiring, only on interpretation, which the e2e bus-balance gate checks. +//! +//! ## Soundness ledger (DESIGN.md §7, adapted to the syscall variant) +//! +//! 1. Every eval constraint is μ-gated; padding rows are all-zero (except the +//! keccak-style PTR pad) with μ=0. +//! 2. 3-op adds use TWO summed committed carry bits + the explicit sum +//! identity (a ternary carry would be degree 4 after gating). +//! 3. 2-op adds use the `emit_add_pair`-style expression carry (no committed +//! cell) with μ-gated booleanity; the output's bytes are range-checked by +//! the downstream XOR lookup that consumes them. +//! 4. Every add/shift output feeds a downstream `ByteAlu` XOR — that lookup is +//! its only byte range check. The last-round outputs are consumed by the +//! feed-forward XORs, closing the chain. +//! 5. The message words `m` are never XORed, so their 64 bytes get explicit +//! `AreBytes` sends. Same for the 64 `OLD_OUT` bytes (the previous memory +//! content of the out region, which appear only on the Memw bus) and the 8 +//! address bytes (aliasing — see keccak.rs's addr comment). +//! 6. rotr16/rotr8 are free byte relabels `[b2,b3,b0,b1]` / `[b1,b2,b3,b0]`. +//! 7. rotr12/rotr7 are inline μ-gated shift identities with `AreBytes` on all +//! four shift halfwords (`SLL_lo/SLLC_lo/SLL_hi/SLLC_hi`); soundness needs +//! the tight bound on the `SLL` pair (2^16 invertible mod p — the audited +//! Euclidean-division argument). +//! 8. The message schedule is `permute^r` wired from the ORIGINAL M columns. +//! 9. All identities stay < 2^35 ≪ p (non-overflow side conditions), given +//! byte-range operands and boolean carries. +//! 10. (Internal-bus binding — N/A here: the syscall variant has no `Blake3` +//! bus; a row's inputs and outputs are tied by being the same row.) +//! +//! ⚠ This chip implements the **6-round internal variant** — NOT standard +//! 7-round BLAKE3. Its collision resistance is a named assumption +//! (DESIGN.md "If this is picked up again"). + +use executor::vm::instruction::execution::{ + BLAKE3_IV, BLAKE3_MSG_PERMUTATION, BLAKE3_ROUNDS, BLAKE3_SYSCALL_NUMBER, +}; +use stark::lookup::{BusInteraction, BusValue, LinearTerm, Multiplicity, Packing}; +use stark::trace::TraceTable; + +use stark::constraints::builder::{ConstraintBuilder, ConstraintSet}; + +use super::types::{BusId, FE, GoldilocksExtension, GoldilocksField, VmTable, alu_op}; +use crate::constraints::templates::{AddOperand, INV_SHIFT_32}; + +/// G-instances per compression: 8 per round × 6 rounds. +pub const NUM_G: usize = BLAKE3_ROUNDS * 8; + +/// Dwords in the state region: 14 input (h|m|t|len_flags) + 8 output. +pub const STATE_DWORDS: usize = 22; +/// Input dwords (read-only). +pub const IN_DWORDS: usize = 14; + +/// The (a, b, c, d) state indices of the 8 G-calls of one round: +/// 4 column mixes then 4 diagonal mixes (BLAKE3 spec §2.1). +const G_INDICES: [(usize, usize, usize, usize); 8] = [ + (0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14), +]; + +/// Shift amounts of the two non-free rotations, as `rotl` inner shifts: +/// rotr12 = rotl20 = rotl16∘rotl4 (r=4); rotr7 = rotl25 = rotl16∘rotl9 (r=9). +const ROT_SHIFT_R: [u32; 2] = [4, 9]; + +// ========================================================================= +// Column indices +// ========================================================================= + +pub mod cols { + use super::NUM_G; + + pub const TIMESTAMP_0: usize = 0; + pub const TIMESTAMP_1: usize = 1; + + /// State address as 8 bytes (DWordBL). + pub const ADDR: usize = 2; + + /// Per-dword pointers [22][4] halfwords (DWordHL), ptr[k] = addr + 8k. + pub const PTR: usize = ADDR + 8; // 10 + + /// Input bytes: h[32] | m[64] | t_lo[4] | t_hi[4] | block_len[4] | flags[4]. + pub const IN: usize = PTR + super::STATE_DWORDS * 4; // 98 + + /// 48 G-blocks × 60 cells (56 bytes + 4 carry bits) — see `g` accessors. + pub const G: usize = IN + 112; // 210 + pub const G_SIZE: usize = 60; + + /// Feed-forward output bytes out[0..16] (64 bytes). + pub const OUT: usize = G + NUM_G * G_SIZE; // 3090 + + /// Previous memory content of the out region (64 bytes). Appears only in + /// the Memw write ops' `old` field; range-checked by AreBytes. + pub const OLD_OUT: usize = OUT + 64; // 3154 + + /// Multiplicity / gate flag. + pub const MU: usize = OLD_OUT + 64; // 3218 + + pub const NUM_COLUMNS: usize = MU + 1; // 3219 + + // ------------------------------------------------------------------------- + // Index helpers + // ------------------------------------------------------------------------- + + #[inline] + pub const fn addr(byte: usize) -> usize { + ADDR + byte + } + + /// ptr[k][hw] — halfword hw of the pointer to dword k. + #[inline] + pub const fn ptr(k: usize, hw: usize) -> usize { + PTR + k * 4 + hw + } + + /// Input word i (0..28: h[0..8], m[8..24], t_lo=24, t_hi=25, len=26, flags=27), + /// byte b. + #[inline] + pub const fn in_word(i: usize, b: usize) -> usize { + IN + i * 4 + b + } + + /// Base column of G-block g. + #[inline] + pub const fn g_base(g: usize) -> usize { + G + g * G_SIZE + } + + // Offsets inside one G block (56 byte cells + 4 carry bits = 60): + /// add3 #1 output word (4 bytes). + pub const G_A1: usize = 0; + /// add3 #1 carry bits c1, c2. + pub const G_A1_C: usize = 4; + /// X1 = vd ^ A1 (4 bytes). + pub const G_X1: usize = 6; + /// add2 #1 output word (4 bytes). + pub const G_C1: usize = 10; + /// X2 = vb ^ C1 (4 bytes). + pub const G_X2: usize = 14; + /// rotr12 block: SLL_lo(2) SLLC_lo(2) SLL_hi(2) SLLC_hi(2) Y(4). + pub const G_R1: usize = 18; + /// add3 #2 output word (4 bytes). + pub const G_A2: usize = 30; + /// add3 #2 carry bits. + pub const G_A2_C: usize = 34; + /// X3 = vd ^ A2 (4 bytes). + pub const G_X3: usize = 36; + /// add2 #2 output word (4 bytes). + pub const G_C2: usize = 40; + /// X4 = B1 ^ C2 (4 bytes). + pub const G_X4: usize = 44; + /// rotr7 block: same layout as G_R1. + pub const G_R2: usize = 48; + + /// Feed-forward output word i (0..16), byte b. + #[inline] + pub const fn out_word(i: usize, b: usize) -> usize { + OUT + i * 4 + b + } + + /// Previous-content byte b (0..64) of the out region. + #[inline] + pub const fn old_out(b: usize) -> usize { + OLD_OUT + b + } +} + +// ========================================================================= +// The single dataflow, interpreted twice +// ========================================================================= + +/// The BLAKE3 compression dataflow, abstracted over its word representation. +/// +/// [`run_flow`] is the only place the G-function wiring, message schedule and +/// feed-forward exist; implementors interpret the primitive ops either as +/// column wiring ([`WireFlow`]) or as u32 witness computation ([`ValueFlow`]). +pub(crate) trait Blake3Flow { + type Word: Copy; + + /// h[i] input word. + fn input_h(&mut self, i: usize) -> Self::Word; + /// v[12..16] init words: t_lo, t_hi, block_len, flags. + fn input_v12(&mut self, j: usize) -> Self::Word; + /// IV[i] constant (v[8..12]). + fn iv_const(&mut self, i: usize) -> Self::Word; + + /// 3-operand add `s = a + b + m[m_idx] mod 2^32` (half 0/1 = which add3 of G g). + fn add3( + &mut self, + g: usize, + half: usize, + a: Self::Word, + b: Self::Word, + m_idx: usize, + ) -> Self::Word; + /// 2-operand add `s = a + b mod 2^32`. + fn add2(&mut self, g: usize, half: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// XOR (slot 0..4 = X1..X4 of G g). Operand order is part of the wire format. + fn xor(&mut self, g: usize, slot: usize, a: Self::Word, b: Self::Word) -> Self::Word; + /// rotr16: free byte relabel [b2,b3,b0,b1]. + fn rotr16(&mut self, w: Self::Word) -> Self::Word; + /// rotr8: free byte relabel [b1,b2,b3,b0]. + fn rotr8(&mut self, w: Self::Word) -> Self::Word; + /// rotr12 (half=0) / rotr7 (half=1) via the inline shift identity. + fn rot_shift(&mut self, g: usize, half: usize, w: Self::Word) -> Self::Word; + /// Feed-forward XOR pair: out[i] = v[i] ^ v[i+8], out[i+8] = v[i+8] ^ h[i]. + fn feed_forward(&mut self, i: usize, vi: Self::Word, vi8: Self::Word, hi: Self::Word); +} + +/// Drive the full 6-round compression through `f`. The message schedule is +/// tracked as indices into the ORIGINAL m (permute^r composition), so both +/// interpretations reference original message words — never copies. +pub(crate) fn run_flow(f: &mut F) { + let h: [F::Word; 8] = core::array::from_fn(|i| f.input_h(i)); + let mut v: [F::Word; 16] = core::array::from_fn(|i| { + if i < 8 { + h[i] + } else if i < 12 { + f.iv_const(i - 8) + } else { + f.input_v12(i - 12) + } + }); + + // sched[i] = index into the original m of the word consumed at position i + // this round. permute: m'[i] = m[P[i]] ⇒ sched'[i] = sched[P[i]]. + let mut sched: [usize; 16] = core::array::from_fn(|i| i); + + for r in 0..BLAKE3_ROUNDS { + for (j, &(ia, ib, ic, id)) in G_INDICES.iter().enumerate() { + let g = r * 8 + j; + let (va, vb, vc, vd) = (v[ia], v[ib], v[ic], v[id]); + let mx = sched[2 * j]; + let my = sched[2 * j + 1]; + + let a1 = f.add3(g, 0, va, vb, mx); + let x1 = f.xor(g, 0, vd, a1); + let vd1 = f.rotr16(x1); + let c1 = f.add2(g, 0, vc, vd1); + let x2 = f.xor(g, 1, vb, c1); + let b1 = f.rot_shift(g, 0, x2); // rotr12 + let a2 = f.add3(g, 1, a1, b1, my); + let x3 = f.xor(g, 2, vd1, a2); + let vd2 = f.rotr8(x3); + let c2 = f.add2(g, 1, c1, vd2); + let x4 = f.xor(g, 3, b1, c2); + let b2 = f.rot_shift(g, 1, x4); // rotr7 + + v[ia] = a2; + v[ib] = b2; + v[ic] = c2; + v[id] = vd2; + } + if r < BLAKE3_ROUNDS - 1 { + let prev = sched; + for (i, &p) in BLAKE3_MSG_PERMUTATION.iter().enumerate() { + sched[i] = prev[p]; + } + } + } + + for i in 0..8 { + f.feed_forward(i, v[i], v[i + 8], h[i]); + } +} + +// ========================================================================= +// Wire interpretation (columns) +// ========================================================================= + +/// A 32-bit word as wiring: four byte columns (LSB first) or a constant. +/// Constants only ever appear as the IV `v[c]` operands of round-0 add2s. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WordRef { + Cols([usize; 4]), + Const(u32), +} + +impl WordRef { + fn byte(self, b: usize) -> ByteRef { + match self { + WordRef::Cols(c) => ByteRef::Col(c[b]), + WordRef::Const(w) => ByteRef::Const(((w >> (8 * b)) & 0xFF) as u8), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ByteRef { + Col(usize), + Const(u8), +} + +/// One recorded 3-op add: operands (a, b, m columns), output columns, carries. +pub(crate) struct Add3Wire { + pub a: WordRef, + pub b: WordRef, + pub m: [usize; 4], + pub s: [usize; 4], + pub c1: usize, + pub c2: usize, +} + +/// One recorded 2-op add: operands, output columns (carry is an expression). +pub(crate) struct Add2Wire { + pub a: WordRef, + pub b: WordRef, + pub s: [usize; 4], +} + +/// One recorded XOR: per-byte operands and output columns. +pub(crate) struct XorWire { + pub a: WordRef, + pub b: WordRef, + pub out: [usize; 4], +} + +/// One recorded shift rotation: input word, the 8 shift-halfword byte columns +/// (SLL_lo, SLLC_lo, SLL_hi, SLLC_hi — 2 bytes each), output columns, r. +pub(crate) struct RotWire { + pub input: WordRef, + pub sll_lo: [usize; 2], + pub sllc_lo: [usize; 2], + pub sll_hi: [usize; 2], + pub sllc_hi: [usize; 2], + pub y: [usize; 4], + pub r: u32, +} + +/// The full wiring of one compression row: everything the constraints and the +/// bus senders need, recorded in canonical order by [`run_flow`]. +pub(crate) struct WireFlow { + pub add3s: Vec, + pub add2s: Vec, + pub xors: Vec, + pub rots: Vec, +} + +impl WireFlow { + pub(crate) fn build() -> Self { + let mut w = WireFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + 16), + rots: Vec::with_capacity(NUM_G * 2), + }; + run_flow(&mut w); + w + } +} + +#[inline] +fn word_cols(start: usize) -> [usize; 4] { + [start, start + 1, start + 2, start + 3] +} + +impl Blake3Flow for WireFlow { + type Word = WordRef; + + fn input_h(&mut self, i: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(i, 0))) + } + fn input_v12(&mut self, j: usize) -> WordRef { + WordRef::Cols(word_cols(cols::in_word(24 + j, 0))) + } + fn iv_const(&mut self, i: usize) -> WordRef { + WordRef::Const(BLAKE3_IV[i]) + } + + fn add3(&mut self, g: usize, half: usize, a: WordRef, b: WordRef, m_idx: usize) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_A1 } else { cols::G_A2 }; + let cbase = cols::g_base(g) + + if half == 0 { + cols::G_A1_C + } else { + cols::G_A2_C + }; + let s = word_cols(base); + self.add3s.push(Add3Wire { + a, + b, + m: word_cols(cols::in_word(8 + m_idx, 0)), + s, + c1: cbase, + c2: cbase + 1, + }); + WordRef::Cols(s) + } + + fn add2(&mut self, g: usize, half: usize, a: WordRef, b: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_C1 } else { cols::G_C2 }; + let s = word_cols(base); + self.add2s.push(Add2Wire { a, b, s }); + WordRef::Cols(s) + } + + fn xor(&mut self, g: usize, slot: usize, a: WordRef, b: WordRef) -> WordRef { + let off = match slot { + 0 => cols::G_X1, + 1 => cols::G_X2, + 2 => cols::G_X3, + _ => cols::G_X4, + }; + let out = word_cols(cols::g_base(g) + off); + self.xors.push(XorWire { a, b, out }); + WordRef::Cols(out) + } + + fn rotr16(&mut self, w: WordRef) -> WordRef { + match w { + WordRef::Cols([b0, b1, b2, b3]) => WordRef::Cols([b2, b3, b0, b1]), + WordRef::Const(v) => WordRef::Const(v.rotate_right(16)), + } + } + fn rotr8(&mut self, w: WordRef) -> WordRef { + match w { + WordRef::Cols([b0, b1, b2, b3]) => WordRef::Cols([b1, b2, b3, b0]), + WordRef::Const(v) => WordRef::Const(v.rotate_right(8)), + } + } + + fn rot_shift(&mut self, g: usize, half: usize, w: WordRef) -> WordRef { + let base = cols::g_base(g) + if half == 0 { cols::G_R1 } else { cols::G_R2 }; + let y = word_cols(base + 8); + self.rots.push(RotWire { + input: w, + sll_lo: [base, base + 1], + sllc_lo: [base + 2, base + 3], + sll_hi: [base + 4, base + 5], + sllc_hi: [base + 6, base + 7], + y, + r: ROT_SHIFT_R[half], + }); + WordRef::Cols(y) + } + + fn feed_forward(&mut self, i: usize, vi: WordRef, vi8: WordRef, hi: WordRef) { + let out_lo = word_cols(cols::out_word(i, 0)); + let out_hi = word_cols(cols::out_word(i + 8, 0)); + self.xors.push(XorWire { + a: vi, + b: vi8, + out: out_lo, + }); + self.xors.push(XorWire { + a: vi8, + b: hi, + out: out_hi, + }); + } +} + +// ========================================================================= +// Value interpretation (u32 witness) +// ========================================================================= + +/// Everything the trace filler and the BITWISE collector need for one +/// compression, recorded cell-exactly in the same canonical order as +/// [`WireFlow`]. `xor_ops` carries (a, b) operand VALUES per XOR word — the +/// per-byte lookups are `(a_byte, b_byte)` in the same operand order the +/// senders use. +pub(crate) struct ValueFlow { + /// (s, c1, c2) per add3, canonical order. + pub add3s: Vec<(u32, u8, u8)>, + /// s per add2 (the carry is an expression, not a cell). + pub add2s: Vec, + /// (a, b, out) per XOR word, canonical order (Gs then feed-forward). + pub xors: Vec<(u32, u32, u32)>, + /// (sll_lo, sllc_lo, sll_hi, sllc_hi, y) per shift rotation. + pub rots: Vec<(u16, u16, u16, u16, u32)>, + /// The 16-word output. + pub out: [u32; 16], + + h: [u32; 8], + m: [u32; 16], + v12: [u32; 4], +} + +impl ValueFlow { + pub(crate) fn compute(h: &[u32; 8], m: &[u32; 16], t: u64, block_len: u32, flags: u32) -> Self { + let mut f = ValueFlow { + add3s: Vec::with_capacity(NUM_G * 2), + add2s: Vec::with_capacity(NUM_G * 2), + xors: Vec::with_capacity(NUM_G * 4 + 16), + rots: Vec::with_capacity(NUM_G * 2), + out: [0; 16], + h: *h, + m: *m, + v12: [t as u32, (t >> 32) as u32, block_len, flags], + }; + run_flow(&mut f); + f + } +} + +impl Blake3Flow for ValueFlow { + type Word = u32; + + fn input_h(&mut self, i: usize) -> u32 { + self.h[i] + } + fn input_v12(&mut self, j: usize) -> u32 { + self.v12[j] + } + fn iv_const(&mut self, i: usize) -> u32 { + BLAKE3_IV[i] + } + + fn add3(&mut self, _g: usize, _half: usize, a: u32, b: u32, m_idx: usize) -> u32 { + let m = self.m[m_idx]; + let wide = a as u64 + b as u64 + m as u64; + let s = wide as u32; + let carry = (wide >> 32) as u8; // 0, 1 or 2 + // Two summed carry bits: c1 + c2 = carry. + let (c1, c2) = match carry { + 0 => (0, 0), + 1 => (1, 0), + _ => (1, 1), + }; + self.add3s.push((s, c1, c2)); + s + } + + fn add2(&mut self, _g: usize, _half: usize, a: u32, b: u32) -> u32 { + let s = a.wrapping_add(b); + self.add2s.push(s); + s + } + + fn xor(&mut self, _g: usize, _slot: usize, a: u32, b: u32) -> u32 { + let out = a ^ b; + self.xors.push((a, b, out)); + out + } + + fn rotr16(&mut self, w: u32) -> u32 { + w.rotate_right(16) + } + fn rotr8(&mut self, w: u32) -> u32 { + w.rotate_right(8) + } + + fn rot_shift(&mut self, _g: usize, half: usize, w: u32) -> u32 { + let r = ROT_SHIFT_R[half]; + let xlo = w & 0xFFFF; + let xhi = w >> 16; + // xlo·2^r = SLLC_lo·2^16 + SLL_lo (and same for hi): Euclidean split. + let sll_lo = ((xlo << r) & 0xFFFF) as u16; + let sllc_lo = ((xlo << r) >> 16) as u16; + let sll_hi = ((xhi << r) & 0xFFFF) as u16; + let sllc_hi = ((xhi << r) >> 16) as u16; + // Recombine + halfword swap: Ylo = SLL_hi + SLLC_lo, Yhi = SLL_lo + SLLC_hi. + let ylo = sll_hi as u32 + sllc_lo as u32; + let yhi = sll_lo as u32 + sllc_hi as u32; + let y = ylo | (yhi << 16); + debug_assert_eq!(y, w.rotate_right(if r == 4 { 12 } else { 7 })); + self.rots.push((sll_lo, sllc_lo, sll_hi, sllc_hi, y)); + y + } + + fn feed_forward(&mut self, i: usize, vi: u32, vi8: u32, hi: u32) { + let lo = vi ^ vi8; + let hi_w = vi8 ^ hi; + self.xors.push((vi, vi8, lo)); + self.xors.push((vi8, hi, hi_w)); + self.out[i] = lo; + self.out[i + 8] = hi_w; + } +} + +// ========================================================================= +// Operation struct + trace generation +// ========================================================================= + +#[derive(Debug, Clone)] +pub struct Blake3Operation { + pub timestamp: u64, + pub state_addr: u64, + pub h: [u32; 8], + pub m: [u32; 16], + pub t: u64, + pub block_len: u32, + pub flags: u32, + /// Previous memory content of the 64-byte out region (for the Memw `old`). + pub old_out: [u8; 64], + /// The 16-word compression output (recomputed by the trace builder). + pub out: [u32; 16], +} + +/// Write a 32-bit word as 4 byte cells at `col..col+4`. +#[inline] +fn set_word_bytes(table: &mut T, row: usize, col: usize, w: u32) { + for b in 0..4 { + table.set_u64(row, col + b, ((w >> (8 * b)) & 0xFF) as u64); + } +} + +pub fn generate_blake3_trace( + ops: &[Blake3Operation], +) -> TraceTable { + let n = ops.len(); + let num_rows = n.next_power_of_two().max(4); + let mut trace = TraceTable::new_main( + crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), + cols::NUM_COLUMNS, + 1, + ); + let table = &mut trace.main_table; + + for (row, op) in ops.iter().enumerate() { + table.set_dword_wl(row, cols::TIMESTAMP_0, op.timestamp); + table.set_dword_bl(row, cols::addr(0), op.state_addr); + + // Pointers ptr[k] = addr + 8k. + for k in 0..STATE_DWORDS { + let ptr = op + .state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + table.set_dword_hl(row, cols::ptr(k, 0), ptr); + } + + // Input words: h | m | t_lo t_hi len flags. + for i in 0..8 { + set_word_bytes(table, row, cols::in_word(i, 0), op.h[i]); + } + for i in 0..16 { + set_word_bytes(table, row, cols::in_word(8 + i, 0), op.m[i]); + } + set_word_bytes(table, row, cols::in_word(24, 0), op.t as u32); + set_word_bytes(table, row, cols::in_word(25, 0), (op.t >> 32) as u32); + set_word_bytes(table, row, cols::in_word(26, 0), op.block_len); + set_word_bytes(table, row, cols::in_word(27, 0), op.flags); + + // The mixing core, cell-exactly in canonical order. + let flow = ValueFlow::compute(&op.h, &op.m, op.t, op.block_len, op.flags); + debug_assert_eq!( + flow.out, op.out, + "trace-builder output must match the executor" + ); + + let mut a3 = flow.add3s.iter(); + let mut a2 = flow.add2s.iter(); + let mut xo = flow.xors.iter(); + let mut ro = flow.rots.iter(); + for g in 0..NUM_G { + let base = cols::g_base(g); + for half in 0..2 { + let (s_off, c_off, x_off, c2_off, x2_off, r_off) = if half == 0 { + ( + cols::G_A1, + cols::G_A1_C, + cols::G_X1, + cols::G_C1, + cols::G_X2, + cols::G_R1, + ) + } else { + ( + cols::G_A2, + cols::G_A2_C, + cols::G_X3, + cols::G_C2, + cols::G_X4, + cols::G_R2, + ) + }; + let &(s, c1, c2) = a3.next().expect("add3 count"); + set_word_bytes(table, row, base + s_off, s); + table.set_u64(row, base + c_off, c1 as u64); + table.set_u64(row, base + c_off + 1, c2 as u64); + + let &(_, _, x) = xo.next().expect("xor count"); + set_word_bytes(table, row, base + x_off, x); + + let &c = a2.next().expect("add2 count"); + set_word_bytes(table, row, base + c2_off, c); + + let &(_, _, x2) = xo.next().expect("xor count"); + set_word_bytes(table, row, base + x2_off, x2); + + let &(sll_lo, sllc_lo, sll_hi, sllc_hi, y) = ro.next().expect("rot count"); + table.set_u64(row, base + r_off, (sll_lo & 0xFF) as u64); + table.set_u64(row, base + r_off + 1, (sll_lo >> 8) as u64); + table.set_u64(row, base + r_off + 2, (sllc_lo & 0xFF) as u64); + table.set_u64(row, base + r_off + 3, (sllc_lo >> 8) as u64); + table.set_u64(row, base + r_off + 4, (sll_hi & 0xFF) as u64); + table.set_u64(row, base + r_off + 5, (sll_hi >> 8) as u64); + table.set_u64(row, base + r_off + 6, (sllc_hi & 0xFF) as u64); + table.set_u64(row, base + r_off + 7, (sllc_hi >> 8) as u64); + set_word_bytes(table, row, base + r_off + 8, y); + } + } + // Feed-forward outputs (the last 16 entries of flow.xors). + for i in 0..16 { + set_word_bytes(table, row, cols::out_word(i, 0), flow.out[i]); + } + // Previous content of the out region. + for b in 0..64 { + table.set_u64(row, cols::old_out(b), op.old_out[b] as u64); + } + + table.set_fe(row, cols::MU, FE::one()); + } + + // Padding rows: ptr[k][0] = 8k (all fit in the low halfword), matching the + // keccak pad idiom. μ = 0 gates every constraint and interaction. + for row in n..num_rows { + for k in 0..STATE_DWORDS { + table.set_u64(row, cols::ptr(k, 0), (k as u64) * 8); + } + } + + trace +} + +// ========================================================================= +// Bus interactions +// ========================================================================= + +/// Order groups: I/O (Ecall + reg-read + 22 Memw), then the mixing core's +/// ByteAlu XORs (canonical WireFlow order), then the shift AreBytes, then the +/// message/old-out/addr AreBytes, the alignment AND and the pointer IS_HALFs. +pub fn bus_interactions() -> Vec { + let syscall_lo = BLAKE3_SYSCALL_NUMBER & 0xFFFF_FFFF; + let syscall_hi = BLAKE3_SYSCALL_NUMBER >> 32; + let wires = WireFlow::build(); + let mut interactions = Vec::with_capacity(1400); + + let byte_bus_value = |b: ByteRef| -> BusValue { + match b { + ByteRef::Col(c) => BusValue::Packed { + start_column: c, + packing: Packing::Direct, + }, + ByteRef::Const(v) => BusValue::constant(v as u64), + } + }; + + // 1. ECALL receiver: [ts_lo, ts_hi, syscall_lo32, syscall_hi32]. + interactions.push(BusInteraction::receiver( + BusId::Ecall, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }, + BusValue::constant(syscall_lo), + BusValue::constant(syscall_hi), + ], + )); + + // 2. MEMW read of register x10 binding the state address (keccak idiom): + // [old(8), is_register=1, base=20, value(8), ts(2), w2=1, w4=0, w8=0]. + { + let addr_word = |lo_byte: usize| -> BusValue { + BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::addr(lo_byte), + }, + LinearTerm::Column { + coefficient: 256, + column: cols::addr(lo_byte + 1), + }, + LinearTerm::Column { + coefficient: 65536, + column: cols::addr(lo_byte + 2), + }, + LinearTerm::Column { + coefficient: 16777216, + column: cols::addr(lo_byte + 3), + }, + ]) + }; + let mut values = Vec::with_capacity(24); + values.push(addr_word(0)); + values.push(addr_word(4)); + for _ in 2..8 { + values.push(BusValue::constant(0)); + } + values.push(BusValue::constant(1)); // is_register + values.push(BusValue::constant(20)); // x10 → address 2*10 + values.push(BusValue::constant(0)); + values.push(addr_word(0)); + values.push(addr_word(4)); + for _ in 2..8 { + values.push(BusValue::constant(0)); + } + values.push(BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }); + values.push(BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }); + values.push(BusValue::constant(1)); // w2 (register) + values.push(BusValue::constant(0)); + values.push(BusValue::constant(0)); + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::MU), + values, + )); + } + + // 3. MEMW per state dword: [old(8), is_register=0, addr(2), value(8), ts(2), + // w2=0, w4=0, w8=1]. Input dwords are pure reads (old = value = input + // bytes); output dwords write OUT over OLD_OUT. + for k in 0..STATE_DWORDS { + let addr_lo = BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::ptr(k, 0), + }, + LinearTerm::Column { + coefficient: 65536, + column: cols::ptr(k, 1), + }, + ]); + let addr_hi = BusValue::linear(vec![ + LinearTerm::Column { + coefficient: 1, + column: cols::ptr(k, 2), + }, + LinearTerm::Column { + coefficient: 65536, + column: cols::ptr(k, 3), + }, + ]); + + // (old bytes, value bytes) column bases for this dword. + let (old_base, val_base): (Vec, Vec) = if k < IN_DWORDS { + let cols8: Vec = (0..8).map(|b| cols::in_word(2 * k, 0) + b).collect(); + (cols8.clone(), cols8) + } else { + let o = k - IN_DWORDS; + ( + (0..8).map(|b| cols::old_out(o * 8 + b)).collect(), + (0..8).map(|b| cols::out_word(2 * o, 0) + b).collect(), + ) + }; + + let mut values = Vec::with_capacity(24); + for &c in &old_base { + values.push(BusValue::Packed { + start_column: c, + packing: Packing::Direct, + }); + } + values.push(BusValue::constant(0)); // is_register + values.push(addr_lo); + values.push(addr_hi); + for &c in &val_base { + values.push(BusValue::Packed { + start_column: c, + packing: Packing::Direct, + }); + } + values.push(BusValue::Packed { + start_column: cols::TIMESTAMP_0, + packing: Packing::Direct, + }); + values.push(BusValue::Packed { + start_column: cols::TIMESTAMP_1, + packing: Packing::Direct, + }); + values.push(BusValue::constant(0)); + values.push(BusValue::constant(0)); + values.push(BusValue::constant(1)); // w8 + interactions.push(BusInteraction::sender( + BusId::Memw, + Multiplicity::Column(cols::MU), + values, + )); + } + + // 4. Mixing core + feed-forward: ByteAlu[XOR] per byte, canonical order. + for xw in &wires.xors { + for b in 0..4 { + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MU), + vec![ + BusValue::constant(alu_op::XOR as u64), + byte_bus_value(xw.a.byte(b)), + byte_bus_value(xw.b.byte(b)), + BusValue::Packed { + start_column: xw.out[b], + packing: Packing::Direct, + }, + ], + )); + } + } + + // 5. Shift-halfword AreBytes: 4 pairs per rotation + // (SLL_lo, SLLC_lo, SLL_hi, SLLC_hi bytes). + for rw in &wires.rots { + for pair in [rw.sll_lo, rw.sllc_lo, rw.sll_hi, rw.sllc_hi] { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: pair[0], + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: pair[1], + packing: Packing::Direct, + }, + ], + )); + } + } + + // 6. Message AreBytes (m is never XORed — DESIGN §4.7/§7.5): 32 pairs. + for i in 0..16 { + for p in 0..2 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::in_word(8 + i, 2 * p), + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::in_word(8 + i, 2 * p + 1), + packing: Packing::Direct, + }, + ], + )); + } + } + + // 7. OLD_OUT AreBytes: those bytes only ride the Memw bus; without a byte + // range check their packed linear combinations alias (same argument as the + // addr bytes in keccak.rs). + for p in 0..32 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::old_out(2 * p), + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::old_out(2 * p + 1), + packing: Packing::Direct, + }, + ], + )); + } + + // 8. Address byte range checks (4 pairs) + alignment addr[0] & 7 = 0. + for i in 0..4 { + interactions.push(BusInteraction::sender( + BusId::AreBytes, + Multiplicity::Column(cols::MU), + vec![ + BusValue::Packed { + start_column: cols::addr(2 * i), + packing: Packing::Direct, + }, + BusValue::Packed { + start_column: cols::addr(2 * i + 1), + packing: Packing::Direct, + }, + ], + )); + } + interactions.push(BusInteraction::sender( + BusId::ByteAlu, + Multiplicity::Column(cols::MU), + vec![ + BusValue::constant(alu_op::AND as u64), + BusValue::Packed { + start_column: cols::addr(0), + packing: Packing::Direct, + }, + BusValue::constant(7), + BusValue::constant(0), + ], + )); + + // 9. IS_HALF range checks on the 22 pointers' halfwords. + for k in 0..STATE_DWORDS { + for hw in 0..4 { + interactions.push(BusInteraction::sender( + BusId::IsHalfword, + Multiplicity::Column(cols::MU), + vec![BusValue::Packed { + start_column: cols::ptr(k, hw), + packing: Packing::Direct, + }], + )); + } + } + + interactions +} + +// ========================================================================= +// Single-source constraint set +// ========================================================================= + +/// The BLAKE3 table's transition constraints (814 total): +/// - idx 0..44: 22 pointer `ADD` carry pairs (`ptr[k] = addr + 8k`, μ-gated); +/// - idx 44: μ·carry_1 = 0 — top-dword no-overflow (`addr + 168 = ptr[21]`); +/// - idx 45..333: all 96 add3 groups (sum identity + 2 carry booleanities); +/// - idx 333..429: all 96 add2 expression-carry booleanities; +/// - idx 429..813: all 96 rotations (2 shift identities + 2 recombine each). +/// NOTE the grouping is by op type across the whole row, NOT per G — G #g's +/// 16 constraints are scattered across the three bands. +/// - idx 813: `IS_BIT(MU)` — μ·(1−μ) = 0, ungated. The bus argument pins +/// μ to {0,1} indirectly (the Ecall receive anchors μ>0 rows to a CPU ecall +/// whose ECALL flag is IS_BIT; MEMW's width flags are boolean), but that is +/// an inter-table argument — this makes it local, matching ecsm/commit. +/// +/// All μ-gated, max degree 3 (the booleanities; identities are degree 2). +#[derive(Clone, Copy)] +pub struct Blake3Constraints; + +/// Word expression from a [`WordRef`]: b0 + 256·b1 + 2^16·b2 + 2^24·b3. +fn word_expr>( + b: &B, + w: &WordRef, +) -> B::Expr { + match w { + WordRef::Cols(c) => { + b.main(0, c[0]) + + b.main(0, c[1]) * b.const_base(256) + + b.main(0, c[2]) * b.const_base(65536) + + b.main(0, c[3]) * b.const_base(16777216) + } + WordRef::Const(v) => b.const_base(*v as u64), + } +} + +/// Halfword expression from 2 byte columns: b0 + 256·b1. +fn half_expr>( + b: &B, + c: &[usize; 2], +) -> B::Expr { + b.main(0, c[0]) + b.main(0, c[1]) * b.const_base(256) +} + +impl ConstraintSet for Blake3Constraints { + fn max_degree(&self) -> usize { + 3 + } + + fn eval>(&self, b: &mut B) { + use crate::constraints::templates::emit_add_pair; + + let wires = WireFlow::build(); + let mu = |b: &B| b.main(0, cols::MU); + + // idx 0..44: ptr[k] = addr + 8k (μ-gated carry pairs). + for k in 0..STATE_DWORDS { + emit_add_pair( + b, + k * 2, + &[cols::MU], + &AddOperand::from_dword_bl(cols::ADDR), + &AddOperand::constant((k * 8) as i64), + &AddOperand::from_dword_hl(cols::ptr(k, 0)), + ); + } + + // idx 44: top-dword no-overflow — μ·carry_1 of addr + 168 = ptr[21]. + let mut idx = STATE_DWORDS * 2; + { + let c256 = b.const_base(256); + let c65536 = b.const_base(65536); + let c16777216 = b.const_base(16777216); + let addr_lo = b.main(0, cols::addr(0)) + + b.main(0, cols::addr(1)) * c256.clone() + + b.main(0, cols::addr(2)) * c65536.clone() + + b.main(0, cols::addr(3)) * c16777216.clone(); + let addr_hi = b.main(0, cols::addr(4)) + + b.main(0, cols::addr(5)) * c256 + + b.main(0, cols::addr(6)) * c65536.clone() + + b.main(0, cols::addr(7)) * c16777216; + let last = STATE_DWORDS - 1; + let ptr_lo = + b.main(0, cols::ptr(last, 0)) + b.main(0, cols::ptr(last, 1)) * c65536.clone(); + let ptr_hi = b.main(0, cols::ptr(last, 2)) + b.main(0, cols::ptr(last, 3)) * c65536; + + let inv_2_32 = b.const_base(INV_SHIFT_32); + let off = b.const_base((8 * last) as u64); + let carry_0 = (addr_lo + off - ptr_lo) * inv_2_32.clone(); + let carry_1 = (addr_hi + carry_0 - ptr_hi) * inv_2_32; + let m = mu(b); + b.emit_base(idx, m * carry_1); + idx += 1; + } + + // Mixing core. Same canonical order as the wire builder records. + let two_32 = b.const_base(1u64 << 32); + let inv_2_32 = b.const_base(INV_SHIFT_32); + + // add3: μ·(a + b + m − s − 2^32·(c1+c2)) = 0; μ·ci·(1−ci) = 0. + for aw in &wires.add3s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let m_w = word_expr(b, &WordRef::Cols(aw.m)); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let c1 = b.main(0, aw.c1); + let c2 = b.main(0, aw.c2); + let sum_id = a + bb + m_w - s - (c1.clone() + c2.clone()) * two_32.clone(); + let m = mu(b); + b.emit_base(idx, m * sum_id); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c1.clone() * (one - c1)); + idx += 1; + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * c2.clone() * (one - c2)); + idx += 1; + } + + // add2: carry = (a + b − s)·2^−32; μ·carry·(1−carry) = 0. + for aw in &wires.add2s { + let a = word_expr(b, &aw.a); + let bb = word_expr(b, &aw.b); + let s = word_expr(b, &WordRef::Cols(aw.s)); + let carry = (a + bb - s) * inv_2_32.clone(); + let one = b.one(); + let m = mu(b); + b.emit_base(idx, m * carry.clone() * (one - carry)); + idx += 1; + } + + // Rotations: 2 shift identities + 2 recombine identities each. + for rw in &wires.rots { + let (xlo, xhi) = match &rw.input { + WordRef::Cols(c) => (half_expr(b, &[c[0], c[1]]), half_expr(b, &[c[2], c[3]])), + WordRef::Const(_) => unreachable!("shift inputs are always committed XOR outputs"), + }; + let sll_lo = half_expr(b, &rw.sll_lo); + let sllc_lo = half_expr(b, &rw.sllc_lo); + let sll_hi = half_expr(b, &rw.sll_hi); + let sllc_hi = half_expr(b, &rw.sllc_hi); + let ylo = half_expr(b, &[rw.y[0], rw.y[1]]); + let yhi = half_expr(b, &[rw.y[2], rw.y[3]]); + let two_r = b.const_base(1u64 << rw.r); + let two_16 = b.const_base(65536); + + // μ·(xlo·2^r − SLLC_lo·2^16 − SLL_lo) = 0 (and hi). + let m = mu(b); + b.emit_base( + idx, + m * (xlo * two_r.clone() - sllc_lo.clone() * two_16.clone() - sll_lo.clone()), + ); + idx += 1; + let m = mu(b); + b.emit_base( + idx, + m * (xhi * two_r - sllc_hi.clone() * two_16 - sll_hi.clone()), + ); + idx += 1; + // μ·(Ylo − SLL_hi − SLLC_lo) = 0; μ·(Yhi − SLL_lo − SLLC_hi) = 0. + let m = mu(b); + b.emit_base(idx, m * (ylo - sll_hi - sllc_lo)); + idx += 1; + let m = mu(b); + b.emit_base(idx, m * (yhi - sll_lo - sllc_hi)); + idx += 1; + } + + // idx 813: IS_BIT(MU) — ungated booleanity, degree 2. See the struct + // doc for why this is emitted even though the bus argument already + // pins μ indirectly. + crate::constraints::templates::emit_is_bit(b, idx, cols::MU, None); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The value interpretation must reproduce the executor's compression — + /// same function the canonical oracle vectors validate. + #[test] + fn value_flow_matches_executor() { + use executor::vm::instruction::execution::blake3_compress_6round; + let h: [u32; 8] = core::array::from_fn(|i| 0x9E3779B9u32.wrapping_mul(i as u32 + 1)); + let m: [u32; 16] = core::array::from_fn(|i| 0x85EBCA6Bu32.wrapping_mul(i as u32 + 7)); + let t = 0x0123_4567_89AB_CDEFu64; + let (bl, fl) = (64u32, 11u32); + let flow = ValueFlow::compute(&h, &m, t, bl, fl); + assert_eq!(flow.out, blake3_compress_6round(&h, &m, t, bl, fl)); + } + + /// Canonical op counts: 96 add3s, 96 add2s, 96 rotations, 192+16 XORs. + #[test] + fn wire_flow_counts() { + let w = WireFlow::build(); + assert_eq!(w.add3s.len(), NUM_G * 2); + assert_eq!(w.add2s.len(), NUM_G * 2); + assert_eq!(w.rots.len(), NUM_G * 2); + assert_eq!(w.xors.len(), NUM_G * 4 + 16); + // Every output column lands exactly once, and inside the row. + use std::collections::HashSet; + let mut seen = HashSet::new(); + let mut claim = |c: usize| { + assert!(c < cols::NUM_COLUMNS, "column {c} out of range"); + assert!(seen.insert(c), "column {c} written twice"); + }; + for aw in &w.add3s { + for c in aw.s { + claim(c); + } + claim(aw.c1); + claim(aw.c2); + } + for aw in &w.add2s { + for c in aw.s { + claim(c); + } + } + for xw in &w.xors { + for c in xw.out { + claim(c); + } + } + for rw in &w.rots { + for c in rw + .sll_lo + .iter() + .chain(&rw.sllc_lo) + .chain(&rw.sll_hi) + .chain(&rw.sllc_hi) + .chain(&rw.y) + { + claim(*c); + } + } + // 48 G-blocks × 60 cells + 64 out bytes, all distinct. + assert_eq!(seen.len(), NUM_G * cols::G_SIZE + 64); + } +} + +/// ★ The executor's compression and `crypto`'s shared primitive are the same +/// function. +/// +/// #903 landed a second host transcription of the BLAKE3 compression: +/// `executor::vm::instruction::execution::blake3_compress_6round`, with its own +/// `blake3_g`, its own `BLAKE3_IV` and its own `BLAKE3_ROUNDS = 6`. `crypto`'s +/// `blake3_compress_rounds` is the one P-a Stage 1 hoisted precisely so there +/// would be a single definition — the same treatment the CUDA reference got. +/// +/// Two independently written encodings of one function is what PA-PLAN §1.4 +/// forbids ("do not prove that two … coincide; make them one function"), and +/// merging #903 reintroduced it. Until they are unified, this is the gate: the +/// executor is what the guest's syscall actually runs, so a divergence here is +/// a guest that hashes differently from the host prover — R5's invisible +/// failure, which surfaces only as in-guest proof rejection. +/// +/// Checked over the message schedule's structural edge cases plus a pseudo-random +/// sweep, at every `(t, block_len, flags)` shape the chain framing produces. +#[cfg(test)] +mod executor_primitive_parity { + use crypto::hash::blake3::{BLAKE3_SIX_ROUNDS, blake3_compress_rounds}; + use executor::vm::instruction::execution::blake3_compress_6round; + + #[test] + fn the_executor_compression_is_the_shared_primitive() { + // A cheap deterministic stream; no rand dependency in this crate. + let mut z = 0x243f_6a88_85a3_08d3u64; + let mut next = move || { + z ^= z << 13; + z ^= z >> 7; + z ^= z << 17; + z as u32 + }; + + // The flag/counter shapes `Blake3Chain` actually emits: first block + // (CHUNK_START), interior, and last (CHUNK_END|ROOT with the true byte + // count as block_len). t is 0 throughout for the chain, but the syscall + // takes a full 64-bit counter, so both halves are exercised. + let shapes: [(u64, u32, u32); 5] = [ + (0, 64, 1), // CHUNK_START + (0, 64, 0), // interior + (0, 7, 2 | 8), // CHUNK_END | ROOT, partial final block + (u64::MAX, 64, 0), // both counter halves set + (1 << 32, 0, 0xffff_ffff), // high half only; degenerate len/flags + ]; + + for (t, block_len, flags) in shapes { + for _ in 0..64 { + let h: [u32; 8] = core::array::from_fn(|_| next()); + let m: [u32; 16] = core::array::from_fn(|_| next()); + + assert_eq!( + blake3_compress_6round(&h, &m, t, block_len, flags), + blake3_compress_rounds(&h, &m, t, block_len, flags, BLAKE3_SIX_ROUNDS), + "executor and crypto disagree at t={t}, block_len={block_len}, flags={flags}" + ); + } + } + } + + /// CONTROL: the two would NOT agree at a different round count, so the test + /// above is comparing round counts as well as wiring. + #[test] + fn the_parity_is_round_count_sensitive() { + let h = [1u32, 2, 3, 4, 5, 6, 7, 8]; + let m: [u32; 16] = core::array::from_fn(|i| (i as u32) * 7 + 1); + assert_ne!( + blake3_compress_6round(&h, &m, 0, 64, 1), + blake3_compress_rounds(&h, &m, 0, 64, 1, BLAKE3_SIX_ROUNDS + 1), + "a 7-round reference must not match the 6-round executor" + ); + } +} + +/// ★ The state the guest hands the accelerator is the layout the executor +/// reads back. +/// +/// `Blake3Chain`'s riscv64 arm marshals a compression into +/// `crypto::hash::blake3::chain::pack_syscall_state`'s 22 dwords and reads the +/// result out of `unpack_syscall_out`. That marshaling is the last unchecked +/// link between the host prover's hash and the guest's: `executor_primitive_parity` +/// above gates the *compression*, and `crypto` gates the chain framing, but a +/// transposed dword or a swapped counter half would leave both of those green +/// and still make the guest hash differently — R5's invisible failure, visible +/// only as in-guest proof rejection. +/// +/// It needs no guest to check. The executor's syscall handler is ordinary host +/// code driven by a real `EcallEbreak`, so this lays the packed dwords into a VM +/// `Memory` exactly as the guest's `&mut [u64; 22]` presents them, runs the +/// instruction, and unpacks the result — closing the loop through the same two +/// functions the guest calls. +#[cfg(test)] +mod executor_syscall_packing { + use crypto::hash::blake3::chain::{SYSCALL_OUT_DWORD, pack_syscall_state, unpack_syscall_out}; + use crypto::hash::blake3::{BLAKE3_SIX_ROUNDS, blake3_compress_rounds}; + use executor::vm::instruction::decoding::Instruction; + use executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER; + use executor::vm::memory::Memory; + use executor::vm::registers::Registers; + + /// Pre-filled into the output dwords, so a packing that pointed the + /// accelerator at the wrong part of the region cannot pass on stale data. + const SENTINEL: u64 = 0xDEAD_BEEF_DEAD_BEEF; + + /// One compression the way a guest performs it: pack, ecall, unpack. + fn through_the_accelerator( + h: &[u32; 8], + m: &[u32; 16], + t: u64, + block_len: u32, + flags: u32, + ) -> [u32; 16] { + let addr = 0x1000u64; + let mut memory = Memory::default(); + let mut registers = Registers::default(); + + let mut state = pack_syscall_state(h, m, t, block_len, flags); + for dword in &mut state[SYSCALL_OUT_DWORD..] { + *dword = SENTINEL; + } + for (k, dword) in state.iter().enumerate() { + memory + .store_doubleword(addr + (k as u64) * 8, *dword) + .unwrap(); + } + + let mut pc = 0; + registers.write(17, BLAKE3_SYSCALL_NUMBER).unwrap(); + registers.write(10, addr).unwrap(); + Instruction::EcallEbreak + .run(&mut pc, &mut registers, &mut memory) + .unwrap(); + + for (k, dword) in state.iter_mut().enumerate() { + *dword = memory.load_doubleword(addr + (k as u64) * 8).unwrap(); + } + for (k, dword) in state.iter().enumerate().skip(SYSCALL_OUT_DWORD) { + assert_ne!(*dword, SENTINEL, "output dword {k} was never written"); + } + unpack_syscall_out(&state) + } + + #[test] + fn the_packed_state_is_what_the_executor_reads() { + // A cheap deterministic stream; no rand dependency in this crate. + let mut z = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + z ^= z << 13; + z ^= z >> 7; + z ^= z << 17; + z as u32 + }; + + // The flag shapes `Blake3Chain` emits — first block, interior, and last + // with a partial `block_len` — plus two counters setting exactly one + // half. The chain always sends `t = 0`, so only these pin the split + // order the packing chose. + let shapes: [(u64, u32, u32); 5] = [ + (0, 64, 1), // CHUNK_START + (0, 64, 0), // interior + (0, 7, 2 | 8), // CHUNK_END | ROOT, partial block + (0x0000_0000_FFFF_FFFF, 64, 0), // low counter half only + (0xFFFF_FFFF_0000_0000, 0, !0u32), // high half; degenerate len/flags + ]; + + for (t, block_len, flags) in shapes { + for _ in 0..16 { + let h: [u32; 8] = core::array::from_fn(|_| next()); + let m: [u32; 16] = core::array::from_fn(|_| next()); + assert_eq!( + through_the_accelerator(&h, &m, t, block_len, flags), + blake3_compress_rounds(&h, &m, t, block_len, flags, BLAKE3_SIX_ROUNDS), + "packed state mismatch at t={t}, block_len={block_len}, flags={flags}" + ); + } + } + } + + /// CONTROL: the check above discriminates the *layout*, not merely the + /// compression. The counter's two halves share one dword and are the + /// likeliest thing to transpose — and the chain, which only ever sends + /// `t = 0`, would never notice. They must be distinguishable, or agreement + /// above would survive the swap. + #[test] + fn the_packing_check_is_layout_sensitive() { + let h: [u32; 8] = core::array::from_fn(|i| (i as u32).wrapping_mul(2_654_435_761)); + let m: [u32; 16] = + core::array::from_fn(|i| (i as u32).wrapping_mul(40_503).wrapping_add(7)); + let t = 0x0000_0001_0000_0000u64; + + assert_ne!( + through_the_accelerator(&h, &m, t, 64, 1), + blake3_compress_rounds(&h, &m, t.rotate_left(32), 64, 1, BLAKE3_SIX_ROUNDS), + "the counter halves must be distinguishable" + ); + } +} diff --git a/prover/src/tables/cpu.rs b/prover/src/tables/cpu.rs index fc4c2f976..ee7200f1f 100644 --- a/prover/src/tables/cpu.rs +++ b/prover/src/tables/cpu.rs @@ -185,6 +185,10 @@ pub struct CpuOperation { pub ecall_keccak: bool, /// For KeccakPermute ECALLs: state address from x10. pub keccak_state_addr: u64, + /// Whether this ECALL is a Blake3Compress syscall. + pub ecall_blake3: bool, + /// For Blake3Compress ECALLs: state address from x10. + pub blake3_state_addr: u64, /// Whether this ECALL is an ECSM (elliptic-curve scalar multiply) syscall pub ecall_ecsm: bool, @@ -236,6 +240,9 @@ impl CpuOperation { let ecall_keccak = f.ecall && log.src1_val == executor::vm::instruction::execution::KECCAK_SYSCALL_NUMBER; let keccak_state_addr = if ecall_keccak { log.src2_val } else { 0 }; + let ecall_blake3 = + f.ecall && log.src1_val == executor::vm::instruction::execution::BLAKE3_SYSCALL_NUMBER; + let blake3_state_addr = if ecall_blake3 { log.src2_val } else { 0 }; // The ECSM operand addresses (x10/x11/x12) are recovered from the register state // in the trace builder. let ecall_ecsm = @@ -259,6 +266,8 @@ impl CpuOperation { commit_count, ecall_keccak, keccak_state_addr, + ecall_blake3, + blake3_state_addr, decode, timestamp, ..Default::default() @@ -359,6 +368,8 @@ impl CpuOperation { commit_count, ecall_keccak, keccak_state_addr, + ecall_blake3, + blake3_state_addr, ecall_ecsm, ecall_hint, } diff --git a/prover/src/tables/mod.rs b/prover/src/tables/mod.rs index f1a899f56..32b9a5a1d 100644 --- a/prover/src/tables/mod.rs +++ b/prover/src/tables/mod.rs @@ -22,6 +22,7 @@ pub mod types; pub mod bitwise; +pub mod blake3; pub mod branch; pub mod bytewise; pub mod commit; diff --git a/prover/src/tables/trace_builder.rs b/prover/src/tables/trace_builder.rs index 29874caef..a2fa896d8 100644 --- a/prover/src/tables/trace_builder.rs +++ b/prover/src/tables/trace_builder.rs @@ -40,6 +40,7 @@ use stark::storage_mode::StorageMode; use stark::trace::TraceTable; use super::bitwise::{self, BitwiseOperation, BitwiseOperationType}; +use super::blake3::{self, Blake3Operation}; use super::branch::{self, BranchOperation}; use super::bytewise; use super::commit::{self, CommitOperation}; @@ -547,6 +548,7 @@ fn collect_ops_from_cpu( Vec, Vec, Vec, + Vec, Vec, Vec, Vec, @@ -559,6 +561,7 @@ fn collect_ops_from_cpu( let mut bitwise_ops = Vec::with_capacity(cpu_ops.len() * 4); let mut commit_ops = Vec::new(); let mut keccak_ops = Vec::new(); + let mut blake3_ops = Vec::new(); let mut cpu32_ops = Vec::new(); let mut ecsm_ops = Vec::new(); let mut ecdas_ops = Vec::new(); @@ -648,6 +651,60 @@ fn collect_ops_from_cpu( }); } + // Collect Blake3Compress ECALL operations + if op.ecall_blake3 { + let state_addr = op.blake3_state_addr; + // 14 input dwords: h | m | t | (block_len, flags), LE words. + let mut words = [0u32; 28]; + for k in 0..14usize { + let dword_addr = state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + let mut dw = 0u64; + for b in 0..8 { + let byte_addr = dword_addr + .checked_add(b as u64) + .expect("blake3 state address range must be validated by the executor"); + let (byte_val, _ts) = memory_state.read_byte(byte_addr); + dw |= (byte_val as u64) << (b * 8); + } + words[2 * k] = dw as u32; + words[2 * k + 1] = (dw >> 32) as u32; + } + let h: [u32; 8] = words[0..8].try_into().unwrap(); + let m: [u32; 16] = words[8..24].try_into().unwrap(); + let t = (words[24] as u64) | ((words[25] as u64) << 32); + let block_len = words[26]; + let flags = words[27]; + let out = executor::vm::instruction::execution::blake3_compress_6round( + &h, &m, t, block_len, flags, + ); + // Previous content of the out region, read BEFORE the write ops + // below advance memory_state. + let mut old_out = [0u8; 64]; + for (b, byte) in old_out.iter_mut().enumerate() { + let byte_addr = state_addr + .checked_add(112 + b as u64) + .expect("blake3 state address range must be validated by the executor"); + let (v, _ts) = memory_state.read_byte(byte_addr); + *byte = v; + } + let blake3_memw_ops = + collect_blake3_memw_ops(op, &words, &out, memory_state, register_state); + memw.extend_ops(blake3_memw_ops); + blake3_ops.push(Blake3Operation { + timestamp: op.timestamp, + state_addr, + h, + m, + t, + block_len, + flags, + old_out, + out, + }); + } + // Collect ECSM ecall operations (memory I/O + the two table row sets) if op.ecall_ecsm { let (ecsm_memw, ecsm_op, ecdas_rows) = @@ -716,6 +773,7 @@ fn collect_ops_from_cpu( bitwise_ops, commit_ops, keccak_ops, + blake3_ops, cpu32_ops, ecsm_ops, ecdas_ops, @@ -1503,6 +1561,79 @@ fn collect_keccak_memw_ops( memw_ops } +/// Collect MEMW operations for a Blake3Compress ECALL. +/// +/// One register read of x10 plus 22 dword ops at the call's timestamp: the 14 +/// input dwords are pure reads (old = value = the input bytes, re-written at +/// `ts` like a LOAD), the 8 output dwords write the compression output over +/// the previous content. +fn collect_blake3_memw_ops( + op: &CpuOperation, + words: &[u32; 28], + out: &[u32; 16], + memory_state: &mut MemoryState, + register_state: &mut RegisterState, +) -> Vec { + let ts = op.timestamp; + let state_addr = op.blake3_state_addr; + let mut memw_ops = Vec::with_capacity(23); // 1 register read + 22 dword ops + + // Read register x10 to bind state_addr (same as keccak:c:read_addr). + { + let reg_value = pack_register_value(state_addr); + let reg_addr = 2 * 10u64; // x10 -> address 20 + let (_old_val, old_ts) = register_state.read(10); + let old_timestamps = [old_ts, old_ts, 0, 0, 0, 0, 0, 0]; + let memw_op = MemwOperation::new(true, reg_addr, reg_value, ts, 2, true) + .with_old(reg_value, old_timestamps); + memw_ops.push(memw_op); + register_state.write(10, state_addr, ts); + } + + for k in 0..22usize { + let dword_addr = state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + + // The dword's new value: input dwords re-write their own bytes, output + // dwords write the compression output. + let dw = if k < 14 { + (words[2 * k] as u64) | ((words[2 * k + 1] as u64) << 32) + } else { + let o = k - 14; + (out[2 * o] as u64) | ((out[2 * o + 1] as u64) << 32) + }; + let mut value_bytes = [0u32; 8]; + for (b, byte) in value_bytes.iter_mut().enumerate() { + *byte = ((dw >> (b * 8)) & 0xFF) as u32; + } + + let mut old_bytes = [0u32; 8]; + let mut old_timestamps = [0u64; 8]; + for b in 0..8 { + let byte_addr = dword_addr + .checked_add(b as u64) + .expect("blake3 state address range must be validated by the executor"); + let (old_val, old_ts) = memory_state.read_byte(byte_addr); + old_bytes[b] = old_val as u32; + old_timestamps[b] = old_ts; + } + + let memw_op = MemwOperation::new(false, dword_addr, value_bytes, ts, 8, true) + .with_old(old_bytes, old_timestamps); + memw_ops.push(memw_op); + + for (b, &val) in value_bytes.iter().enumerate() { + let byte_addr = dword_addr + .checked_add(b as u64) + .expect("blake3 state address range must be validated by the executor"); + memory_state.write_byte(byte_addr, val as u8, ts); + } + } + + memw_ops +} + /// /// From spec memw.md: /// - MEMW-C4 through MEMW-C7: old_timestamp[i] < timestamp (based on width) @@ -2437,6 +2568,97 @@ pub(crate) fn collect_bitwise_from_ecdas(ops: &[ecdas::EcdasOperation]) -> Vec Vec { + let mut ops = Vec::new(); + + for bop in blake3_ops { + let state_addr = bop.state_addr; + + // Alignment: addr[0] & 7 = 0. + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluAnd, + (state_addr & 0xFF) as u8, + 7, + )); + + // Addr byte range checks: (addr[2i], addr[2i+1]) pairs. + for i in 0..4 { + let lo = ((state_addr >> (2 * i * 8)) & 0xFF) as u8; + let hi = ((state_addr >> ((2 * i + 1) * 8)) & 0xFF) as u8; + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + lo, + hi, + )); + } + + // IS_HALF for the 22 pointers' halfwords. + for k in 0..blake3::STATE_DWORDS { + let ptr = state_addr + .checked_add(k as u64 * 8) + .expect("blake3 state address range must be validated by the executor"); + for shift in [0, 16, 32, 48] { + let half = ((ptr >> shift) & 0xFFFF) as u16; + ops.push(BitwiseOperation::halfword( + BitwiseOperationType::IsHalf, + (half & 0xFF) as u8, + ((half >> 8) & 0xFF) as u8, + )); + } + } + + // Mixing core + feed-forward, in the senders' canonical order. + let flow = blake3::ValueFlow::compute(&bop.h, &bop.m, bop.t, bop.block_len, bop.flags); + for &(a, b, _out) in &flow.xors { + for byte in 0..4 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::ByteAluXor, + ((a >> (8 * byte)) & 0xFF) as u8, + ((b >> (8 * byte)) & 0xFF) as u8, + )); + } + } + for &(sll_lo, sllc_lo, sll_hi, sllc_hi, _y) in &flow.rots { + for hw in [sll_lo, sllc_lo, sll_hi, sllc_hi] { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + (hw & 0xFF) as u8, + (hw >> 8) as u8, + )); + } + } + + // Message AreBytes: (byte 2p, byte 2p+1) of each m word. + for m in bop.m { + for p in 0..2 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + ((m >> (16 * p)) & 0xFF) as u8, + ((m >> (16 * p + 8)) & 0xFF) as u8, + )); + } + } + + // OLD_OUT AreBytes pairs. + for p in 0..32 { + ops.push(BitwiseOperation::byte_op( + BitwiseOperationType::AreBytes, + bop.old_out[2 * p], + bop.old_out[2 * p + 1], + )); + } + } + + ops +} + /// Collect BITWISE lookups generated by the keccak chips. /// /// The keccak round chip sends BYTE_ALU and ARE_BYTES interactions (the θ/ρ @@ -2864,6 +3086,9 @@ pub struct Traces { /// KECCAK_RC precomputed round constant table (32 rows) pub keccak_rc: TraceTable, + /// BLAKE3 6-round compression table (one row per compression call) + pub blake3: TraceTable, + /// ECSM core table (one row per scalar-multiplication ecall) pub ecsm: TraceTable, @@ -2907,6 +3132,7 @@ struct CollectedOps { dvrm_ops: Vec<(DvrmOperation, bool)>, commit_ops: Vec, keccak_ops: Vec, + blake3_ops: Vec, // Auxiliary ALU / memory / CPU32 dispatch chips (driven by the CPU ALU/MEMORY dispatch). eq_ops: Vec, bytewise_ops: Vec, @@ -2968,6 +3194,7 @@ fn collect_all_ops( mut bitwise_ops: Vec, commit_ops: Vec, keccak_ops: Vec, + blake3_ops: Vec, cpu32_ops: Vec, ecsm_ops: Vec, ecdas_ops: Vec, @@ -3108,6 +3335,7 @@ fn collect_all_ops( dvrm_ops, commit_ops, keccak_ops, + blake3_ops, eq_ops, bytewise_ops, store_ops, @@ -3152,6 +3380,7 @@ fn build_traces( dvrm_ops, commit_ops, keccak_ops, + blake3_ops, eq_ops, bytewise_ops, store_ops, @@ -3243,6 +3472,7 @@ fn build_traces( Box::new(|h| h.add_ops(&collect_bitwise_from_memw_aligned(&memw_aligned_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_commit(&commit_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_keccak(&keccak_ops))), + Box::new(|h| h.add_ops(&collect_bitwise_from_blake3(&blake3_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_ecsm(&ecsm_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_ecdas(&ecdas_ops))), Box::new(|h| h.add_ops(&collect_bitwise_from_hint(&hint_ops))), @@ -3513,6 +3743,7 @@ fn build_traces( .collect(); keccak_rnd::generate_keccak_rnd_trace(&keccak_rnd_ops) }; + let gen_blake3 = || blake3::generate_blake3_trace(&blake3_ops); let gen_keccak_rc = || { let mut keccak_rc_trace = keccak_rc::generate_keccak_rc_trace(); keccak_rc::update_multiplicities(&mut keccak_rc_trace, keccak_ops.len()); @@ -3542,6 +3773,7 @@ fn build_traces( (None, None, None, None); let (mut commit_slot, mut keccak_slot, mut keccak_rnd_slot, mut keccak_rc_slot) = (None, None, None, None); + let mut blake3_slot = None; let (mut pages_slot, mut register_slot, mut halt_slot) = (None, None, None); let (mut eqs_slot, mut bytewises_slot, mut stores_slot, mut cpu32s_slot) = (None, None, None, None); @@ -3579,6 +3811,7 @@ fn build_traces( spawn_into!(keccak_slot, gen_keccak); spawn_into!(keccak_rnd_slot, gen_keccak_rnd); spawn_into!(keccak_rc_slot, gen_keccak_rc); + spawn_into!(blake3_slot, gen_blake3); spawn_into!(commit_slot, gen_commit); spawn_into!(register_slot, gen_register); spawn_into!(halt_slot, gen_halt); @@ -3607,6 +3840,7 @@ fn build_traces( keccak_slot = Some(gen_keccak()); keccak_rnd_slot = Some(gen_keccak_rnd()); keccak_rc_slot = Some(gen_keccak_rc()); + blake3_slot = Some(gen_blake3()); pages_slot = Some(gen_pages()); register_slot = Some(gen_register()); halt_slot = Some(gen_halt()); @@ -3643,6 +3877,7 @@ fn build_traces( let keccak_trace = keccak_slot.expect(PHASE5_RAN); let keccak_rnd_trace = keccak_rnd_slot.expect(PHASE5_RAN); let keccak_rc_trace = keccak_rc_slot.expect(PHASE5_RAN); + let blake3_trace = blake3_slot.expect(PHASE5_RAN); #[allow(unused_mut)] let (mut pages, page_configs) = pages_slot.expect(PHASE5_RAN); #[allow(unused_mut)] @@ -3716,6 +3951,7 @@ fn build_traces( commit: commit_trace, keccak: keccak_trace, keccak_rnd: keccak_rnd_trace, + blake3: blake3_trace, keccak_rc: keccak_rc_trace, ecsm: ecsm_trace, ecdas: ecdas_trace, @@ -3989,6 +4225,7 @@ impl Traces { pub fn total_field_elements(&self) -> u64 { use super::bitwise::NUM_PRECOMPUTED_COLS as BITWISE_PRECOMPUTED; use super::bitwise::cols::NUM_COLUMNS as BITWISE_COLS; + use super::blake3::cols::NUM_COLUMNS as BLAKE3_COLS; use super::branch::cols::NUM_COLUMNS as BRANCH_COLS; use super::bytewise::cols::NUM_COLUMNS as BYTEWISE_COLS; use super::commit::cols::NUM_COLUMNS as COMMIT_COLS; @@ -4038,6 +4275,7 @@ impl Traces { keccak, keccak_rnd, keccak_rc, + blake3, ecsm, ecdas, hint, @@ -4094,6 +4332,7 @@ impl Traces { total += (keccak.num_rows() * KECCAK_COLS) as u64; total += (keccak_rnd.num_rows() * KECCAK_RND_COLS) as u64; total += (keccak_rc.num_rows() * (KECCAK_RC_COLS - KECCAK_RC_PRECOMPUTED)) as u64; + total += (blake3.num_rows() * BLAKE3_COLS) as u64; for t in eqs { total += (t.num_rows() * EQ_COLS) as u64; } @@ -4144,6 +4383,7 @@ impl Traces { let n_keccak = aux_cols(super::keccak::bus_interactions().len()); let n_keccak_rnd = aux_cols(super::keccak_rnd::bus_interactions().len()); let n_keccak_rc = aux_cols(super::keccak_rc::bus_interactions().len()); + let n_blake3 = aux_cols(super::blake3::bus_interactions().len()); let n_eq = aux_cols(super::eq::bus_interactions().len()); let n_bytewise = aux_cols(super::bytewise::bus_interactions().len()); let n_store = aux_cols(super::store::bus_interactions().len()); @@ -4171,6 +4411,7 @@ impl Traces { keccak, keccak_rnd, keccak_rc, + blake3, ecsm, ecdas, hint, @@ -4227,6 +4468,7 @@ impl Traces { total += (keccak.num_rows() * n_keccak) as u64; total += (keccak_rnd.num_rows() * n_keccak_rnd) as u64; total += (keccak_rc.num_rows() * n_keccak_rc) as u64; + total += (blake3.num_rows() * n_blake3) as u64; for t in eqs { total += (t.num_rows() * n_eq) as u64; } @@ -4592,6 +4834,7 @@ impl Traces { bitwise_ops, commit_ops, keccak_ops, + blake3_ops, cpu32_ops, ecsm_ops, ecdas_ops, @@ -4611,6 +4854,7 @@ impl Traces { bitwise_ops, commit_ops, keccak_ops, + blake3_ops, cpu32_ops, ecsm_ops, ecdas_ops, @@ -4705,6 +4949,7 @@ impl Traces { bitwise_ops, commit_ops, keccak_ops, + blake3_ops, cpu32_ops, ecsm_ops, ecdas_ops, @@ -4720,6 +4965,7 @@ impl Traces { bitwise_ops, commit_ops, keccak_ops, + blake3_ops, cpu32_ops, ecsm_ops, ecdas_ops, diff --git a/prover/src/tables/types.rs b/prover/src/tables/types.rs index fab4aabff..f8da38ca8 100644 --- a/prover/src/tables/types.rs +++ b/prover/src/tables/types.rs @@ -359,6 +359,18 @@ pub enum BusId { /// Cross-epoch memory bus: the local-to-global table's per-cell init/fini /// boundary claims, matched across epochs by the final aggregation LogUp. GlobalMemory = 31, + + // ========================================================================= + // LFM — the Lambda Field Machine (recursion machine; `lfm` module) + // ========================================================================= + /// LFM write-once memory: token `(addr, v0..v3)`. Writes send with the + /// preprocessed static read count; reads receive gated by is_real. + LfmMem = 32, + /// LFM 16-bit range lookup (the `LFM_RANGE` fixed table). + LfmRange = 33, + /// LFM public values: token `(index, v0..v3)`; closed by a + /// consumer-computed balance (the COMMIT-bus pattern). + LfmPublic = 34, } impl BusId { @@ -388,6 +400,9 @@ impl BusId { BusId::Ecdas => "Ecdas", BusId::Bit => "Bit", BusId::GlobalMemory => "GlobalMemory", + BusId::LfmMem => "LfmMem", + BusId::LfmRange => "LfmRange", + BusId::LfmPublic => "LfmPublic", } } } @@ -420,6 +435,9 @@ impl TryFrom for BusId { 28 => Ok(BusId::Ecdas), 30 => Ok(BusId::Bit), 31 => Ok(BusId::GlobalMemory), + 32 => Ok(BusId::LfmMem), + 33 => Ok(BusId::LfmRange), + 34 => Ok(BusId::LfmPublic), other => Err(other), } } diff --git a/prover/src/test_utils.rs b/prover/src/test_utils.rs index d6a8b8608..caa3104e0 100644 --- a/prover/src/test_utils.rs +++ b/prover/src/test_utils.rs @@ -39,6 +39,9 @@ use crate::tables::bitwise::{ BitwiseOperation, BitwiseOperationType, bus_interactions as bitwise_bus_interactions, cols as bitwise_cols, }; +use crate::tables::blake3::{ + Blake3Constraints, bus_interactions as blake3_bus_interactions, cols as blake3_cols, +}; use crate::tables::branch::{ BranchConstraints, bus_interactions as branch_bus_interactions, cols as branch_cols, }; @@ -143,6 +146,7 @@ where transcript, #[cfg(feature = "disk-spill")] StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) } @@ -972,6 +976,18 @@ pub fn create_keccak_air(proof_options: &ProofOptions) -> ConcreteVmAir ConcreteVmAir { + build_air( + blake3_cols::NUM_COLUMNS, + blake3_bus_interactions(), + proof_options, + 1, + Blake3Constraints, + "BLAKE3", + ) +} + /// Create KECCAK_RND AIR with pi constraints and bus interactions. pub fn create_keccak_rnd_air(proof_options: &ProofOptions) -> ConcreteVmAir { build_air( @@ -1019,3 +1035,128 @@ pub fn create_ecdas_air(proof_options: &ProofOptions) -> ConcreteVmAir>, +); + +/// The page base used wherever a test needs a concrete PAGE or GLOBAL_MEMORY +/// AIR. +/// +/// See [`production_airs`] on why a base has to be pinned at all. +pub const PAGE_TEST_BASE: u64 = 0x1000; + +/// The epoch label used wherever a test needs a concrete L2G AIR. A 1-based fini +/// epoch — `l2g_memory_air` debug-asserts `>= 1`. +pub const EPOCH_TEST_LABEL: u64 = 1; + +/// Every production table AIR, constructed and boxed behind the common trait +/// object. +/// +/// AIR-construction only — no ELF, no program execution — so this runs +/// anywhere. There is no ELF-free registry to iterate instead (`VmAirs::n` needs +/// a real ELF plus preprocessed-commitment builds, and omits zero-count tables), +/// so this list is hand-maintained — but exactly once. It previously existed as +/// three hand-copied per-suite lists, and all three shared the same blind spot: +/// the continuation-only tables at the end were in none of them. +/// +/// # This list includes the CONTINUATION tables +/// +/// The last three come from `crate::continuation` and appear in no monolithic +/// proof. They are not optional extras: a continuation proof is what the +/// recursion path actually verifies, so a per-table sweep that stops at the +/// monolithic 25 is complete for a proof shape we do not care about. +/// +/// # Four of these AIRs are PARAMETERIZED +/// +/// `PAGE` and `GLOBAL_MEMORY` fold a `page_base` into constant bus terms; the +/// two `L2G` tables fold an `epoch_label` into constant terms +/// (`BusValue::constant(epoch_label)` and `LinearTerm::Constant(epoch_label-1)` +/// respectively). Those constants reach the captured IR, so each of these four +/// has a DIFFERENT constraint program per parameter value — there is no single +/// "the PAGE program". This list pins one value each +/// ([`PAGE_TEST_BASE`], [`EPOCH_TEST_LABEL`]) so the per-table suites have +/// something concrete to check; anything reasoning about a whole continuation +/// proof must account for one program per distinct base and per distinct epoch. +/// `constraint_artifact_tests::parameterized_airs_vary_per_parameter_value` +/// characterizes exactly how they differ. +pub fn production_airs(proof_options: &ProofOptions) -> Vec { + vec![ + ("CPU", Box::new(create_cpu_air(proof_options))), + ("BITWISE", Box::new(create_bitwise_air(proof_options))), + ("LT", Box::new(create_lt_air(proof_options))), + ("SHIFT", Box::new(create_shift_air(proof_options))), + ("EQ", Box::new(create_eq_air(proof_options))), + ("BYTEWISE", Box::new(create_bytewise_air(proof_options))), + ("STORE", Box::new(create_store_air(proof_options))), + ("CPU32", Box::new(create_cpu32_air(proof_options))), + ("MEMW", Box::new(create_memw_air(proof_options))), + ("MEMW_A", Box::new(create_memw_aligned_air(proof_options))), + ("MEMW_R", Box::new(create_memw_register_air(proof_options))), + ("LOAD", Box::new(create_load_air(proof_options))), + ("DECODE", Box::new(create_decode_air(proof_options))), + ("MUL", Box::new(create_mul_air(proof_options))), + ("DVRM", Box::new(create_dvrm_air(proof_options))), + ("BRANCH", Box::new(create_branch_air(proof_options))), + ("HALT", Box::new(create_halt_air(proof_options))), + ("COMMIT", Box::new(create_commit_air(proof_options))), + ( + "PAGE", + Box::new(create_page_air(proof_options, PAGE_TEST_BASE)), + ), + ("REGISTER", Box::new(create_register_air(proof_options))), + ("KECCAK", Box::new(create_keccak_air(proof_options))), + ("KECCAK_RND", Box::new(create_keccak_rnd_air(proof_options))), + ("KECCAK_RC", Box::new(create_keccak_rc_air(proof_options))), + ("ECSM", Box::new(create_ecsm_air(proof_options))), + ("ECDAS", Box::new(create_ecdas_air(proof_options))), + ("HINT", Box::new(create_hint_air(proof_options))), + // ---- continuation-only tables (no monolithic proof contains these) ---- + ( + "L2G_GLOBAL", + Box::new(crate::continuation::l2g_global_air( + proof_options, + EPOCH_TEST_LABEL, + )), + ), + ( + "L2G_MEMORY", + Box::new(crate::continuation::l2g_memory_air( + proof_options, + EPOCH_TEST_LABEL, + )), + ), + ( + "GLOBAL_MEMORY", + Box::new(create_global_memory_air(proof_options, PAGE_TEST_BASE)), + ), + ] +} + +/// Create the GLOBAL_MEMORY AIR for one page, as a continuation proof would. +/// +/// The preprocessed commitment is supplied rather than recomputed: it is a +/// blowup-dependent Merkle root over the page's genesis values, and it is NOT +/// part of the constraint artifact (the verifier gets it through the existing +/// static-commitment mechanism). Recomputing it here would cost an FFT plus a +/// Merkle build per call and change nothing any IR suite looks at. +pub fn create_global_memory_air( + proof_options: &ProofOptions, + page_base: u64, +) -> ConcreteVmAir { + let config = crate::tables::page::PageConfig::zero_init(page_base); + crate::continuation::global_memory_air(proof_options, &config, Some([0u8; 32])) +} + +/// The number of production table AIRs [`production_airs`] yields: 26 monolithic +/// plus 3 continuation-only. +/// +/// Every suite that iterates the list asserts against this. That assert is the +/// point: the list is hand-maintained, so without it a table added to +/// `test_utils` or to `continuation.rs` and forgotten here escapes every +/// per-table suite at once and silently — which is exactly what happened to the +/// three continuation tables. Bump it deliberately when a table is genuinely +/// added or removed. +pub const NUM_PRODUCTION_AIRS: usize = 29; diff --git a/prover/src/tests/constraint_artifact_tests.rs b/prover/src/tests/constraint_artifact_tests.rs new file mode 100644 index 000000000..ce5edd58f --- /dev/null +++ b/prover/src/tests/constraint_artifact_tests.rs @@ -0,0 +1,1300 @@ +//! Serialization bit-exactness: for every production table, the constraint +//! artifact SERIALIZED at build time and read back evaluates identically to the +//! compiled folders. +//! +//! `constraint_program_tests` already pins the in-memory capture against both +//! folders. This suite pins the extra hop that "constraints as data" adds — the +//! bytes: +//! +//! ```text +//! capture → artifact → to_bytes → from_bytes → lift → evaluate +//! ``` +//! +//! Every stage in that chain is a place a program can change meaning: a +//! truncated `u32` index, a constant that loses canonical form through raw +//! limbs, a metadata list that reorders, an op tag that decodes to a different +//! operation. None of it is visible to a test that only exercises the in-memory +//! program, which is why this suite runs the deserialized artifact — never the +//! captured object — against the folders. +//! +//! The folders are the oracle: they are the production prove/verify path, +//! independently pinned by the prove→verify suites and cross-version +//! verification. All three evaluation paths are checked against them: +//! `eval_program` (prover shape), `eval_program_verifier` (OOD shape, the +//! recursion path), and `eval_device_program` (the flat blob). + +use math::field::element::FieldElement; +use stark::constraint_ir::{ + ConstraintArtifact, eval_device_program, eval_program, eval_program_verifier, +}; +use stark::frame::Frame; +use stark::proof::options::GoldilocksCubicProofOptions; +use stark::table::TableView; +use stark::traits::{AIR, TransitionEvaluationContext}; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; +use crate::test_utils::*; + +type Gl = GoldilocksField; +type Ext3 = GoldilocksExtension; +type Fp = FieldElement; +type Fp3 = FieldElement; + +const TRIALS: usize = 100; + +/// Deterministic SplitMix64 (no `rand` dependency). +struct SplitMix64(u64); +impl SplitMix64 { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn fp3(&mut self) -> Fp3 { + Fp3::new([ + Fp::from(self.next_u64()), + Fp::from(self.next_u64()), + Fp::from(self.next_u64()), + ]) + } +} + +/// Extension element → raw `[u64; 3]` limbs (the device representation). +fn enc(x: &Fp3) -> [u64; 3] { + let limbs = x.value(); + [*limbs[0].value(), *limbs[1].value(), *limbs[2].value()] +} + +/// One production AIR's serialized-artifact differential. Returns the artifact's +/// measured size for the size report. +fn check_air_artifact( + air: &dyn AIR, + label: &str, +) -> ArtifactSize { + let n = air.context().num_transition_constraints; + let num_base = air.num_base_transition_constraints(); + let (n_main, n_aux) = air.trace_layout(); + + let artifact = ConstraintArtifact::capture(air); + artifact + .validate_against(air) + .unwrap_or_else(|e| panic!("[{label}] freshly captured artifact rejected: {e}")); + + // The wire hop. Everything below runs the DESERIALIZED artifact, so a + // codec bug cannot hide behind the in-memory object. + let bytes = artifact + .to_bytes() + .unwrap_or_else(|e| panic!("[{label}] serialize failed: {e}")); + let artifact = ConstraintArtifact::from_bytes(&bytes) + .unwrap_or_else(|e| panic!("[{label}] deserialize failed: {e}")); + artifact + .validate_against(air) + .unwrap_or_else(|e| panic!("[{label}] deserialized artifact rejected: {e}")); + + let prog = artifact.program(); + let dev = artifact.device_program(); + + // Structural identity against the capture the AIR itself would produce. + let captured = air.constraint_program(); + assert_eq!( + prog.nodes, captured.nodes, + "[{label}] nodes changed on the wire" + ); + assert_eq!( + prog.dims, captured.dims, + "[{label}] dims changed on the wire" + ); + assert_eq!( + prog.roots, captured.roots, + "[{label}] roots changed on the wire" + ); + assert_eq!( + prog.num_base, captured.num_base, + "[{label}] num_base changed" + ); + assert_eq!( + prog.base_consts, captured.base_consts, + "[{label}] base constants changed on the wire" + ); + assert_eq!( + prog.ext_consts, captured.ext_consts, + "[{label}] ext constants changed on the wire" + ); + assert_eq!(prog.roots.len(), n, "[{label}] one root per constraint"); + + // Release-safe exact-once backstop, as in `constraint_program_tests`: root + // id 0 is the reserved base-zero sentinel and no production constraint is + // identically zero, so a root still at the sentinel means that constraint + // was never captured. + for (i, &root) in prog.roots.iter().enumerate() { + assert_ne!(root, 0, "[{label}] constraint {i} was never captured"); + } + + let mut rng = SplitMix64(0x5EED_1234 ^ label.len() as u64); + for trial in 0..TRIALS { + let mk_step = |rng: &mut SplitMix64| { + let main: Vec = (0..n_main).map(|_| Fp::from(rng.next_u64())).collect(); + let aux: Vec = (0..n_aux).map(|_| rng.fp3()).collect(); + TableView::new(vec![main], vec![aux]) + }; + let frame = Frame::::new(vec![mk_step(&mut rng), mk_step(&mut rng)]); + let challenges = vec![rng.fp3(), rng.fp3()]; // [z, alpha] + let alphas: Vec = (0..air.max_bus_elements() + 2).map(|_| rng.fp3()).collect(); + let offset = rng.fp3(); + + let ctx = TransitionEvaluationContext::new_prover( + frame.as_row_frame(), + &challenges, + &alphas, + &offset, + ); + + // --- oracle: the compiled prover folder --- + let mut f_base = vec![Fp::zero(); num_base]; + let mut f_ext = vec![Fp3::zero(); n]; + air.compute_transition_prover(&ctx, &mut f_base, &mut f_ext); + + // --- path 1: generic interpreter over the deserialized program --- + let mut i_base = vec![Fp::zero(); num_base]; + let mut i_ext = vec![Fp3::zero(); n]; + eval_program(&prog, &ctx, &mut i_base, &mut i_ext); + + for c in 0..num_base { + assert_eq!( + f_base[c], i_base[c], + "[{label}] prover folder vs serialized program, base constraint {c}, trial {trial}" + ); + } + for c in num_base..n { + assert_eq!( + f_ext[c], i_ext[c], + "[{label}] prover folder vs serialized program, ext constraint {c}, trial {trial}" + ); + } + + // --- path 2: flat device walk over the deserialized blob --- + let main_raw: Vec> = (0..2) + .map(|off| { + let step = frame.get_evaluation_step(off); + (0..n_main) + .map(|c| *step.get_main_evaluation_element(0, c).value()) + .collect() + }) + .collect(); + let aux_raw: Vec> = (0..2) + .map(|off| { + let step = frame.get_evaluation_step(off); + (0..n_aux) + .map(|c| enc(step.get_aux_evaluation_element(0, c))) + .collect() + }) + .collect(); + let rap_raw: Vec<[u64; 3]> = challenges.iter().map(enc).collect(); + let alpha_raw: Vec<[u64; 3]> = alphas.iter().map(enc).collect(); + + let mut d_base = vec![0u64; num_base]; + let mut d_ext = vec![[0u64; 3]; n]; + eval_device_program( + &dev, + &main_raw, + &aux_raw, + &rap_raw, + &alpha_raw, + enc(&offset), + &mut d_base, + &mut d_ext, + ); + for c in 0..num_base { + assert_eq!( + d_base[c], + *f_base[c].value(), + "[{label}] prover folder vs serialized device blob, base constraint {c}, trial {trial}" + ); + } + for c in num_base..n { + assert_eq!( + d_ext[c], + enc(&f_ext[c]), + "[{label}] prover folder vs serialized device blob, ext constraint {c}, trial {trial}" + ); + } + + // --- path 3: the verifier/OOD shape, i.e. the recursion path --- + let embed = |step: &TableView| -> TableView { + let main: Vec = (0..n_main) + .map(|c| step.get_main_evaluation_element(0, c).to_extension()) + .collect(); + let aux: Vec = (0..n_aux) + .map(|c| *step.get_aux_evaluation_element(0, c)) + .collect(); + TableView::new(vec![main], vec![aux]) + }; + let vframe: Frame = Frame::new(vec![ + embed(frame.get_evaluation_step(0)), + embed(frame.get_evaluation_step(1)), + ]); + let vctx = TransitionEvaluationContext::::new_verifier( + &vframe, + &challenges, + &alphas, + &offset, + ); + + let v_folder = air.compute_transition(&vctx); + let mut v_interp = vec![Fp3::zero(); n]; + eval_program_verifier(&prog, &vctx, &mut v_interp); + for c in 0..n { + assert_eq!( + v_folder[c], v_interp[c], + "[{label}] verifier folder vs serialized program, constraint {c}, trial {trial}" + ); + } + } + + ArtifactSize { + label: label.to_string(), + constraints: n, + nodes: artifact.nodes.len(), + base_consts: artifact.base_consts.len(), + ext_consts: artifact.ext_consts.len(), + bytes: bytes.len(), + } +} + +/// One AIR's measured artifact size. +struct ArtifactSize { + label: String, + constraints: usize, + nodes: usize, + base_consts: usize, + ext_consts: usize, + bytes: usize, +} + +/// Every production table's serialized artifact evaluates bit-identically to +/// the compiled folders, on the prover shape, the verifier/OOD shape, and the +/// flat device blob. +#[test] +fn all_table_artifacts_roundtrip_and_match_folders() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "the production AIR list changed size; every per-table suite's coverage moved with it" + ); + + let mut sizes: Vec = Vec::with_capacity(airs.len()); + for (label, air) in &airs { + sizes.push(check_air_artifact(&**air, label)); + } + + report_sizes(&sizes); +} + +/// Print the per-AIR and total artifact sizes — the recursion machine's +/// program-length budget — and hold a ceiling so runaway growth is noticed. +fn report_sizes(sizes: &[ArtifactSize]) { + let total_bytes: usize = sizes.iter().map(|s| s.bytes).sum(); + let total_nodes: usize = sizes.iter().map(|s| s.nodes).sum(); + let total_constraints: usize = sizes.iter().map(|s| s.constraints).sum(); + + println!("\nconstraint artifact sizes (blowup=2)"); + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + "table", "constr", "nodes", "bconst", "econst", "bytes" + ); + for s in sizes { + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + s.label, s.constraints, s.nodes, s.base_consts, s.ext_consts, s.bytes + ); + } + println!( + "{:<12} {:>7} {:>9} {:>7} {:>7} {:>10}", + "TOTAL", total_constraints, total_nodes, "", "", total_bytes + ); + println!( + "total: {total_nodes} nodes, {total_bytes} bytes ({:.1} KiB) across {} tables\n", + total_bytes as f64 / 1024.0, + sizes.len() + ); + + // A loose ceiling: this is a budget signal, not a tight assertion. It exists + // so a change that multiplies the program size fails here instead of being + // discovered when the recursion machine will not fit. + const CEILING_BYTES: usize = 8 * 1024 * 1024; + assert!( + total_bytes < CEILING_BYTES, + "total artifact size {total_bytes} exceeds the {CEILING_BYTES}-byte budget ceiling" + ); +} + +/// Per-AIR instruction census for the recursion machine's constraint leg. +/// +/// The machine is straight-line and cannot interpret, so a serialized program is +/// UNROLLED: one machine instruction per arithmetic IR node. That makes the node +/// census a direct instruction-count estimate for the constraint-evaluation leg, +/// which is the last unmeasured piece of the epoch-verifier budget. +/// +/// The classification that matters, and why: +/// +/// - **Leaves are addresses, not instructions.** `Var` reads an OOD frame value +/// the DEEP/opening leg already placed in memory; `RapChallenge` / `AlphaPow` / +/// `TableOffset` are transcript-derived values computed once per proof. The +/// constraint leg pays nothing marginal for them. +/// - **Constants are `Const` instructions**, one per distinct pooled value. +/// - **Arithmetic nodes are ALU instructions**, and the base/ext split is the +/// expensive distinction: a base node is a `BaseAlu` over one Goldilocks +/// element, an extension node an `ExtAlu` over three. +/// - **A `Mul` with an extension result and exactly one base operand** is the +/// `MulBase` form — 3 base multiplies instead of 9. Counting these separately +/// is the difference between a real estimate and a pessimistic one. +/// +/// # The IR's own `dim` tags are the WRONG split for the machine +/// +/// `Dim` records what the PROVER computes: its frame is base-field, so a +/// trace-only subexpression stays in the base field. The machine runs the +/// VERIFIER's evaluation at the OOD point, where the frame holds only extension +/// elements — `eval_program_verifier` resolves every `Var` to `Value::Ext` +/// regardless of `main`. So a node is base at verify time only if its whole +/// subtree is constants. +/// +/// Both splits are reported because the difference is large and load-bearing, +/// and taking the declared one would badly understate the machine's extension +/// traffic. The verifier-side column is the one to budget against. +/// +/// A consequence worth naming: a base-at-verify-time node is a constant-only +/// subtree, so the emitter can FOLD it at build time into a pooled constant. It +/// costs zero instructions, which is why the instruction estimate below counts +/// only extension work plus the pool. +/// +/// Printed rather than asserted (beyond a loose ceiling): this is an instrument, +/// and pinning exact counts would turn every constraint edit into a test failure. +/// +/// # WHAT THIS INSTRUMENT CANNOT SEE +/// +/// It reads captured IR, one AIR at a time. It knows nothing about how a proof +/// is ASSEMBLED from sub-proofs — not that the split-table families are chunked, +/// not that `FIXED_TABLE_COUNT` forces a sub-proof for a table with zero rows, +/// not which tables a continuation epoch even contains. +/// +/// So any conclusion about workload sensitivity, epoch composition or sub-proof +/// count is outside what these numbers support, however inviting the per-table +/// breakdown makes it. This is not hypothetical: a previous reading of this +/// table concluded the constraint leg was "workload-shaped" because ECDAS, ECSM +/// and KECCAK_RND are 87% of it — and that is false, because those tables are +/// present in every proof whether the workload touches them or not. The leg is +/// workload-INDEPENDENT, and only `epoch_chunk_multiplier` and +/// `continuation_epoch_constraint_leg` can tell you so. +/// +/// Use those two for anything per-proof. Use this one for per-AIR facts only. +#[test] +fn constraint_op_census() { + use stark::constraint_ir::artifact::DIM_BASE; + use stark::constraint_ir::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, + }; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + println!("\nconstraint-leg instruction census (one instruction per arithmetic node)"); + println!(" prover-dim = what the IR declares; verify-dim = what the machine runs"); + println!( + "{:<14} {:>7} {:>7} {:>6} {:>8} {:>8} {:>7} {:>8} {:>8}", + "table", "nodes", "leaves", "const", "pv-base", "fold", "ext", "mulbase", "instr" + ); + + let (mut t_nodes, mut t_leaves, mut t_const) = (0usize, 0usize, 0usize); + let (mut t_pv_base, mut t_fold, mut t_ext, mut t_mulbase) = (0usize, 0usize, 0usize, 0usize); + let mut t_constraints = 0usize; + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let nodes = &artifact.nodes; + + // Verifier-side dim: base ONLY for constant-only subtrees, because the + // OOD frame is all-extension. + let mut v_base = vec![false; nodes.len()]; + + let (mut leaves, mut consts, mut pv_base, mut foldable, mut ext_alu, mut mulbase) = + (0, 0, 0, 0, 0, 0); + for (i, n) in nodes.iter().enumerate() { + match n.op { + OP_VAR | OP_RAP_CHALLENGE | OP_ALPHA_POW | OP_TABLE_OFFSET => { + leaves += 1; + v_base[i] = false; + } + OP_CONST_BASE => { + consts += 1; + v_base[i] = true; + } + OP_CONST_EXT => { + consts += 1; + v_base[i] = false; + } + OP_ADD | OP_SUB | OP_MUL | OP_NEG | OP_EMBED => { + if n.dim == DIM_BASE { + pv_base += 1; + } + let (ba, bb) = match n.op { + OP_NEG => (v_base[n.a as usize], true), + OP_EMBED => (false, false), + _ => (v_base[n.a as usize], v_base[n.b as usize]), + }; + // Mirrors `interp::binop`: base only when both operands are + // base values AND the declared dim is base. + v_base[i] = ba && bb && n.dim == DIM_BASE; + if v_base[i] { + // Constant-only subtree: the emitter folds it at build + // time, so it emits no instruction at all. + foldable += 1; + } else { + let is_mulbase = n.op == OP_MUL && (ba != bb); + if is_mulbase { + mulbase += 1; + } else { + ext_alu += 1; + } + } + } + other => panic!("[{label}] unclassified op tag {other}"), + } + } + + // Instructions the constraint leg actually emits: extension ALU work + // plus one Const per pooled constant. Leaves are addresses, and + // constant-only subtrees fold away at build time. + let instr = ext_alu + mulbase + consts; + println!( + "{:<14} {:>7} {:>7} {:>6} {:>8} {:>8} {:>7} {:>8} {:>8}", + label, + nodes.len(), + leaves, + consts, + pv_base, + foldable, + ext_alu, + mulbase, + instr + ); + + t_nodes += nodes.len(); + t_leaves += leaves; + t_const += consts; + t_pv_base += pv_base; + t_fold += foldable; + t_ext += ext_alu; + t_mulbase += mulbase; + t_constraints += artifact.roots.len(); + + // Every constant node must correspond to exactly one pooled table entry; + // if that ever stopped holding, the Const instruction count above would + // be wrong. + assert_eq!( + consts, + artifact.base_consts.len() + artifact.ext_consts.len(), + "[{label}] constant nodes and pooled constants disagree" + ); + } + + // --- two emitter properties the design depends on, measured --- + // + // 1. MulAdd fusability. `ExtAlu` carries MulAdd as a first-class op, but the + // IR has no MulAdd node — it emits Mul then Add. A peephole can fuse + // `Add(Mul(a,b), c)` into one instruction, but ONLY when the Mul feeds + // exactly one consumer; hash-consing means a shared Mul would have to be + // recomputed, turning a saving into a cost. + // 2. `Op::Embed` usage. It should be zero — the builder documents it as + // unreachable from the single-body capture path — which matters because + // Embed is the one op whose machine lowering depends on the word model. + let (mut t_fusable, mut t_embed, mut t_dead, mut t_maxfan) = (0usize, 0usize, 0usize, 0u32); + // Machine constants are interned PROGRAM-WIDE, keyed on the canonical + // 4-lane word — one `Const` row per distinct value however many AIRs and + // however many nodes use it. Summing the per-AIR pools therefore overcounts, + // and small structural constants (0, 1, byte/halfword shifts) recur across + // every table. + let mut distinct_words: std::collections::BTreeSet<[u64; 4]> = + std::collections::BTreeSet::new(); + for (_, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let nodes = &artifact.nodes; + + let mut uses = vec![0u32; nodes.len()]; + for n in nodes { + match n.op { + OP_ADD | OP_SUB | OP_MUL => { + uses[n.a as usize] += 1; + uses[n.b as usize] += 1; + } + OP_NEG | OP_EMBED => uses[n.a as usize] += 1, + _ => {} + } + } + // A root is a consumer too: fusing away a node that a constraint roots at + // would delete the value the quotient recombination needs. + for &r in &artifact.roots { + uses[r as usize] += 1; + } + + for &c in &artifact.base_consts { + distinct_words.insert([c, 0, 0, 0]); + } + for &e in &artifact.ext_consts { + distinct_words.insert([e[0], e[1], e[2], 0]); + } + + // A node nobody reads is a write with mult = 0 — wasted instructions, and + // the emitter must DCE it rather than emit a zero-multiplicity write. + t_dead += uses.iter().filter(|&&u| u == 0).count(); + t_maxfan = t_maxfan.max(uses.iter().copied().max().unwrap_or(0)); + + for n in nodes { + if n.op == OP_EMBED { + t_embed += 1; + } + if n.op == OP_ADD { + let a_fusable = nodes[n.a as usize].op == OP_MUL && uses[n.a as usize] == 1; + let b_fusable = nodes[n.b as usize].op == OP_MUL && uses[n.b as usize] == 1; + if a_fusable || b_fusable { + t_fusable += 1; + } + } + } + } + + let arith = t_fold + t_ext + t_mulbase; + // One row per instruction, and every constant is one interned row + // program-wide — so the pool is counted once, not once per AIR. + let pool = distinct_words.len(); + let instr = t_ext + t_mulbase + pool; + println!( + "{:<14} {:>7} {:>7} {:>6} {:>8} {:>8} {:>7} {:>8} {:>8}", + "TOTAL", t_nodes, t_leaves, t_const, t_pv_base, t_fold, t_ext, t_mulbase, instr + ); + let unfused = instr + t_constraints; + let fused = unfused - t_fusable; + println!( + "\n arithmetic nodes {arith}\n \ + base by the IR's own dim {t_pv_base} (prover-side; NOT the machine's split)\n \ + base at verify time {t_fold} (constant-only subtrees -> fold at build time)\n \ + extension ALU {t_ext}\n \ + of which MulBase-routed {t_mulbase} (ext x base: 1 XALU row, vs 4+ if lowered by hand)\n \ + per-AIR constant pools {t_const} (sum; NOT the machine's cost)\n \ + interned program-wide {pool} (one Const row per distinct 4-lane word)\n \ + = constraint-leg instr {instr}\n \ + + quotient recombination {t_constraints} beta-folds (one per constraint)\n \ + = upper bound {unfused}\n \ + MulAdd-fusable Add nodes {t_fusable} (Add over a single-use Mul; MulAdd costs the same as Mul, so ALWAYS fuse)\n \ + = ESTIMATE, fused {fused}\n\n \ + leaves (addresses, free) {t_leaves}\n \ + Op::Embed nodes {t_embed} (base->ext is free; would emit nothing)\n \ + dead nodes (mult = 0) {t_dead}\n \ + max fanout (max mult) {t_maxfan}\n" + ); + + // Loose ceiling: the design budget treats this leg as ~1% of the epoch + // program. An order-of-magnitude regression should fail here. + assert!( + instr < 200_000, + "constraint-leg instruction estimate {instr} has grown past the design budget" + ); +} + +/// The constraint leg's per-EPOCH multiplier, measured on real fixtures. +/// +/// `constraint_op_census` counts instructions per distinct AIR. An epoch does +/// not evaluate each AIR once: every SUB-PROOF carries its own trace and needs +/// its own constraint evaluation, and the split-table families are CHUNKED — +/// `chunks = ceil(rows / max_rows[table])`, with `max_rows` sized per table so +/// each chunk costs about the same memory (`tables/mod.rs::max_rows`). +/// +/// So the epoch cost is `Σ over sub-proofs instr(that sub-proof's AIR)`, and the +/// multiplier against the per-AIR total is what this measures. It is the number +/// `lfm-design.md` §5.2 is missing: its ≈69K line reads as per-epoch but is +/// per-distinct-AIR. +/// +/// Measured by building real traces, so the chunk counts are the prover's own +/// splitting rather than a reconstruction of it. +#[test] +fn epoch_chunk_multiplier() { + use crate::tables::MaxRowsConfig; + use crate::tables::trace_builder::Traces; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + // Per-AIR constraint-leg instruction counts, keyed by label. + let instr: std::collections::BTreeMap<&str, usize> = production_airs(&opts) + .iter() + .map(|(label, air)| (*label, leg_instructions(&**air))) + .collect(); + let get = |k: &str| *instr.get(k).unwrap_or_else(|| panic!("no AIR {k}")); + + // Fixtures spanning roughly an epoch's worth of execution. An intermediate + // continuation epoch runs exactly 2^epoch_size_log2 cycles, so a fixture's + // cycle count is the axis to read these against. + for name in [ + "fib_iterative_1M", + "fib_iterative_2M", + "array_multipass_20M", + ] { + let (elf, logs, _) = run_asm_elf(name); + let max_rows = MaxRowsConfig::default(); + let traces = Traces::from_elf_and_logs_minimal(&elf, &logs, &max_rows, &[]) + .expect("trace build succeeds"); + + // (label, chunk count) for every sub-proof the epoch would contain. + let chunked: Vec<(&str, usize)> = vec![ + ("CPU", traces.cpus.len()), + ("LT", traces.lts.len()), + ("SHIFT", traces.shifts.len()), + ("MEMW", traces.memws.len()), + ("MEMW_A", traces.memw_aligneds.len()), + ("LOAD", traces.loads.len()), + ("MUL", traces.muls.len()), + ("DVRM", traces.dvrms.len()), + ("BRANCH", traces.branches.len()), + ("MEMW_R", traces.memw_registers.len()), + ("EQ", traces.eqs.len()), + ("BYTEWISE", traces.bytewises.len()), + ("STORE", traces.stores.len()), + ("CPU32", traces.cpu32s.len()), + ]; + + let chunked_total: usize = chunked.iter().map(|(l, n)| get(l) * n).sum(); + let chunk_count: usize = chunked.iter().map(|(_, n)| *n).sum(); + + // Fixed (unchunked) tables present once per proof, plus one PAGE per + // touched page. A continuation epoch substitutes L2G/GLOBAL_MEMORY for + // PAGE (page_configs is empty there), so this monolithic shape is an + // upper bound on the page contribution. + let pages = traces.pages.len(); + let fixed = get("BITWISE") + + get("DECODE") + + get("REGISTER") + + get("COMMIT") + + get("HALT") + + get("KECCAK") + + get("KECCAK_RND") + + get("KECCAK_RC") + + get("ECSM") + + get("ECDAS"); + let page_total = get("PAGE") * pages; + let epoch_total = chunked_total + fixed + page_total; + + let per_air_total: usize = instr.values().sum(); + println!( + "\n{name}: {} cycles\n \ + chunked sub-proofs {chunk_count} (of 14 families) -> {chunked_total} instr\n \ + fixed tables -> {fixed} instr\n \ + {pages} pages x {} instr -> {page_total} instr\n \ + EPOCH TOTAL {epoch_total} instr vs per-distinct-AIR {per_air_total} \ + multiplier {:.2}x", + logs.len(), + get("PAGE"), + epoch_total as f64 / per_air_total as f64 + ); + for (l, n) in &chunked { + if *n > 1 { + println!(" {l:<10} {n} chunks x {} = {}", get(l), get(l) * n); + } + } + } +} + +/// The constraint leg for a real CONTINUATION EPOCH — the shape we actually +/// recurse. +/// +/// The monolithic measurement above is the wrong shape for the target: a +/// continuation epoch passes `page_configs = &[]`, so PAGE never appears, and it +/// carries an L2G_MEMORY sub-proof instead. Its composition is +/// +/// ```text +/// 14 split-table families (>= 1 chunk each) +/// + FIXED_TABLE_COUNT (10 final, 9 intermediate — HALT only on the last) +/// + 1 L2G_MEMORY +/// ``` +/// +/// which gives **24 sub-proofs intermediate, 25 final** — independently measured +/// on the LFM fibonacci epoch fixture. This test asserts that arithmetic so the +/// composition is pinned rather than inferred: if the epoch shape changes, the +/// count here stops matching the measured one and this fails. +/// +/// The instruction total is then a minimum, since it assumes one chunk per +/// family — a larger epoch adds chunks of the CHEAP AIRs (see +/// `epoch_chunk_multiplier`). +#[test] +fn continuation_epoch_constraint_leg() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let instr: std::collections::BTreeMap<&str, usize> = production_airs(&opts) + .iter() + .map(|(label, air)| (*label, leg_instructions(&**air))) + .collect(); + let get = |k: &str| *instr.get(k).unwrap_or_else(|| panic!("no AIR {k}")); + + // The 14 chunked split-table families, at their minimum of one chunk each. + let families = [ + "CPU", "LT", "SHIFT", "MEMW", "MEMW_A", "LOAD", "MUL", "DVRM", "BRANCH", "MEMW_R", "EQ", + "BYTEWISE", "STORE", "CPU32", + ]; + // FIXED_TABLE_COUNT = 10 (`prover/src/lib.rs`): always exactly one sub-proof + // each, REGARDLESS of TableCounts — a zero-row table still needs its proof, + // or its constraints drop out of verification. HALT is the one an + // intermediate epoch omits. + let fixed_final = [ + "BITWISE", + "DECODE", + "HALT", + "COMMIT", + "KECCAK", + "KECCAK_RND", + "KECCAK_RC", + "REGISTER", + "ECSM", + "ECDAS", + ]; + + let families_instr: usize = families.iter().map(|l| get(l)).sum(); + let fixed_final_instr: usize = fixed_final.iter().map(|l| get(l)).sum(); + let fixed_intermediate_instr = fixed_final_instr - get("HALT"); + let l2g = get("L2G_MEMORY"); + + let intermediate = families_instr + fixed_intermediate_instr + l2g; + let final_epoch = families_instr + fixed_final_instr + l2g; + + let n_intermediate = families.len() + fixed_final.len() - 1 + 1; + let n_final = families.len() + fixed_final.len() + 1; + assert_eq!( + (n_intermediate, n_final), + (24, 25), + "epoch sub-proof composition no longer reproduces the measured 24 intermediate / 25 final" + ); + + println!( + "\ncontinuation epoch constraint leg (minimum: one chunk per family)\n \ + 14 split families {families_instr}\n \ + 9 fixed (no HALT) {fixed_intermediate_instr}\n \ + 1 L2G_MEMORY {l2g}\n \ + INTERMEDIATE epoch {intermediate} instr over {n_intermediate} sub-proofs\n \ + FINAL epoch (+HALT) {final_epoch} instr over {n_final} sub-proofs\n \ + fixed share {:.0}% — the leg is workload-INDEPENDENT\n", + 100.0 * fixed_intermediate_instr as f64 / intermediate as f64 + ); + + // The global proof is one L2G_GLOBAL per epoch plus one GLOBAL_MEMORY per + // touched page — negligible at any plausible page count, which is what + // settles the page-base question as identity-only rather than size. + println!( + " global proof: {} instr/epoch (L2G_GLOBAL) + {} instr/page (GLOBAL_MEMORY)\n", + get("L2G_GLOBAL"), + get("GLOBAL_MEMORY") + ); +} + +/// The chunk counts of a REAL continuation epoch, measured first-hand. +/// +/// `epoch_chunk_multiplier` measures monolithic runs, which is the wrong shape: +/// a monolithic proof covers the whole execution, while an epoch covers exactly +/// `2^epoch_size_log2` cycles and carries a different table set. This drives the +/// actual continuation path — `Executor::resume_with_limit` for one epoch's +/// cycles, then `Traces::from_image_and_logs` — so the chunk counts are the +/// prover's own, for an epoch. +/// +/// Only epoch 0 is measured, and that is sufficient rather than a shortcut: +/// epoch 0's `register_init` comes from the entry point, so it needs no previous +/// epoch, and every INTERMEDIATE epoch runs exactly `epoch_size` cycles by +/// construction (`continuation.rs` errors otherwise). Later epochs differ only +/// in which instructions those cycles execute. +/// +/// Proving is deliberately not run: the register chaining that would require it +/// (`prev_fini` comes out of `prove_epoch`) has no bearing on table sizes. +#[test] +fn continuation_epoch_chunk_counts_measured() { + use crate::tables::MaxRowsConfig; + use crate::tables::register; + use crate::tables::trace_builder::{Traces, build_initial_image_paged}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let instr: std::collections::BTreeMap<&str, usize> = production_airs(&opts) + .iter() + .map(|(label, air)| (*label, leg_instructions(&**air))) + .collect(); + let get = |k: &str| *instr.get(k).unwrap_or_else(|| panic!("no AIR {k}")); + + // 2^20 cycles: past CPU's 2^19 chunk bound, so chunking is actually + // exercised, while staying cheap enough to build traces for. + const EPOCH_SIZE_LOG2: u32 = 20; + let epoch_size = 1usize << EPOCH_SIZE_LOG2; + + for name in ["fib_iterative_2M", "array_multipass_20M"] { + let elf_bytes = asm_elf_bytes(name); + let elf = Elf::load(&elf_bytes).expect("load elf"); + let mut executor = Executor::new(&elf, vec![]).expect("executor"); + let image = build_initial_image_paged(&elf, &[]); + let register_init = register::register_init_from_entry_point(elf.entry_point); + + let logs = executor + .resume_with_limit(epoch_size) + .expect("resume") + .expect("program runs at least one epoch") + .to_vec(); + let is_final = executor.pc() == 0; + assert!( + !is_final && logs.len() == epoch_size, + "[{name}] wanted a full intermediate epoch, got {} cycles (final={is_final})", + logs.len() + ); + + let traces = Traces::from_image_and_logs( + &elf, + &image, + ®ister_init, + &logs, + &MaxRowsConfig::default(), + &[], + is_final, + true, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("epoch trace build"); + + let chunked: Vec<(&str, usize)> = vec![ + ("CPU", traces.cpus.len()), + ("LT", traces.lts.len()), + ("SHIFT", traces.shifts.len()), + ("MEMW", traces.memws.len()), + ("MEMW_A", traces.memw_aligneds.len()), + ("LOAD", traces.loads.len()), + ("MUL", traces.muls.len()), + ("DVRM", traces.dvrms.len()), + ("BRANCH", traces.branches.len()), + ("MEMW_R", traces.memw_registers.len()), + ("EQ", traces.eqs.len()), + ("BYTEWISE", traces.bytewises.len()), + ("STORE", traces.stores.len()), + ("CPU32", traces.cpu32s.len()), + ]; + + // An epoch never builds PAGE — the continuation path passes + // `page_configs = &[]`. Pin that here rather than trusting the comment. + assert!( + traces.page_configs.is_empty(), + "[{name}] a continuation epoch must not build PAGE tables" + ); + + let families: usize = chunked.iter().map(|(l, n)| get(l) * n).sum(); + let n_chunks: usize = chunked.iter().map(|(_, n)| *n).sum(); + // Intermediate epoch: 9 fixed tables (no HALT) + 1 L2G_MEMORY. + let fixed = get("BITWISE") + + get("DECODE") + + get("COMMIT") + + get("KECCAK") + + get("KECCAK_RND") + + get("KECCAK_RC") + + get("REGISTER") + + get("ECSM") + + get("ECDAS"); + let total = families + fixed + get("L2G_MEMORY"); + + println!( + "\n{name}, epoch 0 @ 2^{EPOCH_SIZE_LOG2} cycles\n \ + {n_chunks} chunked sub-proofs -> {families} instr\n \ + 9 fixed + L2G_MEMORY -> {} instr\n \ + EPOCH TOTAL {total} instr over {} sub-proofs", + fixed + get("L2G_MEMORY"), + n_chunks + 10 + ); + for (l, n) in &chunked { + if *n > 1 { + println!(" {l:<10} {n} chunks x {} = {}", get(l), get(l) * n); + } + } + } +} + +/// Constraint-leg instructions for one AIR: extension ALU plus MulBase-routed +/// multiplies. Shared by `constraint_op_census` and `epoch_chunk_multiplier` so +/// the two cannot drift apart. +fn leg_instructions(air: &dyn AIR) -> usize { + use stark::constraint_ir::artifact::DIM_BASE; + use stark::constraint_ir::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, + }; + let artifact = ConstraintArtifact::capture(air); + let nodes = &artifact.nodes; + let mut v_base = vec![false; nodes.len()]; + let mut count = 0usize; + for (i, n) in nodes.iter().enumerate() { + match n.op { + OP_VAR | OP_RAP_CHALLENGE | OP_ALPHA_POW | OP_TABLE_OFFSET | OP_CONST_EXT => { + v_base[i] = false + } + OP_CONST_BASE => v_base[i] = true, + OP_ADD | OP_SUB | OP_MUL | OP_NEG | OP_EMBED => { + let (ba, bb) = match n.op { + OP_NEG => (v_base[n.a as usize], true), + OP_EMBED => (false, false), + _ => (v_base[n.a as usize], v_base[n.b as usize]), + }; + v_base[i] = ba && bb && n.dim == DIM_BASE; + if !v_base[i] { + count += 1; + } + } + other => panic!("unclassified op tag {other}"), + } + } + count +} + +/// The captured artifact does not depend on the proof options. +/// +/// This is the premise behind leaving `ProofOptions` OUT of the artifact — if it +/// failed, one artifact per table would not be enough and the whole scheme would +/// need an artifact per (table, blowup) pair. `AirContext` carries the options +/// alongside the shape scalars, so the independence is worth pinning rather than +/// assuming from a reading of the constructor. +#[test] +fn artifacts_are_invariant_across_proof_options() { + let opts2 = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let opts4 = GoldilocksCubicProofOptions::with_blowup(4).expect("blowup=4 valid"); + + let airs2 = production_airs(&opts2); + let airs4 = production_airs(&opts4); + + for ((label, a2), (_, a4)) in airs2.iter().zip(airs4.iter()) { + let art2 = ConstraintArtifact::capture(&**a2); + let art4 = ConstraintArtifact::capture(&**a4); + assert_eq!( + art2, art4, + "[{label}] the constraint artifact differs between blowup 2 and 4; it would have to \ + be stored per (table, blowup) pair" + ); + assert_eq!( + art2.to_bytes().expect("serialize"), + art4.to_bytes().expect("serialize"), + "[{label}] artifact bytes differ across blowup factors" + ); + } +} + +/// The captured artifact does not depend on the trace length either. +/// +/// Same failure mode as the proof-options axis, different variable: if anything +/// in a captured program folded a domain-size-dependent constant, artifacts +/// would multiply per epoch shape. +/// +/// The axis is structurally absent — no AIR constructor takes a trace length — +/// so the only route by which one could reach the artifact is +/// `composition_poly_degree_bound(n)`, the single trace-length-dependent method +/// on the trait, whose value the artifact stores divided through by `n`. That +/// division is only sound if the bound is exactly linear, so this sweeps a wide +/// range of `n` per table rather than trusting the two probe points +/// `ConstraintArtifact::capture` checks. A table whose bound had any constant +/// term or any non-linearity would be misrepresented by the stored multiplier, +/// and would show up here. +#[test] +fn artifacts_are_invariant_across_trace_length() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!(airs.len(), NUM_PRODUCTION_AIRS); + + for (label, air) in &airs { + let artifact = ConstraintArtifact::capture(&**air); + let k = artifact.shape.composition_degree_multiplier as usize; + + for log_n in 4usize..=24 { + let n = 1usize << log_n; + assert_eq!( + air.composition_poly_degree_bound(n), + k * n, + "[{label}] composition_poly_degree_bound is not k·n at n=2^{log_n}; the artifact \ + stores only the linear coefficient, so a trace-length-dependent AIR would need \ + an artifact per epoch shape" + ); + } + + // Nothing else on the artifact can vary with the trace length, but pin + // capture determinism so a future source of nondeterminism (map + // iteration order in the constant tables, say) is caught here. + let again = ConstraintArtifact::capture(&**air); + assert_eq!(artifact, again, "[{label}] capture is not deterministic"); + } +} + +/// The four PARAMETERIZED tables produce a different program per parameter +/// value. +/// +/// `PAGE` / `GLOBAL_MEMORY` fold a page base into constant bus terms; the two +/// `L2G` tables fold an epoch label. This test does not assert that away — it +/// characterizes it, because it is a real property of the current constraints +/// and the recursion machine has to plan around it. +/// +/// # The variation is NOT confined to constant VALUES +/// +/// The obvious guess is that two parameter values give the same node array with +/// one constant swapped. That is what `PAGE` and `GLOBAL_MEMORY` do, and it is +/// wrong in general: the builder interns constants by value, so a parameter +/// whose value happens to already be in the constant table costs no new node, +/// while a fresh value appends one — which shifts every later node id and hence +/// the constraint ROOTS. `L2G_GLOBAL` at `epoch_label = 1` reuses the existing +/// `1` constant; at `epoch_label = 7` it appends. Same algebra, different node +/// count and different root ids. +/// +/// This matters for the machine-side fix: "swap one constant per page" would be +/// a cheap patch and it is not available. Promoting the parameter to a runtime +/// uniform is, because the ALGEBRA is invariant — which is what the shape and +/// metadata assertions below pin. +#[test] +fn parameterized_airs_vary_per_parameter_value() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + // (label, artifact at parameter A, artifact at parameter B) + let cases: Vec<(&str, ConstraintArtifact, ConstraintArtifact)> = vec![ + ( + "PAGE", + ConstraintArtifact::capture(&create_page_air(&opts, 0x1000)), + ConstraintArtifact::capture(&create_page_air(&opts, 0x9000)), + ), + ( + "GLOBAL_MEMORY", + ConstraintArtifact::capture(&create_global_memory_air(&opts, 0x1000)), + ConstraintArtifact::capture(&create_global_memory_air(&opts, 0x9000)), + ), + ( + "L2G_GLOBAL", + ConstraintArtifact::capture(&crate::continuation::l2g_global_air(&opts, 1)), + ConstraintArtifact::capture(&crate::continuation::l2g_global_air(&opts, 7)), + ), + ( + "L2G_MEMORY", + ConstraintArtifact::capture(&crate::continuation::l2g_memory_air(&opts, 1)), + ConstraintArtifact::capture(&crate::continuation::l2g_memory_air(&opts, 7)), + ), + ]; + + println!("\nparameterized tables: how two parameter values differ"); + for (label, a, b) in &cases { + assert_ne!( + a, b, + "[{label}] is documented as parameterized but two parameter values gave the same \ + artifact; either the parameter stopped reaching the IR or the test picked two \ + values that collide" + ); + + // Invariant: the ALGEBRA. Same widths, same constraint count, same + // zerofier shapes, same degree bound — only the embedded parameter + // moves. This is the property that makes the parameter promotable to a + // runtime uniform. + assert_eq!(a.shape, b.shape, "[{label}] shape must not vary"); + assert_eq!(a.meta, b.meta, "[{label}] metadata must not vary"); + assert_eq!(a.num_base, b.num_base, "[{label}] num_base must not vary"); + assert_eq!( + a.roots.len(), + b.roots.len(), + "[{label}] constraint count must not vary" + ); + + // Variable: node count and root ids — an artifact of the builder's + // hash-consing, not of the constraints. A parameter value already in the + // constant table costs no new ConstBase node while a fresh one appends, + // which shifts every later node id. + // + // MEASURED, and note the counts are not all +1: L2G_GLOBAL moves 1 node + // for 1 constant, L2G_MEMORY moves 2 for 1. The second node is some + // further CSE difference downstream of the reused constant (L2G_MEMORY + // at epoch_label = 1 contributes the constant 0, which IS node id 0, so + // expressions over it have more chance to coincide with existing ones) — + // that specific explanation is inferred, not verified, so the bound + // below is deliberately loose. What is being pinned is only that the + // delta stays local rather than the algebra changing shape. + let node_delta = a.nodes.len().abs_diff(b.nodes.len()); + let const_delta = a.base_consts.len().abs_diff(b.base_consts.len()); + assert!( + node_delta <= 4 && const_delta <= 1, + "[{label}] two parameter values changed the program by {node_delta} nodes and \ + {const_delta} constants — too much to be the parameter's own interned constant and \ + its enclosing ops; the variation is structural, not just parametric" + ); + let roots_moved = a.roots != b.roots; + + println!( + " {label:<14} nodes {:>3} vs {:>3} consts {:>2} vs {:>2} roots moved: {}", + a.nodes.len(), + b.nodes.len(), + a.base_consts.len(), + b.base_consts.len(), + roots_moved + ); + } + println!(); +} + +/// GLOBAL_MEMORY has a second, ENUMERABLE axis: private-input pages preprocess +/// OFFSET only, which changes the artifact's SHAPE rather than a constant. +/// +/// Worth separating from the parameter axis above because the two have very +/// different consequences. A page base is an arbitrary address, so its artifact +/// set is unbounded; `is_private_input` is a boolean, so GLOBAL_MEMORY simply has +/// two shape variants and both can be enumerated. This pins that the difference +/// is confined to the preprocessed-column fields and does not touch the program. +#[test] +fn global_memory_private_input_is_a_second_shape_not_a_second_program() { + use crate::tables::page::PageConfig; + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + let elf_page = PageConfig::zero_init(PAGE_TEST_BASE); + let mut private_page = PageConfig::zero_init(PAGE_TEST_BASE); + private_page.is_private_input = true; + + let elf = ConstraintArtifact::capture(&crate::continuation::global_memory_air( + &opts, + &elf_page, + Some([0u8; 32]), + )); + let private = ConstraintArtifact::capture(&crate::continuation::global_memory_air( + &opts, + &private_page, + Some([0u8; 32]), + )); + + // Same constraints, same metadata: the bus interactions depend only on the + // page base, which is equal here. + assert_eq!(elf.nodes, private.nodes, "the program must not vary"); + assert_eq!(elf.base_consts, private.base_consts); + assert_eq!(elf.roots, private.roots); + assert_eq!(elf.meta, private.meta); + + // The shape does vary, in exactly the preprocessed fields. Both variants are + // preprocessed; they differ in HOW MANY columns. An ELF page commits OFFSET + // and INIT, so the verifier recomputes the genesis values from the ELF. A + // private-input page commits OFFSET alone — INIT is the private input and + // stays a main-trace column, but OFFSET is the row's address and leaving it + // prover-chosen would let a genesis token name an arbitrary address. + assert!(elf.shape.is_preprocessed, "an ELF page is preprocessed"); + assert!( + private.shape.is_preprocessed, + "a private-input page still preprocesses OFFSET" + ); + assert_eq!( + elf.shape.num_precomputed_columns, + crate::tables::global_memory::NUM_PREPROCESSED_COLS as u32, + "an ELF page commits OFFSET and INIT" + ); + assert_eq!( + private.shape.num_precomputed_columns, + crate::tables::page::NUM_PREPROCESSED_COLS_PRIVATE as u32, + "a private-input page commits OFFSET only" + ); + + let mut normalized = private.shape.clone(); + normalized.num_precomputed_columns = elf.shape.num_precomputed_columns; + assert_eq!( + normalized, elf.shape, + "the two variants must differ ONLY in the preprocessed-column fields" + ); +} + +/// An artifact captured from one table must not validate against another. +/// +/// The suite above only ever shows the shape check ACCEPTING. Without this, a +/// `validate_against` that returned `Ok(())` unconditionally would pass +/// everything here. +#[test] +fn an_artifact_does_not_validate_against_a_different_table() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + + let mut checked = 0usize; + for (i, (label_i, air_i)) in airs.iter().enumerate() { + let artifact = ConstraintArtifact::capture(&**air_i); + for (j, (label_j, air_j)) in airs.iter().enumerate() { + if i == j { + continue; + } + if artifact.validate_against(&**air_j).is_ok() { + // Two tables can legitimately share every shape scalar (several + // are bus-only tables with identical layouts), so an accept is + // only a failure when the programs actually differ. + let prog_i = air_i.constraint_program(); + let prog_j = air_j.constraint_program(); + assert_eq!( + (&prog_i.nodes, &prog_i.roots), + (&prog_j.nodes, &prog_j.roots), + "[{label_i}] artifact validated against [{label_j}], whose constraint \ + program is different — the shape check cannot tell them apart" + ); + } else { + checked += 1; + } + } + } + assert!( + checked > 0, + "validate_against never rejected any cross-table pairing; the check is not live" + ); +} + +/// A pre-captured program can be supplied to a production AIR and is used +/// without capture — the scoped verify-path unban, on real tables. +#[test] +fn production_airs_accept_a_precaptured_program() { + let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + + // Build the artifact from one instance, install it into a fresh one. + let artifact = ConstraintArtifact::capture(&create_cpu_air(&opts)); + let air = create_cpu_air(&opts).with_precaptured(artifact.program()); + + let supplied = air + .precaptured_constraint_program() + .expect("the supplied program must be visible on the guest-safe accessor"); + assert!( + std::ptr::eq(air.constraint_program(), supplied), + "constraint_program() must return the supplied program, not a fresh capture" + ); + + // And it still evaluates like the folder. + let n = air.context().num_transition_constraints; + let num_base = air.num_base_transition_constraints(); + let (n_main, n_aux) = air.trace_layout(); + let mut rng = SplitMix64(0x00A1_1CE5); + for _ in 0..16 { + let mk_step = |rng: &mut SplitMix64| { + let main: Vec = (0..n_main).map(|_| Fp::from(rng.next_u64())).collect(); + let aux: Vec = (0..n_aux).map(|_| rng.fp3()).collect(); + TableView::new(vec![main], vec![aux]) + }; + let frame = Frame::::new(vec![mk_step(&mut rng), mk_step(&mut rng)]); + let challenges = vec![rng.fp3(), rng.fp3()]; + let alphas: Vec = (0..air.max_bus_elements() + 2).map(|_| rng.fp3()).collect(); + let offset = rng.fp3(); + let ctx = TransitionEvaluationContext::new_prover( + frame.as_row_frame(), + &challenges, + &alphas, + &offset, + ); + + let mut f_base = vec![Fp::zero(); num_base]; + let mut f_ext = vec![Fp3::zero(); n]; + air.compute_transition_prover(&ctx, &mut f_base, &mut f_ext); + + let mut i_base = vec![Fp::zero(); num_base]; + let mut i_ext = vec![Fp3::zero(); n]; + eval_program(supplied, &ctx, &mut i_base, &mut i_ext); + + assert_eq!(f_base, i_base); + for c in num_base..n { + assert_eq!(f_ext[c], i_ext[c]); + } + } +} diff --git a/prover/src/tests/constraint_program_device_tests.rs b/prover/src/tests/constraint_program_device_tests.rs index a29a7cb49..ab315fe62 100644 --- a/prover/src/tests/constraint_program_device_tests.rs +++ b/prover/src/tests/constraint_program_device_tests.rs @@ -155,31 +155,14 @@ fn check_air_device( #[test] fn all_table_programs_lower_and_match_folders() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "production AIR list changed size" + ); - check_air_device(&create_cpu_air(&opts), "CPU"); - check_air_device(&create_bitwise_air(&opts), "BITWISE"); - check_air_device(&create_lt_air(&opts), "LT"); - check_air_device(&create_shift_air(&opts), "SHIFT"); - check_air_device(&create_eq_air(&opts), "EQ"); - check_air_device(&create_bytewise_air(&opts), "BYTEWISE"); - check_air_device(&create_store_air(&opts), "STORE"); - check_air_device(&create_cpu32_air(&opts), "CPU32"); - check_air_device(&create_memw_air(&opts), "MEMW"); - check_air_device(&create_memw_aligned_air(&opts), "MEMW_A"); - check_air_device(&create_memw_register_air(&opts), "MEMW_R"); - check_air_device(&create_load_air(&opts), "LOAD"); - check_air_device(&create_decode_air(&opts), "DECODE"); - check_air_device(&create_mul_air(&opts), "MUL"); - check_air_device(&create_dvrm_air(&opts), "DVRM"); - check_air_device(&create_branch_air(&opts), "BRANCH"); - check_air_device(&create_halt_air(&opts), "HALT"); - check_air_device(&create_commit_air(&opts), "COMMIT"); - check_air_device(&create_page_air(&opts, 0x1000), "PAGE"); - check_air_device(&create_register_air(&opts), "REGISTER"); - check_air_device(&create_keccak_air(&opts), "KECCAK"); - check_air_device(&create_keccak_rnd_air(&opts), "KECCAK_RND"); - check_air_device(&create_keccak_rc_air(&opts), "KECCAK_RC"); - check_air_device(&create_ecsm_air(&opts), "ECSM"); - check_air_device(&create_ecdas_air(&opts), "ECDAS"); - check_air_device(&create_hint_air(&opts), "HINT"); + for (label, air) in &airs { + check_air_device(&**air, label); + } } diff --git a/prover/src/tests/constraint_program_tests.rs b/prover/src/tests/constraint_program_tests.rs index e227da53d..c04ed4e6d 100644 --- a/prover/src/tests/constraint_program_tests.rs +++ b/prover/src/tests/constraint_program_tests.rs @@ -153,31 +153,14 @@ fn check_air(air: &dyn AIR #[test] fn all_table_programs_match_folders() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); - - check_air(&create_cpu_air(&opts), "CPU"); - check_air(&create_bitwise_air(&opts), "BITWISE"); - check_air(&create_lt_air(&opts), "LT"); - check_air(&create_shift_air(&opts), "SHIFT"); - check_air(&create_eq_air(&opts), "EQ"); - check_air(&create_bytewise_air(&opts), "BYTEWISE"); - check_air(&create_store_air(&opts), "STORE"); - check_air(&create_cpu32_air(&opts), "CPU32"); - check_air(&create_memw_air(&opts), "MEMW"); - check_air(&create_memw_aligned_air(&opts), "MEMW_A"); - check_air(&create_memw_register_air(&opts), "MEMW_R"); - check_air(&create_load_air(&opts), "LOAD"); - check_air(&create_decode_air(&opts), "DECODE"); - check_air(&create_mul_air(&opts), "MUL"); - check_air(&create_dvrm_air(&opts), "DVRM"); - check_air(&create_branch_air(&opts), "BRANCH"); - check_air(&create_halt_air(&opts), "HALT"); - check_air(&create_commit_air(&opts), "COMMIT"); - check_air(&create_page_air(&opts, 0x1000), "PAGE"); - check_air(&create_register_air(&opts), "REGISTER"); - check_air(&create_keccak_air(&opts), "KECCAK"); - check_air(&create_keccak_rnd_air(&opts), "KECCAK_RND"); - check_air(&create_keccak_rc_air(&opts), "KECCAK_RC"); - check_air(&create_ecsm_air(&opts), "ECSM"); - check_air(&create_ecdas_air(&opts), "ECDAS"); - check_air(&create_hint_air(&opts), "HINT"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "production AIR list changed size" + ); + + for (label, air) in &airs { + check_air(&**air, label); + } } diff --git a/prover/src/tests/mod.rs b/prover/src/tests/mod.rs index 9288cf2ac..539064109 100644 --- a/prover/src/tests/mod.rs +++ b/prover/src/tests/mod.rs @@ -15,6 +15,8 @@ pub mod commit_tests; #[cfg(test)] pub mod compute_commit_bus_offset_tests; #[cfg(test)] +pub mod constraint_artifact_tests; +#[cfg(test)] pub mod constraint_emit_tests; #[cfg(test)] pub mod constraint_program_device_tests; diff --git a/prover/src/tests/ood_window_ir_tests.rs b/prover/src/tests/ood_window_ir_tests.rs index 29d224627..72d0c4e31 100644 --- a/prover/src/tests/ood_window_ir_tests.rs +++ b/prover/src/tests/ood_window_ir_tests.rs @@ -21,10 +21,12 @@ //! the real constraints rather than a copy of the declaration. //! //! It only CONSTRUCTS AIRs (no program execution, no ELF), so it runs anywhere. -//! The table list mirrors the enumeration in `constraint_program_tests.rs` — the -//! canonical per-table `create_*_air` constructors from `test_utils`; there is no -//! ELF-free registry to iterate (`VmAirs::air_refs` needs a real ELF plus -//! preprocessed-commitment builds), so a new table must be added here. +//! The table list is `test_utils::production_airs` — shared with the other +//! per-table IR suites, since three hand-maintained copies of it meant a new +//! table could be added to one and silently skipped by the others. There is +//! still no ELF-free registry to iterate (`VmAirs::air_refs` needs a real ELF +//! plus preprocessed-commitment builds), so a new table must be added to +//! `production_airs`. use stark::proof::options::GoldilocksCubicProofOptions; use stark::traits::AIR; @@ -88,31 +90,14 @@ fn assert_ood_window_matches_ir( #[test] fn all_table_windows_match_captured_ir() { let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 valid"); + let airs = production_airs(&opts); + assert_eq!( + airs.len(), + NUM_PRODUCTION_AIRS, + "production AIR list changed size" + ); - assert_ood_window_matches_ir(&create_cpu_air(&opts), true, "CPU"); - assert_ood_window_matches_ir(&create_bitwise_air(&opts), true, "BITWISE"); - assert_ood_window_matches_ir(&create_lt_air(&opts), true, "LT"); - assert_ood_window_matches_ir(&create_shift_air(&opts), true, "SHIFT"); - assert_ood_window_matches_ir(&create_eq_air(&opts), true, "EQ"); - assert_ood_window_matches_ir(&create_bytewise_air(&opts), true, "BYTEWISE"); - assert_ood_window_matches_ir(&create_store_air(&opts), true, "STORE"); - assert_ood_window_matches_ir(&create_cpu32_air(&opts), true, "CPU32"); - assert_ood_window_matches_ir(&create_memw_air(&opts), true, "MEMW"); - assert_ood_window_matches_ir(&create_memw_aligned_air(&opts), true, "MEMW_A"); - assert_ood_window_matches_ir(&create_memw_register_air(&opts), true, "MEMW_R"); - assert_ood_window_matches_ir(&create_load_air(&opts), true, "LOAD"); - assert_ood_window_matches_ir(&create_decode_air(&opts), true, "DECODE"); - assert_ood_window_matches_ir(&create_mul_air(&opts), true, "MUL"); - assert_ood_window_matches_ir(&create_dvrm_air(&opts), true, "DVRM"); - assert_ood_window_matches_ir(&create_branch_air(&opts), true, "BRANCH"); - assert_ood_window_matches_ir(&create_halt_air(&opts), true, "HALT"); - assert_ood_window_matches_ir(&create_commit_air(&opts), true, "COMMIT"); - assert_ood_window_matches_ir(&create_page_air(&opts, 0x1000), true, "PAGE"); - assert_ood_window_matches_ir(&create_register_air(&opts), true, "REGISTER"); - assert_ood_window_matches_ir(&create_keccak_air(&opts), true, "KECCAK"); - assert_ood_window_matches_ir(&create_keccak_rnd_air(&opts), true, "KECCAK_RND"); - assert_ood_window_matches_ir(&create_keccak_rc_air(&opts), true, "KECCAK_RC"); - assert_ood_window_matches_ir(&create_ecsm_air(&opts), true, "ECSM"); - assert_ood_window_matches_ir(&create_ecdas_air(&opts), true, "ECDAS"); - assert_ood_window_matches_ir(&create_hint_air(&opts), true, "HINT"); + for (label, air) in &airs { + assert_ood_window_matches_ir(&**air, true, label); + } } diff --git a/prover/src/tests/page_offset_forgery_poc.rs b/prover/src/tests/page_offset_forgery_poc.rs index 5e2e24d78..c1bbc9f63 100644 --- a/prover/src/tests/page_offset_forgery_poc.rs +++ b/prover/src/tests/page_offset_forgery_poc.rs @@ -208,6 +208,7 @@ fn craft_proof( &mut transcript, #[cfg(feature = "disk-spill")] stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, )?; Ok(VmProof { @@ -712,6 +713,7 @@ fn craft_proof_with_duplicate_page( &mut transcript, #[cfg(feature = "disk-spill")] stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) // The injected duplicate page writes only FINI, a main-trace column, and every // page's OFFSET/INIT stays honest — so the preprocessed check cannot fire and diff --git a/prover/src/tests/prove_elfs_tests.rs b/prover/src/tests/prove_elfs_tests.rs index bbc8d2c63..397685b2f 100644 --- a/prover/src/tests/prove_elfs_tests.rs +++ b/prover/src/tests/prove_elfs_tests.rs @@ -1104,6 +1104,55 @@ fn test_prove_elfs_keccak_multi_call() { ); } +#[test] +fn test_prove_elfs_blake3() { + let _ = env_logger::builder().is_test(true).try_init(); + + let elf_bytes = crate::test_utils::asm_elf_bytes("test_blake3"); + let elf = Elf::load(&elf_bytes).expect("Failed to load ELF"); + let executor = + executor::vm::execution::Executor::new(&elf, vec![]).expect("Failed to create executor"); + let result = executor.run().expect("Failed to run program"); + + // The guest seeds the 14 input dwords with k+1, compresses, copies out over + // m and compresses again. Cross-check the committed output against a direct + // replay of the executor's compression function. + use executor::vm::instruction::execution::blake3_compress_6round; + let words: [u32; 28] = core::array::from_fn(|i| { + let dw = (i / 2 + 1) as u64; + if i % 2 == 0 { + dw as u32 + } else { + (dw >> 32) as u32 + } + }); + let h: [u32; 8] = words[0..8].try_into().unwrap(); + let m: [u32; 16] = words[8..24].try_into().unwrap(); + let t = (words[24] as u64) | ((words[25] as u64) << 32); + let (block_len, flags) = (words[26], words[27]); + let out1 = blake3_compress_6round(&h, &m, t, block_len, flags); + let out2 = blake3_compress_6round(&h, &out1, t, block_len, flags); + let expected_bytes: Vec = out2.iter().flat_map(|w| w.to_le_bytes()).collect(); + + assert_eq!( + result.return_values.memory_values, expected_bytes, + "committed output must match two chained 6-round compressions" + ); + + // Must use from_elf_and_logs (stack RAM needs PAGE tables, like keccak). + let mut traces = + Traces::from_elf_and_logs_minimal(&elf, &result.logs, &Default::default(), &[]).unwrap(); + assert_eq!( + traces.public_output_bytes, + result.return_values.memory_values + ); + + assert!( + prove_and_verify_vm_minimal(&elf, &mut traces), + "blake3 prove/verify failed" + ); +} + #[test] fn test_prove_elfs_ecsm() { let _ = env_logger::builder().is_test(true).try_init(); diff --git a/prover/src/tests/recursion_soundness_gap_poc.rs b/prover/src/tests/recursion_soundness_gap_poc.rs index 73410ff62..28a1b5b64 100644 --- a/prover/src/tests/recursion_soundness_gap_poc.rs +++ b/prover/src/tests/recursion_soundness_gap_poc.rs @@ -184,6 +184,7 @@ fn custom_prove_with_statement_elf( &mut transcript, #[cfg(feature = "disk-spill")] stark::storage_mode::StorageMode::Ram, + stark::residency_mode::ResidencyMode::Retain, ) .expect("multi_prove failed"); diff --git a/prover/tests/d0_king_gate.rs b/prover/tests/d0_king_gate.rs new file mode 100644 index 000000000..b146f12d3 --- /dev/null +++ b/prover/tests/d0_king_gate.rs @@ -0,0 +1,122 @@ +//! Cross-version proof oracle for the D0 commitment-hash migration. +//! +//! A prove/verify round trip inside one build cannot see a self-consistent +//! drift: a version that changes how it commits still accepts its own proofs. +//! This exchanges proof *bytes* across versions instead — generate at the ref +//! before a change, verify at the ref after — so a moved leaf layout, transcript +//! or wire format fails loudly. It is the LFM-side counterpart of +//! `scripts/cross_verify_vm.sh`, which does the same for RV64 ELF proofs in both +//! directions. +//! +//! `#[ignore]`d because it is an oracle, not a regression test: it needs two +//! builds, an out-of-tree byte store, and an operator deciding which two refs +//! are being compared. +//! +//! ```text +//! # at the OLD ref +//! KING_GATE=generate KING_GATE_DIR=/some/dir \ +//! cargo test --release -p lambda-vm-prover --test d0_king_gate -- --ignored --nocapture +//! # at the NEW ref, same directory +//! KING_GATE=verify KING_GATE_DIR=/some/dir \ +//! cargo test --release -p lambda-vm-prover --test d0_king_gate -- --ignored --nocapture +//! ``` +//! +//! This is the gate for D0 steps 3-7 (`thoughts/shared/block-compression/`): +//! the Blake3 leaf/pair backends, the B1 transcript, the `lfm_prove` wiring and +//! the registry rows all have to keep Test-hasher proofs verifying, and this is +//! what says they do. Steps that deliberately move the format re-generate the +//! bytes and say so. +//! +//! Two things worth keeping true of this file. It must compile *unchanged* +//! across the refs being compared — that is itself the API-stability half of the +//! test, and editing it to make it build defeats the purpose. And it must be +//! able to fail: flipping one byte of the stored archive has to make `verify` +//! reject. + +use lambda_vm_prover::lfm::programs::trivial_program; +use lambda_vm_prover::lfm::registry::{LfmProgramKind, build_artifacts}; +use lambda_vm_prover::lfm::word::LfmWord; +use lambda_vm_prover::lfm::{lfm_prove, lfm_verify}; +use lambda_vm_prover::tables::types::FE; +use stark::proof::options::{GoldilocksCubicProofOptions, ProofOptions}; +use stark::proof::stark::MultiProof; + +type F = lambda_vm_prover::tables::types::GoldilocksField; +type E = lambda_vm_prover::tables::types::GoldilocksExtension; + +fn options() -> ProofOptions { + GoldilocksCubicProofOptions::with_blowup(2).expect("options") +} + +fn arenas() -> Vec> { + vec![ + (0..4u64) + .map(|i| core::array::from_fn(|j| FE::from(1_000 * (i + 1) + j as u64))) + .collect(), + ] +} + +fn dir() -> std::path::PathBuf { + std::path::PathBuf::from(std::env::var("KING_GATE_DIR").expect("KING_GATE_DIR")) +} + +#[test] +#[ignore = "cross-version oracle: needs KING_GATE=generate|verify and KING_GATE_DIR"] +fn lfm_trivial_v0_cross_version() { + let mode = std::env::var("KING_GATE").expect( + "set KING_GATE=generate (at the old ref) or KING_GATE=verify (at the new one), \ + plus KING_GATE_DIR pointing at a directory that outlives both builds", + ); + let opts = options(); + let proof_path = dir().join("lfm_trivial_v0.rkyv"); + let words_path = dir().join("lfm_trivial_v0.words.rkyv"); + + match mode.as_str() { + "generate" => { + let program = trivial_program(); + let artifacts = build_artifacts(&program, &opts); + let proved = lfm_prove(&program, &artifacts, &arenas(), &opts).expect("prove"); + assert!( + lfm_verify( + LfmProgramKind::TrivialV0, + &proved.proof, + &proved.public_words, + &opts + ) + .expect("registered"), + "the freshly built proof must verify where it was built" + ); + let bytes = rkyv::to_bytes::(&proved.proof).expect("archive"); + let words = + rkyv::to_bytes::(&proved.public_words).expect("archive words"); + std::fs::write(&proof_path, &bytes).expect("write proof"); + std::fs::write(&words_path, &words).expect("write words"); + eprintln!( + "GENERATED {} ({} bytes) + {} ({} bytes)", + proof_path.display(), + bytes.len(), + words_path.display(), + words.len() + ); + } + "verify" => { + let bytes = std::fs::read(&proof_path).expect("read proof"); + let words = std::fs::read(&words_path).expect("read words"); + let proof = rkyv::from_bytes::, rkyv::rancor::Error>(&bytes) + .expect("the archive from the other ref must still deserialize"); + let public_words = rkyv::from_bytes::, rkyv::rancor::Error>(&words) + .expect("words deserialize"); + assert!( + lfm_verify(LfmProgramKind::TrivialV0, &proof, &public_words, &opts) + .expect("registered"), + "proof bytes from the other ref must verify under this build" + ); + eprintln!( + "VERIFIED {} ({} bytes) under this build", + proof_path.display(), + bytes.len() + ); + } + other => panic!("KING_GATE must be generate|verify, got {other:?}"), + } +} diff --git a/scripts/gen_blake3_bench.sh b/scripts/gen_blake3_bench.sh new file mode 100755 index 000000000..ce640739d --- /dev/null +++ b/scripts/gen_blake3_bench.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# +# gen_blake3_bench.sh — generate + compile a blake3-saturated guest. +# +# The guest seeds a 176-byte BLAKE3 state region (layout: h[4 dwords] | m[8] | +# t[1] | len,flags[1] | out[8]), fires the BLAKE3 6-round compression ecall N +# times IN PLACE on that region, commits the 64-byte output and halts. +# +# The calls are deliberately NOT chained (out is not copied over m): chaining +# costs an 8-dword copy loop = ~82 of ~87 cycles per compression, and those +# copy cycles become CPU/MEMW rows that dilute the very table being measured. +# The prover's cost per ecall is identical either way — the executor runs every +# call, and no layer dedupes rows (timestamps differ per call) — so dropping +# the chain buys a ~5-cycle loop body and a >90% blake3-saturated trace, the +# same shape as gen_keccak_bench.sh. (The e2e correctness test, which is about +# values rather than cost, does chain: prover/src/tests test_prove_elfs_blake3.) +# +# The BLAKE3 table commits ONE row per compression (fully unrolled layout), so +# padding-flush sweep points are simply powers of two: N = 2^k. At ~5 +# cycles/compression a 2^17-row table costs ~0.7M cycles — inside one 2^20 +# epoch; use --epoch-size-log2 21 from N = 2^18 up. +# +# ABI (executor/src/vm/instruction/execution.rs BLAKE3_SYSCALL_NUMBER): +# a7 = u64::MAX - 2, written as the sign-extended -3 +# a0 = 8-byte-aligned pointer to the 176-byte region +# +# Usage: scripts/gen_blake3_bench.sh N OUT.elf +# Honors CLANG / ASM_CFLAGS / ASM_LDFLAGS like the Makefile's asm rule. + +set -euo pipefail + +N="${1:?usage: gen_blake3_bench.sh N out.elf}" +OUT="${2:?usage: gen_blake3_bench.sh N out.elf}" + +if ! [[ "$N" =~ ^[0-9]+$ ]] || [ "$N" -lt 1 ]; then + echo "gen_blake3_bench.sh: N must be a positive integer, got '$N'" >&2 + exit 1 +fi + +CLANG="${CLANG:-clang}" +ASM_CFLAGS="${ASM_CFLAGS:---target=riscv64 -march=rv64im -mabi=lp64}" +ASM_LDFLAGS="${ASM_LDFLAGS:--fuse-ld=lld -nostdlib -Wl,-e,main}" + +if ! command -v "$CLANG" >/dev/null 2>&1; then + echo "gen_blake3_bench.sh: '$CLANG' not found; run 'make deps' or set CLANG=..." >&2 + exit 1 +fi + +SRC="$(mktemp "${TMPDIR:-/tmp}/blake3_bench.XXXXXX.s")" +trap 'rm -f "$SRC"' EXIT + +cat > "$SRC" < + +One row per compression call, fully unrolled: 6 rounds × 8 G-functions in +SSA form. The message schedule is a compile-time index permutation (the +`sched` array in `run_flow`, composed from `BLAKE3_MSG_PERMUTATION`), so +every round references the 16 original committed message words — there is +no state or message handoff between rows. I/O follows the KECCAK core idiom: an `ECALL` receiver binds +(timestamp, syscall number), a `MEMW` register read binds the `x10` +pointer, and 22 per-dword `MEMW` operations carry the reads and writes. + +Key constraint-design decisions (full rationale: +`thoughts/blake3/blake3-chip/DESIGN.md`, deltas in `IMPLEMENTATION.md`): + +- every eval constraint is gated by the multiplicity column $mu$, and the + maximum constraint degree *including* the $times mu$ factor is 3; +- 3-operand adds commit *two summed carry bits* with an explicit sum + identity (a ternary carry would be degree 4 after gating); +- 2-operand adds use an expression carry (no committed cell) with a + $mu$-gated booleanity; +- `rotr16`/`rotr8` are free byte relabels; `rotr12`/`rotr7` are inline + $mu$-gated Euclidean shift identities whose soundness rests on the + tight $[0, 2^16)$ bound of the `SLL` halfwords ($2^16$ is invertible + mod $p$); +- every add/shift output feeds a downstream `BYTE_ALU[XOR]` lookup, which + is its only byte range check; the message words, the previous + out-region content and the address bytes are never XOR-consumed and + carry explicit `ARE_BYTES` checks instead. + +The chip's wiring is single-sourced: the compression dataflow is written +once in `prover/src/tables/blake3.rs` (`run_flow`) and interpreted both as +column wiring (constraints + bus senders) and as the u32 witness (trace +fill + lookup multiplicities), so the two cannot diverge structurally. + +== Formalized constraints + +#render_constraint_table(chip, config, groups: "io") +#render_constraint_table(chip, config, groups: "addr") +#render_constraint_table(chip, config, groups: "range") +#render_constraint_table(chip, config, groups: "mu") + += Verification evidence + +The design was taken to a z3-gated model *before* the Rust implementation +(`thoughts/blake3/blake3-chip/z3_blake_verify.py`): the G quarter-round +and the init/feed-forward layout are UNSAT under free inputs, five +negative controls and two field-level bound-necessity controls are SAT, +and the concrete 6- and 7-round pipelines reproduce the oracle's pinned +vectors. Two independent transcription audits +(`thoughts/blake3/TRANSCRIPTION-AUDIT.md`, +`GATE-TRANSCRIPTION-AUDIT.md`) checked the gate against the oracle. The +Rust chip is additionally pinned by the 10 canonical 6-round vectors at +the syscall level and by an end-to-end prove+verify of chained +compressions. + += The 6-round assumption + +*A6R.* The BLAKE3 compression function restricted to 6 rounds is +collision-resistant and suitable as a 2-to-1 compression for Merkle +hashing and as a PRF for Fiat–Shamir, in the same sense the full 7-round +function is believed to be. (Precedent: KangarooTwelve's reduced-round +Keccak. Best public cryptanalysis of BLAKE3 reaches far fewer rounds; the +margin removed here is one round of seven.) + +*External review (2026-08).* The round-count choice was reviewed with +external symmetric-cryptography experts consulted by the project: removing +*one* round (7 → 6) was judged comfortable; removing *two* (7 → 5) was +explicitly not. Accordingly, 6 rounds is the endorsed floor. Variants +below 6 rounds are not formally ruled out, but they are not available on +the project's own authority: adopting one would require the external +experts to study the reduced-round margin specifically — a dedicated +cryptanalytic review, not an engineering or configuration decision. + +Any use of #blake3 as a Merkle or transcript hash *invokes this +assumption*. The z3 gate proves the chip computes 6-round BLAKE3 +correctly; it neither proves nor addresses whether 6 rounds are secure. + +*The assumption-free alternative.* The chip design is round-parameterised; +a 7-round instantiation (standard BLAKE3 compression, bit-compatible with +official parent-node merges) costs roughly 10–12% more per merge +end-to-end and requires no assumption beyond standard BLAKE3. + +*Ordering, reversed 2026-08-10.* The 7-round variant is the primary +target; the 6-round variant is the measured performance variant, kept +behind the round parameter and adopted only if that 10–12% is judged worth +signing A6R for. This reverses the ordering this section recorded before, +and the argument is the reference chain rather than cryptanalysis: at 7 +rounds the official crate is a direct external test vector for both the +primitive and its framing, and there is no assumption left to ratify or +defend at audit. It retracts nothing from the external review above — 6 +rounds remains the endorsed floor — and it is not a signature on A6R, +which falls due only if the default moves back to 6 +(`thoughts/shared/lfm-real-hash/A6R-signoff.md`). The chip specified on +this page is still the 6-round instantiation, so the above is recorded +intent and not a change that has landed here. If both are instantiated +they are distinct chips with distinct ECALL numbers. + += Cost + +Measured on the CPU bench box (32 cores, blowup 2, single-epoch +continuations): ≈5,473 compressions/s at ≥#raw("2^17") table rows, ≈7,194 +committed cell-equivalents per compression end-to-end (≈5,316 table-only) +— ≈12× the keccak-f permutation per 2-to-1 merge at equal wall time and +memory. Details and methodology: PR \#903. diff --git a/spec/book.typ b/spec/book.typ index 8bf8612af..7faf8ef1b 100644 --- a/spec/book.typ +++ b/spec/book.typ @@ -49,6 +49,7 @@ ("commit.typ", [`COMMIT` chip], ), ("sha256.typ", [`SHA256` accelerator], ), ("keccak.typ", [`KECCAK` accelerator], ), + ("blake3.typ", [`BLAKE3_6R` accelerator], ), )) ) ) diff --git a/spec/src/blake3.toml b/spec/src/blake3.toml new file mode 100644 index 000000000..bb04388c0 --- /dev/null +++ b/spec/src/blake3.toml @@ -0,0 +1,345 @@ +# BLAKE3_6R — the 6-round internal-variant BLAKE3 compression accelerator. +# +# ⚠ NORMATIVE SOURCE NOTE. This spec documents the shipped chip +# (`prover/src/tables/blake3.rs`); the chip's wiring is single-sourced in Rust +# (`run_flow` interpreted as columns and as witness) and formally gated by the +# z3 model in `thoughts/blake3/blake3-chip/z3_blake_verify.py`. Where this file +# and those artifacts disagree, THEY are normative and this file has a bug. +# Cross-checked totals at spec-writing time: 3,219 main columns, 1,397 +# interactions, 814 constraints, max degree 3 (incl. the ×μ gating factor). +# +# One row = one compression call: 6 rounds × 8 G-functions fully unrolled in +# SSA form. Message schedule = the literal per-round index table `SCHED` below +# (permute^r of the identity under MSG_PERMUTATION = +# [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8]); round r position i consumes +# original message word SCHED[r][i]. +# +# Security: this chip computes 6-round BLAKE3, NOT the standard 7-round +# function. Its use as a Merkle / Fiat–Shamir hash rests on the named +# assumption A6R (see blake3.typ). No external system will ever agree on +# these digests. + +name = "BLAKE3" + +# ------------------------------------------------------------------------- +# Inputs (read from memory at addr .. addr+112; see the ECALL ABI) +# ------------------------------------------------------------------------- + +[[variables.input]] +name = "timestamp" +type = "DWordWL" +desc = "timestamp at which the compression is performed" +pad = 0 + +[[variables.input]] +name = "addr" +type = "DWordBL" +desc = "8-aligned base address of the 176-byte state region (h|m|t|len,flags|out)" +pad = 0 + +[[variables.input]] +name = "h" +type = [["Byte", 4], 8] +desc = "input chaining value h[0..8], 8 little-endian u32 words" +pad = 0 + +[[variables.input]] +name = "m" +type = [["Byte", 4], 16] +desc = "message block m[0..16] = left_cv ‖ right_cv for a 2-to-1 merge" +pad = 0 + +[[variables.input]] +name = "t_lo" +type = ["Byte", 4] +desc = "low u32 of the 64-bit counter t → v[12]" +pad = 0 + +[[variables.input]] +name = "t_hi" +type = ["Byte", 4] +desc = "high u32 of the 64-bit counter t → v[13] (split order is load-bearing)" +pad = 0 + +[[variables.input]] +name = "block_len" +type = ["Byte", 4] +desc = "input byte count of this block → v[14]" +pad = 0 + +[[variables.input]] +name = "flags" +type = ["Byte", 4] +desc = "domain-separation flags → v[15]" +pad = 0 + +# ------------------------------------------------------------------------- +# Outputs (written to memory at addr+112 .. addr+176) +# ------------------------------------------------------------------------- + +[[variables.output]] +name = "out" +type = [["Byte", 4], 16] +desc = "full 16-word compression output; the truncated CV is out[0..8]" +pad = 0 + +# ------------------------------------------------------------------------- +# Auxiliary +# ------------------------------------------------------------------------- + +[[variables.auxiliary]] +name = "state_ptr" +type = ["DWordHL", 22] +desc = "per-dword pointers state_ptr[k] = addr + 8k over the 22-dword region" +pad = ["*", 8, ["arr", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]] + +[[variables.auxiliary]] +name = "old_out" +type = [["Byte", 8], 8] +desc = "previous memory content of the out region, dword o = bytes old_out[o]; the MEMW writes' old value" +pad = 0 + +# Per-G SSA cells, 48 instances (g = 8r + j, round r ∈ 0..6, call j ∈ 0..8). +# Each G block: A1(4B) c1,c2(2 bits) X1(4B) C1(4B) X2(4B) +# R1: SLL_lo(2B) SLLC_lo(2B) SLL_hi(2B) SLLC_hi(2B) Y1(4B) +# A2(4B) c3,c4(2 bits) X3(4B) C2(4B) X4(4B) +# R2: same shape as R1 +# = 56 byte cells + 4 carry bits = 60 cells; column base = 210 + 60g +# (see prover/src/tables/blake3.rs `cols` for the exact offsets). + +[[variables.auxiliary]] +name = "g_add3_out" +type = [[["Byte", 4], 2], 48] +desc = "A1, A2 per G: the two 3-operand add outputs (v[a] after each half)" +pad = 0 + +[[variables.auxiliary]] +name = "g_add3_carry" +type = [[["Bit", 2], 2], 48] +desc = "two summed carry bits per 3-operand add: carry = c1 + c2 ∈ {0,1,2}" +pad = 0 + +[[variables.auxiliary]] +name = "g_xor_out" +type = [[["Byte", 4], 4], 48] +desc = "X1..X4 per G: the four 32-bit XOR outputs (rotr16/rotr8 are free byte relabels of X1/X3)" +pad = 0 + +[[variables.auxiliary]] +name = "g_add2_out" +type = [[["Byte", 4], 2], 48] +desc = "C1, C2 per G: the two 2-operand add outputs (v[c]); carries are expressions, not cells" +pad = 0 + +[[variables.auxiliary]] +name = "g_rot" +type = [[["Byte", 12], 2], 48] +desc = "per rotation (rotr12 then rotr7): SLL_lo, SLLC_lo, SLL_hi, SLLC_hi (2 B each) and the output word Y (4 B)" +pad = 0 + +[[variables.multiplicity]] +name = "μ" +type = "Bit" +desc = "1 on real rows, 0 on padding; gates every constraint and interaction; pinned boolean by an ungated IS_BIT" +pad = 0 + +# ------------------------------------------------------------------------- +# Constants +# ------------------------------------------------------------------------- + +[[constants]] +name = "IV" +desc = "BLAKE3 IV[0..4] (SHA-256 IV words), inlined into round-0 arithmetic — not columns" +value = ["arr", 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A] + +[[constants]] +name = "SCHED" +desc = "per-round message-schedule index table: round r position i reads m[SCHED[r][i]]; SCHED[r] = permute^r(identity), MSG_PERMUTATION = [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8]; the trailing permute after round 5 is never consumed" +value = ["arr", + ["arr", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ["arr", 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8], + ["arr", 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1], + ["arr", 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6], + ["arr", 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4], + ["arr", 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7], +] + +[[constants]] +name = "G_INDICES" +desc = "the (a,b,c,d) working-state slots of the 8 G-calls per round: 4 column mixes then 4 diagonal mixes" +value = ["arr", + ["arr", 0, 4, 8, 12], ["arr", 1, 5, 9, 13], ["arr", 2, 6, 10, 14], ["arr", 3, 7, 11, 15], + ["arr", 0, 5, 10, 15], ["arr", 1, 6, 11, 12], ["arr", 2, 7, 8, 13], ["arr", 3, 4, 9, 14], +] + +# ------------------------------------------------------------------------- +# Constraint groups. All eval constraints are μ-gated; padding rows are +# all-zero (except the keccak-idiom state_ptr pad) with μ = 0. +# ------------------------------------------------------------------------- + +# ------------------------------------------------------------------------- +# Constraint groups. +# +# ⚠ SCOPE. This file machine-formalizes the chip's I/O AND RANGE surface — +# the ECALL binding, the x10 register read, all 22 MEMW dword operations, +# the pointer arithmetic, and every explicit ARE_BYTES/IS_HALF/IS_BIT — i.e. +# exactly the surface the z3 gate does NOT model (DESIGN.md §1.1: "the gate +# cannot check this", §7 items 4/5/10). The unrolled 6-round MIXING CORE +# (96 add3 sum identities + carry booleanities, 96 add2 expression-carry +# booleanities, 96 inline rotation identity groups, 832 BYTE_ALU[XOR] +# lookups wired per the SSA dataflow) is NOT re-formalized here: its +# normative sources are the single-source Rust dataflow +# (prover/src/tables/blake3.rs `run_flow`, interpreted once as columns and +# once as witness) and the z3 gate that proves that dataflow equal to the +# reference function under free inputs. Totals for cross-checking: 814 +# constraints, 1,397 interactions, max degree 3 including ×μ. +# ------------------------------------------------------------------------- + +[[constraint_groups]] +name = "io" + +# ECALL receive: [timestamp, -3 as u64 = 2^64 - 3] +[[constraints.io]] +kind = "interaction" +tag = "ECALL" +input = ["timestamp", ["cast", ["-", ["^", 2, 64], 3], "DWordWL"]] +multiplicity = ["-", "μ"] + +# MEMW register read of x10 binding addr (keccak:c:read_addr idiom) +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [1, ["cast", ["*", 2, 10], "DWordWL"], "addr", "timestamp", 1, 0, 0] +output = "addr" +multiplicity = "μ" +ref = "blake3:c:read_addr" + +# The 22 dword MEMW operations at `timestamp`, per region. Input dwords +# (k < 14) are reads in the combined read+write encoding: old = value = the +# input words, timestamps advance. Output dwords (k >= 14) write `out` over +# `old_out`. + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", "k"], "DWordWL"], ["idx", "h", ["*", 2, "k"]], ["idx", "h", ["+", ["*", 2, "k"], 1]], "timestamp", 0, 0, 1] +output = ["arr", ["idx", "h", ["*", 2, "k"]], ["idx", "h", ["+", ["*", 2, "k"], 1]]] +iters = [["k", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:load_h" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", ["+", "k", 4]], "DWordWL"], ["idx", "m", ["*", 2, "k"]], ["idx", "m", ["+", ["*", 2, "k"], 1]], "timestamp", 0, 0, 1] +output = ["arr", ["idx", "m", ["*", 2, "k"]], ["idx", "m", ["+", ["*", 2, "k"], 1]]] +iters = [["k", 0, 7]] +multiplicity = "μ" +ref = "blake3:c:load_m" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", 12], "DWordWL"], "t_lo", "t_hi", "timestamp", 0, 0, 1] +output = ["arr", "t_lo", "t_hi"] +multiplicity = "μ" +ref = "blake3:c:load_t" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", 13], "DWordWL"], "block_len", "flags", "timestamp", 0, 0, 1] +output = ["arr", "block_len", "flags"] +multiplicity = "μ" +ref = "blake3:c:load_len_flags" + +[[constraints.io]] +kind = "interaction" +tag = "MEMW" +input = [0, ["cast", ["idx", "state_ptr", ["+", "o", 14]], "DWordWL"], ["idx", "out", ["*", 2, "o"]], ["idx", "out", ["+", ["*", 2, "o"], 1]], "timestamp", 0, 0, 1] +output = ["idx", "old_out", "o"] +iters = [["o", 0, 7]] +multiplicity = "μ" +ref = "blake3:c:store_out" + +[[constraint_groups]] +name = "addr" + +# state_ptr[k] = addr + 8k via the shared ADD template (μ-gated carry pair); +# the top dword k = 21 additionally forbids wraparound (addr + 168 < 2^64). +[[constraints.addr]] +kind = "template" +tag = "ADD" +input = [["cast", "addr", "DWordWL"], ["cast", ["*", 8, "k"], "DWordWL"]] +output = ["cast", ["idx", "state_ptr", "k"], "DWordWL"] +iters = [["k", 0, 21]] +ref = "blake3:c:state_ptr" + +# alignment: addr[0] & 7 = 0 +[[constraints.addr]] +kind = "interaction" +tag = "BYTE_ALU" +input = [0, ["idx", "addr", 0], 7, 0] +multiplicity = "μ" +ref = "blake3:c:alignment" + +# addr byte range checks — without them the addr_lo/addr_hi linear +# combinations alias non-byte encodings (keccak:c:range_addr rationale). +[[constraints.addr]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", "addr", ["*", 2, "i"]], ["idx", "addr", ["+", ["*", 2, "i"], 1]]] +iters = [["i", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_addr" + +[[constraints.addr]] +kind = "interaction" +tag = "IS_HALF" +input = [["idx", ["cast", ["idx", "state_ptr", "k"], "DWordHL"], "hw"]] +iters = [["k", 0, 21], ["hw", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_state_ptr" + +[[constraint_groups]] +name = "range" + +# m is never XOR-consumed — explicit byte checks (DESIGN §4.7 / §7.5). +[[constraints.range]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", ["idx", "m", "i"], ["*", 2, "p"]], ["idx", ["idx", "m", "i"], ["+", ["*", 2, "p"], 1]]] +iters = [["i", 0, 15], ["p", 0, 1]] +multiplicity = "μ" +ref = "blake3:c:range_m" + +# old_out rides only the MEMW bus — same aliasing argument as the address. +[[constraints.range]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", ["idx", "old_out", "o"], ["*", 2, "p"]], ["idx", ["idx", "old_out", "o"], ["+", ["*", 2, "p"], 1]]] +iters = [["o", 0, 7], ["p", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_old_out" + +# The four shift halfwords of each inline rotation (SLL_lo, SLLC_lo, +# SLL_hi, SLLC_hi — bytes 0..8 of each rotation block). The tight SLL +# bounds are load-bearing for the rotation identities (DESIGN §4.2). +[[constraints.range]] +kind = "interaction" +tag = "ARE_BYTES" +input = [["idx", ["idx", ["idx", "g_rot", "g"], "half"], ["*", 2, "p"]], ["idx", ["idx", ["idx", "g_rot", "g"], "half"], ["+", ["*", 2, "p"], 1]]] +iters = [["g", 0, 47], ["half", 0, 1], ["p", 0, 3]] +multiplicity = "μ" +ref = "blake3:c:range_rot" + +[[constraint_groups]] +name = "mu" + +# μ boolean, ungated — the ECALL receive anchors μ>0 rows to a CPU ecall +# whose ECALL flag is boolean; this constraint makes the argument local. +[[constraints.mu]] +kind = "template" +tag = "IS_BIT" +input = ["μ"] +ref = "blake3:c:range_mu" diff --git a/syscalls/src/syscalls.rs b/syscalls/src/syscalls.rs index 5228455ea..578c3b067 100644 --- a/syscalls/src/syscalls.rs +++ b/syscalls/src/syscalls.rs @@ -29,6 +29,10 @@ pub enum SyscallNumbers { #[cfg(target_arch = "riscv64")] const KECCAK_SYSCALL_NUMBER: usize = usize::MAX - 1; +/// Syscall number for the BLAKE3 6-round compression accelerator (u64::MAX - 2). +#[cfg(target_arch = "riscv64")] +const BLAKE3_SYSCALL_NUMBER: usize = usize::MAX - 2; + /// Syscall number for the ECSM secp256k1 scalar-multiply accelerator (-11 as usize). #[cfg(target_arch = "riscv64")] const ECSM_SYSCALL_NUMBER: usize = usize::MAX - 10; @@ -175,6 +179,30 @@ pub fn keccak_permute(_state: &mut [u64; 25]) { unimplemented!("syscalls are only implemented for riscv64 targets"); } +#[cfg(target_arch = "riscv64")] +/// BLAKE3 **6-round** compression via the accelerator (internal variant — NOT +/// standard 7-round BLAKE3; see `thoughts/blake3/blake3-chip/DESIGN.md`). +/// +/// `state` is the 176-byte region as 22 dwords: `h[8 words] | m[16 words] | +/// t | (block_len, flags) | out[16 words]`, all words little-endian, two per +/// dword. The accelerator reads dwords 0..14 and writes `out` to dwords 14..22. +/// Using `[u64; 22]` guarantees the 8-byte alignment the ecall requires. +pub fn blake3_compress_6round(state: &mut [u64; 22]) { + unsafe { + asm!( + "ecall", + in("a0") state.as_mut_ptr(), + in("a7") BLAKE3_SYSCALL_NUMBER, + ) + } +} + +#[cfg(not(target_arch = "riscv64"))] +/// BLAKE3 6-round compression via the accelerator (internal variant). +pub fn blake3_compress_6round(_state: &mut [u64; 22]) { + unimplemented!("syscalls are only implemented for riscv64 targets"); +} + #[cfg(target_arch = "riscv64")] /// Compute `xR = (k·G)_x` on secp256k1 via the ECSM accelerator. All values are 32-byte /// little-endian. Requires `0 < k < N` and a canonical valid `xG` curve coordinate. diff --git a/thoughts/blake3/.gitignore b/thoughts/blake3/.gitignore new file mode 100644 index 000000000..7fcf8256c --- /dev/null +++ b/thoughts/blake3/.gitignore @@ -0,0 +1,4 @@ +target/ +Cargo.lock +__pycache__/ +venv/ diff --git a/thoughts/blake3/GATE-TRANSCRIPTION-AUDIT.md b/thoughts/blake3/GATE-TRANSCRIPTION-AUDIT.md new file mode 100644 index 000000000..5d8c7ba17 --- /dev/null +++ b/thoughts/blake3/GATE-TRANSCRIPTION-AUDIT.md @@ -0,0 +1,471 @@ +# Transcription audit — does the BLAKE3 gate assert more than the design delivers? + +Adversarial, one-directional audit of `blake3-chip/z3_blake_verify.py` against +(a) the oracle `blake3-oracle/blake3_ref.py` and (b) the constraint system +`blake3-chip/DESIGN.md` specifies. Branch `spike/blake3-recovered`. + +Only one direction is dangerous. A model **weaker** than the object yields a +spurious SAT — a false alarm. A model **stronger** yields UNSAT on an object +that is genuinely forgeable — false assurance — and no positive control can see +it, because an honest witness satisfies a correct model and an over-strong model +equally well. The three positive controls and the 35-vector oracle anchor are +blind to exactly this. + +Reproduce: `python3 audit_gate_transcription.py` (~4 min, 73 checks) or +`--slow` (+ the gate's own BV UNSATs, ~8 min). Every check is paired with a +tamper that must make it fail; a check that does not bite is itself reported as +a failure. No file outside this audit was modified — tampers are applied to +in-memory copies and reverted. + +--- + +## Verdict + +**One over-strong assertion, and it is the load-bearing one: the gate declares +byte-ness rather than deriving it.** Every committed cell in the model is a +`BitVec(…, 8)`, so the range check that DESIGN §4.3/§4.4/§5/§7.4 make the whole +soundness argument rest on is an *axiom of the model*, not something the model +can observe being present or absent. The gate proves the same UNSAT for the +designed chip and for a chip with **no range checks at all**. + +I checked whether that gap is live. It is not, today: I traced every SSA value +in `build_g` and `build_compress` mechanically and the design's premise holds — +all 288 add/shift outputs of a 6-round compression are consumed by a `ByteAlu` +XOR, for ROUNDS ∈ {1,2,6,7}. So the gate's UNSAT is *correct*, for a reason the +gate does not model. It would stay UNSAT under three specific deviations, one of +them invited by DESIGN §1.1 itself. + +Transcription (a) — `bref_*` vs the oracle — is **exact**, mechanically, on every +element the brief named. + +| # | severity | what | +|---|---|---| +| **F1** | **high** | **Model stronger.** Byte-ness is the `BitVec(…,8)` declaration. The gate cannot distinguish a chip that range-checks an add output from one that does not, nor a chip with §4.7's 32 message `AreBytes` sends from one without them. The premise holds today; nothing in the gate would notice it becoming false. | +| **F2** | medium | **Model weaker / evidence over-stated.** The width audit's `DROP SLL bound → SAT` control is run on a *single-halfword fragment*. Composed with the second identity, the recombine and the downstream byte check, dropping one `SLL` bound is **UNSAT** and dropping both is forgeable at **exactly one input**, `X = 0xFFFFFFFF`. §7.3's stated mechanism is also backwards. | +| **F3** | medium | The "covers every G, hence every round" chaining argument (§9, MAIN 0) **is** the free-range-check argument in disguise, and the gate models neither. It is prose on both counts. | +| F4 | low | μ, padding rows, the degree ledger, and the bus layer are not carried at all. Scope, not error — but §7.1's "μ-gate every eval constraint" and §4.8's degree table are cited as gate-backed and are not. | +| F5 | cosmetic | Three documentation defects (§7.3's mechanism, §4.8's recombine degree row, §3-vs-§4.3 on whether add2 commits a carry column). | + +Along the way, two results the gate did not have: + +* The rotation is pinned to `rotr12` / `rotr7` **for all 2³² inputs in the + field**, not just in BV and not just at one concrete halfword. +* The necessary *and sufficient* `AreBytes` set for a rotation is **one of + `{SLL_lo, SLL_hi}`**. The two `SLLC` bounds and one of the two `SLL` bounds + are not load-bearing — 3 of the 4 sends per rotation, i.e. 288 of the design's + 1,250 sends per compression. + +--- + +## F1 — the free range check is declared, not derived + +This is the claim DESIGN §7.4 flags as soundness-critical and the one the gate is +cited to discharge: + +> §4.3 `s`'s bytes are range-checked **free** by the next XOR that consumes `s` +> §5 Every committed add/shift word is an operand of a later XOR ⇒ its bytes +> are byte-range-checked for free by that `ByteAlu` lookup. +> §7.4 **Every add/shift output must actually feed a downstream XOR** (its only +> range check). If a future refactor reorders so an add output is *last* +> with no XOR consumer, add an explicit AreBytes or the carry argument is +> unsound. + +The model's answer to "where does `s ∈ [0,2^32)` come from" is +`z3_blake_verify.py:133-134`: + +```python +def fresh_word(self): + return [self._fresh(8) for _ in range(4)] +``` + +and `:128-129`, `self._fresh(w=8)` → `BitVec(f"{tag}_v{n}", w)`. `add2` +(`:176-184`) and `add3` (`:186-197`) both begin `s = self.fresh_word()`. So `s` +is four 8-bit bitvectors *by construction*. The audit confirms it mechanically: +every cell the model creates is width 8, and every constraint it emits is `=` +(xor / sum / shift / recombine) or `or` (carry booleanity) — 1,408 and 288 +respectively for a 6-round compression. **There is no range-check object in the +model to be present or absent.** + +**In fairness, the gate says it is doing this.** The `Circuit` class header +(`:118-119`) reads *"A `word` is a list of 4 free 8-bit BVs … Byte width == the +ByteAlu/AreBytes range-check contract"*, and the module docstring (`:18-21`) +lists `ByteAlu[XOR]` and `AreBytes` under "Chip contracts assumed". So this is a +**documented assumption, not a hidden one** — unlike the EC audit's F2, nothing +here is detected from a comment. Inside the class, "AreBytes" occurs only in +those two comments (`:119`, `:212`); the only place the gate *models* an +`AreBytes` bound is `field_shift_bound` (`:388`), and only for the isolated +fragment F2 is about. + +The problem is that the equation "byte width == the range-check contract" is +applied **uniformly to every column**, including the ones for which the design +must separately arrange the contract (§4.2's and §4.7's explicit sends, §5's +downstream-XOR argument) — and DESIGN §9 then cites the gate's UNSAT as +establishing what the comment assumed. The gate's own docstring (`:5`) states +the method as "Every committed column of the designed chip is a FREE bitvector"; +it is a free *byte*, and byte-ness is the property under proof. + +### Which gate rows this invalidates + +| DESIGN §9 row | as written | what it actually establishes | +|---|---|---| +| MAIN 0, one G, free inputs, **UNSAT** | "the quarter-round … is **correctly & tightly constrained**" | correctly & tightly constrained **given that its inputs and its add outputs are byte-range-checked** — which is §7.4/§7.5's obligation on the Rust, not a result | +| MAIN 1, rounds=0, **UNSAT** | "`v` layout … and the feed-forward are correct" | same caveat for `h`, `t_lo/t_hi`, `block_len`, `flags` | +| §9 "Proven (symbolic, all inputs): the G quarter-round … hence, by the chaining argument, the full N-round compression" | unconditional | conditional on the same premise — see F3 | + +### Where the missing range check actually bites + +The gate declares byte-ness on *every* cell, so I checked, per SSA class, +whether the range check is load-bearing at all. Three of the six classes do not +need one; the model's blanket declaration hides that distinction too. + +| SSA class | per G | model's source of byte-ness | chip's source (DESIGN) | is it load-bearing? | +|---|---:|---|---|---| +| `add2`/`add3` output `s` (A1, C1, A2, C2) | 4 words | `BitVec(…,8)` | downstream `ByteAlu` (§4.3/§4.4/§5) | **YES** — without it the chip may commit the **unreduced** sum | +| `ByteAlu` XOR output (X1..X4) | 4 words | `BitVec(…,8)` | the lookup itself | n/a — pinned by contract | +| `SLL_lo`, `SLL_hi` | 2 halfwords/rot | `BitVec(…,8)` | explicit `AreBytes` (§4.2) | **YES, but only one of the two** (F2) | +| `SLLC_lo`, `SLLC_hi` | 2 halfwords/rot | `BitVec(…,8)` | explicit `AreBytes` (§4.2) | **no** — proved | +| rotation output `Y` (B1, B2) | 1 word/rot | `BitVec(…,8)` | downstream `ByteAlu` (§5) | **no** — the two recombine identities pin its *value* even with free field cells | +| message `m` | 16 words | `BitVec(…,8)` | explicit `AreBytes`, §4.7 | **YES** for cell-binding | +| `h`, `t_lo/t_hi/block_len/flags` | 12 words | `BitVec(…,8)` | XOR consumers (§4.7) | for cell-binding | + +Each row is an executable check in `audit_gate_transcription.py` §B/§C. + +### The forgery it hides — construction + +**(1) An add output without its ByteAlu consumer.** This is verbatim the §7.4 +deviation. Model the design's add in the field instead of in BV: `s` is four +Goldilocks cells, `carry ∈ {0,1}`, and the only constraint is §4.3's sum +identity. + +``` +a = b = 0x80000000 +honest : carry = 1, s = 0 cells [0, 0, 0, 0] +forged : carry = 0, s = 2^32 cells [2^32, 0, 0, 0] +``` + +Both satisfy `a + b ≡ s + 2^32·carry (mod p)` and the booleanity. The add no +longer computes mod 2^32; the chip's `v[c]` is off by exactly 2^32, and the +error propagates through every downstream add. `audit_gate_transcription.py` §C +runs it and reports both `add2` and `add3` FORGEABLE with the range check +removed, PINNED with it, symbolically over all operands. The BV model reports +UNSAT in both worlds because there is nothing to remove. + +**(2) §4.7's message range check — the one explicit input `AreBytes` in the whole +design (32 sends).** The gate models `m = [cir.fresh_word() for _ in range(16)]` +(`:332`), i.e. 64 declared bytes. Drop the sends and the 64 cells are free field +elements entering only through `wval(M) = Σ m_i·2^{8i}`, so the chip binds the +*value*, not the bytes: + +``` +m0 honest : [0x9A, 0x00, 0x13, 0x7F] +m0 forged : [0x19A, p−1, 0x13, 0x7F] same value mod p +``` + +Every constraint in the circuit is satisfied identically and the compression +output is bit-identical. The chip proves "the compression of these 64 cells" and +there are `p^3` cell-vectors per message word. Whether this becomes a Merkle +collision depends on the caller — for the §1.1 internal bus the receive tuple +carries the cells, so a byte-constrained counterparty pins them, and for the +§1.2 memory variant MEMW does. **The two gaps compose**: the gate models neither +the `AreBytes` nor the bus, so the obligation is discharged by nothing. + +**(3) The CV-only feed-forward trim — the deviation the design invites.** +`h[0..4]` land in round-0 `a` slots, and G uses `v[a]` only as an **add +operand**; it is never XORed. Their sole `ByteAlu` consumer is the *upper* +feed-forward half `out[i+8] = v[i+8] ⊕ h[i]`. The audit measures this: consumer +counts `[1,1,1,1,2,2,2,2]`, and for `h[0]` that single consumer is op #577 of 592 +— i.e. the feed-forward, not a round XOR. + +DESIGN §1.1 says: *"CV-only call sites read `out[0:8]`; the chip always produces +all 16 (the XOF root needs them)"*. For the internal 2-to-1 Merkle variant — the +primary target — an implementer dropping `out[8:16]` saves 32 committed cells and +32 sends (≈80 cell-equiv, 1.6%) and removes `h[0..4]`'s only range check. §7.4 +does not cover it, because it speaks of *add/shift outputs* and `h` is an input. +The gate reports the same UNSAT. + +### Fix + +Give the model a range-check object. Concretely: have `Circuit` allocate cells +that are *not* byte-bounded by default and add an explicit `are_bytes(word)` / +`byte_alu_xor()` that imposes the bound, so that "this word has no consumer" is +representable and shows up as SAT. That is a rewrite of the model into the field +(z3 `Int` mod p, as `audit_gate_transcription.py` §C does per-op) — but a cheaper +90% is available: keep BV and add the **structural** check this audit implements, +asserting that every `add`/`rotr` output word appears as a `xor` operand and that +`m` is explicitly range-checked. Twenty lines, and it is the check the design's +§7.4 actually asks for. + +--- + +## F2 — the width audit's shift control is run on a fragment + +`field_shift_bound` (`:380-395`) models **one halfword identity in isolation**: +`in_hw·2^r ≡ SLLC·2^16 + SLL (mod p)`, `SLLC ∈ [0,2^16)`, `SLL` unbounded, at one +concrete `in_hw = 0x9C3A` and one `r = 9`. It reports SAT and §9 renders that as + +> audit: **DROP `SLL` bound** (field neg ctrl) | **SAT** | without it the +> rotation is forgeable + +The chip does not contain that fragment. It contains **two** shift identities, +**two** recombine identities, and `Y` byte-range-checked by the downstream XOR. +Composing them (`audit_gate_transcription.py` §C2, symbolic over all 2³² inputs) +gives a different picture: + +| bounds kept | r=4 (rotr12) | r=9 (rotr7) | +|---|---|---| +| all four | PINNED | PINNED | +| `SLL_lo` only | **PINNED** | **PINNED** | +| `SLL_hi` only | **PINNED** | **PINNED** | +| `SLLC_lo` and/or `SLLC_hi`, no `SLL` | FORGEABLE | FORGEABLE | +| none | FORGEABLE | FORGEABLE | + +All 32 configurations were also checked non-vacuous (the honest witness +satisfies each). So: + +* **dropping one `SLL` bound is not exploitable at all** — the gate's control + claims it is; +* **dropping both** is exploitable at **exactly one input**. Enumerated + exhaustively, for both `r`: + +``` +X = 0xFFFFFFFF honest Y = 0xFFFFFFFF forged Y = 0x00000000 +SLL_lo = SLL_hi = p − 2^r (i.e. honest − 2^16 as a field element) +SLLC_lo = SLLC_hi = 2^r (i.e. honest + 1) +``` + +Every other `X` is UNSAT. The defence is still necessary — a prover can grind an +intermediate XOR output to `0xFFFFFFFF` cheaply, and there are 96 rotation slots +per compression — but "forgeable" at one point is not what the control shows, and +the control shows it for a chip that does not exist. + +**§7.3's mechanism is backwards.** It says *"dropping it makes the rotation +forgeable (a wrong `SLL` admits a **large field SLLC**)"*. `SLLC` is bounded to +`[0,2^16)` by its own `AreBytes` and stays small in the forgery (`2^r`); it is +`SLL` that goes large (`p − 2^r`). The witness above is the counterexample to the +prose, not to the conclusion. + +**Cost consequence, flagged not pursued:** §4.2 spends 4 `AreBytes` sends per +rotation. One suffices in this composed model. At 96 rotations that is 288 of +the design's ~1,250 sends per compression → ≈432 aux cells ≈ **8.6% of the 5,030 +cell-equiv budget**. Before acting on that, note it depends on `Y` being +byte-checked by its consumer — i.e. on F1's premise — and on `2^{-16} mod p` +being large, which is exactly the kind of implicit structural fact this audit +exists to distrust. It wants its own gate row, not a code change. + +By contrast `field_add_carry` (`:398-413`) **is** faithful: dropping the carry +booleanity is forgeable even with `s` byte-range-checked (verified composed), and +its UNSAT direction holds for all `(a,b,m)`, not just the one concrete triple it +tests. Both width-audit positives were re-derived symbolically and hold +universally. + +--- + +## F3 — "covers every G, hence every round" is the same argument as the free range check + +§9's MAIN 0 row carries the whole default run: + +> **covers every G, hence every round** (a round is a fixed composition of 8 +> G-calls). + +MAIN 0 proves: *for all byte-valued `v[a],v[b],v[c],v[d],mx,my`, the G's four +outputs equal `bref_g`*. Composing it needs each G's **inputs** to be +byte-valued, which holds because they are the previous G's outputs, which are +byte-valued because of the downstream-XOR range check. **The chaining argument +and the free-range-check argument are one argument.** The gate models neither: +the chaining is prose in §9, and the range check is the `BitVec(…,8)` +declaration. + +What *is* checkable, and what this audit checks mechanically because the gate +does not: + +* `build_round` calls `build_g` on the 8 quadruples of `G_CALLS`, in order, all + 7 rounds — compared against the quadruples recovered by instrumenting the + **oracle's** `round_fn`, not against the gate's own constant. +* `build_compress` feeds every G the original message column under `permute^r` + — all 7 rounds × 8 calls = **56 index pairs**, compared against the oracle's + permutation composition. Both tampers (a swapped `MSG_PERMUTATION` entry, a + swapped `mx/my` in `G_CALLS[5]`) are detected. +* Every G quadruple has four distinct state indices, the 8 calls touch each of + the 16 slots exactly twice and consume each message index exactly once — so + MAIN 0's `a,b,c,d = 0,1,2,3` instance really is general. + +README finding 2 ("in Rust, 48 G instances are emitted separately and a wrong +column index in instance #37 is not covered") stands and is out of scope: there +is no Rust. + +--- + +## F4 — what the model does not carry at all + +Not errors; scope. Listed because DESIGN cites the gate for some of them. + +| DESIGN claim | modelled? | consequence | +|---|---|---| +| §4.5 / §7.1 "every eval constraint is μ-gated; padding rows all-zero" | **no** — no μ variable exists; the docstring says "here mu=1 (a real row), so mu drops out" | exact for a live row. The gate says nothing about padding rows, so §7.1 is unbacked. Note the *ungated* system is strictly **stronger** as a system over all rows — it is only the single-live-row scope that makes this safe. | +| §4.8 degree ledger; the O1 (a)-vs-(c) decision | **no** — the model has no degree notion | `check_g` would be equally UNSAT for the rejected ternary-carry option (a). (a) is rejected for degree, not soundness, so this is harmless — but §4.8 is not gate-backed. | +| §1.1 `Blake3` bus, `Multiplicity::Column(MU)`, `TIMESTAMP_0/1` | **no** — confirmed absent: no `Multiplicity`, `TIMESTAMP`, `receive`/`send` anywhere in the file | README finding 1 (the missing input↔output timestamp binding) is invisible to the gate. Confirmed, not re-derived. | +| §4.1 "operands may be linear combos (sum ≤ 255)" | unexercised | `rotr16`/`rotr8` are pure index relabels, so every modelled operand is a single cell. Consistent with the design's actual use. | +| `block_len ∈ [0,64]`, `flags ∈ [0,128)` | modelled as free 32-bit words | model **weaker** — safe, and DESIGN specifies no such constraint either. | + +The BITWISE contracts the gate *assumes* were cross-checked and are real: +`prover/src/tables/bitwise.rs:351-364` enumerates `x,y ∈ [0,256)` and sets +`cols::XOR = x ^ y`; the `ByteAlu` receiver is at `:903-920` and the `AreBytes` +receiver at `:781-796`, both over that domain. DESIGN's cites (`:903`, `:783`) +are accurate. + +--- + +## The assertion tables + +Verdicts: **match** = model = object; **stronger** = model asserts more; +**weaker** = model omits; **not modelled** = outside the model entirely. +`file:line` refers to `blake3-chip/z3_blake_verify.py` unless noted. + +### (a) `blake3_ref.py` → `bref_*` + +Checked mechanically, not by eye: the G schedule and rotation amounts are +**recovered from the oracle by instrumentation** and compared, and every function +is differentially tested. + +| element | oracle | gate | verdict | +|---|---|---|---| +| `IV`, 8 words | `blake3_ref.py:29-32` | `:50-51` | match, element-wise | +| `MSG_PERMUTATION`, 16 indices | `:37` | `:52` | match, element-wise; and is a permutation of 0..15 | +| `MASK32` | `:54` | `:53` | match | +| G body: add order, XOR order, rotation amounts **16,12,8,7 in that order** | `:96-103` | `bref_g` `:72-80` | match — amounts recovered by patching `rotr`/`RotateRight` on both sides | +| G argument order (`v[a]+v[b]+mx` first, `my` second half) | `:96-100` | `:73,77` | match (differential, 300 random + 18 edge inputs) | +| `G_CALLS` 8 quadruples + message indices, **including the 4 diagonals** | `round_fn` `:113-121` | `:58-67` | **match, recovered from `round_fn` by instrumenting `g`** | +| `bref_round` iterates `G_CALLS` in order | `:113-121` | `:83-85` | match | +| `bref_permute` | `permute` `:126` | `:88-89` | match (index-identical) | +| initial `v`: `h[0..8]`, `IV[0..4]`, `t_lo`→v[12], `t_hi`→v[13], `block_len`→v[14], `flags`→v[15] | `compress` `:167-172` | `bref_compress` `:101-104` | match — probed slot by slot at rounds=0, and across 7 counters incl. `2^32−1`, `2^32` | +| counter split `t_lo = t mod 2^32`, `t_hi = t >> 32` | `:164-165` | `:509` (caller) | match | +| permutation applied `r < rounds−1`, i.e. **rounds−1 times** | `:181-182` | `:106-109` | match — counted by instrumentation for rounds 0..8 on both sides: `0,0,1,2,3,4,5,6,7` | +| feed-forward `out[i]=v[i]^v[i+8]`, `out[i+8]=v[i+8]^h[i]` with `h` the **original** CV | `:186-188` | `:110-114` | match | +| rounds parameterisation (6 vs 7 is the loop bound only) | `:176-182` | `:106-109` | match — differential over rounds {0,1,2,5,6,7,8} × 25 vectors | + +No discrepancy. Tampers on `IV`, `G_CALLS` and `MSG_PERMUTATION` are all +detected by the differentials. + +### (b) `DESIGN.md` → `build_g` / `build_round` / `build_compress` + +| DESIGN element | design | model | verdict | +|---|---|---|---| +| §4.1 XOR = 4 per-byte `ByteAlu[XOR]` sends, output pinned + operands range-checked | §4.1 | `xor` `:161-166`, 4 equalities | match (contract verified against `bitwise.rs:351-364`) | +| §4.2 `rotr16` = byte relabel `[b2,b3,b0,b1]`, free | §4.2/§7.6 | `:168-170` | **match — an actual index permutation of the source XOR's byte objects**, not a BV rotate; commits 0 columns, emits 0 constraints; and value-equal to `RotateRight(...,16)` over all 2³² | +| §4.2 `rotr8` = `[b1,b2,b3,b0]`, free | §4.2/§7.6 | `:172-174` | match, same evidence | +| §4.2 `rotr12 = rotl16∘rotl4` (r=4), `rotr7 = rotl16∘rotl9` (r=9) | §4.2 | `:207` `{12:4, 7:9}` | match | +| §4.2 two shift identities `hw·2^r = SLLC·2^16 + SLL` | §4.2 | `:222-223` | match | +| §4.2 recombine `Ylo = SLL_hi + SLLC_lo`, `Yhi = SLL_lo + SLLC_hi` | §4.2 | `:226-227` | match | +| §4.2 `SLL_*`, `SLLC_*` are 16-bit (2 bytes each) | §4.2 | `fresh_word()[:2]` `:213-216` | match | +| §4.2 4 `AreBytes` sends/rotation | §4.2 | **not modelled** | **stronger** (F1); and 3 of the 4 are not load-bearing (F2) | +| §4.3 2-op add: `a+b = s + 2^32·carry`, carry boolean | §4.3 | `add2` `:176-184` | match — the design's *derived* carry and the model's *committed* boolean carry proved equivalent over F_p | +| §4.4 3-op add: `a+b+m = s + 2^32·(c1+c2)`, `c1,c2` boolean | §4.4 | `add3` `:186-197` | match | +| §4.3/§4.4/§5 `s` byte-range-checked free by the next XOR | §4.3/§4.4/§5/§7.4 | `s = self.fresh_word()` (declared bytes) | **stronger** (F1) | +| §4.6 feed-forward, 16 XORs, `out[i+8] = v[i+8] ⊕ h[i]` | §4.6 | `:274-277` | match | +| §4.7 `m` needs explicit `AreBytes`; `h`,`t`,`bl`,`fl` are free | §4.7 | **not modelled** | **stronger** (F1). The *dataflow* half of the claim is verified here mechanically: `h`,`t_lo`,`t_hi`,`bl`,`fl` each feed an XOR, `m` does not | +| §5 every add/shift output feeds a downstream XOR | §5/§7.4 | **not modelled** | **stronger** (F1) — premise verified true here for ROUNDS ∈ {1,2,6,7}: 288 add/shift outputs, 0 unchecked | +| §3 per-G budget 56 byte-cells + 6 carry bits | §3 | counted from the model | match (56 / 6) | +| §2 per-G op mix: 2 add3, 2 add2, 4 xor, 2 shift-rotations, 1 rotr16, 1 rotr8 | §2/§5 | counted from the model | match | +| §7.7 `permute^r` wired from the original `M` columns | §7.7 | `:266-272` | match — 56 index pairs vs the oracle's composition | +| §7.8 IV inlined as constants at `v[8..12]` | §1.1/§7.8 | `const_word` `:258-261` | match | +| §7.9 all field expressions `< 2^35 ≪ p` | §7.9 | `WIDE = 48` | match — the ℤ identity and the mod-p identity proved equivalent under the byte bounds | +| §4.5 μ-gating, all-zero padding | §4.5/§7.1 | **not modelled** | not modelled (F4) | +| §4.8 degree ≤ 3 | §4.8 | **not modelled** | not modelled (F4) | +| §1.1 `Blake3` bus, μ multiplicity, timestamps | §1.1/§3 | **not modelled** | not modelled (F4; README finding 1) | + +### Gate hygiene + +| check | result | +|---|---| +| the G circuit's constraints are satisfiable on their own — MAIN 0's UNSAT is not vacuous | pass | +| the 6-round circuit's constraints are satisfiable on their own | pass | +| constraints emitted per op: xor 4, add2 2, add3 3, rotr12 4, rotr16 0 | pass | +| all 10 canonical 6-round fixtures reproduce from the live oracle (the positive controls are not anchored to a stale file) | pass | +| …and none of them equals the 7-round compression of the same input | pass | +| `gen_7round_vector` returns the oracle's own 7-round output | pass | +| the assumed `ByteAlu[XOR]` / `AreBytes` contracts match `prover/src/tables/bitwise.rs` | pass | +| `check_g()` UNSAT, `check_compress(0)` UNSAT, `check_g(swap_g_operand)` SAT (`--slow`) | pass | + +--- + +## Documentation defects (F5) + +Not soundness; a reader following the citations is misled. + +* **§7.3's mechanism is backwards.** "a wrong `SLL` admits a large field `SLLC`" + — the forgery keeps `SLLC` small (`2^r`, inside its own bound) and makes `SLL` + large (`p − 2^r`). Witness above. +* **§4.8's recombine row over-states its degree.** `μ·(Ylo − SLL_hi − SLLC_lo)` + is linear in committed columns → body degree 1, ×μ = 2. The table says 2 → 3. + Safe-side wrong; the "no constraint exceeds 3" verdict is unaffected. +* **§3 and §4.3 disagree on whether `add2` commits a carry column.** §3's per-G + table counts 1 carry bit per `add2` (6 per G); §4.3 makes it a *derived linear + expression* `(a+b−s)·INV_SHIFT_32` with no column. Semantically equivalent + (proved), but 96 cells per compression hang on the reading, in a design whose + headline is a cell count. +* The gate's docstring "Every committed column … is a FREE bitvector" should say + "a free **byte**" — the distinction is F1. + +--- + +## Could not determine + +Stated so the boundary is explicit rather than implied. + +1. **Anything about a Rust chip.** There is none. README finding 2 (48 G + instances emitted separately; a wrong column index in instance #37) is + unauditable until it exists, and F1's forgeries are all statements about what + a future implementation must not do. +2. **The bus layer.** Not modelled by the gate, not audited here. F1's message + and `h` constructions become live or benign depending on it; README finding 1 + (the missing input↔output timestamp binding) sits in the same place. + Confirmed absent from the gate, per the brief — not re-derived. +3. **The `--full` monolithic UNSATs** (`check_round`, `check_compress(2/6/7)`) + were **not run** — 30-40 min timeouts each. I audited the model they run on, + not their verdicts. Note that they inherit F1 in full: a monolithic 6-round + UNSAT is still an UNSAT about a model in which every cell is a declared byte. +4. **The `HWSL` inline soundness proof** the design defers to + (`../keccak-verify/hwsl_inline_test.py` Part 2) — that directory is not in + this artifact. §C's composed field model re-derives the shift-identity result + independently, so the conclusion does not rest on the missing file, but the + cited proof was not read. +5. **Completeness.** Every result here is about soundness (can a wrong witness + pass). Whether an honest trace generator can *produce* the witnesses — the + `AreBytes` send layout, the carry values — is unchecked; a mismatch there is + an unprovable honest witness, not a forgery. +6. **Whether the recovered artifact is byte-identical to the 2026-07-23 + original** (README's own open item). Unchanged by this audit. + +--- + +## Regression suite + +`audit_gate_transcription.py`, 73 checks (76 with `--slow`), all passing, every +one paired with a tamper that must break it: + +``` +A reference transcription (a): constants element-wise; G_CALLS and rotation + amounts RECOVERED from the oracle by instrumentation; differential g / + round / permute / compress over rounds {0,1,2,5,6,7,8}; counter split + across 2^32; permute-application count 0,0,1,2,3,4,5,6,7; v-layout probed + slot by slot. Tampers: IV, G_CALLS, MSG_PERMUTATION — all detected. +B circuit transcription (b): rotr16/rotr8 are index relabels by object + identity, commit no columns, value-equal to RotateRight; per-G cell and op + census; SSA range-check provenance over one G and over ROUNDS 1/2/6/7; + h[0..4]'s single feed-forward consumer; message indexing under permute^r, + 56 pairs; what the model does not represent (no range object, no mu, no + bus). Tampers: a wrong relabel, a G whose add output loses its XOR + consumer, a swapped MSG_PERMUTATION entry, a swapped G_CALLS message pair + — all detected. +C the dangerous direction, in the field: add2/add3 pinned with the range + check and FORGEABLE without it (concrete witness a=b=0x80000000 -> s=2^32); + the rotation output needs no range check of its own; the 32-configuration + bound lattice with non-vacuity; the composed forgery at X=0xFFFFFFFF, + enumerated exhaustively; both width-audit positives re-derived symbolically + for all inputs; the message-cell collision. +D hygiene: non-vacuity of MAIN 0 and the 6-round model; per-op constraint + counts; derived-vs-committed carry equivalence; canonical fixtures + reproduce from the live oracle; Z-vs-F_p equivalence of the WIDE=48 model; + the BITWISE contracts checked against prover/src/tables/bitwise.rs; + (--slow) the gate's own BV verdicts. +``` diff --git a/thoughts/blake3/README.md b/thoughts/blake3/README.md new file mode 100644 index 000000000..1892bbf47 --- /dev/null +++ b/thoughts/blake3/README.md @@ -0,0 +1,215 @@ +# BLAKE3 accelerator — oracle + gate-proved chip design (RECOVERED, re-validated) + +**Provenance: recovered 2026-07-29 from subagent transcripts, not from a +backup.** The original work (2026-07-23) was written to a session scratchpad +under `/private/tmp/...`, never committed, and the scratchpad was gone by the +time anyone looked. The files here were reconstructed by replaying the `Write` +and `Edit` tool calls out of +`.claude/projects/.../1c23da47-.../subagents/agent-ablake3-{oracle,chip-design}-*.jsonl` +(5 Writes + 11 Edits, every Edit applied cleanly — no partial replays). + +Committing them is the point: this is the second campaign whose verification +artifacts were nearly lost to a scratchpad. Anything worth keeping belongs in +the repo. + +## What this is + +A BLAKE3 compression-function accelerator taken to a **gate-proved design**, +and — as of 2026-08-05 — **implemented in Rust** (PR #903: executor syscall +`u64::MAX-2`, chip `prover/src/tables/blake3.rs`, adversarially reviewed, e2e +prove+verify green, measured 12.2× keccak merges/s). The named "6-round +collision resistance" assumption (A6R) is recorded in the spec +(`spec/blake3.typ`) and in `blake3-chip/IMPLEMENTATION.md`; production use as +a Merkle/FS hash still requires ratifying it (or shipping the assumption-free +7-round instantiation, which costs ~10-12% more per merge). + +Purpose is **internal** (Merkle / Fiat–Shamir replacement candidate; the 6-round +variant is the primary target, K12 as precedent). The EVM has no BLAKE3 — only +the BLAKE2b-F precompile at 0x09 (EIP-152), variable-round and rarely used, so +that stays guest code. + +## Contents + +| file | what it is | +|---|---| +| `blake3-oracle/blake3_ref.py` | independent reference implementation | +| `blake3-oracle/test_oracle.py` | three-anchor validation harness; emits the canonical 6-round vectors | +| `blake3-oracle/ORACLE.md` | anchor results and the contract map | +| `blake3-oracle/official_test_vectors.json` | 35-case vector set — **see provenance note below** | +| `blake3-oracle/canonical_6round_vectors.json` | 10 pinned 6-round vectors, regenerated by the harness | +| `blake3-chip/DESIGN.md` | chip design + §7 risk ledger | +| `blake3-chip/IMPLEMENTATION.md` | Rust-implementation notes: deltas from the design + gates run | +| `blake3-chip/z3_blake_verify.py` | the soundness gate | +| `TRANSCRIPTION-AUDIT.md` | audit: oracle → gate transcription | +| `GATE-TRANSCRIPTION-AUDIT.md` | audit: gate constraint-model transcription | +| `audit_gate_transcription.py` | the executable half of the gate audit | +| `poseidon2-cost-study.md` | poseidon2-vs-blake3 cost study (2026-08-05) | +| `ground-truth/` | tiny Rust generator that produced the vector set from the official `blake3` crate | + +## Re-validation, 2026-07-29 — everything runs and passes + +Both fixtures were missing from the recovery (they had been downloaded or +generated, so no tool call held them). Both are now restored, and **every claim +in `DESIGN.md` §9 reproduces**: + +``` +oracle: [1] official vector set PASS 35/35 x 3 modes + [2] blake3 PyPI package SKIP (not installed) + [3] Plonky3 blake3-air PASS 20,000 compressions +gate: G-function UNSAT (covers all G) : True + init+feed-forward UNSAT (rounds=0): True + negative controls all SAT : True (5/5) + positive controls all SAT : True (6-round seeds 0,1,2 + 7-round) + width audit (bound necessity) : True + OVERALL: PASS +``` + +Independently of the harness, `blake3_ref.py` reproduces the published +known-answer vectors exactly: `blake3("")` = `af1349b9f5f9…41f3262` and +`blake3("abc")` = `6437b3ac38…d5bd9d85`. That single check exercises the IV, the +G function, all four rotations, the permutation *and its count*, the +feed-forward, the flag bit values and little-endian packing at once. + +Two independent reviews (different models, no coordination) found **no +discrepancy in the primitive**. One wrote a from-scratch BLAKE3 structured +deliberately differently and differentially tested 100k random compressions, all +128 flag values × {6,7} rounds, a rounds sweep 0..8, and whole-hash over 227 +lengths × 4 modes — zero mismatches — and confirmed `r < rounds−1`, i.e. **6 +permutes for 7 rounds**, so the classic off-by-one is absent. + +### ⚠ Provenance of `official_test_vectors.json` + +It was **regenerated from the official `blake3` Rust crate v1.8.5** +(`ground-truth/`), not downloaded from the upstream repo. It carries the +official parameters — key `whats the Elvish word for friend`, context +`BLAKE3 2019-12-27 16:29:52 test vectors context`, the same 35 input lengths — +and case 0 matches the independently-known published digest. + +This is a **genuine, non-circular anchor**: the Rust crate is the BLAKE3 +authors' reference implementation and is entirely independent of +`blake3_ref.py`. But it is *not* the published artifact, and `test_oracle.py` +still labels it "Official test_vectors.json". Read it as "checked against the +official reference implementation using the official vector parameters". + +### What the two reviews pinned that no anchor covers + +- **Counter split order at `t ≥ 2^32` — confirmed.** `t_lo = t mod 2^32 → v[12]`, + `t_hi = t >> 32 → v[13]`. Verified *behaviourally* against the official crate + through two independent counter paths (`OutputReader::set_position` and + `hazmat::HasherExt::set_input_offset`), over counters 0 … 2^47 including + 2^32−1, 2^32, 2^32+1: **44/44**. Negative control — swapping the halves — + breaks 5 of 6 chunk cases, the sixth being `counter = 0`, correctly invariant. + This closes ORACLE.md's own open question O5. +- **Message schedule count *and direction*.** Iterating `permute` from the + identity reproduces **all seven rows** of the crate's precomputed + `MSG_SCHEDULE`. Three mutants (permute before round 0, skip the 0→1 permute, + inverse direction) are all caught. A fourth — permuting *after* the last + round — is provably a no-op, so the trailing-permute guard is an optimisation + and cannot hide an off-by-one either way. +- `compress` does not mutate its arguments; incremental `update()` equals the + one-shot path over 60 random split patterns. + +### Harness defects — FIXED + +1. ~~`main()` printed `VALIDATION STATUS: VALIDATED … anchored on official test + vectors + official PyPI package + Plonky3` **even when anchor 2 SKIPped**~~ — + the `status` dict was written and never read. **Fixed:** the banner now + reports what actually ran (`VALIDATED` / `PARTIALLY VALIDATED` / `NOT + VALIDATED`) and names the anchors it is *not* anchored on. Verified by + running with a fixture removed. +2. ~~The missing-file failure **cascaded**~~ — one `FileNotFoundError` killed + anchors 2 and 3 *and* the canonical-vector emitter, which is why the gate's + positive controls were blocked on an unrelated download. **Fixed:** anchors + are independent; a missing fixture SKIPs only itself. Verified — with + `official_test_vectors.json` removed, anchor 3 still runs and the vectors are + still emitted. +3. ~~Anchor 1 was labelled "Official test_vectors.json"~~ — it is regenerated + from the crate. **Fixed:** relabelled "Official-parameter vectors" and the + run prints its provenance. + +### Known harness defects — still open (low severity) + +4. `test_internal_consistency` carries a comment describing a feed-forward + recomputation (*"recompute v to check"*) that **is not implemented** — it only + checks output length and the CV prefix. +5. **`test_6round_derivation`'s first assertion is a tautology** — + `compress_6round`'s body *is* `compress(rounds=6)`. ORACLE.md §2.6 calls it + the "Code-diff anchor"; it establishes nothing. The differs-from-7r half is + real. +6. **Footgun for the Rust phase:** `compress(...)` defaults to `rounds=7`, so a + 6-round caller that omits the kwarg silently gets 7. Trace generators must + call `compress_6round`. Left as-is deliberately: changing the validated + oracle's signature would invalidate the anchors it just passed. +7. ORACLE.md §5's closing ratio is internally inconsistent: ~5–6k cell-equiv + against 24×1480 = 35,520 is ≈1/6, not the "¼–⅓" its prose claims. Superseded + by DESIGN.md §6's ≈1/15 against a 77,000 baseline — which is the number that + was actually derived. + +## DESIGN findings (review, 2026-07-29) — FIXED IN THE DESIGN + +1. **The internal `Blake3` bus had no input↔output binding.** §1.1 defined a + receive of `(h, m, t, block_len, flags)` and a separate send of `out[0..16]`, + both at multiplicity μ, while §3 listed `TIMESTAMP_0/1` as "bus binding + (internal variant **may omit**)". Omit it and, with two compressions in a + trace, row A can receive inputs_A and send out_B while row B does the + reverse: every tuple appears once on each side, **the bus balances**, and + both callers read a wrong result. The design's own cited precedent does not + do this — keccak carries `TIMESTAMP_0, TIMESTAMP_1` in *both* halves of its + internal bus (send at round 0, receive at round 24). + **Fixed:** §1.1 now states the binding is mandatory in both tuples, with the + attack and the keccak precedent spelled out; §3's "may omit" is gone; and it + is item 10 of §7's soundness-critical list. Also recorded there: **the gate + cannot catch a violation**, since it models arithmetic with no bus layer. +2. **"Covers every G, hence every round" is a model argument.** MAIN 0 proves + one G under free inputs; in Rust the 48 instances are emitted separately, so + a wrong column index in instance #37 is invisible to it. + **Fixed:** now item 11 of §7, pointing at the concrete positive controls as + the thing that covers it and requiring `--full`'s monolithic UNSAT before + Rust ships. The controls themselves were unrunnable at review time and now + run and pass 4/4, so the residual risk is materially lower than when the + finding was written. +3. **The 3-op add carry encoding is ambiguous** — `(c1,c2) = (1,0)` and `(0,1)` + both encode carry 1. Checked: it does not admit a wrong `s`, so this is a + note rather than a defect, recorded so nobody "fixes" it into a bug. **No + change made, deliberately.** + +## Still unaudited — where to send the next reviewer + +*(2026-08-05 update: the transcription audit this section asks for has since +been done — see `TRANSCRIPTION-AUDIT.md` and `GATE-TRANSCRIPTION-AUDIT.md`, +which found and fixed the issues recorded in DESIGN.md §4.2/§7. The section is +kept for its account of WHY that audit mattered.)* + +Two independent reviews established that **the oracle defines the right +function**, so the gate's UNSATs are about the right function. They did *not* +audit the step after that: **nobody has checked the z3 gate's transcription of +the oracle into constraints.** Only its constants block +(`z3_blake_verify.py:50-80`) was spot-checked, and it matches exactly. + +That is the highest-value next pass, and the EC campaign is the reason to take +it seriously: the equivalent audit there +(`thoughts/ec-recover-opt/gate/TRANSCRIPTION-AUDIT.md`) found three premises the +gate asserted about the chip and never read, one of them hiding a working +forgery. The dangerous direction is a model **stronger** than the thing it +models — it yields UNSAT where the real object is forgeable, and a positive +anchor cannot catch it, because honest inputs satisfy a correct model and an +over-strong one equally well. + +Also still thin: neither review verified the recovery is *byte-identical to the +original* — only that the artifact is correct BLAKE3, which is a different and +weaker claim; and the historical counts ("35/35×3", "92/92" against PyPI +v1.0.9) remain unreproduced as recorded. + +## If this is picked up again + +*(2026-08-05: it was picked up — see the top of this file. The A6R assumption +is now written down in `spec/blake3.typ`; formal ratification remains open.)* + +The blocking item is a **protocol decision, not an engineering one**: the gate +proves the chip matches the reference, *not* that 6 rounds are secure. That +needs a named, signed assumption in the spec. + +Note for anyone citing precedent: the EC `lincomb2` design study justified its +NUMS assumption with "like blake3's 6-round assumption" — but no such assumption +was ever recorded in the spec, because this work never shipped. It was a +precedent for something that had not happened. diff --git a/thoughts/blake3/TRANSCRIPTION-AUDIT.md b/thoughts/blake3/TRANSCRIPTION-AUDIT.md new file mode 100644 index 000000000..e96cc8b67 --- /dev/null +++ b/thoughts/blake3/TRANSCRIPTION-AUDIT.md @@ -0,0 +1,225 @@ +# Transcription audit — does the BLAKE3 gate assert what the design and oracle say? + +Auditor: independent pass, 2026-07-29, branch `spike/blake3-recovered`. +Objects audited: + +- **oracle**: `blake3-oracle/blake3_ref.py` + `test_oracle.py` (does it define the right function?) +- **gate**: `blake3-chip/z3_blake_verify.py` against `blake3-chip/DESIGN.md` and the oracle + (is the constraint transcription faithful? can the model be stronger than the chip?) +- **the uncommitted fixes** on `DESIGN.md` / `test_oracle.py` (the 3 design findings + + harness defects #1/#3) — verified, see §5. + +Method mirrors `../ec-recover-opt/gate/TRANSCRIPTION-AUDIT.md`: only one direction is +dangerous. A model **weaker** than the chip yields spurious SAT (false alarm); a model +**stronger** than the chip yields UNSAT on a forgeable chip (false assurance), and no +positive anchor can see that, because an honest witness satisfies a correct model and an +over-strong one equally well. Here there is no Rust chip yet — the gate is the only +executable statement of the design — so the audit is gate ↔ design + oracle, and every +place the gate *cannot see* is a place the future Rust must get right by construction. + +Reproduce: everything below ran with `blake3/venv` (z3 5.0.0, blake3 PyPI 1.x), +`ground-truth` (official `blake3` crate v1.8.5, pure-Rust), and the vendored +`others/Plonky3/blake3-air`. Mutant/tamper scripts were scratch files, not committed. + +--- + +## Verdict + +**No over-strong or mis-transcribed premise found.** Every equation in the gate matches +the design it encodes (§2 table), the gate's reference is behaviourally identical to the +externally-anchored oracle (§3), the gate is *sensitive* to every wiring-bug class we +could construct — including classes with no shipped negative control (§4, 7/7 mutants +fire) — and the width analysis holds with slack (expressions ≤ ~2^41 vs the 2^48 model +width). The two re-runs reproduce the recorded board: default run **OVERALL: PASS**; +`--full` monolithic UNSATs: **SEE §6**. + +The honest map of what a green board does NOT cover (§2, "no automated check" rows) +is where the remaining risk lives: μ-gating/padding, input range checks, the degree-3 +ledger, the bus layer, and the precomputed-table contracts. All are documented in +DESIGN §7; the uncommitted fixes added items 10–11. None is new. + +## §1 — Oracle re-validation (does the oracle define the right function?) + +Re-ran and independently re-derived, all green: + +| check | result | +|---|---| +| harness `test_oracle.py`, anchor 1 (official-parameter vectors) | PASS 35/35 × 3 modes | +| anchor 2 (official `blake3` PyPI pkg) — **live this time** (was SKIP) | PASS 92/92 | +| anchor 3 (Plonky3 `blake3-air` port, direct compression) | PASS 20 000/20 000 | +| banner honesty (defect #1 fix) | reads VALIDATED only because all three ran (see §5) | +| known-answer: `blake3("")`, `blake3("abc")` | exact match to published digests | +| differential vs PyPI: 140 lengths × {default, keyed, derive} + 48 XOF-length checks | 468/468 | +| counter split `t_lo/t_hi` vs official crate, XOF `set_position` path, t ∈ {0,1,2, 2^32−2, 2^32−1, **2^32**, **2^32+1**, 2^40, 2^47} | **9/9** (scratch `counter_probe.rs` + `blake3_ref.compress`) | +| swapped-halves negative control | breaks 7/9; the 2 invariants are t=0 and t=0x1_0000_0001 (t_lo==t_hi), both correctly invariant | +| message schedule count+direction: `permute^r` from identity vs the crate's precomputed `MSG_SCHEDULE` | all 7 rows exact | + +The historical counts ("35/35×3", "92/92") now reproduce as recorded. ORACLE.md O5 +(counter width) remains closed — re-confirmed against the crate at t ≥ 2^32. + +## §2 — Per-premise transcription table (gate ↔ DESIGN ↔ oracle) + +| gate premise | source | verified | how | +|---|---|---|---| +| `IV`, `MSG_PERMUTATION` constants | DESIGN §1, oracle §2.1, Plonky3 `constants.rs` | ✅ exact | 3-way diff | +| `G_CALLS` (8 index tuples + msg order) | oracle `round_fn` | ✅ exact | diff | +| `bref_*` reference independent of circuit wiring | DESIGN §8 | ✅ | 200 concrete trials vs `blake3_ref.compress` (rounds 6+7), 0 mismatch; leading-permute mutant differs ⇒ `r < rounds−1` guard direction correct | +| init layout `v = h ‖ IV[0..4] ‖ t_lo,t_hi,bl,fl` | oracle §2.4, DESIGN §7.8 | ✅ | MAIN 1 UNSAT + `wrong_iv` control | +| feed-forward `out[i]=v[i]⊕v[i+8]`, `out[i+8]=v[i+8]⊕h[i]` | oracle §2.4, DESIGN §4.6 | ✅ | MAIN 1 UNSAT + `drop_ff_xor` control | +| schedule = `permute^r` of the ORIGINAL `M` | DESIGN §7.7 | ✅ | `wrong_msg_index` control + `permute_inverse` mutant + positive controls | +| `add2`: `a+b = s + 2^32·c`, c boolean | DESIGN §4.3 | ✅ | equation exact; field-level necessity of booleanity confirmed (this audit, §4) | +| `add3`: `a+b+m = s + 2^32·(c1+c2)`, c1,c2 boolean | DESIGN §4.4 (O1 option c) | ✅ | equation exact; width audit drop→SAT | +| `rotr16=[b2,b3,b0,b1]`, `rotr8=[b1,b2,b3,b0]` free relabels | DESIGN §4.2/§7.6 | ✅ | relabel mutants flip check to SAT (§4) | +| `rotr12/rotr7` shift identity `hw·2^r = SLLC·2^16 + SLL`, r=4/9 | DESIGN §4.2 | ✅ | equation exact; `rot_wrong_amount` control | +| recombine `Ylo=SLL_hi+SLLC_lo`, `Yhi=SLL_lo+SLLC_hi` | DESIGN §4.2 | ✅ | recombine mutants flip to SAT (§4) | +| ByteAlu[XOR] / AreBytes table contracts | `prover/src/tables/bitwise.rs` | ⚠ assume-guarantee | documented; same assumption keccak gate makes; **no automated check here** | +| μ-gating / all-zero padding (μ=1 modelled) | DESIGN §4.5, §7.1 | ⚠ gate cannot see | no bus/multiplicity layer; on the implementer | +| input range checks (h,t,bl,fl free via XOR; **m needs explicit AreBytes**) | DESIGN §4.7, §7.5 | ⚠ gate cannot see | gate inputs are bytes by construction; a dropped `AreBytes(m)` in Rust is invisible here | +| degree ≤ 3 ledger | DESIGN §4.8 | ⚠ no automated check | manual ledger; gate models equations, not degrees | +| `Blake3` bus TIMESTAMP binding (findings fix) | DESIGN §1.1/§7.10 | ⚠ gate cannot see | no bus layer; verified by construction, see §5 | +| 48 G instances wired as MAIN 0 models | DESIGN §7.11 | ✅ concrete | positive controls run all 48; `--full` monolithic UNSATs (§6); per-instance index mutant fires (§4) | +| WIDE=48 model cannot wrap | gate internals | ✅ | worst expression ≈ 2^41 (add3 with 8-bit carries) ≪ 2^48 | + +## §3 — Reference (`bref_*`) independence + +The gate's soundness rests on `bref_*` being an independent statement of BLAKE3. It is +structurally independent (32-bit BV `RotateRight`/`+`/`^` vs the byte-level circuit) and +behaviourally identical to the oracle: 200 random concrete inputs, rounds ∈ {6,7}, +0 mismatches. The permute guard `r < rounds−1` matches the oracle's (a leading-extra-permute +variant provably differs). The one structural mirror both share with the oracle — the +constants and `G_CALLS` table — is pinned by the *external* anchors (crate, PyPI, Plonky3), +so a common-mode bug there would have to be a bug in BLAKE3 itself. + +## §4 — Gate sensitivity: shipped controls + mutation sweep + +Shipped controls all reproduced (default run): 5/5 structural SAT, width audit 4/4, +positive controls 4/4 SAT. + +Mutation sweep (scratch, not committed) — bug classes with **no shipped negative control**; +each was injected into a copy of the circuit builders and must flip its check to SAT: + +| mutant | class | result | +|---|---|---| +| `rotr16_bad_relabel` | free-rotation byte order (DESIGN §7.6) | **sat — detected** | +| `rotr8_bad_relabel` | free-rotation byte order | **sat — detected** | +| `rotr12_bad_recombine` | carry paired to wrong halfword | **sat — detected** | +| `swap_mx_my` | message operand order in G | **sat — detected** | +| `permute_inverse` | schedule direction | **sat — detected** | +| `bad_diag_index` | one wrong column in G instance #7 (per-instance wiring, §7.11) | **sat — detected** | +| `rounds_off_by_one` | round-loop bound | **sat — detected** | + +Field-level addition: the shipped width audit demonstrates bound-necessity only for the +**3-op** add. This audit verified the same for the **2-op** add: booleanity present → +UNSAT (pinned), dropped → SAT (forgeable mod p). Same class, now demonstrated for both. + +## §5 — Verdict on the uncommitted fixes + +- **Finding 1 (bus input↔output binding) — FIX REAL.** `DESIGN.md` §1.1/§3/§7.10 now + mandate `TIMESTAMP_0/1` in both `Blake3` receive and send. The cited precedent checks + out: `prover/src/tables/keccak.rs:264-319` sends `(ts, 0, input_state)` and receives + `(ts, 24, output_state)` on the internal `Keccak` bus with `TIMESTAMP_0/1` in *both* + tuples (`BusValue::Packed` at `cols::TIMESTAMP_0/1`). The swap-attack reasoning is + sound: with no common key, rows A/B exchanging output tuples keeps every tuple + appearing once per side, so LogUp balances while both callers read wrong results. + Correctly documented as gate-invisible (no bus layer). +- **Finding 2 ("covers every G" is a model argument) — FIX REAL.** §7.11 records it; + the positive controls do run the full 48-instance pipeline concretely, and this audit's + per-instance index mutant backs it. (Superseded in part: this bullet originally also + cited the `--full` monolithic UNSATs as backing. They were run on 2026-08-06 and came + back `unknown` on all four queries — see §6 — so they support nothing either way. The + argument rests on the concrete positive controls and the index mutant.) +- **Finding 3 (carry encoding ambiguity `(1,0)`/`(0,1)`) — correctly classified + harmless.** The sum identity constrains only `c1+c2`; `s` is pinned regardless. +- **Harness defect #1 (banner overstatement) — FIX REAL.** The banner now reads from + the status dict; exercised live (below). +- **Harness defect #3 (missing-fixture cascade) — FIX REAL.** With + `official_test_vectors.json` renamed away: anchor 1 SKIPs alone, anchors 2/3 PASS, + the canonical-vector emitter still runs (it is now unconditional), banner reads + "PARTIALLY VALIDATED … NOT anchored on: official-parameter vectors". Fixture restored + afterwards; regenerated `canonical_6round_vectors.json` is byte-identical. + +## §6 — Gate re-runs + +- default (`z3_blake_verify.py`), z3 5.0.0: **OVERALL: PASS** (board identical to §9 of + DESIGN.md). +- `--full` (monolithic symbolic round / rounds=2 / 6-round / 7-round UNSATs), run + 2026-08-06: **ATTEMPTED-INCONCLUSIVE — no pass, and no counterexample.** The run took + ~145 min and exited 1 (`OVERALL: FAIL`), but all four monolithic queries returned + `unknown`, not `sat`: + + ``` + round (clean) -> unknown (want unsat) + compress rounds=2 -> unknown (want unsat) + compress rounds=6 -> unknown (want unsat) + compress rounds=7 -> unknown (want unsat) + ``` + + `unknown` is z3's resource-limit return (`s.set("timeout", timeout_ms)` then + `s.check()`, `z3_blake_verify.py:320-321`/`:340-341`); the verdict tests `== unsat` + (line 553), so a timeout is scored `False` and pulls OVERALL to FAIL. The four + budgets sum to 140 min against ~145 min wall, i.e. every check burned its full + allowance. **Nothing was disproven; nothing was proven monolithically.** The fast + board is unchanged and green: + + ``` + G-function UNSAT (covers all G) : True + init+feed-forward UNSAT (rounds=0): True + negative controls all SAT : True + positive controls all SAT : True (full 6-/7-round pipeline, concrete) + ``` + + Consequence for §5's Finding 2 above: the "`--full` monolithic UNSATs (§6)" cited + there as backing the per-instance coverage argument did **not** land, so that + argument currently rests on the concrete positive controls and the per-instance + index mutant alone. Remediation: rerun with a much larger timeout budget on a + server (single-threaded, CPU-bound), and/or restructure the monolithic query as + round-by-round induction. + +## §6b — Reconciliation with the second, independent audit (`audit_gate_transcription.py`) + +A separately-authored executable audit (74/74 checks pass, run this session) agrees with +every verdict above and sharpens three points this audit stated more coarsely: + +1. **Rotation bound necessity, refined.** The load-bearing bound set is *at least one of* + `{SLL_lo, SLL_hi}` — every configuration with neither is forgeable, every one with + either is pinned; the `SLLC` bounds are not load-bearing. DESIGN §4.2's "the tight + SLL bound" should read "a tight bound on at least one SLL halfword". The composed + (whole-rotation) forgery with both SLL bounds dropped exists for exactly **one** + input, `X=0xFFFFFFFF` (forged `Y=0`), not for arbitrary inputs. +2. **Doc note (safe direction):** DESIGN §4.8's degree-ledger row for the recombine + identity overstates (claims body 2 → 3 after ×μ; the body is linear, so 1 → 2). + The "no constraint exceeds 3" verdict is unaffected. +3. **Doc/cost inconsistency:** DESIGN §3's per-G table commits 1 carry *column* per add2, + while §4.3 makes that carry a *derived* linear expression (`(a+b−s)·INV_SHIFT_32`). + The two are equivalent over F_p (proven by that suite) but differ by 96 cells per + compression in the §6 cost table. The gate models the committed form. + +It also independently confirms this report's two "gate cannot see" rows with explicit +forgeries: the missing `AreBytes(m)` (§2, DESIGN §4.7) and the declared-not-derived +input range checks. + +## §7 — Still open (pre-existing, report-only per audit scope) + +1. Harness defect #2: `test_internal_consistency`'s comment promises a feed-forward + recomputation that is not implemented (`test_oracle.py:227`). Comment lies; check is + shallow (length + CV prefix only). +2. Harness defect #4: `test_6round_derivation`'s first assertion is a tautology + (`compress_6round` *is* `compress(rounds=6)`). The differs-from-7r half is the real + content. +3. Harness defect #5 (footgun for the Rust phase): `compress(...)` defaults `rounds=7`. + Trace generators must call `compress_6round` / pass `rounds=` explicitly. +4. ORACLE.md §5 prose "¼–⅓ of a keccak permutation" is inconsistent with its own + ~5–6k figure (≈1/6 of 24×1480); superseded by DESIGN §6's derived ≈1/15. Doc-only. +5. `ground-truth/Cargo.toml` could not build inside this repo ("believes it's in a + workspace"). Fixed with an empty `[workspace]` table — **this audit touched that one + committed file**; without it the documented regeneration flow fails out of the box. +6. Suggestion (not a defect): fold the §4 mutant sweep and the add2 field check into the + shipped gate as regression controls, so future edits to the gate are held to the same + sensitivity. + +## §8 — Scratch artifacts left in the tree (untracked; commit or delete, user's call) + +- `thoughts/blake3/venv/` (z3 5.0.0 + official `blake3` PyPI pkg) +- `thoughts/blake3/ground-truth/src/bin/counter_probe.rs` (the t≥2^32 counter probe) +- `thoughts/blake3/ground-truth/target/`, `thoughts/blake3/blake3-oracle/__pycache__/` +- mutant sweep + add2 field check: `/tmp/blake_mutants.py` and heredocs (not in tree) diff --git a/thoughts/blake3/audit_gate_transcription.py b/thoughts/blake3/audit_gate_transcription.py new file mode 100644 index 000000000..6cd9b1000 --- /dev/null +++ b/thoughts/blake3/audit_gate_transcription.py @@ -0,0 +1,1198 @@ +""" +Transcription audit of `blake3-chip/z3_blake_verify.py` — an EXECUTABLE +regression suite for GATE-TRANSCRIPTION-AUDIT.md. + +Two transcriptions are under test, and only one direction is dangerous. + + (a) blake3-oracle/blake3_ref.py -> the gate's `bref_*` BV reference. + If these diverge, every UNSAT the gate reports proves the chip matches + the WRONG function. + + (b) blake3-chip/DESIGN.md -> the gate's `build_g/build_round/ + build_compress` circuit model. + A model WEAKER than the designed chip yields a spurious SAT — a false + alarm, safe. A model STRONGER yields UNSAT where the real object is + forgeable — false assurance, and no positive control can see it, + because an honest witness satisfies a correct model and an over-strong + model equally well. + +Every check below is paired with a TAMPER that must make it fail; a check +that does not bite is itself reported as a failure. Nothing outside this +file is modified: tampers are applied to in-memory copies and reverted. + +Run: python3 audit_gate_transcription.py (fast sections) + python3 audit_gate_transcription.py --slow (+ the BV UNSATs, ~5 min) +""" +import importlib.util +import itertools +import os +import random +import sys + +from z3 import ( + And, BitVec, BitVecVal, Concat, Int, IntVal, Or, RotateRight, Solver, + is_bv, sat, simplify, unsat, +) + +HERE = os.path.dirname(os.path.abspath(__file__)) +P = 2**64 - 2**32 + 1 # Goldilocks +MASK32 = 0xFFFFFFFF + + +def _load(name, relpath): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, relpath)) + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +ORA = _load("blake3_ref", "blake3-oracle/blake3_ref.py") +GATE = _load("z3_blake_verify", "blake3-chip/z3_blake_verify.py") + + +# --------------------------------------------------------------------------- +# result bookkeeping +# --------------------------------------------------------------------------- +RESULTS = [] + + +def record(section, name, ok, detail=""): + RESULTS.append((section, name, bool(ok), detail)) + flag = "PASS" if ok else "**FAIL**" + print(f" [{flag}] {name}" + (f" {detail}" if detail else "")) + return ok + + +def note(text): + """An observation that is reported but is not a pass/fail check.""" + print(f" [note] {text}") + + +def ora_G_CALLS(): + """The G-call schedule RECOVERED from the oracle (blake3_ref.round_fn), + with sentinel message words so the mx/my indices come back too.""" + calls = [] + orig = ORA.g + + def spy(state, a, b, c, d, mx, my): + calls.append((a, b, c, d, mx - 1000, my - 1000)) + ORA.g = spy + try: + ORA.round_fn([0] * 16, [1000 + i for i in range(16)]) + finally: + ORA.g = orig + return calls + + +def bites(section, name, tamper_fn): + """A check must FAIL under its tamper, or the check is decorative.""" + try: + detected = tamper_fn() + except Exception as exc: # a crash is also detection + detected = True + record(section, f"tamper bites: {name}", True, f"(raised {type(exc).__name__})") + return True + return record(section, f"tamper bites: {name}", detected, + "" if detected else "TAMPER NOT DETECTED — the check is vacuous") + + +# =========================================================================== +# SECTION A — transcription (a): blake3_ref.py -> bref_* +# =========================================================================== +def section_A(slow): + print("\n" + "=" * 74) + print("A REFERENCE TRANSCRIPTION — blake3_ref.py -> bref_* (the BV oracle)") + print("=" * 74) + + # -- A1 constant tables, element by element ---------------------------- + record("A", "IV identical (8 words, element-wise)", + list(GATE.IV) == list(ORA.IV), f"{[hex(x) for x in GATE.IV[:2]]}...") + record("A", "MSG_PERMUTATION identical (16 indices, element-wise)", + list(GATE.MSG_PERMUTATION) == list(ORA.MSG_PERMUTATION), + str(GATE.MSG_PERMUTATION)) + record("A", "MSG_PERMUTATION is a permutation of 0..15", + sorted(GATE.MSG_PERMUTATION) == list(range(16))) + record("A", "MASK32 identical", GATE.MASK32 == ORA.MASK32) + + def tamper_iv(): + old = GATE.IV[3] + GATE.IV[3] ^= 1 + bad = list(GATE.IV) != list(ORA.IV) + GATE.IV[3] = old + return bad + bites("A", "IV comparison", tamper_iv) + + # -- A2 the G-call schedule, RECOVERED from the oracle ------------------ + ora_calls = ora_G_CALLS() + record("A", "G_CALLS == the oracle's round_fn call sequence (recovered)", + [tuple(x) for x in GATE.G_CALLS] == ora_calls, + f"{len(ora_calls)} calls") + record("A", "every G quadruple has 4 DISTINCT state indices " + "(so check_g's a,b,c,d=0,1,2,3 instance is general)", + all(len({a, b, c, d}) == 4 for a, b, c, d, _, _ in GATE.G_CALLS)) + record("A", "the 8 G-calls touch each of the 16 state slots exactly twice", + sorted(i for q in GATE.G_CALLS for i in q[:4]) == + sorted(list(range(16)) * 2)) + record("A", "the 8 G-calls consume message indices 0..15 exactly once", + sorted(i for q in GATE.G_CALLS for i in q[4:]) == list(range(16))) + + def tamper_gcalls(): + old = GATE.G_CALLS[4] + GATE.G_CALLS[4] = (0, 5, 10, 15, 9, 8) # mx/my swapped + bad = [tuple(x) for x in GATE.G_CALLS] != ora_calls + GATE.G_CALLS[4] = old + return bad + bites("A", "G_CALLS comparison", tamper_gcalls) + + # -- A3 rotation amounts and their ORDER, recovered from both sides ----- + def oracle_rot_amounts(): + seen = [] + orig = ORA.rotr + + def spy(x, n): + seen.append(n) + return orig(x, n) + ORA.rotr = spy + try: + ORA.g([0] * 4, 0, 1, 2, 3, 0, 0) + finally: + ORA.rotr = orig + return seen + + def bref_rot_amounts(): + seen = [] + orig = GATE.RotateRight + + def spy(x, n): + seen.append(n) + return orig(x, n) + GATE.RotateRight = spy + try: + GATE.bref_g([BitVec(f"a{i}", 32) for i in range(4)], 0, 1, 2, 3, + BitVec("mx", 32), BitVec("my", 32)) + finally: + GATE.RotateRight = orig + return seen + + ora_rots, bref_rots = oracle_rot_amounts(), bref_rot_amounts() + record("A", "bref_g rotation amounts and order == oracle g", + ora_rots == bref_rots == [16, 12, 8, 7], f"{bref_rots}") + + # -- A4 differential: bref_* vs the oracle on concrete values ----------- + rng = random.Random(0xB1A3E) + + def w32(v): + return BitVecVal(v & MASK32, 32) + + def as_int(bv): + return simplify(bv).as_long() + + def diff_g(n): + for _ in range(n): + st = [rng.randrange(1 << 32) for _ in range(4)] + mx, my = rng.randrange(1 << 32), rng.randrange(1 << 32) + ref = list(st) + ORA.g(ref, 0, 1, 2, 3, mx, my) + bv = [w32(x) for x in st] + GATE.bref_g(bv, 0, 1, 2, 3, w32(mx), w32(my)) + if [as_int(x) for x in bv] != ref: + return False, (st, mx, my) + return True, None + + ok, cex = diff_g(300) + record("A", "bref_g == oracle g (300 random + carry/rotate edge inputs)", ok, + "" if ok else f"counterexample {cex}") + + # edge cases: all-zero, all-ones, single bits (exercise every carry and + # every rotate boundary) + edges = [[0] * 4, [MASK32] * 4, [1, 0, 0, 0], [0, 0, 0, MASK32], + [0x80000000] * 4, [0x0000FFFF, 0xFFFF0000, 0xF0F0F0F0, 0x0F0F0F0F]] + ok_edge = True + for st in edges: + for msg in ([0, 0], [MASK32, MASK32], [0x80000000, 1]): + ref = list(st) + ORA.g(ref, 0, 1, 2, 3, msg[0], msg[1]) + bv = [w32(x) for x in st] + GATE.bref_g(bv, 0, 1, 2, 3, w32(msg[0]), w32(msg[1])) + ok_edge &= ([as_int(x) for x in bv] == ref) + record("A", "bref_g == oracle g (edge inputs: 0, 2^32-1, MSB, split words)", + ok_edge) + + def diff_round(n): + for _ in range(n): + st = [rng.randrange(1 << 32) for _ in range(16)] + m = [rng.randrange(1 << 32) for _ in range(16)] + ref = list(st) + ORA.round_fn(ref, m) + got = GATE.bref_round_only([w32(x) for x in st], [w32(x) for x in m]) + if [as_int(x) for x in got] != ref: + return False + return True + record("A", "bref_round_only == oracle round_fn (60 random states+messages)", + diff_round(60)) + + m0 = [rng.randrange(1 << 32) for _ in range(16)] + record("A", "bref_permute == oracle permute (and is index-identical)", + [as_int(x) for x in GATE.bref_permute([w32(x) for x in m0])] + == ORA.permute(m0)) + + def diff_compress(n, rounds_list): + for _ in range(n): + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + t = rng.randrange(1 << 64) + bl = rng.randrange(65) + fl = rng.randrange(128) + for r in rounds_list: + ref = ORA.compress(h, m, t, bl, fl, rounds=r) + got = GATE.bref_compress([w32(x) for x in h], [w32(x) for x in m], + w32(t & MASK32), w32((t >> 32) & MASK32), + w32(bl), w32(fl), r) + if [as_int(x) for x in got] != ref: + return False, (h, m, t, bl, fl, r) + return True, None + + ok, cex = diff_compress(25, [0, 1, 2, 5, 6, 7, 8]) + record("A", "bref_compress == oracle compress (25 vectors x rounds " + "{0,1,2,5,6,7,8}) — pins the rounds parameterisation", ok, + "" if ok else f"counterexample {cex}") + + # counters straddling 2^32 — the split order t_lo=v[12], t_hi=v[13] + ok_ctr = True + for t in (0, 1, 2**32 - 1, 2**32, 2**32 + 1, 2**47 + 12345, 2**64 - 1): + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + ref = ORA.compress(h, m, t, 64, 3, rounds=6) + got = GATE.bref_compress([w32(x) for x in h], [w32(x) for x in m], + w32(t & MASK32), w32((t >> 32) & MASK32), + w32(64), w32(3), 6) + ok_ctr &= ([as_int(x) for x in got] == ref) + record("A", "counter split order matches across 2^32 " + "(t_lo->v[12], t_hi->v[13]; 7 counters incl. 2^32-1, 2^32)", ok_ctr) + + def tamper_bref_perm(): + old = GATE.MSG_PERMUTATION[:] + GATE.MSG_PERMUTATION[0], GATE.MSG_PERMUTATION[1] = old[1], old[0] + bad = not diff_compress(3, [2, 6])[0] + GATE.MSG_PERMUTATION[:] = old + return bad + bites("A", "bref_compress differential (permutation)", tamper_bref_perm) + + def tamper_bref_iv(): + old = GATE.IV[0] + GATE.IV[0] ^= 1 + bad = not diff_compress(2, [1, 6])[0] + GATE.IV[0] = old + return bad + bites("A", "bref_compress differential (IV)", tamper_bref_iv) + + # -- A5 how many times the permutation is applied ---------------------- + def ora_permute_count(rounds): + n = [0] + orig = ORA.permute + + def spy(m): + n[0] += 1 + return orig(m) + ORA.permute = spy + try: + ORA.compress([0] * 8, [0] * 16, 0, 64, 0, rounds=rounds) + finally: + ORA.permute = orig + return n[0] + + def bref_permute_count(rounds): + n = [0] + orig = GATE.bref_permute + + def spy(m): + n[0] += 1 + return orig(m) + GATE.bref_permute = spy + try: + GATE.bref_compress([w32(0)] * 8, [w32(0)] * 16, w32(0), w32(0), + w32(0), w32(0), rounds) + finally: + GATE.bref_permute = orig + return n[0] + + counts = [(r, ora_permute_count(r), bref_permute_count(r)) for r in range(9)] + record("A", "permute applications per rounds r == max(r-1,0), oracle == bref " + "(the classic off-by-one)", + all(o == b == max(r - 1, 0) for r, o, b in counts), + " ".join(f"r{r}:{b}" for r, _, b in counts)) + + # -- A6 the initial-state layout, probed slot by slot ------------------- + # rounds=0 makes out[i]=v[i]^v[i+8] and out[i+8]=v[i+8]^h[i] read the + # initial state directly, so each slot is individually observable. + h = [0] * 8 + m = [0] * 16 + tlo, thi, bl, fl = 0xA1A2A3A4, 0xB1B2B3B4, 0xC1C2C3C4, 0xD1D2D3D4 + out0 = [as_int(x) for x in GATE.bref_compress( + [w32(x) for x in h], [w32(x) for x in m], w32(tlo), w32(thi), + w32(bl), w32(fl), 0)] + layout_ok = ( + out0[0] == GATE.IV[0] and out0[1] == GATE.IV[1] and + out0[2] == GATE.IV[2] and out0[3] == GATE.IV[3] and + out0[4] == tlo and out0[5] == thi and out0[6] == bl and out0[7] == fl and + out0[8] == GATE.IV[0] and out0[12] == tlo and out0[13] == thi) + record("A", "initial v layout: v[8..12]=IV, v[12]=t_lo, v[13]=t_hi, " + "v[14]=block_len, v[15]=flags (probed slot by slot)", layout_ok, + f"out[4..8]={[hex(x) for x in out0[4:8]]}") + + hh = [0x11111111 * (i + 1) for i in range(8)] + out1 = [as_int(x) for x in GATE.bref_compress( + [w32(x) for x in hh], [w32(x) for x in m], w32(0), w32(0), w32(0), + w32(0), 0)] + ff_ok = (all(out1[i] == (hh[i] ^ [GATE.IV[0], GATE.IV[1], GATE.IV[2], + GATE.IV[3], 0, 0, 0, 0][i]) + for i in range(8)) and + all(out1[i + 8] == ([GATE.IV[0], GATE.IV[1], GATE.IV[2], GATE.IV[3], + 0, 0, 0, 0][i] ^ hh[i]) for i in range(8))) + record("A", "feed-forward: out[i]=v[i]^v[i+8] and out[i+8]=v[i+8]^h[i] " + "with h the ORIGINAL input CV (not the mutated state)", ff_ok) + + +# =========================================================================== +# SECTION B — transcription (b): DESIGN.md -> build_g / build_round / +# build_compress. Structural, by instrumenting the model. +# =========================================================================== +class Traced(GATE.Circuit): + """Circuit subclass that records the SSA dataflow the model builds. + + Words are lists of z3 byte expressions; rotr16/rotr8 return the SAME byte + objects (they are relabels), so resolving an operand's bytes to their + producing word automatically follows a free rotation back to its source + XOR — which is exactly the provenance question DESIGN 4.3/4.4/5 raises. + """ + + def __init__(self, tag, bug=None): + super().__init__(tag, bug) + self.words = [] # wid -> {kind, cells, used} + self.owner = {} # byte-expr name -> wid + self.ops = [] # {kind, ins:[wid], outs:[wid], perm:[..]} + self._pending = [] + + # -- registration ------------------------------------------------------ + def fresh_word(self): + w = super().fresh_word() + wid = len(self.words) + self.words.append({"kind": "unassigned", "cells": w, "used": 4}) + for c in w: + self.owner[str(c)] = wid + self._pending.append(wid) + return w + + def const_word(self, val): + w = super().const_word(val) + wid = len(self.words) + self.words.append({"kind": "const", "cells": w, "used": 4}) + return w + + def _wids(self, word): + return sorted({self.owner[str(c)] for c in word if str(c) in self.owner}) + + def _op(self, kind, ins, out_kinds, perm=None): + outs = self._pending[:] + self._pending = [] + for wid, k in zip(outs, out_kinds): + self.words[wid]["kind"] = k + self.ops.append({"kind": kind, + "ins": [self._wids(w) for w in ins], + "in_words": ins, "outs": outs, "perm": perm}) + return outs + + # -- the operations under contract ------------------------------------ + def xor(self, A, B): + self._pending = [] + out = super().xor(A, B) + self._op("xor", [A, B], ["xor_out"]) + return out + + def add2(self, A, B, drop_bool=False): + self._pending = [] + out = super().add2(A, B, drop_bool) + self._op("add2", [A, B], ["add_out"]) + return out + + def add3(self, A, B, M, drop_bool=False): + self._pending = [] + out = super().add3(A, B, M, drop_bool) + self._op("add3", [A, B, M], ["add_out"]) + return out + + def rotr(self, A, n, wrong_amount=False): + self._pending = [] + out = super().rotr(A, n, wrong_amount) + # fresh_word order inside Circuit.rotr: sll_lo, sllc_lo, sll_hi, + # sllc_hi, Y (the first four are used two bytes wide) + outs = self._op("rotr", [A], ["sll", "sllc", "sll", "sllc", "rot_out"], + perm=n) + for wid in outs[:4]: + self.words[wid]["used"] = 2 + return out + + def rotr16(self, A): + out = super().rotr16(A) + self.ops.append({"kind": "relabel16", "ins": [self._wids(A)], + "in_words": [A], "outs": [], + "perm": [A.index(c) if c in A else None for c in out]}) + return out + + def rotr8(self, A): + out = super().rotr8(A) + self.ops.append({"kind": "relabel8", "ins": [self._wids(A)], + "in_words": [A], "outs": [], + "perm": [A.index(c) if c in A else None for c in out]}) + return out + + # -- provenance analysis ---------------------------------------------- + def xor_consumed(self): + """wids that appear as an operand of at least one ByteAlu[XOR].""" + s = set() + for op in self.ops: + if op["kind"] == "xor": + for group in op["ins"]: + s.update(group) + return s + + def unchecked(self): + """SSA words whose byte-range DESIGN 4.3/4.4/5 sources from a + downstream XOR, but which no XOR in this scope consumes.""" + xc = self.xor_consumed() + return [wid for wid, w in enumerate(self.words) + if w["kind"] in ("add_out", "rot_out") and wid not in xc] + + +def _build_one_g(cls=Traced, build=None): + cir = cls("aud") + v = [None] * 16 + va, vb, vc, vd = (cir.fresh_word(), cir.fresh_word(), + cir.fresh_word(), cir.fresh_word()) + mx, my = cir.fresh_word(), cir.fresh_word() + for w in (va, vb, vc, vd, mx, my): + cir.words[cir.owner[str(w[0])]]["kind"] = "input" + v[0], v[1], v[2], v[3] = va, vb, vc, vd + (build or GATE.build_g)(cir, v, 0, 1, 2, 3, mx, my, None, False) + return cir, v, (va, vb, vc, vd, mx, my) + + +def _build_compress(rounds): + cir = Traced("audc") + h = [cir.fresh_word() for _ in range(8)] + m = [cir.fresh_word() for _ in range(16)] + tlo, thi, bl, fl = (cir.fresh_word(), cir.fresh_word(), + cir.fresh_word(), cir.fresh_word()) + for w in h + m + [tlo, thi, bl, fl]: + cir.words[cir.owner[str(w[0])]]["kind"] = "input" + out = GATE.build_compress(cir, h, m, tlo, thi, bl, fl, rounds) + return cir, out, dict(h=h, m=m, tlo=tlo, thi=thi, bl=bl, fl=fl) + + +def section_B(slow): + print("\n" + "=" * 74) + print("B CIRCUIT TRANSCRIPTION — DESIGN.md -> build_g / build_round / " + "build_compress") + print("=" * 74) + + # -- B1 the free rotations are byte relabels, not BV rotates ----------- + cir = GATE.Circuit("rel") + A = cir.fresh_word() + n_before = cir.n + r16, r8 = cir.rotr16(A), cir.rotr8(A) + record("B", "rotr16 is the index relabel [b2,b3,b0,b1] on the SOURCE bytes " + "(object identity, DESIGN 4.2/7.6)", + [c is A[i] for c, i in zip(r16, (2, 3, 0, 1))] == [True] * 4) + record("B", "rotr8 is the index relabel [b1,b2,b3,b0] on the SOURCE bytes", + [c is A[i] for c, i in zip(r8, (1, 2, 3, 0))] == [True] * 4) + record("B", "the free rotations commit NO new columns and emit NO " + "constraints (DESIGN 3: 'produce no columns')", + cir.n == n_before and cir.C == []) + + s = Solver() + s.add(Or(cir.word32(r16) != RotateRight(cir.word32(A), 16), + cir.word32(r8) != RotateRight(cir.word32(A), 8))) + record("B", "and the relabels are VALUE-equal to RotateRight 16 / 8 " + "(z3, all 2^32 inputs)", s.check() == unsat) + + def tamper_relabel(): + w = GATE.Circuit("t") + B = w.fresh_word() + wrong = [B[1], B[2], B[3], B[0]] # rotr8 pattern used for 16 + s2 = Solver() + s2.add(w.word32(wrong) != RotateRight(w.word32(B), 16)) + return s2.check() == sat + bites("B", "relabel value check", tamper_relabel) + + # -- B2 range-check provenance: the load-bearing claim ------------------ + print("\n -- B2 where does each SSA word's byte range actually come from? --") + gcir, gv, _ = _build_one_g() + kinds = {} + for w in gcir.words: + kinds[w["kind"]] = kinds.get(w["kind"], 0) + 1 + record("B", "one G commits 56 byte-cells + 6 carry bits (DESIGN 3 table)", + sum(w["used"] for w in gcir.words + if w["kind"] in ("add_out", "xor_out", "sll", "sllc", "rot_out")) == 56 + and sum(1 for c in gcir.C if "Or" in str(c)[:3] or str(c).startswith("Or")) == 6, + f"cells={sum(w['used'] for w in gcir.words if w['kind'] not in ('input','unassigned','const'))}, " + f"bool-constraints={sum(1 for c in gcir.C if str(c).startswith('Or'))}") + record("B", "one G = 2 add3 + 2 add2 + 4 xor + 2 shift-rotations + " + "1 rotr16 + 1 rotr8 (DESIGN 2/5)", + [sum(1 for o in gcir.ops if o["kind"] == k) + for k in ("add3", "add2", "xor", "rotr", "relabel16", "relabel8")] + == [2, 2, 4, 2, 1, 1]) + + xc_g = gcir.xor_consumed() + add_outs = [wid for wid, w in enumerate(gcir.words) if w["kind"] == "add_out"] + record("B", "all FOUR add outputs of a G (A1, C1, A2, C2) are consumed by a " + "ByteAlu INSIDE the same G — so MAIN 0's byte declaration is " + "derivable for them (the class where the range check is " + "load-bearing; see section C)", + len(add_outs) == 4 and all(w in xc_g for w in add_outs)) + + unchecked_in_g = gcir.unchecked() + detail = ", ".join(f"w{wid}({gcir.words[wid]['kind']})" for wid in unchecked_in_g) + record("B", "INSIDE one G, exactly one SSA output has no ByteAlu consumer: " + "the final rotr7 result B2 (its range check lives in the NEXT " + "G / the feed-forward — outside MAIN 0's scope)", + len(unchecked_in_g) == 1 + and gcir.words[unchecked_in_g[0]]["kind"] == "rot_out" + and unchecked_in_g[0] == gcir.owner[str(gv[1][0])], + f"unchecked in G-scope: [{detail}]") + + ccir, cout, cin = _build_compress(6) + unchecked_full = ccir.unchecked() + record("B", "in the FULL 6-round compression every add/shift output IS " + "consumed by a ByteAlu[XOR] — DESIGN 5/7.4's premise verified " + "mechanically (the gate never checks it)", + unchecked_full == [], + f"{sum(1 for w in ccir.words if w['kind'] in ('add_out','rot_out'))} " + f"add/shift outputs, {len(unchecked_full)} unchecked") + other_rounds = {r: len(_build_compress(r)[0].unchecked()) for r in (1, 2, 7)} + record("B", "…and for ROUNDS = 1, 2 and 7 too, so the premise is a property " + "of the layout, not of the round count", + set(other_rounds.values()) == {0}, str(other_rounds)) + + xc = ccir.xor_consumed() + ins_xored = {k: all(ccir.owner[str(w[0])] in xc for w in + (cin[k] if isinstance(cin[k], list) and + isinstance(cin[k][0], list) else [cin[k]])) + for k in ("h", "m", "tlo", "thi", "bl", "fl")} + record("B", "DESIGN 4.7 input claim: h, t_lo, t_hi, block_len, flags each " + "feed an XOR; m does NOT (so m is the one input needing an " + "explicit AreBytes)", + ins_xored["h"] and ins_xored["tlo"] and ins_xored["thi"] + and ins_xored["bl"] and ins_xored["fl"] and not ins_xored["m"], + str(ins_xored)) + + # which XOR is h[i]'s range check? h[0..4] land in round-0 'a' slots, + # which G only ever uses as an ADD operand — so their sole ByteAlu is the + # UPPER feed-forward half, the half a CV-only caller (DESIGN 1.1) would + # naturally drop. + h_consumers = [] + for i in range(8): + wid = ccir.owner[str(cin["h"][i][0])] + cons = [oi for oi, o in enumerate(ccir.ops) + if o["kind"] == "xor" and any(wid in gp for gp in o["ins"])] + h_consumers.append(len(cons)) + ff_start = min(oi for oi, o in enumerate(ccir.ops) + if o["kind"] == "xor" and + any(ccir.owner[str(cin["h"][0][0])] in gp for gp in o["ins"])) + record("B", "h[0..4] have exactly ONE ByteAlu consumer (the upper " + "feed-forward out[i+8]=v[i+8]^h[i]) while h[4..8] have two " + "(they are round-0 'b' slots, so X2 xors them too)", + h_consumers == [1, 1, 1, 1, 2, 2, 2, 2], + f"consumer counts {h_consumers}") + record("B", "…and that single consumer really is a feed-forward XOR, not a " + "round XOR (it is among the last 24 ops of the circuit)", + ff_start >= len(ccir.ops) - 24, + f"op #{ff_start} of {len(ccir.ops)}") + + def tamper_provenance(): + """A G-variant in which an add output has NO ByteAlu consumer — + exactly the deviation DESIGN 7.4 warns about. The detector must see + it.""" + def build_g_leaky(cir, v, a, b, c, d, mx, my, bug, gflag): + v[a] = cir.add3(v[a], v[b], mx) + v[d] = cir.rotr16(cir.xor(v[d], v[a])) + v[c] = cir.add2(v[c], v[d]) + v[b] = cir.rotr(cir.xor(v[b], v[c]), 12) + v[a] = cir.add3(v[a], v[b], my) + v[d] = cir.rotr8(cir.xor(v[d], v[a])) + v[c] = cir.add2(v[c], v[d]) # C2: consumer removed below + v[b] = cir.rotr(cir.xor(v[b], v[a]), 7) # reads v[a], not v[c] + cir2, _, _ = _build_one_g(build=build_g_leaky) + return len(cir2.unchecked()) == 2 + bites("B", "range-check provenance detector", tamper_provenance) + + # -- B3 the message schedule as build_compress actually wires it ------- + print("\n -- B3 message indexing under permute^r --") + + def capture_wiring(rounds=7): + """What build_compress ACTUALLY feeds each G: the original message + column index, recovered by tagging the committed message words.""" + seen = [] + orig_bg = GATE.build_g + + def spy_g(cir, v, a, b, c, d, mx, my, bug, gflag): + seen.append((a, b, c, d, mx[0], my[0])) + return orig_bg(cir, v, a, b, c, d, mx, my, bug, gflag) + + GATE.build_g = spy_g + try: + cir3 = Traced("sched") + h3 = [cir3.fresh_word() for _ in range(8)] + m3 = [cir3.fresh_word() for _ in range(16)] + tg = {str(m3[i][0]): i for i in range(16)} + t3 = [cir3.fresh_word() for _ in range(4)] + GATE.build_compress(cir3, h3, m3, t3[0], t3[1], t3[2], t3[3], rounds) + finally: + GATE.build_g = orig_bg + wired = [[(tg.get(str(mx)), tg.get(str(my))) + for (_, _, _, _, mx, my) in seen[r * 8:(r + 1) * 8]] + for r in range(rounds)] + return wired, [q[:4] for q in seen] + + wired, quads = capture_wiring(7) + # what the ORACLE says round r must consume: permute^r applied to the + # identity schedule, then indexed by round_fn's own message positions + expected = [] + sched = list(range(16)) + for r in range(7): + expected.append([(sched[ix], sched[iy]) for (_, _, _, _, ix, iy) + in ora_G_CALLS()]) + sched = ORA.permute(sched) + record("B", "build_compress feeds every G the ORIGINAL message column " + "under permute^r, for all 7 rounds x 8 G-calls (56 index " + "pairs), matching the oracle's permutation composition", + wired == expected, + f"round0 {wired[0][:2]}... round6 {wired[6][:2]}...") + record("B", "the state quadruples build_round passes match the oracle's " + "round_fn quadruples in order, all 7 rounds", + quads == [tuple(c[:4]) for c in ora_G_CALLS()] * 7) + + def tamper_sched(): + old = GATE.MSG_PERMUTATION[:] + GATE.MSG_PERMUTATION[3], GATE.MSG_PERMUTATION[4] = old[4], old[3] + try: + w2, _ = capture_wiring(7) + finally: + GATE.MSG_PERMUTATION[:] = old + return w2 != expected + bites("B", "message-schedule index check", tamper_sched) + + def tamper_quads(): + old = GATE.G_CALLS[5] + GATE.G_CALLS[5] = (1, 6, 11, 12, 11, 10) # mx/my swapped + try: + w2, q2 = capture_wiring(7) + finally: + GATE.G_CALLS[5] = old + return w2 != expected + bites("B", "G-call wiring check", tamper_quads) + + # -- B4 what the model does NOT carry ---------------------------------- + print("\n -- B4 what the circuit model does not represent --") + src = open(os.path.join(HERE, "blake3-chip/z3_blake_verify.py")).read() + + # Every variable the model creates is 8 bits wide, and every constraint it + # emits is an equation or a carry booleanity. There is no range-check + # OBJECT, so "AreBytes present" and "AreBytes absent" are the same model. + widths = set() + for w in ccir.words: + widths.update(c.size() for c in w["cells"]) + kinds = {} + for c in ccir.C: + kinds[c.decl().name()] = kinds.get(c.decl().name(), 0) + 1 + record("B", "every committed cell the model creates is BitVec(...,8): " + "byte-ness is the DECLARATION, never a modelled lookup", + widths == {8}, f"cell widths {sorted(widths)}") + record("B", "and every emitted constraint is '=' (xor / sum / shift / " + "recombine) or 'or' (carry booleanity) — no range constraint " + "object exists to be present or absent", + set(kinds) <= {"=", "or"}, str(kinds)) + cls_body = src[src.index("# Chip circuit model"):src.index("def build_g")] + record("B", "inside the Circuit class, 'AreBytes' occurs only in comments " + "(the class header ':118-119' and rotr ':212') — the gate " + "DOCUMENTS the assumption ('Byte width == the ByteAlu/AreBytes " + "range-check contract') but has no object for it", + all(ln.strip().startswith("#") for ln in cls_body.split("\n") + if "AreBytes" in ln)) + record("B", "the model has NO mu column: every eval identity is asserted " + "ungated, which is exact for a live row and blind to padding " + "rows (DESIGN 4.5 / 7.1 are therefore outside the gate)", + not any("mu" in str(c).lower() for c in ccir.C) + and "mu" not in "".join(str(w["cells"][0]) for w in ccir.words)) + record("B", "the model has no bus / multiplicity / timestamp layer at all " + "(confirming README finding 1, not re-deriving it)", + not any(k in src for k in ("Multiplicity", "TIMESTAMP", "bus_interaction", + "receive(", "send("))) + nunused = sum(1 for w in ccir.words if w["kind"] in ("sll", "sllc")) + record("B", "each shift-rotation allocates 4 x fresh_word() but uses only " + "2 bytes of each (fresh_word()[:2]) — 8 free unconstrained BVs " + "per rotation, unread and harmless", + nunused == 4 * 96, f"{nunused} halfword slots over 96 rotations") + + +# =========================================================================== +# SECTION C — the dangerous direction, in the field: where byte-ness +# actually comes from, and the forgery the model cannot see. +# =========================================================================== +def _field_word(s, name, ranged): + cells = [Int(f"{name}_{i}") for i in range(4)] + for c in cells: + s.add(c >= 0, c < (256 if ranged else P)) + return cells, sum(cells[i] * 2**(8 * i) for i in range(4)) + + +def add_pinned(nops, out_ranged, ops_concrete=None, want_model=False): + """DESIGN 4.3/4.4 add, modelled in the FIELD. Is the committed sum word + pinned to (sum of operands) mod 2^32? unsat = pinned, sat = forgeable.""" + s = Solver() + ops = [] + for k in range(nops): + if ops_concrete: + ops.append(IntVal(ops_concrete[k])) + else: + _, v = _field_word(s, f"in{k}", True) + ops.append(v) + scells, S = _field_word(s, "S", out_ranged) + if nops == 2: + c = Int("c") + s.add(Or(c == 0, c == 1)) + csum = c + else: + c1, c2 = Int("c1"), Int("c2") + s.add(Or(c1 == 0, c1 == 1), Or(c2 == 0, c2 == 1)) + csum = c1 + c2 + s.add((sum(ops) - S - 2**32 * csum) % P == 0) + T, K = Int("T"), Int("K") + s.add(K >= 0, K <= nops - 1, T >= 0, T < 2**32, sum(ops) == K * 2**32 + T) + s.add((S - T) % P != 0) # a wrong FIELD VALUE, not just cells + res = s.check() + if res == sat and want_model: + mo = s.model() + g = lambda e: mo.eval(e, model_completion=True).as_long() + return res, dict(operands=[hex(g(o)) for o in ops], honest=hex(g(T)), + forged=hex(g(S) % P), cells=[g(x) for x in scells], + carries=g(csum)) + return res, None + + +def rot_pinned(r, kept, want_model=False): + """DESIGN 4.2 rotation in the FIELD, COMPOSED: both shift identities + + both recombine identities + the byte range on Y that the downstream + ByteAlu gives. `kept` = which halfwords carry their AreBytes bound.""" + s = Solver() + xlo, xhi = Int("xlo"), Int("xhi") + s.add(xlo >= 0, xlo < 2**16, xhi >= 0, xhi < 2**16) + hw = {} + for n in ("SLL_lo", "SLLC_lo", "SLL_hi", "SLLC_hi"): + if n in kept: + lo, hi = Int(n + "_b0"), Int(n + "_b1") + s.add(lo >= 0, lo < 256, hi >= 0, hi < 256) + hw[n] = lo + 256 * hi + else: + v = Int(n) + s.add(v >= 0, v < P) + hw[n] = v + s.add((xlo * 2**r - hw["SLLC_lo"] * 2**16 - hw["SLL_lo"]) % P == 0) + s.add((xhi * 2**r - hw["SLLC_hi"] * 2**16 - hw["SLL_hi"]) % P == 0) + Y = [Int(f"Y{i}") for i in range(4)] + for y in Y: + s.add(y >= 0, y < 256) + Ylo, Yhi = Y[0] + 256 * Y[1], Y[2] + 256 * Y[3] + s.add((Ylo - hw["SLL_hi"] - hw["SLLC_lo"]) % P == 0) + s.add((Yhi - hw["SLL_lo"] - hw["SLLC_hi"]) % P == 0) + X = xlo + 2**16 * xhi + Q, R = Int("Q"), Int("R") + s.add(Q >= 0, Q < 2**r, R >= 0, R < 2**32, X * 2**r == Q * 2**32 + R) + wlo, whi = Int("wlo"), Int("whi") + s.add(wlo >= 0, wlo < 2**16, whi >= 0, whi < 2**16, R + Q == wlo + 2**16 * whi) + honest = whi + 2**16 * wlo + s.push() + s.add(Ylo + 2**16 * Yhi != honest) + res = s.check() + mdl = None + if res == sat and want_model: + mo = s.model() + g = lambda e: mo.eval(e, model_completion=True).as_long() + mdl = dict(X=hex(g(X)), honest_Y=hex(g(honest)), + forged_Y=hex(g(Ylo + 2**16 * Yhi)), + SLL_lo=hex(g(hw["SLL_lo"])), SLLC_lo=hex(g(hw["SLLC_lo"])), + SLL_hi=hex(g(hw["SLL_hi"])), SLLC_hi=hex(g(hw["SLLC_hi"]))) + s.pop() + # non-vacuity: the honest witness must satisfy the model + s.add(Ylo + 2**16 * Yhi == honest) + live = s.check() == sat + return res, mdl, live + + +def section_C(slow): + print("\n" + "=" * 74) + print("C FIELD-LEVEL — what the byte range checks actually buy, and the " + "forgery the\n BV model cannot express") + print("=" * 74) + + for n in (2, 3): + res, _ = add_pinned(n, True) + record("C", f"add{n}: WITH the output's byte range check the sum is " + f"pinned to (a+b{'+m' if n == 3 else ''}) mod 2^32, for " + f"ALL operands (symbolic, mod p)", res == unsat) + for n in (2, 3): + res, mdl = add_pinned(n, False, want_model=True) + record("C", f"add{n}: WITHOUT it the committed sum is FORGEABLE — the " + f"prover commits the UNREDUCED sum with carry 0", + res == sat, str(mdl)) + + res, mdl = add_pinned(2, False, ops_concrete=[0x80000000, 0x80000000], + want_model=True) + record("C", "concrete witness: a=b=0x80000000, honest s=0, forged s=2^32 " + "with carry=0 (cells [2^32,0,0,0]) — every modelled constraint " + "satisfied", res == sat and mdl["forged"] == hex(2**32), str(mdl)) + + # the gate cannot tell the two apart: its `s` is 4 BitVec(8)s either way + src = open(os.path.join(HERE, "blake3-chip/z3_blake_verify.py")).read() + record("C", "…and the gate models BOTH chips identically: add2/add3 return " + "`self.fresh_word()`, i.e. 4x BitVec(...,8), so the range check " + "is DECLARED, never derived from a modelled lookup", + "s = self.fresh_word()" in src and "AreBytes" not in + src[src.index("def add2"):src.index("def rotr")]) + + print("\n -- C2 the rotation, composed (the gate tests it in isolation) --") + lattice = {} + for r in (4, 9): + for k in range(4, -1, -1): + for kept in itertools.combinations( + ("SLL_lo", "SLLC_lo", "SLL_hi", "SLLC_hi"), k): + res, _, live = rot_pinned(r, set(kept)) + lattice[(r, kept)] = (res, live) + all_live = all(live for _, live in lattice.values()) + record("C", "non-vacuity: the honest rotation witness satisfies the " + "composed field model in all 32 bound configurations", + all_live) + record("C", "rotation with all four AreBytes bounds: Y is pinned to " + "rotr12/rotr7 for ALL 2^32 inputs (symbolic, mod p — the gate " + "only ever checked one concrete halfword in the field)", + lattice[(4, ("SLL_lo", "SLLC_lo", "SLL_hi", "SLLC_hi"))][0] == unsat + and lattice[(9, ("SLL_lo", "SLLC_lo", "SLL_hi", "SLLC_hi"))][0] == unsat) + one_sll = all(lattice[(r, k)][0] == unsat for r in (4, 9) + for k in (("SLL_lo",), ("SLL_hi",))) + no_sll = all(lattice[(r, k)][0] == sat for r in (4, 9) + for k in ((), ("SLLC_lo",), ("SLLC_hi",), ("SLLC_lo", "SLLC_hi"))) + record("C", "necessary AND sufficient bound set = at least one of " + "{SLL_lo, SLL_hi}; every configuration with neither is " + "forgeable, every configuration with either is pinned — the " + "SLLC bounds are not load-bearing at all", + one_sll and no_sll) + res, mdl, _ = rot_pinned(9, set(("SLLC_lo", "SLLC_hi")), want_model=True) + record("C", "the composed rotation forgery (both SLL bounds dropped) exists " + "for exactly ONE input, X=0xFFFFFFFF -> forged Y=0 instead of " + "0xFFFFFFFF — not 'any input', as the gate's isolated control " + "suggests", res == sat and mdl["X"] == hex(0xFFFFFFFF), str(mdl)) + + # exhaustively: is X = 0xFFFFFFFF the only one? + def enumerate_bad_X(r, limit=4): + found = [] + seen = set() + for _ in range(limit): + s = Solver() + xlo, xhi = Int("xlo"), Int("xhi") + s.add(xlo >= 0, xlo < 2**16, xhi >= 0, xhi < 2**16) + hw = {} + for n in ("SLL_lo", "SLL_hi"): + v = Int(n) + s.add(v >= 0, v < P) + hw[n] = v + for n in ("SLLC_lo", "SLLC_hi"): + lo, hi = Int(n + "_b0"), Int(n + "_b1") + s.add(lo >= 0, lo < 256, hi >= 0, hi < 256) + hw[n] = lo + 256 * hi + s.add((xlo * 2**r - hw["SLLC_lo"] * 2**16 - hw["SLL_lo"]) % P == 0) + s.add((xhi * 2**r - hw["SLLC_hi"] * 2**16 - hw["SLL_hi"]) % P == 0) + Y = [Int(f"Y{i}") for i in range(4)] + for y in Y: + s.add(y >= 0, y < 256) + Ylo, Yhi = Y[0] + 256 * Y[1], Y[2] + 256 * Y[3] + s.add((Ylo - hw["SLL_hi"] - hw["SLLC_lo"]) % P == 0) + s.add((Yhi - hw["SLL_lo"] - hw["SLLC_hi"]) % P == 0) + X = xlo + 2**16 * xhi + Q, R = Int("Q"), Int("R") + s.add(Q >= 0, Q < 2**r, R >= 0, R < 2**32, X * 2**r == Q * 2**32 + R) + wlo, whi = Int("wlo"), Int("whi") + s.add(wlo >= 0, wlo < 2**16, whi >= 0, whi < 2**16, + R + Q == wlo + 2**16 * whi) + s.add(Ylo + 2**16 * Yhi != whi + 2**16 * wlo) + for x in seen: + s.add(X != x) + if s.check() != sat: + break + xv = s.model().eval(X, model_completion=True).as_long() + seen.add(xv) + found.append(hex(xv)) + return found + bad4, bad9 = enumerate_bad_X(4), enumerate_bad_X(9) + record("C", "exhaustive: X=0xFFFFFFFF is the ONLY forgeable input for both " + "r=4 and r=9 (all other X enumerated away -> unsat)", + bad4 == bad9 == ["0xffffffff"], f"r=4 {bad4} r=9 {bad9}") + + # …but the rotation OUTPUT does not need its own byte range check: both + # recombine identities together pin its VALUE regardless of how its cells + # decompose. So the free-range-check argument is load-bearing for the add + # outputs and for one SLL per rotation — and for nothing else. + s = Solver() + xlo, xhi = Int("xlo"), Int("xhi") + s.add(xlo >= 0, xlo < 2**16, xhi >= 0, xhi < 2**16) + hw = {} + for n in ("SLL_lo", "SLLC_lo", "SLL_hi", "SLLC_hi"): + lo, hi = Int(n + "_b0"), Int(n + "_b1") + s.add(lo >= 0, lo < 256, hi >= 0, hi < 256) + hw[n] = lo + 256 * hi + r = 9 + s.add((xlo * 2**r - hw["SLLC_lo"] * 2**16 - hw["SLL_lo"]) % P == 0) + s.add((xhi * 2**r - hw["SLLC_hi"] * 2**16 - hw["SLL_hi"]) % P == 0) + Ycells = [Int(f"Yf{i}") for i in range(4)] + for c in Ycells: + s.add(c >= 0, c < P) # NO range check on Y + Ylo, Yhi = Ycells[0] + 256 * Ycells[1], Ycells[2] + 256 * Ycells[3] + s.add((Ylo - hw["SLL_hi"] - hw["SLLC_lo"]) % P == 0) + s.add((Yhi - hw["SLL_lo"] - hw["SLLC_hi"]) % P == 0) + X = xlo + 2**16 * xhi + Q, R = Int("Q"), Int("R") + s.add(Q >= 0, Q < 2**r, R >= 0, R < 2**32, X * 2**r == Q * 2**32 + R) + wlo, whi = Int("wlo"), Int("whi") + s.add(wlo >= 0, wlo < 2**16, whi >= 0, whi < 2**16, R + Q == wlo + 2**16 * whi) + s.add((Ylo + 2**16 * Yhi - (whi + 2**16 * wlo)) % P != 0) + record("C", "the rotation OUTPUT needs no range check of its own: the two " + "recombine identities pin its value even with free field cells " + "— so the 'free range check' is load-bearing only for the add " + "outputs and one SLL halfword per rotation", s.check() == unsat) + + print("\n -- C3 the width audit's two claims, re-derived symbolically --") + # the gate proves each on ONE concrete input; prove them for all inputs + s = Solver() + inhw = Int("in_hw") + s.add(inhw >= 0, inhw < 2**16) + lo, hi = Int("lo"), Int("hi") + s.add(lo >= 0, lo < 256, hi >= 0, hi < 256) + SLL = lo + 256 * hi + SLLC = Int("SLLC") + s.add(SLLC >= 0, SLLC < 2**16) + r = 9 + s.add((inhw * 2**r - SLLC * 2**16 - SLL) % P == 0) + ref = Int("ref") + s.add(ref >= 0, ref < 2**16, (inhw * 2**r - ref) % 2**16 == 0) + s.add(SLL != ref) + record("C", "field_shift_bound's UNSAT holds for ALL in_hw, not just " + "0x9C3A (the gate tests one point)", s.check() == unsat) + + s = Solver() + a, b, m3 = Int("a"), Int("b"), Int("m") + for x in (a, b, m3): + s.add(x >= 0, x < 2**32) + S = Int("S") + s.add(S >= 0, S < 2**32) + c1, c2 = Int("c1"), Int("c2") + s.add(Or(c1 == 0, c1 == 1), Or(c2 == 0, c2 == 1)) + s.add((a + b + m3 - S - 2**32 * (c1 + c2)) % P == 0) + K, T = Int("K"), Int("T") + s.add(K >= 0, K <= 2, T >= 0, T < 2**32, a + b + m3 == K * 2**32 + T) + s.add(S != T) + record("C", "field_add_carry's UNSAT holds for ALL (a,b,m), not just " + "3x0xF0000000", s.check() == unsat) + + # dropping the booleanity really does free s, composed with s's byte range + s = Solver() + a, b, m3 = IntVal(0x12345678), IntVal(0x9ABCDEF0), IntVal(0x0F0F0F0F) + scells, S = _field_word(s, "S", True) # s STILL byte-range-checked + k = Int("k") + s.add(k >= 0, k < P) # booleanity dropped + s.add((a + b + m3 - S - 2**32 * k) % P == 0) + honest = (0x12345678 + 0x9ABCDEF0 + 0x0F0F0F0F) % 2**32 + s.add(S != honest) + res = s.check() + mdl = None + if res == sat: + mo = s.model() + mdl = dict(honest=hex(honest), + forged=hex(mo.eval(S, model_completion=True).as_long()), + k=mo.eval(k, model_completion=True).as_long()) + record("C", "dropping the carry booleanity is forgeable even WITH the byte " + "range check on s (so this control is faithful to the composed " + "chip, unlike the shift one)", res == sat, str(mdl)) + + print("\n -- C4 the message columns (DESIGN 4.7): AreBytes vs the model --") + # without AreBytes on m the cells bind only sum(m_i 2^8i): exhibit two + # distinct cell vectors that satisfy every constraint identically. + honest_cells = [0x9A, 0x00, 0x13, 0x7F] + forged_cells = [0x9A + 256, 0x00 - 1, 0x13, 0x7F] + same_value = (sum(honest_cells[i] * 2**(8 * i) for i in range(4)) % P == + sum(forged_cells[i] * 2**(8 * i) for i in range(4)) % P) + record("C", "without the explicit AreBytes, a message word has many cell " + "representations with the same value (here [0x9A,0,0x13,0x7F] " + "and [0x19A,-1,0x13,0x7F] = [.., p-1, ..]): the chip binds " + "sum(m_i 2^8i), not the 64 bytes", same_value, + f"forged cells over F_p: {[c % P for c in forged_cells]}") + record("C", "the gate declares m as 16 x 4 BitVec(...,8), so it proves the " + "SAME UNSAT for a chip with and without those 32 AreBytes sends", + "m = [cir.fresh_word() for _ in range(16)]" in src) + + +# =========================================================================== +# SECTION D — gate hygiene: are the UNSATs non-vacuous, and is the model's +# carry encoding the one DESIGN.md specifies? +# =========================================================================== +def section_D(slow): + print("\n" + "=" * 74) + print("D GATE HYGIENE") + print("=" * 74) + + cir, v, _ = _build_one_g() + s = Solver() + s.add(And(*cir.C)) + record("D", "the G circuit's constraint set is SATISFIABLE on its own — " + "MAIN 0's UNSAT is not vacuous", s.check() == sat) + counts = {} + for opname, call in (("xor", lambda c: c.xor(c.fresh_word(), c.fresh_word())), + ("add2", lambda c: c.add2(c.fresh_word(), c.fresh_word())), + ("add3", lambda c: c.add3(c.fresh_word(), c.fresh_word(), + c.fresh_word())), + ("rotr12", lambda c: c.rotr(c.fresh_word(), 12)), + ("rotr16", lambda c: c.rotr16(c.fresh_word()))): + c = GATE.Circuit("cnt") + call(c) + counts[opname] = len(c.C) + record("D", "constraint counts per op match DESIGN 4.1-4.4: xor 4 (pure " + "lookup, modelled as 4 byte equalities), add2 2 (sum + 1 " + "booleanity), add3 3 (sum + 2 booleanities), rotr12 4 " + "(2 shift + 2 recombine), rotr16 0 (free relabel)", + counts == {"xor": 4, "add2": 2, "add3": 3, "rotr12": 4, "rotr16": 0}, + str(counts)) + + ccir, cout, _ = _build_compress(6) + s = Solver() + s.add(And(*ccir.C)) + record("D", "the 6-round circuit's constraint set is SATISFIABLE on its own", + s.check() == sat) + + # DESIGN 4.3 commits NO carry column for the 2-op add (carry is the linear + # expression (a+b-s)*2^-32); the model commits a boolean column instead. + # Prove the two are equivalent. + s = Solver() + A, B, S = Int("A"), Int("B"), Int("S") + for x in (A, B, S): + s.add(x >= 0, x < 2**32) + c_derived = Int("cd") + lhs = Or(And((A + B - S - 2**32 * 0) % P == 0), + And((A + B - S - 2**32 * 1) % P == 0)) # committed-boolean form + rhs = ((A + B - S) * pow(2**32, -1, P) % P == 0) + # derived form: carry := (A+B-S)*2^-32 mod p, booleanity carry*(carry-1)=0 + cd = ((A + B - S) * pow(2**32, -1, P)) % P + rhs = Or(cd == 0, cd == 1) + s.add(lhs != rhs) + record("D", "DESIGN 4.3's DERIVED carry (linear expr x INV_SHIFT_32, " + "booleanity) and the model's COMMITTED boolean carry are " + "equivalent over F_p — the difference is 1 column per add2, " + "not a semantic one", s.check() == unsat) + + note("DESIGN 4.8's ledger row for the recombine identity says body degree " + "2 -> 3 after x mu; the body mu*(Ylo - SLL_hi - SLLC_lo) is LINEAR in " + "committed columns, so it is 1 -> 2. Over-stated in the safe " + "direction; the 'no constraint exceeds 3' verdict is unaffected.") + note("DESIGN 3's per-G table counts 1 committed carry bit for each add2, " + "while DESIGN 4.3 makes that carry a DERIVED linear expression " + "(a+b-s)*INV_SHIFT_32 with no column. The gate models the committed " + "form. Equivalent as constraints (proved above); the two readings " + "differ by 96 cells/compression in the DESIGN 6 cost table.") + + # The positive controls are the gate's only external anchor. They pin the + # circuit's output to a RECORDED vector, so a stale fixture would silently + # anchor the gate to nothing. + vecs = GATE.load_canonical_6round() + ok_vec = all(ORA.compress_6round(v["h"], v["m"], v["t"], v["block_len"], + v["flags"]) == v["out"] for v in vecs) + record("D", f"all {len(vecs)} canonical 6-round fixture vectors reproduce " + "from the oracle's compress_6round — the positive controls " + "anchor to the live oracle, not a stale file", ok_vec) + record("D", "…and they are genuinely 6-round: none of them equals the " + "7-round compression of the same input", + all(ORA.compress(v["h"], v["m"], v["t"], v["block_len"], v["flags"], + rounds=7) != v["out"] for v in vecs)) + h7, m7, tlo7, thi7, bl7, fl7, out7 = GATE.gen_7round_vector() + record("D", "gen_7round_vector's output is the oracle's 7-round " + "compression of its own inputs", + ORA.compress(h7, m7, tlo7 | (thi7 << 32), bl7, fl7, rounds=7) == out7) + + # WIDE = 48 must be wide enough that the BV identities are integer + # identities, and small-enough values that they coincide with mod-p. + s = Solver() + a, b, m3, S = Int("a"), Int("b"), Int("m"), Int("S") + for x in (a, b, m3, S): + s.add(x >= 0, x < 2**32) + c1, c2 = Int("c1"), Int("c2") + s.add(Or(c1 == 0, c1 == 1), Or(c2 == 0, c2 == 1)) + s.add((a + b + m3 == S + 2**32 * (c1 + c2)) != + (((a + b + m3 - S - 2**32 * (c1 + c2)) % P) == 0)) + record("D", "the 3-op sum identity over Z (what WIDE=48 BV computes) and " + "over F_p (what the chip computes) are equivalent under the " + "byte bounds — no wraparound is available on either side, " + "confirming DESIGN 7.9", s.check() == unsat) + # The gate ASSUMES the BITWISE contracts (assume-guarantee). They are + # cheap to verify against the real table, so verify them. + bw = os.path.join(HERE, "..", "..", "prover", "src", "tables", "bitwise.rs") + if os.path.exists(bw): + rs = open(bw).read() + record("D", "the assumed ByteAlu[XOR] contract is real: bitwise.rs " + "enumerates x,y in 0..256 and sets cols::XOR = x^y, and the " + "receiver pins (XOR, X, Y) -> XOR", + "for x in 0u32..256 {" in rs and "for y in 0u32..256 {" in rs + and "table.set_byte(row_idx, cols::XOR, (x ^ y) as u8);" in rs + and "Multiplicity::Column(cols::MU_BYTE_ALU_XOR)" in rs) + record("D", "the assumed AreBytes contract is real: an AreBytes " + "receiver over the same 0..256 x 0..256 domain", + "BusId::AreBytes," in rs + and "ARE_BYTES[X, Y] - range check two byte values" in rs) + else: + note(f"bitwise.rs not found at {bw}; the ByteAlu/AreBytes contracts " + "were not cross-checked in this run.") + + note("block_len and flags are modelled as free 32-bit words; the design " + "says 0..64 and 0..127. The model is WEAKER there, which is the safe " + "direction, and DESIGN.md specifies no such constraint either.") + note("DESIGN 4.1 allows a ByteAlu operand to be a linear combination of " + "cells ('sum <= 255'); the model never uses one — rotr16/rotr8 are " + "pure index relabels, so every operand is a single cell. Consistent, " + "but the linear-combo contract is therefore unexercised.") + + if slow: + print("\n -- D2 the gate's own BV verdicts, re-run --") + record("D", "check_g() == unsat (MAIN 0)", GATE.check_g() == unsat) + record("D", "check_compress(0) == unsat (MAIN 1)", + GATE.check_compress(0) == unsat) + record("D", "check_g(bug='swap_g_operand') == sat", + GATE.check_g(bug="swap_g_operand") == sat) + + +# =========================================================================== +def main(): + slow = "--slow" in sys.argv + print("=" * 74) + print("BLAKE3 GATE TRANSCRIPTION AUDIT — regression suite") + print("=" * 74) + section_A(slow) + section_B(slow) + section_C(slow) + section_D(slow) + + print("\n" + "=" * 74) + fails = [(s, n) for s, n, ok, _ in RESULTS if not ok] + print(f"SUMMARY: {len(RESULTS) - len(fails)}/{len(RESULTS)} checks pass") + for s, n in fails: + print(f" FAIL [{s}] {n}") + print("=" * 74) + sys.exit(1 if fails else 0) + + +if __name__ == "__main__": + main() diff --git a/thoughts/blake3/blake3-chip/DESIGN.md b/thoughts/blake3/blake3-chip/DESIGN.md new file mode 100644 index 000000000..9a04d72f5 --- /dev/null +++ b/thoughts/blake3/blake3-chip/DESIGN.md @@ -0,0 +1,489 @@ +# BLAKE3 compression chip — constraint-system & bus design (Phase 2) + +**Status.** Model-level design + z3 equivalence gate, done **before** any Rust. +Ground truth = the phase-1 oracle (`../blake3-oracle/`, VALIDATED against 3 +external anchors). Cost model = the verified one in `../keccak-verify/tier2_cost_model.md` +(a committed cell is expensive; each bus send ≈ 1.5 base cells of aux; **hard** +max constraint degree 3 *including* the ×μ gating factor). + +> **⚠ Every `../keccak-verify/` citation in this document is DEAD.** That +> directory lived in the same 2026-07-23 session scratchpad this design was +> recovered from and was never committed — it exists on no branch. Two +> conclusions were deferred to it, and both have since been re-established +> independently, so nothing here rests on the missing files: +> * the **cost model** above (1.5 aux cells/send, degree ≤ 3 incl. ×μ) — the +> per-G and per-compression arithmetic in §2/§3/§6 was recomputed from +> scratch and checks out; +> * the **shift-identity bound necessity** cited at §4.2 and §9 — re-derived +> symbolically over all 2^32 inputs by the 2026-07-29 transcription audits, +> which is stronger than the single-point check the lost file made. +> +> Do not go looking for them; read the citations as historical. + +**Verdict (numbers derived below, gate in `z3_blake_verify.py`):** +* **Layout: B — one row per compression, fully unrolled.** Chosen by arithmetic + (≈5,030 cell-equiv vs ≈5,510 for one-row-per-round), and it deletes the + state+message handoff bus entirely. Table below. +* **O1 (3-operand add carry): option (c) — two summed carry bits.** Cheaper than + both options the oracle listed and stays degree ≤3 after μ-gating. +* **Rotations: rotr16/rotr8 free (byte relabel); rotr12/rotr7 inlined** as the + μ-gated linear shift identity (no HWSL sends), saving 4 sends/G. +* **Every eval constraint is μ-gated, padding is all-zero, every constraint ≤3.** +* **≈5,030 cell-equiv per 6-round compression (≈5,810 for 7-round)** — about + **1/15 of a keccak-f permutation** (≈77,000). + +--- + +## 1. Scope & I/O interface + +The chip implements the compression function `f` (oracle §2.4), **not** the tree. +Primary target is the **6-round internal variant** (Merkle 2-to-1 / Fiat–Shamir); +the design is `ROUNDS`-parameterised so 7-round is the same layout with one more +unrolled round. + +### 1.1 Lean internal interface (the one we build first): 2-to-1 compression + +Exposed on a dedicated **`Blake3` bus**. A parent-node caller supplies the two +child chaining values as the message and reads back the truncated CV. + +**Receive** (multiplicity μ) — the compression inputs: + +| field | words | bytes | source | +|---|---|---|---| +| `h[0..8]` chaining value / key | 8 | 32 | caller | +| `m[0..16]` message = `left_cv ‖ right_cv` | 16 | 64 | caller | +| `t_lo, t_hi` counter split | 2 | 8 | caller (t=0 for parents) | +| `block_len` | 1 | 4 | caller (64 for parents) | +| `flags` | 1 | 4 | caller (PARENT ∣ … for parents) | + +**Send** (multiplicity μ) — the output `out[0..16]` (16 words = 64 B). CV-only +call sites read `out[0:8]`; the chip always produces all 16 (the XOF root needs +them, oracle §2.4). + +**Both tuples MUST lead with `TIMESTAMP_0, TIMESTAMP_1` — this is mandatory, +not optional.** The receive and the send are two separate interactions, so +without a key present in *both* nothing ties a row's inputs to its own outputs. +With two compressions in a trace a prover could then have row A receive +`inputs_A` and send `out_B` while row B does the reverse: every tuple still +appears exactly once on each side, **so the bus balances**, and both callers +read a result that is not the compression of their own input. + +This is not a hypothetical hardening: `keccak.rs` — the chip this design copies +its I/O idiom from (§1.2) — carries `TIMESTAMP_0, TIMESTAMP_1` in *both* halves +of its internal `Keccak` bus (send at `round = 0`, receive at `round = 24`) +for exactly this reason. Do not deviate from it. + +**The gate cannot check this.** `z3_blake_verify.py` models arithmetic only and +has no bus-interaction layer at all, so a missing binding leaves every UNSAT on +the board unchanged. It has to be got right by construction. + +`IV[0..4]` (v[8..11]) are **compile-time constants inlined** into the round-0 +arithmetic — not columns, not on the bus. + +### 1.2 General syscall / memory variant (sketched, not built here) + +Same core; replaces the internal `Blake3` receive/send with the keccak I/O +idiom (`prover/src/tables/keccak.rs:160-449`): an `Ecall` receiver binding +(timestamp, syscall#), a `Memw` read of `x10` binding the state pointer, then +per-word `Memw` reads/writes of `h`,`m`,`t`,`block_len`,`flags`,`out`. Adds +~1 Ecall + ~(112+64)/8 ≈ 22 Memw interactions and the pointer-arith columns; +**orthogonal to the mixing core designed here** (open questions O5/O6 live here). + +--- + +## 2. Row-layout decision (by arithmetic) + +Per-compression work (6 rounds): each round = 8 G-functions; each G = **2 +three-operand adds, 2 two-operand adds, 4 XORs, 2 free rotations (rotr16/8), +2 shift rotations (rotr12/7)**. Committed cells and bus sends per G (SSA form, +derivation in §5): + +* committed: **56 byte-cells + 6 carry-bit cells** per G +* sends: **24** per G (16 ByteAlu[XOR] + 8 AreBytes for the two shift rotations) + +| per compression | **A: 1 row / round (6 rows)** | **B: unrolled (1 row)** | +|---|---:|---:| +| logic committed (8 G × 6) | 2,976 | 2,976 | +| feed-forward committed | 64 | 64 | +| I/O input columns | 112 (×6 carried!) = 672 | 112 (once) | +| state+message handoff columns | +128 B/row × 6 = 768 | 0 | +| round-index / selector bookkeeping | ~18 | 0 | +| **committed total** | **≈ 3,760** | **≈ 3,150** | +| bus sends N (logic 192/round) | 1,152 + 6 handoff + 32 msg-rc ≈ 1,190 | 1,152 + 64 ff + 34 I/O = 1,250 | +| **aux = 3·⌈N/2⌉** | **≈ 1,750** | **≈ 1,875** | +| **total cell-equiv** | **≈ 5,510** | **≈ 5,030** | +| handoff bus | `Blake3Round` carries state(64B)+**msg**(64B)/row | none | +| structural cost | per-row state+msg reconstruction, permute-on-bus | pure compile-time wiring | + +**Decision: B.** It wins on total cells (the handoff re-commits the 16-word +state *and* the 16-word message on every one of the 6 rows — BLAKE3, unlike +keccak, must carry the message down the rounds, which is the single biggest +extra cost of A) and it is structurally far simpler: the message schedule is a +compile-time permutation, so unrolling makes every round reference the original +16 committed message words under `permute^r` with **zero** runtime handoff. The +concentration of all sends into one row makes B's aux marginally higher, but the +committed-column saving dominates. B also removes round-index bookkeeping and the +`Blake3Round` bus wholesale. (Matches the oracle's recommendation, now with the +numbers behind it.) + +Only reason to revisit A: if the ~3,150-wide single row's LDE/Merkle width ever +dominates trace area for tiny proofs — not the case here (keccak's per-row width +is already ~1,480+aux and BLAKE3 has 1/4 the rounds). + +--- + +## 3. Column layout (Layout B) + +One row = one compression call. Names group by role; counts are for `ROUNDS=6`. +"SSA word" = a fresh 4-byte committed word produced by one op. + +| block | columns | count | notes | +|---|---|---:|---| +| `TIMESTAMP_0/1` | 2 | 2 | bus binding — **mandatory in both the receive and the send** (§1.1); omitting it lets two rows swap outputs with the bus still balancing | +| `MU` | 1 | 1 | multiplicity / gate flag | +| `H[0..8]` | 8 words | 32 | input CV bytes | +| `M[0..16]` | 16 words | 64 | input message bytes | +| `T_LO,T_HI,BLEN,FLAGS` | 4 words | 16 | counter split, block_len, flags | +| per-G logic × 48 G | see §5 | 2,976 | add/xor/shift SSA words + carry bits | +| feed-forward `OUT[0..16]` | 16 words | 64 | XOR outputs | +| **main columns total** | | **≈ 3,155** | | +| aux (LogUp) `= 3·⌈1250/2⌉` | | **1,875** | degree-3 ext columns | + +Per-G committed breakdown (each of the 48 G-instances): + +| sub-op | SSA output | bytes | carry bits | +|---|---|---:|---:| +| `add3` v[a]=v[a]+v[b]+mx | `A1` | 4 | 2 | +| `xor` v[d]^v[a] (→rotr16 free) | `X1` | 4 | – | +| `add2` v[c]+v[d] | `C1` | 4 | 1 | +| `xor` v[b]^v[c] | `X2` | 4 | – | +| `rotr12`(X2) | `SLLlo,SLLClo,SLLhi,SLLChi,B1` | 12 | – | +| `add3` v[a]=v[a]+v[b]+my | `A2` | 4 | 2 | +| `xor` v[d]^v[a] (→rotr8 free) | `X3` | 4 | – | +| `add2` v[c]+v[d] | `C2` | 4 | 1 | +| `xor` v[b]^v[c] | `X4` | 4 | – | +| `rotr7`(X4) | `SLLlo,SLLClo,SLLhi,SLLChi,B2` | 12 | – | +| **per G** | | **56** | **6** | + +`rotr16`/`rotr8` produce **no columns** — the next consumer reads the XOR-output +bytes in relabeled order (see §4.2). + +--- + +## 4. Constraints & bus interactions + +All arithmetic reduces to the existing precomputed-BITWISE receivers +(`prover/src/tables/bitwise.rs`); all eval constraints are **μ-gated**, so +degree = (μ:1) × (body). Padding rows are all-zero and μ=0. + +### 4.1 XOR — `ByteAlu[XOR]` send (per byte) + +For each 32-bit XOR, 4 sends `ByteAlu[XOR, a_byte, b_byte] → out_byte` +(`bitwise.rs:903`). The lookup **simultaneously** byte-range-checks both operands +and pins `out` to the exact XOR — no separate range check. Operands may be linear +combos (the byte contract requires `sum ≤ 255`), which lets a free rotation be +read in-place. Eval constraints: none (pure lookup). Degree: n/a. + +### 4.2 Rotations + +* **rotr16 / rotr8 — free.** rotr16 = byte relabel `[b0,b1,b2,b3]→[b2,b3,b0,b1]`; + rotr8 = `[b1,b2,b3,b0]` (oracle §3.1, exhaustively verified). No columns, no + lookups, no constraints — the consumer indexes the source XOR's bytes in + rotated order. +* **rotr12 / rotr7 — inline shift identity (chosen over HWSL sends).** + `rotr12 = rotl20 = rotl16∘rotl4` (inner `r=4`); `rotr7 = rotl25 = rotl16∘rotl9` + (`r=9`). For input word `X = xlo + 2^16·xhi` (halfwords `xlo,xhi`, 2 bytes each): + + **Shift identities (eval, degree 2 after ×μ):** + ``` + μ·( xlo·2^r − SLLC_lo·2^16 − SLL_lo ) = 0 + μ·( xhi·2^r − SLLC_hi·2^16 − SLL_hi ) = 0 + ``` + **Recombine + halfword swap (eval, degree 2 after ×μ):** + ``` + μ·( Ylo − SLL_hi − SLLC_lo ) = 0 # output low halfword = Y[0]+256·Y[1] + μ·( Yhi − SLL_lo − SLLC_hi ) = 0 # output high halfword = Y[2]+256·Y[3] + ``` + **Range checks (sends):** `AreBytes` on the 8 bytes of `SLL_lo,SLLC_lo,SLL_hi, + SLLC_hi` = 4 sends/rotation (`bitwise.rs:783`). `Y` is range-checked *free* by + the downstream XOR that consumes it. + + Soundness (originally deferred to `../keccak-verify/hwsl_inline_test.py` + Part 2 — **that file is lost, see the banner at the top; the result was + re-derived independently and more strongly by the 2026-07-29 audits** — and by + the width audit in the gate): given `SLL_* ∈ [0,2^16)` (the tight remainder bound + from AreBytes) and `2^16` invertible mod p, the identity **uniquely** pins + `SLL = (xlo·2^r) mod 2^16` and `SLLC = (xlo·2^r) >> 16`; the loose 16-bit bound + on `SLLC` suffices because it is the quotient, not the remainder. The two + recombination sums are over non-overlapping bit ranges, so `+` = `OR` and each + is an exact 16-bit halfword. + + **Refined by the transcription audits (2026-07-29), symbolically over all + 2^32 inputs — the earlier wording was coarser than the truth:** + * The load-bearing bound set is **at least one of `{SLL_lo, SLL_hi}`**. Every + configuration with neither is forgeable; every configuration with either is + pinned. **The two `SLLC` bounds are not load-bearing at all** — so of the 4 + `AreBytes` sends per rotation, only the `SLL` pair carries soundness weight. + Read the sentence above as "a tight bound on at least one `SLL` halfword", + not "the tight `SLL` bound". + * The *composed* forgery (both `SLL` bounds dropped) exists for exactly **one** + input, `X = 0xFFFFFFFF` → forged `Y = 0`, exhaustively confirmed for both + `r = 4` and `r = 9`. The gate's isolated control makes it look reachable for + arbitrary inputs; it is not. Narrow, but a forgery is a forgery. + * The rotation **output** needs no range check of its own: the two recombine + identities pin its value even with free field cells. So the §4.7 "free range + check" argument is load-bearing for the **add** outputs and one `SLL` + halfword per rotation — not for the rotation output. + + **HWSL alternative, priced:** replace each shift identity with an `Hwsl` send + (`bitwise.rs:831`). Cost/rotation: +2 Hwsl sends, same AreBytes, same columns. + Per compression that is +4 sends/G × 48 = +192 sends → +288 aux cells (≈6%). + Inline wins because the eval identity is free of columns/sends; it costs only + degree budget (2 ≤ 3). **Use inline.** + +### 4.3 Two-operand add — `emit_add_pair` low half (eval, degree 3 after ×μ) + +`s = (a+b) mod 2^32`; one carry bit. Following `templates.rs:334`: +``` +carry = (a + b − s)·2^-32 # linear expression, INV_SHIFT_32 = (2^32)^-1 +μ · carry·(1 − carry) = 0 # degree (1)×(1)×(1 body)=2, ×μ = 3 +``` +`s`'s bytes are range-checked **free** by the next XOR that consumes `s` +(every add output in G feeds a subsequent XOR — see §5). Booleanity + `s∈[0,2^32)` +⇒ `s` unique. + +### 4.4 Three-operand add — **O1 resolved: option (c), two summed carry bits** + +`s = (a+b+m) mod 2^32`, carry ∈ {0,1,2}. Commit two carry **bits** `c1,c2` +(2 cells, no intermediate word): +``` +μ·( a + b + m − s − 2^32·(c1+c2) ) = 0 # sum identity, linear → ×μ = degree 2 +μ · c1·(1 − c1) = 0 # ×μ = degree 3 +μ · c2·(1 − c2) = 0 # ×μ = degree 3 +``` +`s`'s bytes range-checked free downstream. `c1+c2 ∈ {0,1,2}` covers the carry; +`s∈[0,2^32)` + the sum identity pin `s = (a+b+m) mod 2^32` uniquely (proof in the +gate's width audit). + +**Why (c):** + +| O1 option | extra committed / 3-op add | degree (ungated → ×μ) | legal under ×μ? | +|---|---|---|---| +| (a) one ternary carry `k(k−1)(k−2)=0` | 1 bit | 3 → **4** | ❌ (μ-gating mandatory, §4.5) | +| (b) two chained binary adds | 1 word (4 B) + 2 AreBytes | 2 → 3 | ✅ but +4B +2 sends | +| **(c) two summed carry bits** | **2 bits** | 2 (bool) / 1 (sum) → 3 / 2 | ✅ **cheapest** | + +Over a compression, (c) vs (b): saves (4B−2bit) per 3-op add × 96 three-op adds +≈ **300 committed cells + 192 AreBytes sends**. (c) is a strict refinement of the +oracle's two options. + +### 4.5 μ-gating & padding — **O2 resolved: gate everything, all-zero padding** + +Every eval constraint is multiplied by `μ` (the `MU` column, 1 on the real row, +0 on padding), exactly like `keccak_rnd`'s IS_BIT (`keccak_rnd.rs:914`). Padding +rows are **all-zero**: +* bus interactions carry `Multiplicity::Column(MU)` ⇒ 0 contribution on padding; +* eval constraints are `μ·(…)` ⇒ 0 on padding regardless of the (zero) cells. + +This is why O1 must be (b) or (c): the ternary carry (a) is degree 3 *ungated*, +and ×μ pushes it to 4. Inlined `IV` constants are fine because the round-0 add +that consumes them is itself μ-gated (its carry expression is nonsense on an +all-zero padding row, but ×μ=0 kills it). **The μ-gating requirement is what +forecloses option (a) — this is the single tightest coupling in the design.** + +### 4.6 Feed-forward (16 XORs, all `ByteAlu[XOR]`) + +``` +out[i] = v[i] ⊕ v[i+8] i = 0..8 (v[i+8] = final state word) +out[i+8] = v[i+8] ⊕ h[i] i = 0..8 (h = original input CV column) +``` +64 sends, 64 committed output bytes (the XOR outputs), range-checked free by the +lookup. Output bytes are shipped on the `Blake3` send. + +### 4.7 Range checks that are NOT free + +The message `m` enters **only** through adds (never XORed), so its 64 bytes need +explicit `AreBytes` (32 sends/compression). `h` and `t/block_len/flags` all feed +an XOR (feed-forward / round-0 diagonal), so they are free. Every add/shift/xor +output feeds a downstream XOR ⇒ free. + +### 4.8 Degree ledger (the hard gate) + +| constraint | body degree | × μ | ≤ 3? | +|---|---:|---:|:--:| +| 2-op add carry booleanity | 2 | 3 | ✅ | +| 3-op add sum identity | 1 | 2 | ✅ | +| 3-op add carry booleanity ×2 | 2 | 3 | ✅ | +| shift identity (×2) | 1 | 2 | ✅ | +| recombine identity (×2) | 1 (was stated as 2) | 2 | ✅ | +| (rejected) ternary carry | 3 | **4** | ❌ | + +Worst legal constraint = 3. **No constraint exceeds 3.** + +--- + +## 5. Per-G dataflow, SSA + free range-checks + +``` +A1 = add3(v[a], v[b], mx) # v[a] ; 2 carry bits ; range-checked by X1 +X1 = xor(v[d], A1) ; v[d] = rotr16(X1) # free relabel +C1 = add2(v[c], v[d]=rotr16(X1)) # v[c] ; 1 carry bit ; range-checked by X2 +X2 = xor(v[b], C1) +B1 = rotr12(X2) # v[b] ; range-checked by X4 / next round +A2 = add3(A1, B1, my) # v[a] ; 2 carry bits ; range-checked by X3 +X3 = xor(v[d]=rotr16(X1), A2) ; v[d]=rotr8(X3) +C2 = add2(C1, v[d]=rotr8(X3)) # v[c] ; 1 carry bit ; range-checked by X4 +X4 = xor(B1, C2) +B2 = rotr7(X4) # v[b] ; range-checked next round / FF +``` +Every committed add/shift word is an operand of a later XOR ⇒ its bytes are +byte-range-checked for free by that `ByteAlu` lookup. Confirmed: no add/shift +output needs its own AreBytes. (Only `m` does — §4.7.) + +--- + +## 6. Cost & comparison + +| quantity (6-round) | value | +|---|---:| +| committed main columns | ≈ 3,150 | +| bus sends N | ≈ 1,250 (832 XOR incl. 64 feed-forward + 384 shift-AreBytes + 32 msg-AreBytes + 2 I/O) | +| aux base cells (3·⌈N/2⌉) | ≈ 1,875 | +| **total cell-equiv / compression** | **≈ 5,030** | +| 7-round variant | ≈ 5,810 | +| keccak-f permutation (reference) | ≈ 77,000 | +| **BLAKE3-6r as fraction of keccak-f** | **≈ 1/15 (6.5%)** | + +Dominated by the ~960 byte-XOR lookups, as the oracle predicted. Note: the +oracle's prose "¼–⅓ of a keccak permutation" is inconsistent with its own +5–6k/compression figure; the detailed count here (≈5k vs 77k) puts it at **~1/15**. + +--- + +## 7. Soundness-critical spots a Rust implementation must NOT deviate from + +1. **μ-gate every eval constraint** (carry booleanity, sum identity, shift + identity, recombine). Un-gated ternary carry or an un-gated constraint with + inlined IV constants breaks all-zero padding. (§4.5) +2. **3-op add = two summed carry bits with the explicit sum identity** — not a + single ternary carry (degree 4 after gating), and the sum identity must be + present (without it, `s` is only constrained mod nothing). (§4.4) +3. **Shift identity needs a tight `∈ [0,2^16)` AreBytes bound on at least one of + `SLL_lo`/`SLL_hi`** (the `SLLC` bounds are *not* load-bearing — audited + 2026-07-29, §4.2); dropping it + makes the rotation forgeable (a wrong `SLL` admits a large field `SLLC`). + Soundness relies on `2^16` invertible mod p — a BV model cannot see this; + verify in the field (gate width audit + `hwsl_inline_test.py`). (§4.2) +4. **Every add output must actually feed a downstream XOR** (its only range + check). If a future refactor reorders so an add output is *last* with no XOR + consumer, add an explicit AreBytes or the carry argument is unsound. (§5) + + ⚠ **THE GATE CANNOT CHECK THIS, and both 2026-07-29 audits confirmed it with + explicit forgeries.** `build_g` returns each add output as `fresh_word()` = + 4×`BitVec(...,8)`, so byte range is **declared by construction, never derived + from a modelled lookup**. The gate therefore proves the identical UNSAT for a + chip that has the downstream XOR and for one that does not. Drop it and the + sum is forgeable — witness `a = b = 0x80000000`, honest `s = 0`, forged + `s = 2^32` with `carry = 0`, satisfying every modelled constraint. This + invariant rests entirely on the implementer, and a green board is not + evidence for it. +5. **Message `m` needs explicit AreBytes** — it is never XORed. (§4.7) + ⚠ Same blind spot: the gate declares `m` as 16×4 `BitVec(...,8)`, so it proves + the same UNSAT with or without those 32 `AreBytes` sends. Without them a + message word has many cell representations of one value over `F_p` (e.g. + `[0x9A,0,0x13,0x7F]` and `[0x19A,p−1,0x13,0x7F]`), because the chip binds + `Σ m_i·2^(8i)`, not the 64 bytes. +6. **rotr16/rotr8 byte order** exactly `[b2,b3,b0,b1]` / `[b1,b2,b3,b0]` + (little-endian). A wrong relabel silently corrupts. (§4.2) +7. **Message permutation `permute^r`** wired per round from the *original* 16 + `M` columns; MSG_PERMUTATION = `[2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8]`. The + trailing permute after the last round is unused (oracle §2.4). (Gate control + `wrong_msg_index`.) +8. **IV / feed-forward / counter split** exactly per oracle §2.4: + `v[8..12]=IV[0..4]` inlined, `v[12]=t_lo, v[13]=t_hi, v[14]=block_len, + v[15]=flags`; `out[i]=v[i]⊕v[i+8]`, `out[i+8]=v[i+8]⊕h[i]`. (Controls + `wrong_iv`, `drop_ff_xor`.) +9. **Non-overflow side conditions (width audit):** all add/shift field + expressions stay `< 2^35 ≪ p`, so `≡0 mod p` ⇒ `=0` as integers; the whole + soundness argument depends on operands being genuine ≤32-bit (byte columns) + and carries being genuine bits. +10. **`TIMESTAMP_0/1` in BOTH the `Blake3` receive and send** (§1.1). Without a + key in both tuples nothing binds a row's inputs to its own outputs, and two + compressions can swap results while the bus still balances. `keccak.rs` does + this correctly and is the pattern to copy. **The gate cannot catch a + violation** — it models arithmetic only, with no bus layer — so this one is + on the implementer, not on a green board. +11. **Every G instance must be wired as MAIN 0 models it.** MAIN 0 proves *one* + G under free inputs; the 48 unrolled instances are emitted separately, so a + wrong column or message index in a single instance is invisible to it. The + concrete positive controls are what cover that — keep them runnable, and run + `--full`'s monolithic UNSAT before shipping Rust. + +--- + +## 8. Gate + +`z3_blake_verify.py` — free-variable model of every column, every lookup/eval +constraint as an equation, `assert output ≠ oracle-reference`, ask z3 for a +counterexample. Reference (`bref_*`) is an independent 32-bit-BV port of +`blake3_ref.py` (RotateRight / + / ^), structurally independent of the byte-level +shift wiring. Results are appended to §9 after the run (`run.log`). +``` +python3 z3_blake_verify.py # round + wrapper + controls + audit (fast) +python3 z3_blake_verify.py --full # + monolithic 6- and 7-round UNSAT +``` + +## 9. Gate results + +Default run (`python3 z3_blake_verify.py`, ~2 min) — **OVERALL: PASS**: + +| check | result | meaning | +|---|---|---| +| **MAIN 0** — one G-function, free inputs | **UNSAT** | the quarter-round (byte-XOR + inline rotr12/rotr7 shift identities + 2-op & 3-op adds) is correctly & tightly constrained; **covers every G, hence every round** (a round is a fixed composition of 8 G-calls). | +| **MAIN 1** — init-state + feed-forward (rounds=0) | **UNSAT** | `v` layout (`h`/IV/counter-split/block_len/flags) and `out[i]=v[i]⊕v[i+8]`, `out[i+8]=v[i+8]⊕h[i]` are correct. | +| neg `rot_wrong_amount` | **SAT** | wrong rotation amount detected. | +| neg `swap_g_operand` | **SAT** | swapped G-function operand detected. | +| neg `wrong_iv` | **SAT** | wrong IV constant detected. | +| neg `drop_ff_xor` | **SAT** | dropped feed-forward XOR detected. | +| neg `wrong_msg_index` | **SAT** | wrong message-schedule index detected (permutation is load-bearing). | +| **pos** 6-round seeds 0,1,2 (canonical vectors) | **SAT** | full 6-round pipeline reproduces the oracle's recorded output for concrete inputs. | +| **pos** 7-round (oracle-generated) | **SAT** | full 7-round pipeline reproduces the oracle's `compress(…,rounds=7)`. | +| audit: shift `SLL` 16-bit bound present | **UNSAT** | with AreBytes the shift output is pinned. | +| audit: **DROP `SLL` bound** (field neg ctrl) | **SAT** | without it the rotation is forgeable (needs `2^16` invertible mod p). | +| audit: 3-add carry booleanity present | **UNSAT** | with booleanity the sum `s` is pinned. | +| audit: **DROP carry booleanity** (field neg ctrl #4) | **SAT** | without it `s` is forgeable in the prime field. | + +**The 6th team-lead control — "dropped carry booleanity" — lives in the width +audit, not the BV controls, and this is correct.** Dropping a committed carry +column's booleanity is a *field-level* soundness bug: the column becomes a full +Goldilocks element, but a *bounded-BV* model keeps the 8-bit carry + `s∈[0,2^32)` +byte-range, which still pins `s`, so BV reports UNSAT (verified: the BV version +does). Only the mod-p model exhibits the forgery — exactly the phenomenon +`../keccak-verify/hwsl_inline_test.py` Part 2 documents (`2^16`/`2^32` are zero +divisors mod `2^n`). The gate deliberately separates BV-observable logic bugs +from field-only soundness bugs; both classes fire. + +**`--full`** additionally runs the heavy monolithic symbolic UNSATs (one round; +compression rounds=2 for the permutation; full 6- and 7-round). These are *bonus* +confirmations — the G-unsat + fixed-composition chaining argument + rounds=0 + +the concrete full-pipeline positive controls already establish full-compression +correctness. (The direct 6-round symbolic UNSAT is large; it is not required for +the verdict and may take a long time / be run offline.) + +### What is and isn't proven +* **Proven (symbolic, all inputs):** the G quarter-round; the init-state layout; + the feed-forward — hence, by the chaining argument, the full N-round + compression for **both ROUNDS=6 and ROUNDS=7**. +* **Proven (concrete, external anchor):** the *entire* unrolled pipeline + (init + 6/7 rounds + message permutation + feed-forward) reproduces the + oracle's validated vectors. +* **Proven (field-level):** the AreBytes shift bound and the add-carry booleanity + are each *necessary* (dropping either is a forgery mod p). +* **Assumed (assume-guarantee, not re-proven here):** the precomputed BITWISE + table contracts themselves (ByteAlu[XOR], AreBytes) — these are existing, + separately-audited chips (`prover/src/tables/bitwise.rs`). Same assumption the + keccak gate makes. +* **Not modeled here:** the memory/syscall I/O variant (§1.2) — orthogonal; + open questions O5 (counter width, already covered by the Plonky3 anchor) and + O6 (endianness at the MEMW boundary) live there and must be pinned when that + interface is wired. diff --git a/thoughts/blake3/blake3-chip/IMPLEMENTATION.md b/thoughts/blake3/blake3-chip/IMPLEMENTATION.md new file mode 100644 index 000000000..fda2cd634 --- /dev/null +++ b/thoughts/blake3/blake3-chip/IMPLEMENTATION.md @@ -0,0 +1,138 @@ +# BLAKE3 chip — implementation notes (syscall variant) + +Companion to `DESIGN.md`: what the Rust implementation +(`prover/src/tables/blake3.rs` + the executor syscall) does differently from +the internal-variant design, and why. The design's §7 soundness ledger is +reproduced in the chip's module docs with per-item dispositions. + +## Variant + +DESIGN.md §1.1 designs the **lean internal interface** (a `Blake3` bus with a +parent-node caller). No in-circuit caller exists yet, so what is built is the +**§1.2 general syscall variant**: `Ecall` receiver + `Memw` register read of +x10 + per-dword `Memw` I/O, copied idiom-for-idiom from `keccak.rs`. The +internal bus — and with it §7 item 10 (TIMESTAMP binding in both bus tuples) — +does not exist in this variant: a row's inputs and outputs are tied by being +committed on the same row. + +ABI: `x10` → 8-aligned 176-byte region, `h[32] | m[64] | t[8] | +block_len,flags[8] | out[64]` (see `BLAKE3_SYSCALL_NUMBER` docs). Syscall +number `u64::MAX - 2`. + +## Deltas from the design's cell accounting + +| item | DESIGN §3 | implemented | why | +|---|---|---|---| +| add2 carries | 1 committed bit each (§3 table; 6 bits/G) | **expression carry, no cell** (4 bits/G) | §4.3's own formula is the `emit_add_pair` expression form; the §3 table double-counts it. Saves 96 cells/row. | +| G block | 62 cells | **60 cells** (56 bytes + 4 bits) | above | +| I/O apparatus | none (internal variant) | +8 addr bytes, +88 ptr halfwords, +64 OLD_OUT | syscall variant | +| OLD_OUT | n/a | 64 committed bytes + 32 AreBytes | the 8 out-dword `Memw` ops need the previous memory content in their `old` field; those bytes ride only the Memw bus, so they get explicit byte checks (same aliasing argument as keccak.rs's addr bytes) | +| columns | ≈3,155 | **3,219** | | +| sends | ≈1,250 | **1,397** (832 XOR + 384 shift-AreBytes + 32 m + 32 old_out + 4 addr + 1 AND + 88 IS_HALF + 24 I/O) | | +| aux (3·⌈N/2⌉) | ≈1,875 | **2,097** | | +| **cell-equiv/compression** | ≈5,030 | **≈5,316** | +5.7% for the syscall I/O | + +Against keccak-f post-#889 (72,672 cell-equiv): **≈ 1/13.7 per call**, ~6.4× +per byte (64 B vs 136 B absorbed). + +## The single-dataflow rule + +The compression dataflow exists once (`run_flow`), interpreted twice: +`WireFlow` (columns → constraints + bus senders) and `ValueFlow` (u32 → +trace filling + BITWISE multiplicities). Wiring divergence between prover +cells, senders and receiver multiplicities is therefore impossible by +construction; only interpretation bugs remain, and those are what the oracle +vectors + the e2e bus-balance gate check. + +## Gates run + +- executor ↔ oracle: the 10 pinned canonical 6-round vectors + (`canonical_6round_vectors.json`, full-width `t` values — the counter-split + order is load-bearing) + syscall-level tests (alignment/overflow rejection, + input-region non-clobbering). +- `ValueFlow` ↔ executor: differential unit test. +- wire audit: every committed mixing cell written exactly once, in-range + (unit test `wire_flow_counts`). +- e2e: `test_prove_elfs_blake3` — two chained compressions (the second + consumes the first's output and overwrites a non-zero out region), prove + + verify, which exercises bus balance across Ecall/Memw/ByteAlu/AreBytes/ + IsHalfword. +- the z3 gate (`z3_blake_verify.py`) proves the *design*; the transcription + design → Rust is covered by the vectors + e2e, per the gate's own + documentation of what it cannot see (§7 items 4, 5, 11). +- **`--full` monolithic UNSATs, run 2026-08-06: ATTEMPTED-INCONCLUSIVE, not + satisfied.** `z3_blake_verify.py --full` ran ~145 min and exited 1 + (`OVERALL: FAIL`). All four monolithic queries hit z3's resource limit: + + ``` + round (clean) -> unknown (want unsat) + compress rounds=2 -> unknown (want unsat) + compress rounds=6 -> unknown (want unsat) + compress rounds=7 -> unknown (want unsat) + ``` + + `unknown` is the timeout return — the checks `s.set("timeout", timeout_ms)` + then `return s.check()` (`z3_blake_verify.py:320-321`, `:340-341`), and the + verdict tests `== unsat` (line 553), so a timeout scores `False` and drags + OVERALL to FAIL. Timing corroborates a clean sweep of timeouts: the budgets + are 30+30+40+40 = 140 min against ~145 min wall. **No counterexample was + found — nothing was disproven — but no monolithic UNSAT was obtained + either.** The fast board is unchanged and fully green: + + ``` + G-function UNSAT (covers all G) : True + init+feed-forward UNSAT (rounds=0): True + negative controls all SAT : True + positive controls all SAT : True (full 6-/7-round pipeline, concrete) + ``` + + DESIGN.md §7 item 11's "run `--full`'s monolithic UNSAT before shipping + Rust" precondition is therefore **attempted but not satisfied**; the + coverage of the 48 unrolled G instances still rests on the concrete + positive controls plus the per-instance index mutant, which do pass. + Remediation: rerun with a much larger timeout budget on a server (the run + is single-threaded and CPU-bound), and/or restructure the monolithic query + as round-by-round induction instead of one flat bit-vector problem. + +## Known costs and open items + +- **Always-on AIR**: `FIXED_TABLE_COUNT` 10 → 11. Every proof now carries a + ≥4-row BLAKE3 table (~3.2k cols) even when unused. This is exactly the + EC-campaign regression shape (PR #871, +3 near-empty AIRs → +25%); one + near-empty table is far smaller, but a real-block ABBA is REQUIRED before + merge. +- The proof wire format changes (one more sub-proof); old proofs do not + verify against this branch. The recursion guest would need a rebuild + (the in-repo recursion PoC is already non-functional, see project notes). +- `count_table_lengths` (disk-spill sizing) does not count the 23 Memw ops a + blake3 ecall contributes; disk-spill runs of blake3-heavy workloads would + size MEMW slightly small. Not exercised by the bench (no disk-spill). +- 7-round variant: `BLAKE3_ROUNDS` is the single knob; columns/constraints/ + sends all derive from it. Standard-BLAKE3 compatibility would also need the + flags/t plumbed per the tree mode (out of scope here). + +## The 6-round assumption (sign-off record) + +The chip implements 6 rounds, not the standard 7. The z3 gate proves the chip +matches the 6-round reference; it does not and cannot prove 6 rounds are +collision-resistant. Adopting this for Merkle/Fiat–Shamir rests on the named +assumption: + +> **A6R**: the BLAKE3 compression function restricted to 6 rounds is +> collision-resistant and suitable as a 2-to-1 compression for Merkle +> hashing and as a PRF for Fiat–Shamir, in the same sense the full 7-round +> function is believed to be (precedent: KangarooTwelve's reduced-round +> Keccak). + +Directed for implementation by the project owner, 2026-08-05 ("trust me" +sign-off in session). Recorded in the spec (`spec/blake3.typ`, A6R section). + +**External review (2026-08-06, relayed by the project owner):** the round +count was reviewed with external symmetric-cryptography experts — removing +one round (7→6) judged comfortable, removing two (7→5) explicitly not. +6 rounds is therefore the endorsed floor. Sub-6 variants are not formally +dead, but they cannot be adopted on the project's own authority — that +would need the experts to sit with the reduced margin specifically +(dedicated cryptanalytic review, not an engineering call). The 7-round +instantiation remains available as the zero-assumption / interop fallback +at ~10-12% more per merge. diff --git a/thoughts/blake3/blake3-chip/z3_blake_verify.py b/thoughts/blake3/blake3-chip/z3_blake_verify.py new file mode 100644 index 000000000..f6a439f98 --- /dev/null +++ b/thoughts/blake3/blake3-chip/z3_blake_verify.py @@ -0,0 +1,561 @@ +""" +Formal (z3 / QF_BV) assume-guarantee gate for the BLAKE3 compression chip design. + +Method (mirrors ../keccak-verify/z3_verify.py): + * Every committed column of the designed chip is a FREE bitvector. + * Every bus lookup (under its precomputed-table contract) and every eval + constraint becomes an equation relating those free vars. + * The chip OUTPUT is whatever the constraints force. We assert + `output != reference(input)` and ask z3 for a counterexample: + UNSAT -> for all constraint-satisfying assignments, output == reference + (the chip is correctly & tightly constrained). + SAT -> the constraints permit a wrong output (under-constrained / mis-wired). + +The reference (`bref_*`) is written directly from the BLAKE3 spec with 32-bit +BV ops (RotateRight / + / ^) — structurally INDEPENDENT of the chip's byte-level +XOR / halfword-shift wiring, exactly like keccak's zref_round vs the byte circuit. + +Chip contracts assumed (assume-guarantee, from prover/src/tables/bitwise.rs): + ByteAlu[XOR](a,b)->c : a,b,c are bytes and c = a ^ b. (8-bit width = byte + range-check; output pinned by the precomputed table.) + AreBytes[a,b] : a,b are bytes (8-bit width). + (HWSL is NOT used: rotations are inlined as the mu-gated linear shift identity + in*2^r == SLLC*2^16 + SLL, whose soundness is proven by ../keccak-verify/ + hwsl_inline_test.py given the AreBytes 16-bit bounds + 2^16 invertible mod p.) + +Add carries and shift decompositions are eval constraints (mu-gated, degree <=3); +here mu=1 (a real row), so mu drops out and we model the ungated equation. + +DESIGN DECISIONS UNDER TEST (see DESIGN.md): + * State stored as bytes; XOR byte-wise via ByteAlu[XOR]. + * rotr16 / rotr8 : FREE byte relabels (no columns, no lookups). + * rotr12 / rotr7 : inner rotl r=4 / r=9 -> two halfword shift-identities + + cross-halfword recombine + halfword swap. + * 2-operand add : one carry bit, a+b == s + 2^32*carry, s range-checked. + * 3-operand add : O1 option (c) -- TWO summed carry bits c1,c2 in {0,1}, + a+b+m == s + 2^32*(c1+c2). (No committed intermediate word; + degree stays <=3 after mu-gating, unlike k(k-1)(k-2).) +""" +import sys +import json +import os +from z3 import ( + BitVec, BitVecVal, Concat, ZeroExt, RotateRight, Or, And, Solver, sat, unsat, + Int, IntVal, +) + +# --------------------------------------------------------------------------- +# BLAKE3 constants (spec; cross-checked against Plonky3 in the oracle) +# --------------------------------------------------------------------------- +IV = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19] +MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] +MASK32 = 0xFFFFFFFF +WIDE = 48 # wide BV width for add / shift identities (honest < 2^35 << 2^48) +P = 2**64 - 2**32 + 1 # Goldilocks prime (used in the width-audit field checks) + +# G-function schedule: (a,b,c,d, mx_index, my_index) for the 8 calls of a round. +G_CALLS = [ + (0, 4, 8, 12, 0, 1), + (1, 5, 9, 13, 2, 3), + (2, 6, 10, 14, 4, 5), + (3, 7, 11, 15, 6, 7), + (0, 5, 10, 15, 8, 9), + (1, 6, 11, 12, 10, 11), + (2, 7, 8, 13, 12, 13), + (3, 4, 9, 14, 14, 15), +] + +# =========================================================================== +# Independent z3-native reference (BLAKE3 spec, 32-bit BV words) +# =========================================================================== +def bref_g(v, a, b, c, d, mx, my): + v[a] = v[a] + v[b] + mx + v[d] = RotateRight(v[d] ^ v[a], 16) + v[c] = v[c] + v[d] + v[b] = RotateRight(v[b] ^ v[c], 12) + v[a] = v[a] + v[b] + my + v[d] = RotateRight(v[d] ^ v[a], 8) + v[c] = v[c] + v[d] + v[b] = RotateRight(v[b] ^ v[c], 7) + + +def bref_round(v, m): + for (a, b, c, d, ix, iy) in G_CALLS: + bref_g(v, a, b, c, d, m[ix], m[iy]) + + +def bref_permute(m): + return [m[MSG_PERMUTATION[i]] for i in range(16)] + + +def bref_round_only(state16, msg16): + """One round, free 16-word state + free 16-word message -> new state.""" + v = list(state16) + bref_round(v, msg16) + return v + + +def bref_compress(h, m, tlo, thi, bl, fl, rounds): + """Full compression. h:8 BV32, m:16 BV32, counter split tlo/thi, bl, fl.""" + v = [h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], + BitVecVal(IV[0], 32), BitVecVal(IV[1], 32), + BitVecVal(IV[2], 32), BitVecVal(IV[3], 32), + tlo, thi, bl, fl] + schedule = list(m) + for r in range(rounds): + bref_round(v, schedule) + if r < rounds - 1: + schedule = bref_permute(schedule) + out = [None] * 16 + for i in range(8): + out[i] = v[i] ^ v[i + 8] + out[i + 8] = v[i + 8] ^ h[i] + return out + + +# =========================================================================== +# Chip circuit model. A "word" is a list of 4 free 8-bit BVs [b0,b1,b2,b3] +# (little-endian). Byte width == the ByteAlu/AreBytes range-check contract. +# =========================================================================== +class Circuit: + def __init__(self, tag, bug=None): + self.C = [] + self.tag = tag + self.bug = bug + self.n = 0 + + def _fresh(self, w=8): + v = BitVec(f"{self.tag}_v{self.n}", w) + self.n += 1 + return v + + def fresh_word(self): + return [self._fresh(8) for _ in range(4)] + + def const_word(self, val): + return [BitVecVal((val >> (8 * i)) & 0xFF, 8) for i in range(4)] + + # -- value helpers ----------------------------------------------------- + def wval(self, word): + """word as a WIDE-bit BV integer (little-endian byte combination).""" + acc = BitVecVal(0, WIDE) + for i in range(4): + acc = acc + ZeroExt(WIDE - 8, word[i]) * BitVecVal(1 << (8 * i), WIDE) + return acc + + def hwval(self, blo, bhi): + """halfword (2 bytes) as a WIDE-bit BV.""" + return ZeroExt(WIDE - 8, blo) + ZeroExt(WIDE - 8, bhi) * BitVecVal(256, WIDE) + + def word32(self, word): + return Concat(word[3], word[2], word[1], word[0]) + + def fresh_bit(self, boolean=True): + v = self._fresh(8) + if boolean: + self.C.append(Or(v == 0, v == 1)) # mu-gated IS_BIT (mu=1 here) + return v + + # -- operations under contract ---------------------------------------- + def xor(self, A, B): + """ByteAlu[XOR]: out byte-wise = A ^ B (auto byte range-check).""" + out = self.fresh_word() + for i in range(4): + self.C.append(out[i] == A[i] ^ B[i]) + return out + + def rotr16(self, A): + # rotate-right 16 == swap halfwords == byte relabel [b2,b3,b0,b1]. FREE. + return [A[2], A[3], A[0], A[1]] + + def rotr8(self, A): + # rotate-right 8 == byte relabel [b1,b2,b3,b0]. FREE. + return [A[1], A[2], A[3], A[0]] + + def add2(self, A, B, drop_bool=False): + """2-operand add mod 2^32: a+b == s + 2^32*carry, carry in {0,1}.""" + s = self.fresh_word() + carry = self.fresh_bit(boolean=not drop_bool) + self.C.append( + self.wval(A) + self.wval(B) + == self.wval(s) + ZeroExt(WIDE - 8, carry) * BitVecVal(1 << 32, WIDE) + ) + return s + + def add3(self, A, B, M, drop_bool=False): + """3-operand add mod 2^32 (O1 option c): TWO summed carry bits. + a+b+m == s + 2^32*(c1+c2), c1,c2 in {0,1}.""" + s = self.fresh_word() + c1 = self.fresh_bit(boolean=not drop_bool) + c2 = self.fresh_bit(boolean=not drop_bool) + csum = ZeroExt(WIDE - 8, c1) + ZeroExt(WIDE - 8, c2) + self.C.append( + self.wval(A) + self.wval(B) + self.wval(M) + == self.wval(s) + csum * BitVecVal(1 << 32, WIDE) + ) + return s + + def rotr(self, A, n, wrong_amount=False): + """rotr12 / rotr7 via inner rotl r + halfword swap. + + r=4 for n=12 (rotl20=rotl16.rotl4); r=9 for n=7 (rotl25=rotl16.rotl9). + Shift identity (inline, mu-gated): hw*2^r == SLLC*2^16 + SLL, with SLL + the tight 16-bit remainder and SLLC the (loose 16-bit) quotient. Then + Y_lo = SLL_hi + SLLC_lo, Y_hi = SLL_lo + SLLC_hi (non-overlapping adds). + """ + r = {12: 4, 7: 9}[n] + if wrong_amount: + r += 1 # negative control: wrong rotation amount + xlo = self.hwval(A[0], A[1]) + xhi = self.hwval(A[2], A[3]) + # SLL / SLLC as free halfwords (each = 2 free bytes -> AreBytes 16-bit). + sll_lo = self.fresh_word()[:2] + sllc_lo = self.fresh_word()[:2] + sll_hi = self.fresh_word()[:2] + sllc_hi = self.fresh_word()[:2] + SLL_lo, SLLC_lo = self.hwval(*sll_lo), self.hwval(*sllc_lo) + SLL_hi, SLLC_hi = self.hwval(*sll_hi), self.hwval(*sllc_hi) + two_r = BitVecVal(1 << r, WIDE) + two_16 = BitVecVal(1 << 16, WIDE) + # shift identities + self.C.append(xlo * two_r == SLLC_lo * two_16 + SLL_lo) + self.C.append(xhi * two_r == SLLC_hi * two_16 + SLL_hi) + # recombine (rotl_r) + halfword swap (rotl16) + Y = self.fresh_word() + self.C.append(self.hwval(Y[0], Y[1]) == SLL_hi + SLLC_lo) # Y low halfword + self.C.append(self.hwval(Y[2], Y[3]) == SLL_lo + SLLC_hi) # Y high halfword + return Y + + +# --------------------------------------------------------------------------- +# Build one round of the chip (free input state + free message). +# --------------------------------------------------------------------------- +def build_g(cir, v, a, b, c, d, mx, my, bug, gflag): + b_first = c if (bug == "swap_g_operand" and gflag) else b # WRONG: v[c] for v[b] + v[a] = cir.add3(v[a], v[b_first], mx) + v[d] = cir.rotr16(cir.xor(v[d], v[a])) + v[c] = cir.add2(v[c], v[d]) + v[b] = cir.rotr(cir.xor(v[b], v[c]), 12, + wrong_amount=(bug == "rot_wrong_amount" and gflag)) + v[a] = cir.add3(v[a], v[b], my, + drop_bool=(bug == "drop_carry_bool" and gflag)) + v[d] = cir.rotr8(cir.xor(v[d], v[a])) + v[c] = cir.add2(v[c], v[d]) + v[b] = cir.rotr(cir.xor(v[b], v[c]), 7) + + +def build_round(cir, v, m, bug=None, bug_first_g_only=True): + for gi, (a, b, c, d, ix, iy) in enumerate(G_CALLS): + gflag = (gi == 0) if bug_first_g_only else True + build_g(cir, v, a, b, c, d, m[ix], m[iy], bug, gflag) + + +def build_compress(cir, h, m, tlo, thi, bl, fl, rounds, bug=None): + iv = list(IV) + if bug == "wrong_iv": + iv[0] ^= 1 # negative control + v = [h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], + cir.const_word(iv[0]), cir.const_word(iv[1]), + cir.const_word(iv[2]), cir.const_word(iv[3]), + tlo, thi, bl, fl] + perm = list(MSG_PERMUTATION) + if bug == "wrong_msg_index": + perm[0], perm[1] = perm[1], perm[0] # negative control + schedule = list(m) + for r in range(rounds): + # only inject round-logic bugs in round 0's first G + rbug = bug if (r == 0 and bug in + ("rot_wrong_amount", "swap_g_operand", "drop_carry_bool")) else None + build_round(cir, v, schedule, bug=rbug) + if r < rounds - 1: + schedule = [schedule[perm[i]] for i in range(16)] + out = [None] * 16 + for i in range(8): + out[i] = cir.xor(v[i], v[i + 8]) + out[i + 8] = cir.xor(v[i + 8], h[i]) + if bug == "drop_ff_xor" and i == 0: + out[0] = cir.fresh_word() # dropped: output left free + return out + + +# =========================================================================== +# Checks +# =========================================================================== +def check_g(bug=None, timeout_ms=0): + """Single G-function vs reference G. Free 4 state words + 2 message words. + UNSAT = the G quarter-round is correctly & tightly constrained. A round is a + fixed composition of 8 G-calls on specified indices, so a correct G under + arbitrary inputs => correct round (the chaining argument).""" + tag = "g" + (f"_{bug}" if bug else "") + cir = Circuit(tag, bug) + va, vb, vc, vd = (cir.fresh_word(), cir.fresh_word(), + cir.fresh_word(), cir.fresh_word()) + mx, my = cir.fresh_word(), cir.fresh_word() + v = [None] * 16 + v[0], v[1], v[2], v[3] = va, vb, vc, vd + build_g(cir, v, 0, 1, 2, 3, mx, my, bug, gflag=True) + rv = [cir.word32(va), cir.word32(vb), cir.word32(vc), cir.word32(vd)] + bref_g(rv, 0, 1, 2, 3, cir.word32(mx), cir.word32(my)) + s = Solver() + if timeout_ms: + s.set("timeout", timeout_ms) + s.add(And(*cir.C)) + s.add(Or(cir.word32(v[0]) != rv[0], cir.word32(v[1]) != rv[1], + cir.word32(v[2]) != rv[2], cir.word32(v[3]) != rv[3])) + return s.check() + + +def check_round(bug=None, timeout_ms=0): + """Round circuit vs reference round. Free state + free message. UNSAT = correct.""" + tag = "rnd" + (f"_{bug}" if bug else "") + cir = Circuit(tag, bug) + state = [cir.fresh_word() for _ in range(16)] + msg = [cir.fresh_word() for _ in range(16)] + v = list(state) + build_round(cir, v, msg, bug=bug) + ref = bref_round_only([cir.word32(w) for w in state], + [cir.word32(w) for w in msg]) + s = Solver() + if timeout_ms: + s.set("timeout", timeout_ms) + s.add(And(*cir.C)) + s.add(Or(*[cir.word32(v[i]) != ref[i] for i in range(16)])) + return s.check() + + +def check_compress(rounds, bug=None, timeout_ms=0): + """Full compression vs reference. UNSAT = correct.""" + tag = f"cmp{rounds}" + (f"_{bug}" if bug else "") + cir = Circuit(tag, bug) + h = [cir.fresh_word() for _ in range(8)] + m = [cir.fresh_word() for _ in range(16)] + tlo, thi, bl, fl = (cir.fresh_word(), cir.fresh_word(), + cir.fresh_word(), cir.fresh_word()) + out = build_compress(cir, h, m, tlo, thi, bl, fl, rounds, bug=bug) + ref = bref_compress([cir.word32(w) for w in h], [cir.word32(w) for w in m], + cir.word32(tlo), cir.word32(thi), cir.word32(bl), + cir.word32(fl), rounds) + s = Solver() + if timeout_ms: + s.set("timeout", timeout_ms) + s.add(And(*cir.C)) + s.add(Or(*[cir.word32(out[i]) != ref[i] for i in range(16)])) + return s.check() + + +def positive_control_compress(rounds, h_i, m_i, tlo_i, thi_i, bl_i, fl_i, out_i): + """Non-vacuity + external anchor: pin inputs to a concrete oracle vector, + assert the chip output == the RECORDED oracle output, expect SAT.""" + tag = f"pos{rounds}" + cir = Circuit(tag) + h = [cir.fresh_word() for _ in range(8)] + m = [cir.fresh_word() for _ in range(16)] + tlo, thi, bl, fl = (cir.fresh_word(), cir.fresh_word(), + cir.fresh_word(), cir.fresh_word()) + out = build_compress(cir, h, m, tlo, thi, bl, fl, rounds) + s = Solver() + s.add(And(*cir.C)) + # pin inputs + for wi, val in zip(h, h_i): + s.add(cir.word32(wi) == BitVecVal(val, 32)) + for wi, val in zip(m, m_i): + s.add(cir.word32(wi) == BitVecVal(val, 32)) + s.add(cir.word32(tlo) == BitVecVal(tlo_i, 32)) + s.add(cir.word32(thi) == BitVecVal(thi_i, 32)) + s.add(cir.word32(bl) == BitVecVal(bl_i, 32)) + s.add(cir.word32(fl) == BitVecVal(fl_i, 32)) + # pin output to the recorded oracle vector + for wi, val in zip(out, out_i): + s.add(cir.word32(wi) == BitVecVal(val, 32)) + return s.check() + + +# =========================================================================== +# WIDTH AUDIT: field-level (mod p) bound-necessity for the shift identity and +# the add carry. A wide-BV model cannot show these (2^16 / 2^32 are zero +# divisors mod 2^n); the prime field is required, exactly as +# ../keccak-verify/hwsl_inline_test.py Part 2 demonstrates. +# =========================================================================== +def field_shift_bound(r, in_hw, drop_sll_bound): + """hw*2^r == SLLC*2^16 + SLL (mod p). SLL bounded to [0,2^16) unless dropped. + Returns 'unsat' if SLL is pinned to the honest value; 'sat' if ambiguous.""" + s = Solver() + if drop_sll_bound: + SLL = Int("SLL"); s.add(SLL >= 0, SLL < P) # UNBOUNDED field elt + else: + lo, hi = Int("sll_lo"), Int("sll_hi") + s.add(lo >= 0, lo < 256, hi >= 0, hi < 256) # AreBytes: 2 bytes + SLL = lo + 256 * hi + SLLC = Int("SLLC") + s.add(SLLC >= 0, SLLC < 2**16) # loose 16-bit is fine + s.add((in_hw * (2 ** r) - SLLC * (2 ** 16) - SLL) % P == 0) + sll_ref = (in_hw * (2 ** r)) % (2 ** 16) + s.add(SLL != sll_ref) # a WRONG SLL admissible? + return str(s.check()) + + +def field_add_carry(a, b, m3, drop_bool): + """3-op: a+b+m == s + 2^32*(c1+c2) (mod p). s in [0,2^32). carries in {0,1} + unless dropped. Returns 'unsat' if s pinned to honest, 'sat' if ambiguous.""" + s = Solver() + S = Int("S"); s.add(S >= 0, S < 2**32) + if drop_bool: + c1 = Int("c1"); s.add(c1 >= 0, c1 < P) # UNBOUNDED + csum = c1 + else: + c1, c2 = Int("c1"), Int("c2") + s.add(Or(c1 == 0, c1 == 1), Or(c2 == 0, c2 == 1)) + csum = c1 + c2 + s.add((a + b + m3 - S - (2**32) * csum) % P == 0) + s_ref = (a + b + m3) % (2**32) + s.add(S != s_ref) + return str(s.check()) + + +# =========================================================================== +def load_canonical_6round(): + here = os.path.dirname(os.path.abspath(__file__)) + path = os.path.join(here, "..", "blake3-oracle", "canonical_6round_vectors.json") + with open(path) as f: + return json.load(f) + + +def gen_7round_vector(): + """Concrete 7-round compression vector from the validated oracle itself.""" + here = os.path.dirname(os.path.abspath(__file__)) + sys.path.insert(0, os.path.join(here, "..", "blake3-oracle")) + import blake3_ref as ora + import random + rng = random.Random(12345) + h = [rng.randrange(0, 1 << 32) for _ in range(8)] + m = [rng.randrange(0, 1 << 32) for _ in range(16)] + t = rng.randrange(0, 1 << 64) + bl = rng.randrange(0, 65) + fl = rng.randrange(0, 128) + out = ora.compress(h, m, t, bl, fl, rounds=7) + return h, m, t & MASK32, (t >> 32) & MASK32, bl, fl, out + + +def main(): + full = "--full" in sys.argv + print("=" * 70) + print("BLAKE3 compression-chip z3 gate") + print("=" * 70) + + # --- MAIN CHECK 0: single G (fundamental unit; covers every G/round) -- + print("\n=== MAIN CHECK 0: one G-function, free inputs (covers every G) ===") + g = check_g() + print(f" G (clean) -> {g} (want unsat)") + g_ok = (g == unsat) + + # --- MAIN CHECK 1: init-state layout + feed-forward (rounds=0) -------- + # Tiny & symbolic: v = initial state, then the feed-forward XORs. Isolates + # the h/IV/counter-split placement and out[i]=v[i]^v[i+8], out[i+8]=v[i+8]^h[i]. + print("\n=== MAIN CHECK 1: init-state + feed-forward (rounds=0, symbolic) ===") + r0 = check_compress(0) + print(f" compress rounds=0 -> {r0} (want unsat)") + wrapper_ok = (r0 == unsat) + + # --- Heavy symbolic multi-round UNSATs: BONUS, gated behind --full ---- + round_ok = None + full6 = full7 = full2 = None + if full: + print("\n=== MAIN CHECK 2 (--full): one round, free state+message ===") + rr = check_round(timeout_ms=1_800_000) + print(f" round (clean) -> {rr} (want unsat)") + round_ok = (rr == unsat) + print("\n=== MAIN CHECK 3 (--full): compression rounds=2 (permutation+chaining) ===") + full2 = check_compress(2, timeout_ms=1_800_000) + print(f" compress rounds=2 -> {full2} (want unsat)") + print("\n=== MAIN CHECK 4 (--full): FULL compression rounds=6 and rounds=7 ===") + full6 = check_compress(6, timeout_ms=2_400_000) + print(f" compress rounds=6 -> {full6} (want unsat)") + full7 = check_compress(7, timeout_ms=2_400_000) + print(f" compress rounds=7 -> {full7} (want unsat)") + else: + print("\n=== Heavy symbolic multi-round UNSATs skipped (pass --full) ===") + print(" G-unsat + fixed G-composition (chaining) already prove every round;") + print(" rounds=0 proves init+feed-forward; the message permutation is") + print(" proven load-bearing by the wrong_msg_index control and exercised") + print(" concretely by the full 6-/7-round positive controls below.") + + # --- NEGATIVE CONTROLS (must all be SAT) ----------------------------- + print("\n=== NEGATIVE CONTROLS — STRUCTURAL bugs (BV-observable, must be SAT) ===") + # NB: 'dropped carry booleanity' is deliberately NOT here. Dropping a carry + # column's booleanity is a FIELD-level soundness bug: an unconstrained + # committed column is a full field element, but in a *bounded BV* model the + # 8-bit carry + the s in [0,2^32) byte-range still pins s, so BV reports + # UNSAT. It is demonstrated correctly in the WIDTH AUDIT below (drop -> SAT), + # exactly as ../keccak-verify/hwsl_inline_test.py Part 2 requires the prime + # field to show HWSL bound-necessity. This is a feature: the gate separates + # BV-observable logic bugs from field-only soundness bugs. + controls = {} + controls["rot_wrong_amount"] = check_g(bug="rot_wrong_amount") # wrong rotation amount + controls["swap_g_operand"] = check_g(bug="swap_g_operand") # swapped G operand + controls["wrong_iv"] = check_compress(1, bug="wrong_iv") # wrong IV constant + controls["drop_ff_xor"] = check_compress(1, bug="drop_ff_xor") # dropped feed-forward XOR + controls["wrong_msg_index"] = check_compress(2, bug="wrong_msg_index") # wrong msg-schedule index + for name, res in controls.items(): + print(f" bug={name:18s} -> {res} (want sat)") + controls_ok = all(res == sat for res in controls.values()) + + # --- POSITIVE CONTROLS (external anchor: pin to oracle vectors) ------- + print("\n=== POSITIVE CONTROLS (pin input+output to oracle vectors -> SAT) ===") + vecs = load_canonical_6round() + pos_ok = True + for vec in vecs[:3]: + res = positive_control_compress( + 6, vec["h"], vec["m"], vec["t"] & MASK32, (vec["t"] >> 32) & MASK32, + vec["block_len"], vec["flags"], vec["out"]) + ok = (res == sat) + pos_ok &= ok + print(f" 6round seed={vec['seed']} (canonical) -> {res} (want sat)") + h7, m7, tlo7, thi7, bl7, fl7, out7 = gen_7round_vector() + res7 = positive_control_compress(7, h7, m7, tlo7, thi7, bl7, fl7, out7) + pos_ok &= (res7 == sat) + print(f" 7round (oracle-generated) -> {res7} (want sat)") + + # --- WIDTH AUDIT (field-level bound-necessity) ----------------------- + # These are the FIELD-level negative controls (BV provably cannot show them, + # since 2^16 / 2^32 are zero divisors mod 2^n). 'DROP -> sat' == the bug is + # exploitable in the prime field; 'present -> unsat' == the range check pins + # the value. Includes the 'dropped carry booleanity' control (team-lead #4). + print("\n=== WIDTH AUDIT + FIELD-LEVEL NEGATIVE CONTROLS (mod p bound necessity) ===") + a_sh = field_shift_bound(9, 0x9C3A, drop_sll_bound=False) + b_sh = field_shift_bound(9, 0x9C3A, drop_sll_bound=True) + print(f" shift r=9 AreBytes SLL bound present -> {a_sh} (want unsat: pinned)") + print(f" shift r=9 DROP SLL bound (neg ctrl) -> {b_sh} (want sat: forgeable)") + a_ad = field_add_carry(0xF0000000, 0xF0000000, 0xF0000000, drop_bool=False) + b_ad = field_add_carry(0xF0000000, 0xF0000000, 0xF0000000, drop_bool=True) + print(f" 3-add carry booleanity present -> {a_ad} (want unsat: pinned)") + print(f" 3-add DROP booleanity (neg ctrl #4) -> {b_ad} (want sat: forgeable)") + audit_ok = (a_sh == "unsat" and b_sh == "sat" and a_ad == "unsat" and b_ad == "sat") + + # --- VERDICT ---------------------------------------------------------- + print("\n" + "=" * 70) + print("VERDICT") + print("=" * 70) + print(f" G-function UNSAT (covers all G) : {g_ok}") + print(f" init+feed-forward UNSAT (rounds=0): {wrapper_ok}") + if full: + print(f" round UNSAT (direct) : {round_ok}") + print(f" compress rounds=2 UNSAT : {full2 == unsat}") + print(f" full 6-round UNSAT : {full6 == unsat}") + print(f" full 7-round UNSAT : {full7 == unsat}") + print(f" negative controls all SAT : {controls_ok}") + print(f" positive controls all SAT : {pos_ok} (full 6-/7-round pipeline, concrete)") + print(f" width audit (bound necessity) : {audit_ok}") + # G correctness + fixed G-composition => round correctness (chaining); + # rounds=0 => init+feed-forward; positive controls run the full pipeline + # concretely; the direct multi-round UNSATs (--full) are bonus confirmation. + base_ok = g_ok and wrapper_ok and controls_ok and pos_ok and audit_ok + full_ok = (not full) or (round_ok and full2 == unsat + and full6 == unsat and full7 == unsat) + ok = base_ok and full_ok + print(f"\n OVERALL: {'PASS' if ok else 'FAIL — investigate above'}") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/thoughts/blake3/blake3-oracle/.gitignore b/thoughts/blake3/blake3-oracle/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/thoughts/blake3/blake3-oracle/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/thoughts/blake3/blake3-oracle/ORACLE.md b/thoughts/blake3/blake3-oracle/ORACLE.md new file mode 100644 index 000000000..aee71d1f0 --- /dev/null +++ b/thoughts/blake3/blake3-oracle/ORACLE.md @@ -0,0 +1,368 @@ +# BLAKE3 Compression-Function Oracle + +**Purpose.** Trust anchor for a future BLAKE3 accelerator (precompile chip) in +the Lambda VM STARK prover. Phase 1 = this oracle (the reference `f` + external +validation + chip-contract reuse map). Phase 2 = chip constraint design, gated +against this oracle. The oracle is the reference the chip's trace generation and +constraints will be checked against; a wrong oracle silently poisons everything +downstream, so the validation section is the load-bearing part. + +**Scope.** The reference is the BLAKE3 **compression function** `f`, NOT the full +tree hash. `blake3_ref.py` also contains a full tree hasher, but that exists +*only* so `f` can be validated against the official whole-hash test vectors. The +chip implements `f`; it does not implement the tree. + +--- + +## 1. Validation status: **VALIDATED** + +`test_oracle.py` passes all of the following (re-run: `./venv/bin/python test_oracle.py`): + +| # | External anchor | Independent of our code? | What it covers | Result | +|---|---|---|---|---| +| 1 | Official **`test_vectors.json`** (BLAKE3 team, `test_vectors/test_vectors.json`, fetched from the BLAKE3 GitHub repo) | Yes — authored by the BLAKE3 authors | 35 input lengths (0 … 102400 B) × 3 modes (default hash, keyed hash, derive-key), extended (131-byte) output | **PASS 35/35 × 3** | +| 2 | Official **`blake3` PyPI package** v1.0.9 (the reference Rust implementation via FFI) | Yes — separate codebase | 23 randomised input lengths (0 … 100000 B) × {default, XOF, keyed, derive-key} = 92 differential checks | **PASS 92/92** | +| 3 | **Plonky3 `blake3-air`** compression, ported in `test_oracle.py` from `others/Plonky3/blake3-air/src/generation.rs` | Yes — Plonky3 team, different codebase | 20 000 random `(h, m, t, block_len)` compared at the **compression-function level** (flags = 0, 7 rounds) | **PASS 20000/20000** | + +Anchors 1–2 validate `f` *indirectly but exhaustively*: the whole-hash path +drives `f` under every flag combination (`CHUNK_START`, `CHUNK_END`, `PARENT`, +`ROOT`, `KEYED_HASH`, `DERIVE_KEY_CONTEXT`, `DERIVE_KEY_MATERIAL` and their +compositions) and a wide range of counters (chunk indices 0…99 for the 102400 B +case, plus XOF output-block counters). Anchor 3 validates `f` **directly** at the +compression level against a second independent implementation (flags = 0 only, +since Plonky3's AIR hardcodes `v[15] = 0`). + +The constants were independently cross-checked: `IV` and `MSG_PERMUTATION` in +`blake3_ref.py` match `others/Plonky3/blake3-air/src/constants.rs` (`IV` stored +there as `[lo16, hi16]` pairs; `MSG_PERMUTATION = [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8]`). + +> Note: the BLAKE3 repo's `reference_impl/reference_impl.py` returned HTTP 404 at +> fetch time (repo layout changed), so it is **not** used. `f` was written from +> the spec's G-function definition; the three anchors above stand on their own. + +--- + +## 2. Precise definition of both variants + +Everything is on 32-bit unsigned words, little-endian. `⊞` = add mod 2³², +`⊕` = XOR, `x ⋙ n` = rotate-right by `n` bits. + +### 2.1 Constants + +``` +IV = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19] + +MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] +``` + +### 2.2 The G function (quarter round) + +`G(v, a, b, c, d, mx, my)` mutates working-state words `v[a], v[b], v[c], v[d]`: + +``` +v[a] = v[a] ⊞ v[b] ⊞ mx +v[d] = (v[d] ⊕ v[a]) ⋙ 16 +v[c] = v[c] ⊞ v[d] +v[b] = (v[b] ⊕ v[c]) ⋙ 12 +v[a] = v[a] ⊞ v[b] ⊞ my +v[d] = (v[d] ⊕ v[a]) ⋙ 8 +v[c] = v[c] ⊞ v[d] +v[b] = (v[b] ⊕ v[c]) ⋙ 7 +``` + +### 2.3 The round + +Given the (already permuted-for-this-round) 16-word schedule `m`: + +``` +# columns +G(v, 0, 4, 8, 12, m[0], m[1]) +G(v, 1, 5, 9, 13, m[2], m[3]) +G(v, 2, 6, 10, 14, m[4], m[5]) +G(v, 3, 7, 11, 15, m[6], m[7]) +# diagonals +G(v, 0, 5, 10, 15, m[8], m[9]) +G(v, 1, 6, 11, 12, m[10], m[11]) +G(v, 2, 7, 8, 13, m[12], m[13]) +G(v, 3, 4, 9, 14, m[14], m[15]) +``` + +### 2.4 The compression function `f` (parameterised by `ROUNDS`) + +Inputs: `h[0..8]` (chaining value, 8×u32), `m[0..16]` (message block, 16×u32), +`t` (u64 counter), `block_len` (u32, 0..64), `flags` (u32). + +``` +v[0..8] = h[0..8] +v[8..12] = IV[0..4] +v[12] = t mod 2³² # counter low +v[13] = t >> 32 # counter high +v[14] = block_len +v[15] = flags + +schedule = m +for r in 0 .. ROUNDS-1: + round(v, schedule) + schedule = permute(schedule) # trailing permute after last round is unused + +# feed-forward (produces the FULL 16-word output) +for i in 0..8: + out[i] = v[i] ⊕ v[i+8] + out[i+8] = v[i+8] ⊕ h[i] +return out[0..16] +``` + +The truncated 8-word chaining value used inside the tree is `out[0:8]`. The XOF +root output uses **all 16** output words — this is why `f` returns 16 words. + +### 2.5 Variant A — standard: `ROUNDS = 7` + +The function above with `ROUNDS = 7`. This is standard BLAKE3, validated by +anchors 1–3. + +### 2.6 Variant B — nonstandard: `ROUNDS = 6` + +**Exactly** the function in §2.4 with `ROUNDS = 6`: rounds 0..5 are applied, +round `r` mixing `permute^r(m)`, followed by the identical feed-forward. The +ONLY difference from Variant A is the loop bound. This is a **NONSTANDARD** +function; **no external test vectors exist**. Its anchoring is derivative: + +* **(a) Code-diff anchor.** In `blake3_ref.py`, `compress_6round(...)` is literally + `compress(..., rounds=6)` — same IV, same initial-state layout, same G, same + message permutation schedule, same feed-forward. `test_oracle.py` + (`test_6round_derivation`) asserts `compress_6round == compress(rounds=6)` and + that it differs from `ROUNDS=7` on 2000/2000 random inputs. +* **(b) Canonical vectors.** 10 deterministic vectors (fixed seeds 0..9) are + generated and recorded below. These are Variant B's canonical reference going + forward. Full inputs/outputs are in `canonical_6round_vectors.json`. + +#### Canonical 6-round vectors (seeds 0..9) + +Each row: 32-hex-digit words. `out` is the full 16-word output concatenated +(`out[0]` first). Inputs `h` (8 words), `m` (16 words), `t`, `block_len`, +`flags` are in `canonical_6round_vectors.json`; a summary fingerprint is shown +here (`out[0]` and `out[15]`) so the doc alone pins the vectors' identity. + +| seed | t | block_len | flags | out[0] | out[15] | +|---|---|---|---|---|---| +| 0 | 0xb4e1357d4a84eb03 | 42 | 0x34 | 0xced9d1ff | 0xb75f3915 | +| 1 | 0xc74803e31ba16215 | 50 | 0x5e | 0xf2a972e9 | 0xdfb91125 | +| 2 | 0x7604e4b4e73695c3 | 58 | 0x7c | 0x5aa6b114 | 0x775f2f92 | +| 3 | 0x92d3043afcf249f3 | 36 | 0x1f | 0xeed92fab | 0xdc293166 | +| 4 | 0x49c7b59b995253fd | 57 | 0x29 | 0xca00bda3 | 0x7561eb37 | +| 5 | 0x6a3753915c76f18a | 18 | 0x43 | 0x14a9f66f | 0xbb7a485d | +| 6 | 0x390567c27bd6aa42 | 26 | 0x03 | 0x32a6ff70 | 0x2a7a62b2 | +| 7 | 0x12bd4acefaecbd38 | 53 | 0x2a | 0xa632ad45 | 0xf3f33689 | +| 8 | 0x329911da9fbd8735 | 19 | 0x5b | 0x913b2ae1 | 0x3c5a654b | +| 9 | 0xeaeb999b8a2e547e | 64 | 0x15 | 0xf5ee9114 | 0xd18a8b94 | + +(To re-derive: `random.Random(seed)` then draw `h=8×u32, m=16×u32, t=u64, +block_len∈[0,65), flags∈[0,128)` in that order — see +`test_oracle.canonical_6round_vectors`.) + +--- + +## 3. Chip-contract reuse map + +Every primitive op of `f` mapped onto the existing precomputed-table contracts. +Citations are to `prover/src/tables/bitwise.rs` (the 2²⁰-row BITWISE table) and +the KECCAK chips, which are the architectural template for a byte-oriented +delegation chip. + +The BITWISE table (`bitwise.rs:97`, `NUM_ROWS = 256·256·16 = 2²⁰`) is indexed by +`(X: byte, Y: byte, Z: 4-bit)` and provides these receivers +(`bitwise.rs:715` `bus_interactions`): + +* `ByteAlu[opsel, X, Y] → out` — byte AND/OR/XOR (`bitwise.rs:865-921`; `opsel` + ∈ {AND, OR, XOR}). The output column is a table column, so a `ByteAlu` send + **simultaneously range-checks X and Y to be bytes and pins `out` to the exact + result** — no separate range check needed on any of the three. +* `ARE_BYTES[X, Y]` — range-check two bytes (`bitwise.rs:783`; pass `Y=0` for a + single byte). +* `IS_HALF[X + 256·Y]` — range-check a 16-bit halfword (`bitwise.rs:798`). +* `IS_B20[...]` — 20-bit range check (`bitwise.rs:813`). +* `HWSL[X + 256·Y, Z] → [SLL, SLLC]` — halfword shift-left (`bitwise.rs:831`), + where `SLL = (hw << Z) & 0xFFFF`, `SLLC = hw >> (16 - Z)` (`bitwise.rs:135-141`), + `Z ∈ [0,16)`. +* `MSB8`, `MSB16`, `ZERO` — not needed by BLAKE3. + +### 3.1 Op-by-op mapping + +| BLAKE3 primitive | Existing contract | How | Cost | +|---|---|---|---| +| **32-bit XOR** (`v[d]⊕v[a]`, `v[b]⊕v[c]`, feed-forward) | `ByteAlu[XOR]` | 4 byte-XOR lookups per 32-bit word, one per byte, exactly as `keccak_rnd` does θ/χ/ι XORs (`keccak_rnd.rs:692-718`). Inputs & output auto-range-checked by the lookup. | 4 sends / 32-bit XOR | +| **`⋙ 16`** | *free* — byte relabeling | rotr16 permutes bytes `[b0,b1,b2,b3] → [b2,b3,b0,b1]`. **VERIFIED** exhaustively (100k random words). No lookup, no column: just re-address the bytes at the next use. | 0 | +| **`⋙ 8`** | *free* — byte relabeling | rotr8 → `[b1,b2,b3,b0]`. **VERIFIED**. | 0 | +| **`⋙ 12`** | `HWSL` (+ `ARE_BYTES`) | rotr12 = rotl20; per the keccak-ρ pattern, HWSL each of the 2 halfwords by `rnc=4`, then a halfword rotate by `rbc=1`, recombining `newlo = SLL_lo + SLLC_hi`, `newhi = SLL_hi + SLLC_lo` (non-overlapping bit ranges ⇒ add = OR), then swap the two halfwords. **VERIFIED** (50k random). Range-check the 4 output bytes with `ARE_BYTES` (as keccak does on ρ outputs, `keccak_rnd.rs:768-790`). | 2 HWSL + 2 ARE_BYTES / rot | +| **`⋙ 7`** | `HWSL` (+ `ARE_BYTES`) | rotr7 = rotl25; same pattern with `rnc=9`, `rbc=1`. **VERIFIED**. `rnc=9 < 16` fits HWSL's 4-bit `Z`. | 2 HWSL + 2 ARE_BYTES / rot | +| **32-bit add mod 2³²** (2-operand `v[c]⊞v[d]`) | carry-bit polynomial constraint + range-check | Exactly `emit_add_pair`'s low half (`templates.rs:334`): with sum `s` committed and range-checked, `carry = (a + b − s)·2⁻³²` is constrained `carry·(1−carry)=0` (`INV_SHIFT_32 = (2³²)⁻¹`, `templates.rs:26`). Sum bytes are range-checked *for free* because `s` immediately feeds an XOR lookup. | 1 poly constraint / add | +| **3-operand add mod 2³²** (`v[a]⊞v[b]⊞mx`) | carry constraint (see §5 open Q) | `a+b+m < 3·2³²` ⇒ carry ∈ {0,1,2}. Either one virtual `k(k−1)(k−2)=0` (deg 3) or two chained `emit_add_pair` steps (deg ≤ 2). See open question O1. | 1–2 poly constraints / add | +| **message schedule** (`permute` between rounds) | *free* — wiring | Fixed compile-time permutation of the 16 input words per round; round `r` references `permute^r`-indexed message columns. No table, exactly like `keccak_rnd` inlines `KECCAK_RHO` offsets as compile-time constants. **Confirmed.** | 0 | +| **IV constants, flags, block_len, counter split** | constants / direct columns | `IV[0..4] → v[8..12]`, `t` split into `v[12]=t mod 2³²`, `v[13]=t>>32`, `v[14]=block_len`, `v[15]=flags`. Constants inlined; counter split is two committed words range-checked. | ~0 | + +**No BLAKE3 op lacks an existing contract.** All arithmetic reduces to +`ByteAlu[XOR]`, `HWSL`, `ARE_BYTES`, and the `emit_add_pair` carry template — +every one already exercised by the KECCAK chips. The 32-bit-add carry range +checks fit `ARE_BYTES`/`IS_HALF` (the sum's bytes/halfwords), and the carry +itself is a `{0,1}` (or `{0,1,2}`) polynomial bit, not a table lookup. + +### 3.2 Why the two "free" rotations are actually free + +`ByteAlu` and `HWSL` operate at byte / halfword granularity, and the working +state is stored as bytes. A rotate-right by a multiple of 8 is a permutation of +byte positions, so the constraint at the *consuming* site simply reads the bytes +in rotated order (the same trick keccak uses implicitly). Only `⋙12` and `⋙7` +cross byte boundaries and therefore need HWSL. This means **half** of BLAKE3's +rotations cost nothing. + +--- + +## 4. I/O column boundary sketch + +Analogous to keccak's 200-byte state handoff (`keccak.rs`), the chip's +bus-facing tuple. Recommended **granularity: bytes** — because XOR (the dominant +op) needs byte operands and the two byte-aligned rotations are free at byte +granularity; adds read bytes as a linear combination (`AddOperand::from_dword_bl`, +`templates.rs:191`) so byte storage costs them nothing. + +**Chip input** (read from guest memory via the ECALL/MEMW interface, exactly the +keccak pattern `keccak.rs:160-449`: ECALL receiver binds the syscall + timestamp, +a MEMW read of `x10` binds the state pointer, then per-word MEMW reads): + +| field | size | granularity | +|---|---|---| +| `h[0..8]` chaining value | 8 words = 32 B | bytes | +| `m[0..16]` message block | 16 words = 64 B | bytes | +| `t` counter | u64 = 8 B | 2 words (lo, hi), byte-stored | +| `block_len` | u32 | 1 word | +| `flags` | u32 | 1 word | + +**Chip output** (written back to memory): + +| field | size | granularity | +|---|---|---| +| `out[0..16]` | 16 words = 64 B | bytes | + +For the truncated (CV-only) call sites the guest reads back `out[0:8]`; the chip +always produces the full 16 words (the XOF root needs them). + +**Internal handoff (if one-row-per-round).** If the chip mirrors keccak's +round-chip split, a `Blake3Round` bus carries `(timestamp, round_index, +state[16 words as 64 bytes], message[16 words])` from row `r` to row `r+1`, +mirroring `keccak_rnd`'s `(timestamp, round, start[200])` handoff +(`keccak_rnd.rs:441-515`). Note BLAKE3 must also carry the (round-permuted) +message down the rounds, unlike keccak whose round chip has no message input. + +--- + +## 5. Cost estimate & recommended granularity + +Cost model (given): a **committed** cell is expensive; each **bus send** ≈ 1.5 +base cells of aux; **max constraint degree 3** is a hard cap. + +### Per-round work (8 G calls; each G = 2 three-operand adds, 2 two-operand adds, +4 XORs, 4 rotations of which 2 are free): + +| resource | per round | note | +|---|---|---| +| `ByteAlu[XOR]` sends | 8·4·4 = **128** | 4 XORs/G × 4 bytes | +| `HWSL` sends | 8·2·2 = **32** | 2 non-free rots/G × 2 halfwords | +| `ARE_BYTES` (rot-output range checks) | ~**32** | 2 rots/G × 4 bytes ÷ 2-per-send | +| add carry constraints | ~**48** | (16 three-op + 16 two-op adds)/round | +| committed byte-cells (state + G intermediates + carries) | ~**450** | ~10 words/G committed × 8 G × 4 B + input state | + +Bus sends/round ≈ 128 + 32 + 32 ≈ **~190**; aux ≈ 190 × 1.5 ≈ **~290** base +cells; committed ≈ **~450**. Total ≈ **~740 cell-equivalents/round**. + +### Per compression (7 rounds + feed-forward + I/O): + +* XOR lookups: 7·128 + 64 (feed-forward) ≈ **~960** +* HWSL lookups: 7·32 ≈ **~224** +* Range-check sends: ~7·32 + I/O ≈ **~250** +* **Total bus sends ≈ ~1450**, aux ≈ ~2200 base cells +* Committed ≈ 7·450 + I/O ≈ **~3300** base cells +* **Grand total ≈ ~5000–6000 cell-equivalents per compression**, dominated by + the ~960 byte-XOR lookups. + +For scale: a keccak-f permutation is ~24 rounds × 1480 cols. A BLAKE3 +compression is roughly **¼–⅓ of one keccak permutation**. + +### Recommended layout + +BLAKE3 has only **7 rounds** (vs keccak's 24). Two viable shapes: + +* **A. One row per round** (keccak-style): ~450–750 columns/row × 7 rows, plus a + `Blake3Round` internal handoff bus carrying state **and** the permuted message. + Fewer columns, but the message-carrying handoff is extra bus traffic keccak + doesn't have. +* **B. One row per compression** (fully unrolled): ~3000–3500 columns in a single + row; no internal handoff bus, no round-index bookkeeping. The message schedule + is pure compile-time wiring so unrolling is natural. + +**Recommendation: start with B (one row per compression).** With only 7 rounds +the column count (~3k) is comparable to keccak's per-round width, and eliminating +the internal state+message handoff bus removes the biggest source of aux cost and +constraint complexity. Revisit A only if the committed width dominates trace-area +budget. Either way the cell total is the same order (~5–6k). + +--- + +## 6. Open questions for the chip phase + +* **O1 — 3-operand add carry granularity (the main one).** `v[a] = v[a] ⊞ v[b] ⊞ + mx` sums three 32-bit values, so the carry-out is in **{0,1,2}**, not {0,1}. + `emit_add_pair` (`templates.rs:334`) only handles a `{0,1}` carry. Options: + 1. **One virtual carry ∈ {0,1,2}:** commit the sum `s` (range-checked), + `k = (a+b+m−s)·2⁻³²`, constrain `k(k−1)(k−2)=0`. This is **degree 3** — at + the cap. It cannot also be `μ`-gated (that would be degree 4). Feasible only + if padding rows satisfy it ungated (all-zero padding ⇒ `k=0` ⇒ satisfied, + the keccak padding convention — verify this holds for BLAKE3 padding). + 2. **Two chained adds:** `t = a ⊞ b` (carry ∈ {0,1}), then `a' = t ⊞ mx` (carry + ∈ {0,1}), each via `emit_add_pair`, at the cost of one extra committed 32-bit + intermediate `t` per 3-operand add (16 extra words/round). Stays degree ≤ 2, + so it can be `μ`-gated to degree 3. Simpler and gate-friendly. + * **Recommendation:** option 2 (chained adds) unless the extra committed width + is measured to hurt — it keeps every add uniformly `{0,1}`-carry and leaves + degree headroom for `μ`-gating. Decide with a bench once the chip exists. + +* **O2 — carry-bit gating & padding.** Decide whether add-carry and rot + constraints are `μ`-gated (like `keccak_rnd`'s IS_BIT, `keccak_rnd.rs:914`) or + rely on all-zero padding rows satisfying them ungated. This interacts with O1's + degree budget. + +* **O3 — one-row-per-round vs unrolled (§5).** Ties to O2 and to whether the + message schedule is carried on a handoff bus or wired per-row at compile time. + +* **O4 — flags coverage of the direct anchor.** Anchor 3 (Plonky3) only checks + `flags = 0` at the compression level; non-zero flags are validated only through + the whole-hash anchors 1–2. If the chip is ever exercised on raw compression + inputs with arbitrary flags outside a valid tree, add a direct differential + check against the PyPI package's low-level API if/when it exposes `compress` + (it currently does not). + +* **O5 — counter (`t`) width.** The whole-hash anchors drive `t` only up to ~99 + (chunk index) plus small XOF counters. The chip must accept a full u64 `t` + (`v[12]/v[13]` split). Constants and the split are validated structurally, but + if the chip supports enormous counters, add a targeted vector. (Plonky3 anchor + already exercises random full-width u64 `t` — so this is **covered**.) + +* **O6 — endianness at the memory boundary.** BLAKE3 words are little-endian; + the byte-granular I/O sketch (§4) assumes LE byte order in memory. Confirm + against the guest's `blake3` calling convention when wiring MEMW. + +--- + +## 7. File manifest + +``` +blake3-oracle/ +├── blake3_ref.py # reference f (ROUNDS-parameterised) + 6-round variant + tree hasher +├── test_oracle.py # anchors 1-3 + 6-round derivation + canonical-vector emitter +├── ORACLE.md # this document +├── official_test_vectors.json # BLAKE3 team vectors (fetched, unmodified) +├── canonical_6round_vectors.json # 10 canonical Variant-B vectors (generated) +└── venv/ # python venv with the official `blake3` pkg (anchor 2) +``` + +No repository files were modified. diff --git a/thoughts/blake3/blake3-oracle/blake3_ref.py b/thoughts/blake3/blake3-oracle/blake3_ref.py new file mode 100644 index 000000000..ee2d48e85 --- /dev/null +++ b/thoughts/blake3/blake3-oracle/blake3_ref.py @@ -0,0 +1,399 @@ +""" +BLAKE3 compression-function ORACLE (reference implementation). + +This is the TRUST ANCHOR for a future BLAKE3 accelerator chip in the Lambda VM +STARK prover. It is written directly from the BLAKE3 specification / reference +design, NOT copied from any implementation, and then validated externally in +`test_oracle.py` against: + - the official BLAKE3 team's `test_vectors.json`, + - the official `blake3` PyPI package (the reference Rust implementation), + - Plonky3's independent `blake3-air` compression implementation. + +Spec sources used while writing this file (all public): + - BLAKE3 paper / spec, section 2.1-2.2 (compression function, G, round). + - The reference message-permutation schedule and IV constants, which also + appear verbatim in the vendored Plonky3 `blake3-air/src/constants.rs` + (IV, MSG_PERMUTATION) — used here only as a cross-check of the constants, + the mixing logic is written from the spec's G-function definition. + +Everything operates on 32-bit unsigned words, little-endian, exactly as BLAKE3 +specifies. +""" + +# --------------------------------------------------------------------------- +# Constants (BLAKE3 spec, section 2.1) +# --------------------------------------------------------------------------- + +# Initialisation vector: the first 8 words of the SHA-256 IV (fractional parts +# of the square roots of the first 8 primes). Identical to SHA-256 / BLAKE2s. +IV = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +] + +# Message word permutation applied between successive rounds. After each round +# the 16 message words are permuted by this index map; round r therefore mixes +# the original message under permutation^r. (BLAKE3 spec / reference schedule.) +MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + +# Domain-separation flags (BLAKE3 spec, table of flags). +CHUNK_START = 1 << 0 # 0x01 +CHUNK_END = 1 << 1 # 0x02 +PARENT = 1 << 2 # 0x04 +ROOT = 1 << 3 # 0x08 +KEYED_HASH = 1 << 4 # 0x10 +DERIVE_KEY_CONTEXT = 1 << 5 # 0x20 +DERIVE_KEY_MATERIAL = 1 << 6 # 0x40 + +# Structural sizes. +BLOCK_LEN = 64 # bytes per compression input block (16 words * 4 bytes) +CHUNK_LEN = 1024 # bytes per chunk (16 blocks) +KEY_LEN = 32 # bytes in a key / chaining value (8 words * 4 bytes) +OUT_LEN = 32 # default output length in bytes + +MASK32 = 0xFFFFFFFF + +# Standard round count for BLAKE3. Variant B is the same function with ROUNDS=6. +DEFAULT_ROUNDS = 7 + + +# --------------------------------------------------------------------------- +# 32-bit word primitives (BLAKE3 spec, section 2.1 "G function") +# --------------------------------------------------------------------------- + +def add32(a, b): + """Addition modulo 2^32 (wrapping).""" + return (a + b) & MASK32 + + +def rotr(x, n): + """Rotate the 32-bit word `x` RIGHT by `n` bits. + + BLAKE3's G uses rotation amounts 16, 12, 8, 7. Rotations by 16 and 8 are + byte-aligned (multiples of 8); 12 and 7 are not. The chip-contract reuse + map in ORACLE.md analyses each of these against the HWSL lookup table. + """ + x &= MASK32 + return ((x >> n) | (x << (32 - n))) & MASK32 + + +def g(state, a, b, c, d, mx, my): + """The BLAKE3 quarter-round mixing function G (spec section 2.1). + + Mixes two message words `mx`, `my` into four state words at indices + a, b, c, d of the 16-word working state. Two "half rounds" of the form + add / xor+rotate: + + v[a] = v[a] + v[b] + mx + v[d] = (v[d] ^ v[a]) >>> 16 + v[c] = v[c] + v[d] + v[b] = (v[b] ^ v[c]) >>> 12 + v[a] = v[a] + v[b] + my + v[d] = (v[d] ^ v[a]) >>> 8 + v[c] = v[c] + v[d] + v[b] = (v[b] ^ v[c]) >>> 7 + """ + state[a] = add32(add32(state[a], state[b]), mx) + state[d] = rotr(state[d] ^ state[a], 16) + state[c] = add32(state[c], state[d]) + state[b] = rotr(state[b] ^ state[c], 12) + state[a] = add32(add32(state[a], state[b]), my) + state[d] = rotr(state[d] ^ state[a], 8) + state[c] = add32(state[c], state[d]) + state[b] = rotr(state[b] ^ state[c], 7) + + +def round_fn(state, m): + """One BLAKE3 round: 4 column mixes then 4 diagonal mixes (spec 2.1). + + `m` is the (already-permuted for this round) 16-word message schedule. + The G calls consume message words m[0..16] in order. + """ + # Mix the columns. + g(state, 0, 4, 8, 12, m[0], m[1]) + g(state, 1, 5, 9, 13, m[2], m[3]) + g(state, 2, 6, 10, 14, m[4], m[5]) + g(state, 3, 7, 11, 15, m[6], m[7]) + # Mix the diagonals. + g(state, 0, 5, 10, 15, m[8], m[9]) + g(state, 1, 6, 11, 12, m[10], m[11]) + g(state, 2, 7, 8, 13, m[12], m[13]) + g(state, 3, 4, 9, 14, m[14], m[15]) + + +def permute(m): + """Apply MSG_PERMUTATION to a 16-word message list, returning a new list.""" + return [m[MSG_PERMUTATION[i]] for i in range(16)] + + +# --------------------------------------------------------------------------- +# The compression function `f` (BLAKE3 spec, section 2.2) +# --------------------------------------------------------------------------- + +def compress(chaining_value, block_words, counter, block_len, flags, + rounds=DEFAULT_ROUNDS): + """BLAKE3 compression function. + + Inputs: + chaining_value : list of 8 u32 words (h[0..8]) + block_words : list of 16 u32 words (m[0..16]) + counter : u64 block counter t + block_len : u32 number of input bytes in this block (0..64) + flags : u32 domain-separation flags + rounds : number of rounds (7 = standard, 6 = variant B) + + Returns a list of 16 u32 words: the full compression output. The truncated + 8-word chaining value used elsewhere in the tree is `output[0:8]`. + + The 16-word initial working state v is: + v[0..8] = chaining_value[0..8] + v[8..12] = IV[0..4] + v[12] = counter mod 2^32 (low 32 bits of t) + v[13] = counter >> 32 (high 32 bits of t) + v[14] = block_len + v[15] = flags + Then `rounds` rounds are applied, permuting the message schedule between + rounds. Finally the feed-forward XOR produces the 16-word output: + output[i] = v[i] ^ v[i+8] for i in 0..8 + output[i+8] = v[i+8] ^ chaining_value[i] for i in 0..8 + """ + assert len(chaining_value) == 8 + assert len(block_words) == 16 + assert 0 <= counter < (1 << 64) + + counter_low = counter & MASK32 + counter_high = (counter >> 32) & MASK32 + + state = [ + chaining_value[0], chaining_value[1], chaining_value[2], chaining_value[3], + chaining_value[4], chaining_value[5], chaining_value[6], chaining_value[7], + IV[0], IV[1], IV[2], IV[3], + counter_low & MASK32, counter_high & MASK32, block_len & MASK32, flags & MASK32, + ] + + # Local copy of the message schedule; permuted between rounds. + m = list(block_words) + for r in range(rounds): + round_fn(state, m) + # Permute between rounds. The permutation after the final round is + # never consumed, so applying it only for r < rounds-1 is equivalent; + # we permute between rounds to keep the loop structure obvious. + if r < rounds - 1: + m = permute(m) + + # Feed-forward XOR producing the full 16-word output. + output = [0] * 16 + for i in range(8): + output[i] = state[i] ^ state[i + 8] + output[i + 8] = state[i + 8] ^ chaining_value[i] + return output + + +def compress_cv(chaining_value, block_words, counter, block_len, flags, + rounds=DEFAULT_ROUNDS): + """The truncated 8-word chaining value: first 8 words of `compress`.""" + return compress(chaining_value, block_words, counter, block_len, flags, + rounds)[:8] + + +# =========================================================================== +# Variant B: 6-round BLAKE3 compression. +# +# This is EXACTLY `compress(..., rounds=6)`. It is a NONSTANDARD function with +# no external test vectors; ORACLE.md documents its canonical vectors. The only +# difference from the validated 7-round function is the loop bound `rounds`. +# =========================================================================== + +def compress_6round(chaining_value, block_words, counter, block_len, flags): + """6-round variant of the BLAKE3 compression function (variant B). + + Rounds 0..5 are applied with message permutations 0..5 (i.e. round r mixes + permute^r(block_words)), then the identical feed-forward XOR finalisation. + Everything else — IV, initial state layout, G function, feed-forward — is + bit-for-bit identical to the 7-round function. + """ + return compress(chaining_value, block_words, counter, block_len, flags, + rounds=6) + + +# =========================================================================== +# Full BLAKE3 tree hash, built ON TOP of `compress`. +# +# This exists ONLY so the compression function can be validated against the +# official whole-hash test vectors (which exercise `compress` under every flag +# combination and many counter values). The chip does NOT implement the tree; +# it implements `compress`. Written from the spec's tree/chunk structure. +# =========================================================================== + +def words_from_le_bytes(b): + """Convert a bytes object (len multiple of 4) into a list of u32 words.""" + assert len(b) % 4 == 0 + return [int.from_bytes(b[i:i + 4], "little") for i in range(0, len(b), 4)] + + +def le_bytes_from_words(words): + return b"".join((w & MASK32).to_bytes(4, "little") for w in words) + + +class _Output: + """A not-yet-finalised node (chunk or parent). Can emit a chaining value + or an extendable root output (spec section 2.3, XOF).""" + + def __init__(self, input_cv, block_words, counter, block_len, flags, rounds): + self.input_cv = input_cv + self.block_words = block_words + self.counter = counter + self.block_len = block_len + self.flags = flags + self.rounds = rounds + + def chaining_value(self): + return compress(self.input_cv, self.block_words, self.counter, + self.block_len, self.flags, self.rounds)[:8] + + def root_output_bytes(self, out_len): + out = bytearray() + counter = 0 + while len(out) < out_len: + words = compress(self.input_cv, self.block_words, counter, + self.block_len, self.flags | ROOT, self.rounds) + # The ROOT output uses ALL 16 output words (this is why compress + # returns 16 words rather than the truncated 8). + out += le_bytes_from_words(words) + counter += 1 + return bytes(out[:out_len]) + + +class _ChunkState: + def __init__(self, key_words, chunk_counter, flags, rounds): + self.cv = list(key_words) + self.chunk_counter = chunk_counter + self.block = b"" + self.blocks_compressed = 0 + self.flags = flags + self.rounds = rounds + + def _start_flag(self): + return CHUNK_START if self.blocks_compressed == 0 else 0 + + def update(self, data): + while data: + if len(self.block) == BLOCK_LEN: + block_words = words_from_le_bytes(self.block) + self.cv = compress(self.cv, block_words, self.chunk_counter, + BLOCK_LEN, self.flags | self._start_flag(), + self.rounds)[:8] + self.blocks_compressed += 1 + self.block = b"" + take = min(BLOCK_LEN - len(self.block), len(data)) + self.block += data[:take] + data = data[take:] + + def output(self): + block_words = words_from_le_bytes(self.block + b"\x00" * (BLOCK_LEN - len(self.block))) + return _Output(self.cv, block_words, self.chunk_counter, len(self.block), + self.flags | self._start_flag() | CHUNK_END, self.rounds) + + +def _parent_output(left_cv, right_cv, key_words, flags, rounds): + block_words = left_cv + right_cv # 16 words + return _Output(list(key_words), block_words, 0, BLOCK_LEN, flags | PARENT, rounds) + + +class Blake3Hasher: + """Minimal BLAKE3 tree hasher over the reference `compress`. + + Supports the three official modes (default hash, keyed hash, derive-key) + and extendable output, so it can be checked against `test_vectors.json`. + """ + + def __init__(self, key_words, flags, rounds=DEFAULT_ROUNDS): + self.key_words = list(key_words) + self.flags = flags + self.rounds = rounds + self.chunk_state = _ChunkState(self.key_words, 0, flags, rounds) + self.cv_stack = [] # list of 8-word chaining values + + @classmethod + def default(cls, rounds=DEFAULT_ROUNDS): + return cls(IV, 0, rounds) + + @classmethod + def keyed(cls, key32, rounds=DEFAULT_ROUNDS): + assert len(key32) == KEY_LEN + return cls(words_from_le_bytes(key32), KEYED_HASH, rounds) + + @classmethod + def derive_key(cls, context_string, rounds=DEFAULT_ROUNDS): + # Phase 1: hash the context string in DERIVE_KEY_CONTEXT mode to get a + # 32-byte context key; Phase 2: keyed-hash the material with that key + # under DERIVE_KEY_MATERIAL. + ctx_hasher = cls(IV, DERIVE_KEY_CONTEXT, rounds) + ctx_hasher.update(context_string.encode("utf-8") if isinstance(context_string, str) else context_string) + context_key = ctx_hasher.finalize(KEY_LEN) + return cls(words_from_le_bytes(context_key), DERIVE_KEY_MATERIAL, rounds) + + def _add_chunk_cv(self, new_cv, total_chunks): + # Merge the CV stack following the binary-tree structure. A completed + # subtree is merged whenever the total chunk count is even at that level. + while total_chunks & 1 == 0: + left = self.cv_stack.pop() + new_cv = _parent_output(left, new_cv, self.key_words, self.flags, + self.rounds).chaining_value() + total_chunks >>= 1 + self.cv_stack.append(new_cv) + + def update(self, data): + data = bytes(data) + while data: + if len(self.chunk_state.block) == BLOCK_LEN and \ + self.chunk_state.blocks_compressed == CHUNK_LEN // BLOCK_LEN - 1: + # current chunk is full: finalise it and start a new one. + chunk_cv = self.chunk_state.output().chaining_value() + total_chunks = self.chunk_state.chunk_counter + 1 + self._add_chunk_cv(chunk_cv, total_chunks) + self.chunk_state = _ChunkState(self.key_words, total_chunks, + self.flags, self.rounds) + # How many bytes still fit in the current chunk. + want = CHUNK_LEN - self._chunk_len() + take = min(want, len(data)) + self.chunk_state.update(data[:take]) + data = data[take:] + + def _chunk_len(self): + return self.chunk_state.blocks_compressed * BLOCK_LEN + len(self.chunk_state.block) + + def finalize(self, out_len=OUT_LEN): + # Walk the current chunk's output up the CV stack, XORing/parenting all + # the way to the root, and emit the root output. + output = self.chunk_state.output() + parent_nodes_remaining = len(self.cv_stack) + while parent_nodes_remaining > 0: + parent_nodes_remaining -= 1 + left = self.cv_stack[parent_nodes_remaining] + output = _parent_output(left, output.chaining_value(), + self.key_words, self.flags, self.rounds) + return output.root_output_bytes(out_len) + + +def blake3_hash(data, out_len=OUT_LEN, rounds=DEFAULT_ROUNDS): + h = Blake3Hasher.default(rounds) + h.update(data) + return h.finalize(out_len) + + +def blake3_keyed_hash(key32, data, out_len=OUT_LEN, rounds=DEFAULT_ROUNDS): + h = Blake3Hasher.keyed(key32, rounds) + h.update(data) + return h.finalize(out_len) + + +def blake3_derive_key(context_string, key_material, out_len=OUT_LEN, rounds=DEFAULT_ROUNDS): + h = Blake3Hasher.derive_key(context_string, rounds) + h.update(key_material) + return h.finalize(out_len) + + +if __name__ == "__main__": + # Tiny smoke test: empty-input default hash (compare to test_oracle.py). + print("blake3('') =", blake3_hash(b"").hex()) diff --git a/thoughts/blake3/blake3-oracle/canonical_6round_vectors.json b/thoughts/blake3/blake3-oracle/canonical_6round_vectors.json new file mode 100644 index 000000000..10ee8cbc1 --- /dev/null +++ b/thoughts/blake3/blake3-oracle/canonical_6round_vectors.json @@ -0,0 +1,522 @@ +[ + { + "seed": 0, + "h": [ + 3626764237, + 1806341205, + 2195908194, + 2046968324, + 3900315155, + 2167613558, + 1210484339, + 3246154361 + ], + "m": [ + 3874773259, + 1332073689, + 3134603515, + 2937688618, + 432508404, + 1864753826, + 3921352636, + 2048741382, + 1118805955, + 60308648, + 3726325546, + 3738645480, + 2437440079, + 4155553746, + 1924014660, + 4006490763 + ], + "t": 13033757608824335107, + "block_len": 42, + "flags": 52, + "out": [ + 3470381567, + 3259559595, + 3171982207, + 2434484470, + 2453496512, + 3624177727, + 1500783166, + 2857307264, + 2908815487, + 3037433307, + 3879152609, + 869521091, + 1118447691, + 3315752744, + 2041348976, + 3076471061 + ] + }, + { + "seed": 1, + "h": [ + 3280387012, + 1095513148, + 1930549411, + 2798570523, + 3387541014, + 403123852, + 3589583794, + 1912923437 + ], + "m": [ + 4059906722, + 3871601465, + 131383004, + 2325348894, + 1001090105, + 92297589, + 2758633299, + 3693442237, + 2878940490, + 1302957853, + 3790218436, + 2170177477, + 148287319, + 3424825176, + 743061144, + 1609337231 + ], + "t": 14359731685826847253, + "block_len": 50, + "flags": 94, + "out": [ + 4071191273, + 2180888812, + 1086656188, + 1268894457, + 2666129712, + 1796871858, + 3910496071, + 2829038646, + 2734036659, + 310856722, + 813072437, + 3759806425, + 3202728316, + 3592162272, + 809631558, + 3753447717 + ] + }, + { + "seed": 2, + "h": [ + 242886303, + 364522461, + 3588440356, + 1323436024, + 2602510382, + 2606193617, + 4077622522, + 117874757 + ], + "m": [ + 1632151663, + 2258090960, + 2407373688, + 1014142328, + 102469680, + 1396478261, + 2191394736, + 3837860530, + 3422057796, + 3276568223, + 1519503515, + 4131333072, + 3238422834, + 2277860467, + 2104593779, + 3972123491 + ], + "t": 8504173462006699459, + "block_len": 58, + "flags": 124, + "out": [ + 1520873748, + 3386274828, + 2268646132, + 2891926386, + 2680601054, + 1060043663, + 2360846610, + 4103578245, + 1023812198, + 2132949004, + 2949933306, + 304921216, + 1147868525, + 2990135490, + 3286938319, + 2002726802 + ] + }, + { + "seed": 3, + "h": [ + 2337446730, + 2593816829, + 3596902313, + 1006443827, + 2045921456, + 646892613, + 2726705791, + 2247046192 + ], + "m": [ + 3183652505, + 275012945, + 2538753386, + 3717411168, + 3774472248, + 3956088670, + 4018314376, + 3774703581, + 418563100, + 583981819, + 931951836, + 1292897679, + 2512874164, + 2509342356, + 3883517040, + 3989790985 + ], + "t": 10579804601021778419, + "block_len": 36, + "flags": 31, + "out": [ + 4007210923, + 328045400, + 2438725180, + 326208257, + 3037127287, + 3191867341, + 897875462, + 3457968278, + 1392116149, + 1252158200, + 2970061409, + 743537389, + 2693293984, + 3933130730, + 1832113072, + 3693687142 + ] + }, + { + "seed": 4, + "h": [ + 1013818839, + 1701057193, + 665600858, + 285680177, + 3942586889, + 3286348376, + 2305023086, + 456053774 + ], + "m": [ + 3983477513, + 3464545456, + 3437897285, + 830799655, + 1330795424, + 3779789200, + 2602114036, + 2884935804, + 2173054921, + 763602979, + 2034044485, + 1289545638, + 3903568191, + 3789523705, + 2183442722, + 1777884721 + ], + "t": 5316417565031027709, + "block_len": 57, + "flags": 41, + "out": [ + 3389046179, + 2216925754, + 3888680557, + 866690006, + 165466574, + 2712732178, + 4102951254, + 2399377685, + 2315607722, + 4284158421, + 3072657499, + 773501543, + 1793536573, + 3003084712, + 1896007841, + 1969351479 + ] + }, + { + "seed": 5, + "h": [ + 2675342405, + 3185950873, + 4051686260, + 2787324501, + 3869338171, + 486215926, + 1059022248, + 2335435112 + ], + "m": [ + 2465058629, + 930847394, + 1200367645, + 3288765765, + 3423720279, + 2651938379, + 544169062, + 3742654890, + 4219466551, + 3746962816, + 1242556253, + 4129516530, + 879521323, + 2966284567, + 3838591282, + 1283288560 + ], + "t": 7653677975526109578, + "block_len": 18, + "flags": 67, + "out": [ + 346682991, + 270262248, + 2601144541, + 3997938779, + 2056340738, + 2008238187, + 1505739028, + 2712480509, + 3247758822, + 2303640909, + 2906048517, + 2417554421, + 375059928, + 1048950168, + 2028430931, + 3145353309 + ] + }, + { + "seed": 6, + "h": [ + 3530265750, + 1123655737, + 1940104, + 1602711601, + 3307725433, + 1171229348, + 3444200791, + 2929389929 + ], + "m": [ + 2945015643, + 3626164985, + 400010022, + 3437188107, + 3456510285, + 1250623880, + 4086115940, + 1547818437, + 3906320867, + 1552099921, + 2584484726, + 1307063374, + 2530408928, + 2255988210, + 2846451649, + 842776239 + ], + "t": 4108804320044427842, + "block_len": 26, + "flags": 3, + "out": [ + 849805168, + 3271909564, + 3519510472, + 4052162593, + 1913105236, + 2673574855, + 3059096669, + 2568909711, + 3012256441, + 251056470, + 2571889841, + 162028814, + 841094977, + 2913193055, + 1533365974, + 712663730 + ] + }, + { + "seed": 7, + "h": [ + 647892279, + 2795742288, + 2301595691, + 2179419893, + 161042648, + 1862494042, + 300026767, + 1823296038 + ], + "m": [ + 4070378921, + 1703729684, + 4192983756, + 3687093963, + 1243862422, + 776213899, + 2744112455, + 1599435267, + 884585951, + 1349251823, + 1946412080, + 1287489453, + 3411833895, + 1048386555, + 2467131055, + 2255701793 + ], + "t": 1350317716114554168, + "block_len": 53, + "flags": 42, + "out": [ + 2788339013, + 315507188, + 3524996285, + 1987664994, + 1810642625, + 3673881822, + 1405781943, + 2464695899, + 2067943261, + 3789991295, + 1966842759, + 3435464740, + 1773068141, + 3149656659, + 2026915971, + 4092802697 + ] + }, + { + "seed": 8, + "h": [ + 973694259, + 4133025703, + 542587089, + 3027165658, + 365867937, + 899355976, + 2756803948, + 1971964490 + ], + "m": [ + 1946188980, + 3567061697, + 384681428, + 1750902959, + 1109633622, + 270963824, + 1620083717, + 2838299811, + 1453582679, + 2969113350, + 3871375977, + 4063259978, + 832596604, + 2486621942, + 3783693026, + 3771309886 + ], + "t": 3645965004013668149, + "block_len": 19, + "flags": 91, + "out": [ + 2436573921, + 3354865794, + 1172422691, + 1864318850, + 548333301, + 3673300372, + 4072793263, + 3573011628, + 1151623047, + 4106489061, + 1631493012, + 147739614, + 1341160100, + 1164702434, + 543615615, + 1012557131 + ] + }, + { + "seed": 9, + "h": [ + 1603362544, + 595022250, + 27638352, + 2159432582, + 347096279, + 1627876803, + 3114132053, + 674984870 + ], + "m": [ + 1022254636, + 476516009, + 2535870938, + 1250600339, + 2895821580, + 901471249, + 1207677876, + 3476821989, + 3807057864, + 3776879099, + 2111885832, + 100859404, + 2563432515, + 2485498850, + 872106831, + 358645241 + ], + "t": 16927792517719413886, + "block_len": 64, + "flags": 21, + "out": [ + 4126052628, + 2238491576, + 700329201, + 1614539036, + 2494029070, + 687619623, + 3058576584, + 757884927, + 1778041274, + 211062928, + 3599623221, + 3465651495, + 3893106709, + 1833234406, + 2278011253, + 3515517844 + ] + } +] \ No newline at end of file diff --git a/thoughts/blake3/blake3-oracle/official_test_vectors.json b/thoughts/blake3/blake3-oracle/official_test_vectors.json new file mode 100644 index 000000000..77cd38adb --- /dev/null +++ b/thoughts/blake3/blake3-oracle/official_test_vectors.json @@ -0,0 +1,334 @@ +{ + "key": "whats the Elvish word for friend", + "context_string": "BLAKE3 2019-12-27 16:29:52 test vectors context", + "cases": [ + { + "input_len": 0, + "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421cce14d", + "keyed_hash": "92b2b75604ed3c761f9d6f62392c8a9227ad0ea3f09573e783f1498a4ed60d26b18171a2f22a4b94822c701f107153dba24918c4bae4d2945c20ece13387627d3b73cbf97b797d5e59948c7ef788f54372df45e45e4293c7dc18c1d41144a9758be58960856be1eabbe22c2653190de560ca3b2ac4aa692a9210694254c371e851bc8f", + "derive_key": "2cc39783c223154fea8dfb7c1b1660f2ac2dcbd1c1de8277b0b0dd39b7e50d7d905630c8be290dfcf3e6842f13bddd573c098c3f17361f1f206b8cad9d088aa4a3f746752c6b0ce6a83b0da81d59649257cdf8eb3e9f7d4998e41021fac119deefb896224ac99f860011f73609e6e0e4540f93b273e56547dfd3aa1a035ba6689d89a0" + }, + { + "input_len": 1, + "hash": "2d3adedff11b61f14c886e35afa036736dcd87a74d27b5c1510225d0f592e213c3a6cb8bf623e20cdb535f8d1a5ffb86342d9c0b64aca3bce1d31f60adfa137b358ad4d79f97b47c3d5e79f179df87a3b9776ef8325f8329886ba42f07fb138bb502f4081cbcec3195c5871e6c23e2cc97d3c69a613eba131e5f1351f3f1da786545e5", + "keyed_hash": "6d7878dfff2f485635d39013278ae14f1454b8c0a3a2d34bc1ab38228a80c95b6568c0490609413006fbd428eb3fd14e7756d90f73a4725fad147f7bf70fd61c4e0cf7074885e92b0e3f125978b4154986d4fb202a3f331a3fb6cf349a3a70e49990f98fe4289761c8602c4e6ab1138d31d3b62218078b2f3ba9a88e1d08d0dd4cea11", + "derive_key": "b3e2e340a117a499c6cf2398a19ee0d29cca2bb7404c73063382693bf66cb06c5827b91bf889b6b97c5477f535361caefca0b5d8c4746441c57617111933158950670f9aa8a05d791daae10ac683cbef8faf897c84e6114a59d2173c3f417023a35d6983f2c7dfa57e7fc559ad751dbfb9ffab39c2ef8c4aafebc9ae973a64f0c76551" + }, + { + "input_len": 2, + "hash": "7b7015bb92cf0b318037702a6cdd81dee41224f734684c2c122cd6359cb1ee63d8386b22e2ddc05836b7c1bb693d92af006deb5ffbc4c70fb44d0195d0c6f252faac61659ef86523aa16517f87cb5f1340e723756ab65efb2f91964e14391de2a432263a6faf1d146937b35a33621c12d00be8223a7f1919cec0acd12097ff3ab00ab1", + "keyed_hash": "5392ddae0e0a69d5f40160462cbd9bd889375082ff224ac9c758802b7a6fd20a9ffbf7efd13e989a6c246f96d3a96b9d279f2c4e63fb0bdff633957acf50ee1a5f658be144bab0f6f16500dee4aa5967fc2c586d85a04caddec90fffb7633f46a60786024353b9e5cebe277fcd9514217fee2267dcda8f7b31697b7c54fab6a939bf8f", + "derive_key": "1f166565a7df0098ee65922d7fea425fb18b9943f19d6161e2d17939356168e6daa59cae19892b2d54f6fc9f475d26031fd1c22ae0a3e8ef7bdb23f452a15e0027629d2e867b1bb1e6ab21c71297377750826c404dfccc2406bd57a83775f89e0b075e59a7732326715ef912078e213944f490ad68037557518b79c0086de6d6f6cdd2" + }, + { + "input_len": 3, + "hash": "e1be4d7a8ab5560aa4199eea339849ba8e293d55ca0a81006726d184519e647f5b49b82f805a538c68915c1ae8035c900fd1d4b13902920fd05e1450822f36de9454b7e9996de4900c8e723512883f93f4345f8a58bfe64ee38d3ad71ab027765d25cdd0e448328a8e7a683b9a6af8b0af94fa09010d9186890b096a08471e4230a134", + "keyed_hash": "39e67b76b5a007d4921969779fe666da67b5213b096084ab674742f0d5ec62b9b9142d0fab08e1b161efdbb28d18afc64d8f72160c958e53a950cdecf91c1a1bbab1a9c0f01def762a77e2e8545d4dec241e98a89b6db2e9a5b070fc110caae2622690bd7b76c02ab60750a3ea75426a6bb8803c370ffe465f07fb57def95df772c39f", + "derive_key": "440aba35cb006b61fc17c0529255de438efc06a8c9ebf3f2ddac3b5a86705797f27e2e914574f4d87ec04c379e12789eccbfbc15892626042707802dbe4e97c3ff59dca80c1e54246b6d055154f7348a39b7d098b2b4824ebe90e104e763b2a447512132cede16243484a55a4e40a85790038bb0dcf762e8c053cabae41bbe22a5bff7" + }, + { + "input_len": 4, + "hash": "f30f5ab28fe047904037f77b6da4fea1e27241c5d132638d8bedce9d40494f328f603ba4564453e06cdcee6cbe728a4519bbe6f0d41e8a14b5b225174a566dbfa61b56afb1e452dc08c804f8c3143c9e2cc4a31bb738bf8c1917b55830c6e65797211701dc0b98daa1faeaa6ee9e56ab606ce03a1a881e8f14e87a4acf4646272cfd12", + "keyed_hash": "7671dde590c95d5ac9616651ff5aa0a27bee5913a348e053b8aa9108917fe070116c0acff3f0d1fa97ab38d813fd46506089118147d83393019b068a55d646251ecf81105f798d76a10ae413f3d925787d6216a7eb444e510fd56916f1d753a5544ecf0072134a146b2615b42f50c179f56b8fae0788008e3e27c67482349e249cb86a", + "derive_key": "f46085c8190d69022369ce1a18880e9b369c135eb93f3c63550d3e7630e91060fbd7d8f4258bec9da4e05044f88b91944f7cab317a2f0c18279629a3867fad0662c9ad4d42c6f27e5b124da17c8c4f3a94a025ba5d1b623686c6099d202a7317a82e3d95dae46a87de0555d727a5df55de44dab799a20dffe239594d6e99ed17950910" + }, + { + "input_len": 5, + "hash": "b40b44dfd97e7a84a996a91af8b85188c66c126940ba7aad2e7ae6b385402aa2ebcfdac6c5d32c31209e1f81a454751280db64942ce395104e1e4eaca62607de1c2ca748251754ea5bbe8c20150e7f47efd57012c63b3c6a6632dc1c7cd15f3e1c999904037d60fac2eb9397f2adbe458d7f264e64f1e73aa927b30988e2aed2f03620", + "keyed_hash": "73ac69eecf286894d8102018a6fc729f4b1f4247d3703f69bdc6a5fe3e0c84616ab199d1f2f3e53bffb17f0a2209fe8b4f7d4c7bae59c2bc7d01f1ff94c67588cc6b38fa6024886f2c078bfe09b5d9e6584cd6c521c3bb52f4de7687b37117a2dbbec0d59e92fa9a8cc3240d4432f91757aabcae03e87431dac003e7d73574bfdd8218", + "derive_key": "1f24eda69dbcb752847ec3ebb5dd42836d86e58500c7c98d906ecd82ed9ae47f6f48a3f67e4e43329c9a89b1ca526b9b35cbf7d25c1e353baffb590fd79be58ddb6c711f1a6b60e98620b851c688670412fcb0435657ba6b638d21f0f2a04f2f6b0bd8834837b10e438d5f4c7c2c71299cf7586ea9144ed09253d51f8f54dd6bff719d" + }, + { + "input_len": 6, + "hash": "06c4e8ffb6872fad96f9aaca5eee1553eb62aed0ad7198cef42e87f6a616c844611a30c4e4f37fe2fe23c0883cde5cf7059d88b657c7ed2087e3d210925ede716435d6d5d82597a1e52b9553919e804f5656278bd739880692c94bff2824d8e0b48cac1d24682699e4883389dc4f2faa2eb3b4db6e39debd5061ff3609916f3e07529a", + "keyed_hash": "82d3199d0013035682cc7f2a399d4c212544376a839aa863a0f4c91220ca7a6dc2ffb3aa05f2631f0fa9ac19b6e97eb7e6669e5ec254799350c8b8d189e8807800842a5383c4d907c932f34490aaf00064de8cdb157357bde37c1504d2960034930887603abc5ccb9f5247f79224baff6120a3c622a46d7b1bcaee02c5025460941256", + "derive_key": "be96b30b37919fe4379dfbe752ae77b4f7e2ab92f7ff27435f76f2f065f6a5f435ae01a1d14bd5a6b3b69d8cbd35f0b01ef2173ff6f9b640ca0bd4748efa398bf9a9c0acd6a66d9332fdc9b47ffe28ba7ab6090c26747b85f4fab22f936b71eb3f64613d8bd9dfabe9bb68da19de78321b481e5297df9e40ec8a3d662f3e1479c65de0" + }, + { + "input_len": 7, + "hash": "3f8770f387faad08faa9d8414e9f449ac68e6ff0417f673f602a646a891419fe66036ef6e6d1a8f54baa9fed1fc11c77cfb9cff65bae915045027046ebe0c01bf5a941f3bb0f73791d3fc0b84370f9f30af0cd5b0fc334dd61f70feb60dad785f070fef1f343ed933b49a5ca0d16a503f599a365a4296739248b28d1a20b0e2cc8975c", + "keyed_hash": "af0a7ec382aedc0cfd626e49e7628bc7a353a4cb108855541a5651bf64fbb28a7c5035ba0f48a9c73dabb2be0533d02e8fd5d0d5639a18b2803ba6bf527e1d145d5fd6406c437b79bcaad6c7bdf1cf4bd56a893c3eb9510335a7a798548c6753f74617bede88bef924ba4b334f8852476d90b26c5dc4c3668a2519266a562c6c8034a6", + "derive_key": "dc3b6485f9d94935329442916b0d059685ba815a1fa2a14107217453a7fc9f0e66266db2ea7c96843f9d8208e600a73f7f45b2f55b9e6d6a7ccf05daae63a3fdd10b25ac0bd2e224ce8291f88c05976d575df998477db86fb2cfbbf91725d62cb57acfeb3c2d973b89b503c2b60dde85a7802b69dc1ac2007d5623cbea8cbfb6b181f5" + }, + { + "input_len": 8, + "hash": "2351207d04fc16ade43ccab08600939c7c1fa70a5c0aaca76063d04c3228eaeb725d6d46ceed8f785ab9f2f9b06acfe398c6699c6129da084cb531177445a682894f9685eaf836999221d17c9a64a3a057000524cd2823986db378b074290a1a9b93a22e135ed2c14c7e20c6d045cd00b903400374126676ea78874d79f2dd7883cf5c", + "keyed_hash": "be2f5495c61cba1bb348a34948c004045e3bd4dae8f0fe82bf44d0da245a060048eb5e68ce6dea1eb0229e144f578b3aa7e9f4f85febd135df8525e6fe40c6f0340d13dd09b255ccd5112a94238f2be3c0b5b7ecde06580426a93e0708555a265305abf86d874e34b4995b788e37a823491f25127a502fe0704baa6bfdf04e76c13276", + "derive_key": "2b166978cef14d9d438046c720519d8b1cad707e199746f1562d0c87fbd32940f0e2545a96693a66654225ebbaac76d093bfa9cd8f525a53acb92a861a98c42e7d1c4ae82e68ab691d510012edd2a728f98cd4794ef757e94d6546961b4f280a51aac339cc95b64a92b83cc3f26d8af8dfb4c091c240acdb4d47728d23e7148720ef04" + }, + { + "input_len": 63, + "hash": "e9bc37a594daad83be9470df7f7b3798297c3d834ce80ba85d6e207627b7db7b1197012b1e7d9af4d7cb7bdd1f3bb49a90a9b5dec3ea2bbc6eaebce77f4e470cbf4687093b5352f04e4a4570fba233164e6acc36900e35d185886a827f7ea9bdc1e5c3ce88b095a200e62c10c043b3e9bc6cb9b6ac4dfa51794b02ace9f98779040755", + "keyed_hash": "bb1eb5d4afa793c1ebdd9fb08def6c36d10096986ae0cfe148cd101170ce37aea05a63d74a840aecd514f654f080e51ac50fd617d22610d91780fe6b07a26b0847abb38291058c97474ef6ddd190d30fc318185c09ca1589d2024f0a6f16d45f11678377483fa5c005b2a107cb9943e5da634e7046855eaa888663de55d6471371d55d", + "derive_key": "b6451e30b953c206e34644c6803724e9d2725e0893039cfc49584f991f451af3b89e8ff572d3da4f4022199b9563b9d70ebb616efff0763e9abec71b550f1371e233319c4c4e74da936ba8e5bbb29a598e007a0bbfa929c99738ca2cc098d59134d11ff300c39f82e2fce9f7f0fa266459503f64ab9913befc65fddc474f6dc1c67669" + }, + { + "input_len": 64, + "hash": "4eed7141ea4a5cd4b788606bd23f46e212af9cacebacdc7d1f4c6dc7f2511b98fc9cc56cb831ffe33ea8e7e1d1df09b26efd2767670066aa82d023b1dfe8ab1b2b7fbb5b97592d46ffe3e05a6a9b592e2949c74160e4674301bc3f97e04903f8c6cf95b863174c33228924cdef7ae47559b10b294acd660666c4538833582b43f82d74", + "keyed_hash": "ba8ced36f327700d213f120b1a207a3b8c04330528586f414d09f2f7d9ccb7e68244c26010afc3f762615bbac552a1ca909e67c83e2fd5478cf46b9e811efccc93f77a21b17a152ebaca1695733fdb086e23cd0eb48c41c034d52523fc21236e5d8c9255306e48d52ba40b4dac24256460d56573d1312319afcf3ed39d72d0bfc69acb", + "derive_key": "a5c4a7053fa86b64746d4bb688d06ad1f02a18fce9afd3e818fefaa7126bf73e9b9493a9befebe0bf0c9509fb3105cfa0e262cde141aa8e3f2c2f77890bb64a4cca96922a21ead111f6338ad5244f2c15c44cb595443ac2ac294231e31be4a4307d0a91e874d36fc9852aeb1265c09b6e0cda7c37ef686fbbcab97e8ff66718be048bb" + }, + { + "input_len": 65, + "hash": "de1e5fa0be70df6d2be8fffd0e99ceaa8eb6e8c93a63f2d8d1c30ecb6b263dee0e16e0a4749d6811dd1d6d1265c29729b1b75a9ac346cf93f0e1d7296dfcfd4313b3a227faaaaf7757cc95b4e87a49be3b8a270a12020233509b1c3632b3485eef309d0abc4a4a696c9decc6e90454b53b000f456a3f10079072baaf7a981653221f2c", + "keyed_hash": "c0a4edefa2d2accb9277c371ac12fcdbb52988a86edc54f0716e1591b4326e72d5e795f46a596b02d3d4bfb43abad1e5d19211152722ec1f20fef2cd413e3c22f2fc5da3d73041275be6ede3517b3b9f0fc67ade5956a672b8b75d96cb43294b9041497de92637ed3f2439225e683910cb3ae923374449ca788fb0f9bea92731bc26ad", + "derive_key": "51fd05c3c1cfbc8ed67d139ad76f5cf8236cd2acd26627a30c104dfd9d3ff8a82b02e8bd36d8498a75ad8c8e9b15eb386970283d6dd42c8ae7911cc592887fdbe26a0a5f0bf821cd92986c60b2502c9be3f98a9c133a7e8045ea867e0828c7252e739321f7c2d65daee4468eb4429efae469a42763f1f94977435d10dccae3e3dce88d" + }, + { + "input_len": 127, + "hash": "d81293fda863f008c09e92fc382a81f5a0b4a1251cba1634016a0f86a6bd640de3137d477156d1fde56b0cf36f8ef18b44b2d79897bece12227539ac9ae0a5119da47644d934d26e74dc316145dcb8bb69ac3f2e05c242dd6ee06484fcb0e956dc44355b452c5e2bbb5e2b66e99f5dd443d0cbcaaafd4beebaed24ae2f8bb672bcef78", + "keyed_hash": "c64200ae7dfaf35577ac5a9521c47863fb71514a3bcad18819218b818de85818ee7a317aaccc1458f78d6f65f3427ec97d9c0adb0d6dacd4471374b621b7b5f35cd54663c64dbe0b9e2d95632f84c611313ea5bd90b71ce97b3cf645776f3adc11e27d135cbadb9875c2bf8d3ae6b02f8a0206aba0c35bfe42574011931c9a255ce6dc", + "derive_key": "c91c090ceee3a3ac81902da31838012625bbcd73fcb92e7d7e56f78deba4f0c3feeb3974306966ccb3e3c69c337ef8a45660ad02526306fd685c88542ad00f759af6dd1adc2e50c2b8aac9f0c5221ff481565cf6455b772515a69463223202e5c371743e35210bbbbabd89651684107fd9fe493c937be16e39cfa7084a36207c99bea3" + }, + { + "input_len": 128, + "hash": "f17e570564b26578c33bb7f44643f539624b05df1a76c81f30acd548c44b45efa69faba091427f9c5c4caa873aa07828651f19c55bad85c47d1368b11c6fd99e47ecba5820a0325984d74fe3e4058494ca12e3f1d3293d0010a9722f7dee64f71246f75e9361f44cc8e214a100650db1313ff76a9f93ec6e84edb7add1cb4a95019b0c", + "keyed_hash": "b04fe15577457267ff3b6f3c947d93be581e7e3a4b018679125eaf86f6a628ecd86bbe0001f10bda47e6077b735016fca8119da11348d93ca302bbd125bde0db2b50edbe728a620bb9d3e6f706286aedea973425c0b9eedf8a38873544cf91badf49ad92a635a93f71ddfcee1eae536c25d1b270956be16588ef1cfef2f1d15f650bd5", + "derive_key": "81720f34452f58a0120a58b6b4608384b5c51d11f39ce97161a0c0e442ca022550e7cd651e312f0b4c6afb3c348ae5dd17d2b29fab3b894d9a0034c7b04fd9190cbd90043ff65d1657bbc05bfdecf2897dd894c7a1b54656d59a50b51190a9da44db426266ad6ce7c173a8c0bbe091b75e734b4dadb59b2861cd2518b4e7591e4b83c9" + }, + { + "input_len": 129, + "hash": "683aaae9f3c5ba37eaaf072aed0f9e30bac0865137bae68b1fde4ca2aebdcb12f96ffa7b36dd78ba321be7e842d364a62a42e3746681c8bace18a4a8a79649285c7127bf8febf125be9de39586d251f0d41da20980b70d35e3dac0eee59e468a894fa7e6a07129aaad09855f6ad4801512a116ba2b7841e6cfc99ad77594a8f2d181a7", + "keyed_hash": "d4a64dae6cdccbac1e5287f54f17c5f985105457c1a2ec1878ebd4b57e20d38f1c9db018541eec241b748f87725665b7b1ace3e0065b29c3bcb232c90e37897fa5aaee7e1e8a2ecfcd9b51463e42238cfdd7fee1aecb3267fa7f2128079176132a412cd8aaf0791276f6b98ff67359bd8652ef3a203976d5ff1cd41885573487bcd683", + "derive_key": "938d2d4435be30eafdbb2b7031f7857c98b04881227391dc40db3c7b21f41fc18d72d0f9c1de5760e1941aebf3100b51d64644cb459eb5d20258e233892805eb98b07570ef2a1787cd48e117c8d6a63a68fd8fc8e59e79dbe63129e88352865721c8d5f0cf183f85e0609860472b0d6087cefdd186d984b21542c1c780684ed6832d8d" + }, + { + "input_len": 1023, + "hash": "10108970eeda3eb932baac1428c7a2163b0e924c9a9e25b35bba72b28f70bd11a182d27a591b05592b15607500e1e8dd56bc6c7fc063715b7a1d737df5bad3339c56778957d870eb9717b57ea3d9fb68d1b55127bba6a906a4a24bbd5acb2d123a37b28f9e9a81bbaae360d58f85e5fc9d75f7c370a0cc09b6522d9c8d822f2f28f485", + "keyed_hash": "c951ecdf03288d0fcc96ee3413563d8a6d3589547f2c2fb36d9786470f1b9d6e890316d2e6d8b8c25b0a5b2180f94fb1a158ef508c3cde45e2966bd796a696d3e13efd86259d756387d9becf5c8bf1ce2192b87025152907b6d8cc33d17826d8b7b9bc97e38c3c85108ef09f013e01c229c20a83d9e8efac5b37470da28575fd755a10", + "derive_key": "74a16c1c3d44368a86e1ca6df64be6a2f64cce8f09220787450722d85725dea59c413264404661e9e4d955409dfe4ad3aa487871bcd454ed12abfe2c2b1eb7757588cf6cb18d2eccad49e018c0d0fec323bec82bf1644c6325717d13ea712e6840d3e6e730d35553f59eff5377a9c350bcc1556694b924b858f329c44ee64b884ef00d" + }, + { + "input_len": 1024, + "hash": "42214739f095a406f3fc83deb889744ac00df831c10daa55189b5d121c855af71cf8107265ecdaf8505b95d8fcec83a98a6a96ea5109d2c179c47a387ffbb404756f6eeae7883b446b70ebb144527c2075ab8ab204c0086bb22b7c93d465efc57f8d917f0b385c6df265e77003b85102967486ed57db5c5ca170ba441427ed9afa684e", + "keyed_hash": "75c46f6f3d9eb4f55ecaaee480db732e6c2105546f1e675003687c31719c7ba4a78bc838c72852d4f49c864acb7adafe2478e824afe51c8919d06168414c265f298a8094b1ad813a9b8614acabac321f24ce61c5a5346eb519520d38ecc43e89b5000236df0597243e4d2493fd626730e2ba17ac4d8824d09d1a4a8f57b8227778e2de", + "derive_key": "7356cd7720d5b66b6d0697eb3177d9f8d73a4a5c5e968896eb6a6896843027066c23b601d3ddfb391e90d5c8eccdef4ae2a264bce9e612ba15e2bc9d654af1481b2e75dbabe615974f1070bba84d56853265a34330b4766f8e75edd1f4a1650476c10802f22b64bd3919d246ba20a17558bc51c199efdec67e80a227251808d8ce5bad" + }, + { + "input_len": 1025, + "hash": "d00278ae47eb27b34faecf67b4fe263f82d5412916c1ffd97c8cb7fb814b8444f4c4a22b4b399155358a994e52bf255de60035742ec71bd08ac275a1b51cc6bfe332b0ef84b409108cda080e6269ed4b3e2c3f7d722aa4cdc98d16deb554e5627be8f955c98e1d5f9565a9194cad0c4285f93700062d9595adb992ae68ff12800ab67a", + "keyed_hash": "357dc55de0c7e382c900fd6e320acc04146be01db6a8ce7210b7189bd664ea69362396b77fdc0d2634a552970843722066c3c15902ae5097e00ff53f1e116f1cd5352720113a837ab2452cafbde4d54085d9cf5d21ca613071551b25d52e69d6c81123872b6f19cd3bc1333edf0c52b94de23ba772cf82636cff4542540a7738d5b930", + "derive_key": "effaa245f065fbf82ac186839a249707c3bddf6d3fdda22d1b95a3c970379bcb5d31013a167509e9066273ab6e2123bc835b408b067d88f96addb550d96b6852dad38e320b9d940f86db74d398c770f462118b35d2724efa13da97194491d96dd37c3c09cbef665953f2ee85ec83d88b88d11547a6f911c8217cca46defa2751e7f3ad" + }, + { + "input_len": 2048, + "hash": "e776b6028c7cd22a4d0ba182a8bf62205d2ef576467e838ed6f2529b85fba24a9a60bf80001410ec9eea6698cd537939fad4749edd484cb541aced55cd9bf54764d063f23f6f1e32e12958ba5cfeb1bf618ad094266d4fc3c968c2088f677454c288c67ba0dba337b9d91c7e1ba586dc9a5bc2d5e90c14f53a8863ac75655461cea8f9", + "keyed_hash": "879cf1fa2ea0e79126cb1063617a05b6ad9d0b696d0d757cf053439f60a99dd10173b961cd574288194b23ece278c330fbb8585485e74967f31352a8183aa782b2b22f26cdcadb61eed1a5bc144b8198fbb0c13abbf8e3192c145d0a5c21633b0ef86054f42809df823389ee40811a5910dcbd1018af31c3b43aa55201ed4edaac74fe", + "derive_key": "7b2945cb4fef70885cc5d78a87bf6f6207dd901ff239201351ffac04e1088a23e2c11a1ebffcea4d80447867b61badb1383d842d4e79645d48dd82ccba290769caa7af8eaa1bd78a2a5e6e94fbdab78d9c7b74e894879f6a515257ccf6f95056f4e25390f24f6b35ffbb74b766202569b1d797f2d4bd9d17524c720107f985f4ddc583" + }, + { + "input_len": 2049, + "hash": "5f4d72f40d7a5f82b15ca2b2e44b1de3c2ef86c426c95c1af0b687952256303096de31d71d74103403822a2e0bc1eb193e7aecc9643a76b7bbc0c9f9c52e8783aae98764ca468962b5c2ec92f0c74eb5448d519713e09413719431c802f948dd5d90425a4ecdadece9eb178d80f26efccae630734dff63340285adec2aed3b51073ad3", + "keyed_hash": "9f29700902f7c86e514ddc4df1e3049f258b2472b6dd5267f61bf13983b78dd5f9a88abfefdfa1e00b418971f2b39c64ca621e8eb37fceac57fd0c8fc8e117d43b81447be22d5d8186f8f5919ba6bcc6846bd7d50726c06d245672c2ad4f61702c646499ee1173daa061ffe15bf45a631e2946d616a4c345822f1151284712f76b2b0e", + "derive_key": "2ea477c5515cc3dd606512ee72bb3e0e758cfae7232826f35fb98ca1bcbdf27316d8e9e79081a80b046b60f6a263616f33ca464bd78d79fa18200d06c7fc9bffd808cc4755277a7d5e09da0f29ed150f6537ea9bed946227ff184cc66a72a5f8c1e4bd8b04e81cf40fe6dc4427ad5678311a61f4ffc39d195589bdbc670f63ae70f4b6" + }, + { + "input_len": 3072, + "hash": "b98cb0ff3623be03326b373de6b9095218513e64f1ee2edd2525c7ad1e5cffd29a3f6b0b978d6608335c09dc94ccf682f9951cdfc501bfe47b9c9189a6fc7b404d120258506341a6d802857322fbd20d3e5dae05b95c88793fa83db1cb08e7d8008d1599b6209d78336e24839724c191b2a52a80448306e0daa84a3fdb566661a37e11", + "keyed_hash": "044a0e7b172a312dc02a4c9a818c036ffa2776368d7f528268d2e6b5df19177022f302d0529e4174cc507c463671217975e81dab02b8fdeb0d7ccc7568dd22574c783a76be215441b32e91b9a904be8ea81f7a0afd14bad8ee7c8efc305ace5d3dd61b996febe8da4f56ca0919359a7533216e2999fc87ff7d8f176fbecb3d6f34278b", + "derive_key": "050df97f8c2ead654d9bb3ab8c9178edcd902a32f8495949feadcc1e0480c46b3604131bbd6e3ba573b6dd682fa0a63e5b165d39fc43a625d00207607a2bfeb65ff1d29292152e26b298868e3b87be95d6458f6f2ce6118437b632415abe6ad522874bcd79e4030a5e7bad2efa90a7a7c67e93f0a18fb28369d0a9329ab5c24134ccb0" + }, + { + "input_len": 3073, + "hash": "7124b49501012f81cc7f11ca069ec9226cecb8a2c850cfe644e327d22d3e1cd39a27ae3b79d68d89da9bf25bc27139ae65a324918a5f9b7828181e52cf373c84f35b639b7fccbb985b6f2fa56aea0c18f531203497b8bbd3a07ceb5926f1cab74d14bd66486d9a91eba99059a98bd1cd25876b2af5a76c3e9eed554ed72ea952b603bf", + "keyed_hash": "68dede9bef00ba89e43f31a6825f4cf433389fedae75c04ee9f0cf16a427c95a96d6da3fe985054d3478865be9a092250839a697bbda74e279e8a9e69f0025e4cfddd6cfb434b1cd9543aaf97c635d1b451a4386041e4bb100f5e45407cbbc24fa53ea2de3536ccb329e4eb9466ec37093a42cf62b82903c696a93a50b702c80f3c3c5", + "derive_key": "72613c9ec9ff7e40f8f5c173784c532ad852e827dba2bf85b2ab4b76f7079081576288e552647a9d86481c2cae75c2dd4e7c5195fb9ada1ef50e9c5098c249d743929191441301c69e1f48505a4305ec1778450ee48b8e69dc23a25960fe33070ea549119599760a8a2d28aeca06b8c5e9ba58bc19e11fe57b6ee98aa44b2a8e6b14a5" + }, + { + "input_len": 4096, + "hash": "015094013f57a5277b59d8475c0501042c0b642e531b0a1c8f58d2163229e9690289e9409ddb1b99768eafe1623da896faf7e1114bebeadc1be30829b6f8af707d85c298f4f0ff4d9438aef948335612ae921e76d411c3a9111df62d27eaf871959ae0062b5492a0feb98ef3ed4af277f5395172dbe5c311918ea0074ce0036454f620", + "keyed_hash": "befc660aea2f1718884cd8deb9902811d332f4fc4a38cf7c7300d597a081bfc0bbb64a36edb564e01e4b4aaf3b060092a6b838bea44afebd2deb8298fa562b7b597c757b9df4c911c3ca462e2ac89e9a787357aaf74c3b56d5c07bc93ce899568a3eb17d9250c20f6c5f6c1e792ec9a2dcb715398d5a6ec6d5c54f586a00403a1af1de", + "derive_key": "1e0d7f3db8c414c97c6307cbda6cd27ac3b030949da8e23be1a1a924ad2f25b9d78038f7b198596c6cc4a9ccf93223c08722d684f240ff6569075ed81591fd93f9fff1110b3a75bc67e426012e5588959cc5a4c192173a03c00731cf84544f65a2fb9378989f72e9694a6a394a8a30997c2e67f95a504e631cd2c5f55246024761b245" + }, + { + "input_len": 4097, + "hash": "9b4052b38f1c5fc8b1f9ff7ac7b27cd242487b3d890d15c96a1c25b8aa0fb99505f91b0b5600a11251652eacfa9497b31cd3c409ce2e45cfe6c0a016967316c426bd26f619eab5d70af9a418b845c608840390f361630bd497b1ab44019316357c61dbe091ce72fc16dc340ac3d6e009e050b3adac4b5b2c92e722cffdc46501531956", + "keyed_hash": "00df940cd36bb9fa7cbbc3556744e0dbc8191401afe70520ba292ee3ca80abbc606db4976cfdd266ae0abf667d9481831ff12e0caa268e7d3e57260c0824115a54ce595ccc897786d9dcbf495599cfd90157186a46ec800a6763f1c59e36197e9939e900809f7077c102f888caaf864b253bc41eea812656d46742e4ea42769f89b83f", + "derive_key": "aca51029626b55fda7117b42a7c211f8c6e9ba4fe5b7a8ca922f34299500ead8a897f66a400fed9198fd61dd2d58d382458e64e100128075fc54b860934e8de2e84170734b06e1d212a117100820dbc48292d148afa50567b8b84b1ec336ae10d40c8c975a624996e12de31abbe135d9d159375739c333798a80c64ae895e51e22f3ad" + }, + { + "input_len": 5120, + "hash": "9cadc15fed8b5d854562b26a9536d9707cadeda9b143978f319ab34230535833acc61c8fdc114a2010ce8038c853e121e1544985133fccdd0a2d507e8e615e611e9a0ba4f47915f49e53d721816a9198e8b30f12d20ec3689989175f1bf7a300eee0d9321fad8da232ece6efb8e9fd81b42ad161f6b9550a069e66b11b40487a5f5059", + "keyed_hash": "2c493e48e9b9bf31e0553a22b23503c0a3388f035cece68eb438d22fa1943e209b4dc9209cd80ce7c1f7c9a744658e7e288465717ae6e56d5463d4f80cdb2ef56495f6a4f5487f69749af0c34c2cdfa857f3056bf8d807336a14d7b89bf62bef2fb54f9af6a546f818dc1e98b9e07f8a5834da50fa28fb5874af91bf06020d1bf0120e", + "derive_key": "7a7acac8a02adcf3038d74cdd1d34527de8a0fcc0ee3399d1262397ce5817f6055d0cefd84d9d57fe792d65a278fd20384ac6c30fdb340092f1a74a92ace99c482b28f0fc0ef3b923e56ade20c6dba47e49227166251337d80a037e987ad3a7f728b5ab6dfafd6e2ab1bd583a95d9c895ba9c2422c24ea0f62961f0dca45cad47bfa0d" + }, + { + "input_len": 5121, + "hash": "628bd2cb2004694adaab7bbd778a25df25c47b9d4155a55f8fbd79f2fe154cff96adaab0613a6146cdaabe498c3a94e529d3fc1da2bd08edf54ed64d40dcd6777647eac51d8277d70219a9694334a68bc8f0f23e20b0ff70ada6f844542dfa32cd4204ca1846ef76d811cdb296f65e260227f477aa7aa008bac878f72257484f2b6c95", + "keyed_hash": "6ccf1c34753e7a044db80798ecd0782a8f76f33563accaddbfbb2e0ea4b2d0240d07e63f13667a8d1490e5e04f13eb617aea16a8c8a5aaed1ef6fbde1b0515e3c81050b361af6ead126032998290b563e3caddeaebfab592e155f2e161fb7cba939092133f23f9e65245e58ec23457b78a2e8a125588aad6e07d7f11a85b88d375b72d", + "derive_key": "b07f01e518e702f7ccb44a267e9e112d403a7b3f4883a47ffbed4b48339b3c341a0add0ac032ab5aaea1e4e5b004707ec5681ae0fcbe3796974c0b1cf31a194740c14519273eedaabec832e8a784b6e7cfc2c5952677e6c3f2c3914454082d7eb1ce1766ac7d75a4d3001fc89544dd46b5147382240d689bbbaefc359fb6ae30263165" + }, + { + "input_len": 6144, + "hash": "3e2e5b74e048f3add6d21faab3f83aa44d3b2278afb83b80b3c35164ebeca2054d742022da6fdda444ebc384b04a54c3ac5839b49da7d39f6d8a9db03deab32aade156c1c0311e9b3435cde0ddba0dce7b26a376cad121294b689193508dd63151603c6ddb866ad16c2ee41585d1633a2cea093bea714f4c5d6b903522045b20395c83", + "keyed_hash": "3d6b6d21281d0ade5b2b016ae4034c5dec10ca7e475f90f76eac7138e9bc8f1dc35754060091dc5caf3efabe0603c60f45e415bb3407db67e6beb3d11cf8e4f7907561f05dace0c15807f4b5f389c841eb114d81a82c02a00b57206b1d11fa6e803486b048a5ce87105a686dee041207e095323dfe172df73deb8c9532066d88f9da7e", + "derive_key": "2a95beae63ddce523762355cf4b9c1d8f131465780a391286a5d01abb5683a1597099e3c6488aab6c48f3c15dbe1942d21dbcdc12115d19a8b8465fb54e9053323a9178e4275647f1a9927f6439e52b7031a0b465c861a3fc531527f7758b2b888cf2f20582e9e2c593709c0a44f9c6e0f8b963994882ea4168827823eef1f64169fef" + }, + { + "input_len": 6145, + "hash": "f1323a8631446cc50536a9f705ee5cb619424d46887f3c376c695b70e0f0507f18a2cfdd73c6e39dd75ce7c1c6e3ef238fd54465f053b25d21044ccb2093beb015015532b108313b5829c3621ce324b8e14229091b7c93f32db2e4e63126a377d2a63a3597997d4f1cba59309cb4af240ba70cebff9a23d5e3ff0cdae2cfd54e070022", + "keyed_hash": "9ac301e9e39e45e3250a7e3b3df701aa0fb6889fbd80eeecf28dbc6300fbc539f3c184ca2f59780e27a576c1d1fb9772e99fd17881d02ac7dfd39675aca918453283ed8c3169085ef4a466b91c1649cc341dfdee60e32231fc34c9c4e0b9a2ba87ca8f372589c744c15fd6f985eec15e98136f25beeb4b13c4e43dc84abcc79cd4646c", + "derive_key": "379bcc61d0051dd489f686c13de00d5b14c505245103dc040d9e4dd1facab8e5114493d029bdbd295aaa744a59e31f35c7f52dba9c3642f773dd0b4262a9980a2aef811697e1305d37ba9d8b6d850ef07fe41108993180cf779aeece363704c76483458603bbeeb693cffbbe5588d1f3535dcad888893e53d977424bb707201569a8d2" + }, + { + "input_len": 7168, + "hash": "61da957ec2499a95d6b8023e2b0e604ec7f6b50e80a9678b89d2628e99ada77a5707c321c83361793b9af62a40f43b523df1c8633cecb4cd14d00bdc79c78fca5165b863893f6d38b02ff7236c5a9a8ad2dba87d24c547cab046c29fc5bc1ed142e1de4763613bb162a5a538e6ef05ed05199d751f9eb58d332791b8d73fb74e4fce95", + "keyed_hash": "b42835e40e9d4a7f42ad8cc04f85a963a76e18198377ed84adddeaecacc6f3fca2f01d5277d69bb681c70fa8d36094f73ec06e452c80d2ff2257ed82e7ba348400989a65ee8daa7094ae0933e3d2210ac6395c4af24f91c2b590ef87d7788d7066ea3eaebca4c08a4f14b9a27644f99084c3543711b64a070b94f2c9d1d8a90d035d52", + "derive_key": "11c37a112765370c94a51415d0d651190c288566e295d505defdad895dae223730d5a5175a38841693020669c7638f40b9bc1f9f39cf98bda7a5b54ae24218a800a2116b34665aa95d846d97ea988bfcb53dd9c055d588fa21ba78996776ea6c40bc428b53c62b5f3ccf200f647a5aae8067f0ea1976391fcc72af1945100e2a6dcb88" + }, + { + "input_len": 7169, + "hash": "a003fc7a51754a9b3c7fae0367ab3d782dccf28855a03d435f8cfe74605e781798a8b20534be1ca9eb2ae2df3fae2ea60e48c6fb0b850b1385b5de0fe460dbe9d9f9b0d8db4435da75c601156df9d047f4ede008732eb17adc05d96180f8a73548522840779e6062d643b79478a6e8dbce68927f36ebf676ffa7d72d5f68f050b119c8", + "keyed_hash": "ed9b1a922c046fdb3d423ae34e143b05ca1bf28b710432857bf738bcedbfa5113c9e28d72fcbfc020814ce3f5d4fc867f01c8f5b6caf305b3ea8a8ba2da3ab69fabcb438f19ff11f5378ad4484d75c478de425fb8e6ee809b54eec9bdb184315dc856617c09f5340451bf42fd3270a7b0b6566169f242e533777604c118a6358250f54", + "derive_key": "554b0a5efea9ef183f2f9b931b7497995d9eb26f5c5c6dad2b97d62fc5ac31d99b20652c016d88ba2a611bbd761668d5eda3e568e940faae24b0d9991c3bd25a65f770b89fdcadabcb3d1a9c1cb63e69721cacf1ae69fefdcef1e3ef41bc5312ccc17222199e47a26552c6adc460cf47a72319cb5039369d0060eaea59d6c65130f1dd" + }, + { + "input_len": 8192, + "hash": "aae792484c8efe4f19e2ca7d371d8c467ffb10748d8a5a1ae579948f718a2a635fe51a27db045a567c1ad51be5aa34c01c6651c4d9b5b5ac5d0fd58cf18dd61a47778566b797a8c67df7b1d60b97b19288d2d877bb2df417ace009dcb0241ca1257d62712b6a4043b4ff33f690d849da91ea3bf711ed583cb7b7a7da2839ba71309bbf", + "keyed_hash": "dc9637c8845a770b4cbf76b8daec0eebf7dc2eac11498517f08d44c8fc00d58a4834464159dcbc12a0ba0c6d6eb41bac0ed6585cabfe0aca36a375e6c5480c22afdc40785c170f5a6b8a1107dbee282318d00d915ac9ed1143ad40765ec120042ee121cd2baa36250c618adaf9e27260fda2f94dea8fb6f08c04f8f10c78292aa46102", + "derive_key": "ad01d7ae4ad059b0d33baa3c01319dcf8088094d0359e5fd45d6aeaa8b2d0c3d4c9e58958553513b67f84f8eac653aeeb02ae1d5672dcecf91cd9985a0e67f4501910ecba25555395427ccc7241d70dc21c190e2aadee875e5aae6bf1912837e53411dabf7a56cbf8e4fb780432b0d7fe6cec45024a0788cf5874616407757e9e6bef7" + }, + { + "input_len": 8193, + "hash": "bab6c09cb8ce8cf459261398d2e7aef35700bf488116ceb94a36d0f5f1b7bc3bb2282aa69be089359ea1154b9a9286c4a56af4de975a9aa4a5c497654914d279bea60bb6d2cf7225a2fa0ff5ef56bbe4b149f3ed15860f78b4e2ad04e158e375c1e0c0b551cd7dfc82f1b155c11b6b3ed51ec9edb30d133653bb5709d1dbd55f4e1ff6", + "keyed_hash": "954a2a75420c8d6547e3ba5b98d963e6fa6491addc8c023189cc519821b4a1f5f03228648fd983aef045c2fa8290934b0866b615f585149587dda2299039965328835a2b18f1d63b7e300fc76ff260b571839fe44876a4eae66cbac8c67694411ed7e09df51068a22c6e67d6d3dd2cca8ff12e3275384006c80f4db68023f24eebba57", + "derive_key": "af1e0346e389b17c23200270a64aa4e1ead98c61695d917de7d5b00491c9b0f12f20a01d6d622edf3de026a4db4e4526225debb93c1237934d71c7340bb5916158cbdafe9ac3225476b6ab57a12357db3abbad7a26c6e66290e44034fb08a20a8d0ec264f309994d2810c49cfba6989d7abb095897459f5425adb48aba07c5fb3c83c0" + }, + { + "input_len": 16384, + "hash": "f875d6646de28985646f34ee13be9a576fd515f76b5b0a26bb324735041ddde49d764c270176e53e97bdffa58d549073f2c660be0e81293767ed4e4929f9ad34bbb39a529334c57c4a381ffd2a6d4bfdbf1482651b172aa883cc13408fa67758a3e47503f93f87720a3177325f7823251b85275f64636a8f1d599c2e49722f42e93893", + "keyed_hash": "9e9fc4eb7cf081ea7c47d1807790ed211bfec56aa25bb7037784c13c4b707b0df9e601b101e4cf63a404dfe50f2e1865bb12edc8fca166579ce0c70dba5a5c0fc960ad6f3772183416a00bd29d4c6e651ea7620bb100c9449858bf14e1ddc9ecd35725581ca5b9160de04060045993d972571c3e8f71e9d0496bfa744656861b169d65", + "derive_key": "160e18b5878cd0df1c3af85eb25a0db5344d43a6fbd7a8ef4ed98d0714c3f7e160dc0b1f09caa35f2f417b9ef309dfe5ebd67f4c9507995a531374d099cf8ae317542e885ec6f589378864d3ea98716b3bbb65ef4ab5e0ab5bb298a501f19a41ec19af84a5e6b428ecd813b1a47ed91c9657c3fba11c406bc316768b58f6802c9e9b57" + }, + { + "input_len": 31744, + "hash": "62b6960e1a44bcc1eb1a611a8d6235b6b4b78f32e7abc4fb4c6cdcce94895c47860cc51f2b0c28a7b77304bd55fe73af663c02d3f52ea053ba43431ca5bab7bfea2f5e9d7121770d88f70ae9649ea713087d1914f7f312147e247f87eb2d4ffef0ac978bf7b6579d57d533355aa20b8b77b13fd09748728a5cc327a8ec470f4013226f", + "keyed_hash": "efa53b389ab67c593dba624d898d0f7353ab99e4ac9d42302ee64cbf9939a4193a7258db2d9cd32a7a3ecfce46144114b15c2fcb68a618a976bd74515d47be08b628be420b5e830fade7c080e351a076fbc38641ad80c736c8a18fe3c66ce12f95c61c2462a9770d60d0f77115bbcd3782b593016a4e728d4c06cee4505cb0c08a42ec", + "derive_key": "39772aef80e0ebe60596361e45b061e8f417429d529171b6764468c22928e28e9759adeb797a3fbf771b1bcea30150a020e317982bf0d6e7d14dd9f064bc11025c25f31e81bd78a921db0174f03dd481d30e93fd8e90f8b2fee209f849f2d2a52f31719a490fb0ba7aea1e09814ee912eba111a9fde9d5c274185f7bae8ba85d300a2b" + }, + { + "input_len": 102400, + "hash": "bc3e3d41a1146b069abffad3c0d44860cf664390afce4d9661f7902e7943e085e01c59dab908c04c3342b816941a26d69c2605ebee5ec5291cc55e15b76146e6745f0601156c3596cb75065a9c57f35585a52e1ac70f69131c23d611ce11ee4ab1ec2c009012d236648e77be9295dd0426f29b764d65de58eb7d01dd42248204f45f8e", + "keyed_hash": "1c35d1a5811083fd7119f5d5d1ba027b4d01c0c6c49fb6ff2cf75393ea5db4a7f9dbdd3e1d81dcbca3ba241bb18760f207710b751846faaeb9dff8262710999a59b2aa1aca298a032d94eacfadf1aa192418eb54808db23b56e34213266aa08499a16b354f018fc4967d05f8b9d2ad87a7278337be9693fc638a3bfdbe314574ee6fc4", + "derive_key": "4652cff7a3f385a6103b5c260fc1593e13c778dbe608efb092fe7ee69df6e9c6d83a3e041bc3a48df2879f4a0a3ed40e7c961c73eff740f3117a0504c2dff4786d44fb17f1549eb0ba585e40ec29bf7732f0b7e286ff8acddc4cb1e23b87ff5d824a986458dcc6a04ac83969b80637562953df51ed1a7e90a7926924d2763778be8560" + } + ], + "random": [ + {"seed": 1, "len": 0, "xof": 16, "key": "4e2873a644ab37671ab25c9962c09661f89625981a15d94c02b696cb27ab6110", "ctx": "lambda-vm oracle review ctx 0/16", "hash": "af1349b9f5f9a1a6a0404dea36dcc949", "keyed": "e2060fc733e3b0c2b258652b301a876b", "derive": "18592c7bb4d1ffeaa4c1a65c9033c1ed"}, + {"seed": 2, "len": 0, "xof": 32, "key": "51f2f6c1699b22240a722f58d8f4e1d847dab43424ab9c7f6cb007e12d4f5575", "ctx": "lambda-vm oracle review ctx 0/32", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", "keyed": "f53925740cc09cd9a37b1d00920479d275d847e0279c49a1b65001943158da28", "derive": "4624d0bf48875f0faa7f0a14e97b564152d1315d6f3777ff5c8bc4fa7df5b51e"}, + {"seed": 3, "len": 0, "xof": 64, "key": "772fa66e71671508c8633f754cdc205bc7ab06a0b8b0b056d1cb550195ec194b", "ctx": "lambda-vm oracle review ctx 0/64", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a", "keyed": "69d6a0893237ef913262d2dc8e2c71f8d6e8346c3745e7f622898791993de30b227fd4e9500b07b0e42b0f7978f7bc668cf0053f27d993bd3b85dd0567fe3549", "derive": "61aa6ed699d26c31cc82858c9df151de16f5459e752b62d1829cdf00dfb555a4c4240102469bc7e1076871b76c1276734dda1f5aa7147b7c7d8613f1cb763001"}, + {"seed": 4, "len": 0, "xof": 131, "key": "0876f38380a77686e29a0c55b4c16d1d07ae429f299d141e3245cc47f35b9d6f", "ctx": "lambda-vm oracle review ctx 0/131", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421cce14d", "keyed": "a5a703baaa710a23e4386586538ff90cffba019e51751a27d98a69e15c372232737faadb85d92412e653759d94aa5abd818b75a9b5108a252ccfdd8b71d92cb9e96b3f52777a355f36d69bc2c495975f889ca448bcf3e099b7940d7fe67038bf2004c04bcd164664285bd69852fbbca42ec60b44adcec2e4ddaea9cb1506320a577b9a", "derive": "a82dfed98f8522164b49badbafa145d39e734f9e868a3c0815b79df69affa65ba9ba052ac0d15b4021676f99dbfb8990a568b77c2c19cbfb5e6fefa1659c08f856b60f8311c676107d56120f9693ee995002dcc4b5e90c4bfdfe7ecbef71e0c40422c015ed6fe5e57432dc6b8c85ffa34b45e43bf94d075042041e47e963e235ea407e"}, + {"seed": 5, "len": 0, "xof": 200, "key": "9d0ff0dd6e45b03425a5b93d7f2f78b817bb8647ea6dfd057fc219ea00d1e2fe", "ctx": "lambda-vm oracle review ctx 0/200", "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262e00f03e7b69af26b7faaf09fcd333050338ddfe085b8cc869ca98b206c08243a26f5487789e8f660afe6c99ef9e0c52b92e7393024a80459cf91f476f9ffdbda7001c22e159b402631f277ca96f2defdf1078282314e763699a31c5363165421cce14d30f8a03e49ee25d2ea3cd48a568957b378a65af65fc35fb3e9e12b81ca2d82cdee16c68908a6772f827564336933c89e6908b2f9c7d1811c0eb795cbd5898fe6f5e8af7633", "keyed": "58265ba18bfcdfa1863b0adc953d516e4a3cec2e2d09c4de65fff6d5ee7cb0b9bbf98267b3435d95521c20397e729b3c0094e9c3bfde552931c3451764b6574c7bd0d058b9cecd209a54df2da39a6160729b8e7df79395d5b48204055973b27f5f1ba83971fde2fdad3e6bbe2729eb3e76859edda23f577ba277546934bfe03e3351349cdcd05c71b04677852ab7cd0992c5faed20eb878fbfd374f5328a92bc8736dce061f787f0a9f435d8d9eab7e1ee5245a2e98a556917d2fa2b996b7ff267965f883cdc73de", "derive": "c49da0d52d5230577d675dd17125b7c9d12ee8ecd2dd6b5c7a3bb87e31249b0817a489137718c47e36f1f55fde71db1f47ea3a16c0e53d7e8f61778a199b664d88d598dc0464883110d1898a53f32c4e72728f774bdf5b949c984d645fc48cffd6fc8b867b36b8c568d91cadc0207aff6677ee79f0ca46b34aba8fbbb4d7caead47c39c50468d51c10a9e2d0a21f95360fdd067f234395e488b3a8110faa3c2d791c3bbb836c820a1f27bf8add1994ba93895fee5766cbd6e6524308800bd6763a18ce98b7716faf"}, + {"seed": 6, "len": 1, "xof": 16, "key": "317c727c3588b8406115f201e6afb5cd7abdc51490aaec9c29278c93d48ab42e", "ctx": "lambda-vm oracle review ctx 1/16", "hash": "f633b185e996eb1973f7b5e81da3add1", "keyed": "2cf20a61154aa7b99ec1e808d64504d3", "derive": "031fac35c559618b2d8df9ba8329cbb5"}, + {"seed": 7, "len": 1, "xof": 32, "key": "c515bba50da1bdc09d42110712b6094ae75f9ad50bc151f1efa27b7bf6a4483b", "ctx": "lambda-vm oracle review ctx 1/32", "hash": "ad821f67cec320b5cabc58516eb59f00ffc47e0a23db617879a2817b7560d081", "keyed": "5c20577f80e9d2525dbf76f370eae9b6f2e040990b06323106ceb3f4c59ca77f", "derive": "ab7326c844abcfd063239f0b96a23b1adec9eb82b0a0a5bff421bdc0567a8813"}, + {"seed": 8, "len": 1, "xof": 64, "key": "c54c0ceab664949da493637296c6e82ccb0cbcd3c0fab16029882933ca84927c", "ctx": "lambda-vm oracle review ctx 1/64", "hash": "ad821f67cec320b5cabc58516eb59f00ffc47e0a23db617879a2817b7560d08140f3fdb8e19afa84cf36653dab48871eee1ae3548a75bff74a0f7268bca72569", "keyed": "4c4897a7500db1cd436be752b3186744701743822d6e4926f7cf662b56749f49d2c4569660f7ca12bbfa1483e42df23f21c90d93fd563010e196bf2f19f01871", "derive": "e7696f2d701338d5725eea71e792aef3fe724ee2e257feedb59123334f8dfaa07d57460273286218154b7722ee7d6a6b2efe055ec942df34eabb325afe39bb42"}, + {"seed": 9, "len": 1, "xof": 131, "key": "eb899ae51ef9827c78b6d1b224ac020a475fcd4387a2dc0afb5ce436c256acbe", "ctx": "lambda-vm oracle review ctx 1/131", "hash": "2ca3a1b761c6412251d07b8a707fb85e306530bc393bf47664172bf5078e278177f5fa73496da85a1f43bcbab552dd7bd29627ef86ef7aa1af9499a15a1bf681c3a273bb00421538827f685d575e2f6040047674fd005d3c3a90e83b34a72f8c12b528c1c7ae262c79b5f9e51c66344eddd3b0f5a1387233bd10247c302f6de304d598", "keyed": "f80116f4d31674dd65a0cca63b2ad7ce5e07c25ab4936eab9532422760778ec7bd59719681aabdddb89e23855cb7c8b28536dd7776d22cfb840531734b1aba68aa70b2ef0e485e729618b12fdf9354081a1dcfeb29ed6e65daf5cc9e46e68ac025c7b4726ddaeebcd3aa8c3ef22190322d350729210c51e5bda090c9c2bc985aac4d04", "derive": "691c85c36007c7270f0608ede75a83818afae06b550aec1698ea81935db7657a35577fb9bd3121aab6c952a0367a3603d8e79072c07ed3796cb0a2e926677a06757d7690d3f66ca7ea5b805abba98c17646c6fe6f1894d4bd0f4519c5c7cf8a9cf2cf6672041432d20da197c01169972d47bd8e3f8bdb34e33d2fe89369964d5aee5ae"}, + {"seed": 10, "len": 1, "xof": 200, "key": "119a6021250e721aa23d6096a7e6f8e9e9de5cc085ec7540afbb588f84476632", "ctx": "lambda-vm oracle review ctx 1/200", "hash": "f17067d6d29eb1bb0208ce84d40ee47504c8656b67e7b663eb1d7182e090b0f4399fdbd2be2d9455eac4e40814d8e5b6ed8fd9090ba9c067a17c959a604ae5501bded22884e9617c71ea944374ce143bb040d8cf377b9a09046e9c769710e755940120d137f08a598f4eed9f69f3961ee13b1f25a661dabde0c6101b3cc7b492a72a9e83f4fb44adc30a86f8124b351dc18d9b7c0c711d3a49ca2b5eee3a341e45970d9981692df5b6b65103f9bc270c1733d30d815ca09b8cf134aef4c8bced5da99519932fdb51", "keyed": "2b3b662e366d5b27c2abfdac897acca87a74bfb3eaa3c44a23bec82ad6c923b80f799d1385fa77cb270689f7a9fde16d6f335b76dc4b16b5457377736ee78c26bd0cd9bbab04f8513bbc98709ab3bf5641d4f6de572607748ddde857df6532cafa4075aae6e0afd5940c62af20e71f78eb8aa4eaa25386cb2aa6a6e619fca07226715e4001230dce1ca4c74365273a2248fbb65decd69337fae6578474bb3a0d04e347f930b5ee431c64ed9b528143becf0e6da9c78a3afba6183a08d85877a6a64a79055738c96d", "derive": "566ebbc1e00fd71b6f35665b60075c8238bf87dd3bb4f58a0b7e94b6599a37a983f27b9274b35c9187160e010a70cd2b3562ddc6b779b1d3ff177114e8a22c827649fcbb26db2a6feddd5fc135c057c3c9e4ab63bb04d57bfd89c3ddec14b7a85cb3b12951ff8c5a0ab7bc983854787bad61ebba20e33f85f79cf8f49c0fada60b62c412e8cae0c2f75e374f24c5a41f274ce041a7379881517c69a72dd631a94ea72dd82ef1c3721004db70a1685330337a30da3678427d1aaaffe3a440f2f6b6b9a5fe912511b3"}, + {"seed": 11, "len": 2, "xof": 16, "key": "36d7a2702afdc652e3931fffedc5e1aaea8215f349f940ebd1eb941dd5e47318", "ctx": "lambda-vm oracle review ctx 2/16", "hash": "0274374489ae986e5475921ba458907b", "keyed": "54ff07b6a6acd7a1f6b0961b318bb08c", "derive": "09ad6852b188894de999199fb6fb74a2"}, + {"seed": 12, "len": 2, "xof": 32, "key": "a5d7f0a5d770f63da9dbd45bed6874726ed533458db65685690376ab47d6fa82", "ctx": "lambda-vm oracle review ctx 2/32", "hash": "ae2ed86f46c2a7fe7d0f242aa6aac5996059df9533838c17c786c04b9ca0f644", "keyed": "de176d34225fe68f469150df8f5ba8cd61bc4a03872962e312b8dda7e83669e2", "derive": "30dfa512613e64ea073c79b3bfde819c269dee858a1af35a9ebe53540098c6c4"}, + {"seed": 13, "len": 2, "xof": 64, "key": "39705c1c83976fb01c0ac14b1f5c5170b5ed292906fe65e483ea6410ac670871", "ctx": "lambda-vm oracle review ctx 2/64", "hash": "c91ff7a53e49408b407f10d8f9c8ed080b5de4811502f0d5f9c5096fb57df4f01b5b2fcda59f9150970abbd8e5bf0f30b5658b5d236b66a70e26a566f77a88c1", "keyed": "5a61e26a1347ee4e3712cdeef297b4a435a896cab11ca069195291f401412a26dbaaf4e03fd66a258bd656f094b4deb643342edf6eb2d237cc5e5c780acdea9a", "derive": "2e6195ec1ba4d7d8f198e6d5e55690bacb567b00bb3c0fee39103463f4353e7b85fd3d70172158b7053da07500906d6d5c3e74e661c46d364f096064bda855f3"}, + {"seed": 14, "len": 2, "xof": 131, "key": "f024acdc8f5356491b8fad84636e34b608fddaac3f4b5c0c67580d0ddd2766a8", "ctx": "lambda-vm oracle review ctx 2/131", "hash": "f997bcf0a91417c7a4a7eedf735d986be1b183d455057a5655fd77147c0db0396da06133facf9dad5523ccb852b43e531f415c4bf0aa170fa9407b1e6667d4e0e3bb59ab2ffabbedcf060af9bda300dcc6fcad1b223c6717b574a1b512777193a9ea7684ee725fa6a93210b782b128d2aecce569918501e61ccc664487174dbf6c390c", "keyed": "0a4090e3375db9ad9dcb59055a53cee9a40b48a7c6f85cb6de6e4b77d626e2540bf035a7e9363b04e0595d82107d0ad723686b76de610d2b32d7155e1874af4763a917b61a47067b0c25461f9a4896fcb1c660ead2410858b7a29aa4ccfdaf49ed28f505085121acf3d75cba353be74a598a8e81560f9e8e7174ed6c84a3f623ff3dbd", "derive": "0a8f4b6d80548fba6628b0f9342e27fee4be72a5498b6fecb46ac4a45b5e320cf94ddaf8c7a4131b6e5b46183cf08dd2ae5310f75dace83b9c5af6d7ce7041e01f40ed2afaf4a38e79409cb8203a5d75301d40a99a45f3f0289757874ef5df2a23ba941b3ba23bd70047c5148bdd45ef502e598a8c3ca0390756fd332cea76d1ccd0da"}, + {"seed": 15, "len": 2, "xof": 200, "key": "85bd87a61d3b05fcfd17c2258ad1a40b37396d6aa816e1127500d5d5928a0a90", "ctx": "lambda-vm oracle review ctx 2/200", "hash": "2c9c567961e877f29cf5469d28f8b1341295df42fb56e5e2c3c240bd63c57778f4bdcde180ae4be83337936fd51f01349e5356b15319852c5b50dffd1c099c34090283b352e8d350263103684047399418cc319d770105f23ca00634cc733f06e320db36d68954c6d44fc6b5853462ea1019a9941fb1041cfc00d590c1c4e487569ad44acac818b9b3ee7d406c736f41f70f72cee3b133b2241a3fb8627a0fe3fa92341a6b6fc5e1222aa688f49d936b6b2ec99cfd1fde1fe6611fdccdce74efaf54a9cabe220723", "keyed": "024283d0b8224be9032765f2cd1076dfc7f4cf6cb947a9c7367ffb4e082cdf137522dad8ec833a35c211797431e9de7d7ddd3fc35692b7cc943400095e978db894b9bc37e1ca7ed271d1edfc0befa95a64a39e71c34b46d4d557e3770f8f01b7c8fa7349ffec6e593e10af3d7bd713c8faef0b00272b347a7baa2a135f006c5f3efdca2ab53b73532f30eb377ee52f9833a556cf76a90e51da5c5e9b42666e09788d4308103f19d596fd961255801a65ce10c85735830e039830ffc36da06ef54eb17dbf3d99e3c2", "derive": "600c0981573e4afba64240a4246475ee49f8b73fd8c0f89a122077676c63e59ec6633c5ce0b851122fb91bbee915b6c929ec653b651986c0a5f9b75b421f7e2813c46f885bf203ff25e511cbaa904d96feb55c336ca224ba11b1bac173072dc69fe3e90fab5bf5b11819479652c5afe014a2e42406191ee4c456a03f7ff7c06ff77c4c9f9569592500ac9f1a5e1491080c163e76e9f4ecd849f6e2818e97db2cf67c7d616d801717002173f4f331da718840597c7b381d1e164a789169ac6c26ea34a470f8692bfa"}, + {"seed": 16, "len": 31, "xof": 16, "key": "a7f5e028fe27073413baf3c095354912dfd380d484e9cfeb548c1492fdc5bbf6", "ctx": "lambda-vm oracle review ctx 31/16", "hash": "e45042ca397670d5d2ef5eb5be0783fa", "keyed": "a5a9e20141df86780d1b27f11b0d1bd4", "derive": "a3bcef201e7ab8174a69e1a97ea66de5"}, + {"seed": 17, "len": 31, "xof": 32, "key": "cda155913dc188c34335e58f73091fe30cce6cf755a62f967c0837a51ddfb283", "ctx": "lambda-vm oracle review ctx 31/32", "hash": "b014f91e00ef5410cfef91b1bd8efd88d398ea2ceaa8a6522ce4157d6d57a5fb", "keyed": "61bad733c76597cf015c4bf9bf6bc766f06369fd79a6b42232946be96b84f719", "derive": "56ca83cbed45c5e4cf5925a22493de95b252ece7d578655be136a03985e2cc96"}, + {"seed": 18, "len": 31, "xof": 64, "key": "d0fca40ee0a6fbbb65f2e91e27643c0d4141d20b6aae333f64ba2fc4661360cd", "ctx": "lambda-vm oracle review ctx 31/64", "hash": "3a9668075cf2350c98e361bf3138f6de927f0c618974a38df8787c440a10fc6e6839bd4613297bf68e1f591f24087f4841a50e79309f96b17b132dd0e1ad627f", "keyed": "b6bde96e47e786800e0f501a77babd92291666f168d05a23cbe57cc7f6384d4e58ec2199f0bc02cb719632403663483a409986f48caef7f1692d5e32668b52e0", "derive": "01e6cfde455c562b5d41ba800be896c0bde376fa3803ce37bce821b92bbfc7e1a08b375bb84eeb5a2c3300dd667ee9212d441cd382b73b97fe0b2172aaf3499a"}, + {"seed": 19, "len": 31, "xof": 131, "key": "f6a765475e789f2922e039484a8f8b1d9da6328f67701ec6ae99c800b5f72b61", "ctx": "lambda-vm oracle review ctx 31/131", "hash": "c42704dfc6c9013843886334233782451a5f5fbb8fccbb8e1b6ce89f3c2c8d82674dc0555a7313e23254a13c195da6c8c36d5adff3ff1da19a739a78efbf324b9d32017bfeb744676230118e5f7714cf22b5ad0eb494bf3b60f67e788b11cb9cba815c3a79a60314ddc7c8be71d6febace72a99b1bb6524446fe6e24735335dfd4de41", "keyed": "ebc1ca503e3ff98fe60319c9464e35b592b1169515178d01a7a4a09b9cdf58d041cd589b7d91438b01cf16453b03917b9ff4f33a9027c6e0fda2d37f526942120908541d46f232981f2927e26bcd9a6a60382ad2f09ffc581331910c02176428c85439817e96262eba404a76705e60cb689c9676779b4945bd25a485f7332951a956a6", "derive": "651fe4214293835fc5aa7353337d3dbee3f9f8f674271c403d41e4f22168a427cb60094adaec81f5f811d54c7facf79d23b7c8090c364d6f25da9263d57b053422c7f6d4524aaeb1e064b5a13ecb76df59e74a28c3af83980717f9c50f4ac1969b57e6d397d3b4f0e36acb2221d3d82779866b9830dbfa0c98968519f5d8ee9f2e7d21"}, + {"seed": 20, "len": 31, "xof": 200, "key": "427d2be2a0a273bcbd83a71c6356ae4db1e6dae70d4ceb93f8d3ceb5b3c2b0f4", "ctx": "lambda-vm oracle review ctx 31/200", "hash": "f2c450b25acf5d07c0eca77f91565db7b1724e825cec605ffa4c3811108a19ce7ab9bedcad7e970e2bdb188f807fcc8d176d7af48528a57f7f7e97b4cf480db33418b8faa8e9e0ec76fa02adafb7751e5ebec1d9a088e7234b3976ca652b2a58fb50402ad4db627b4a3f087295ecb180d75a2ea2cbb99aca5ff1cb01ae0faed7a17692f03d13454834f14247437a9a00749b68b17c633ceacbbf1ed72edea9321d3e3edb41a76acf21dc406224524a1ede3bd7f535bbce38fdf3429798a341245eea76c3d9451fe2", "keyed": "0279a7640de3adf0825a3840dd78c65a80162306b133ee52bc207b2b7bf639959a6fc394a2e23e79ce0c7cc44569aad0d3d24b3c39c010d5839744a86fe1f47a1b55e4dcd2898d13aecc7e9223010097e5ccb05cbaeb2b258c8785fb1d2edbdbbb827c9c5e2592ad901028a5cb5cfda8215acd4f36c03088f8c0a976f410991cc499ea5da7587c671a17d200949e6e801a6f1e8ba9e9bc6e704313f9f577b5a649f94c75ab983708388e232cbaba93097fba3378046efec08279025967db8540dd0470d24c5eee3a", "derive": "03a483217460e212f6c9454f0d3a0e0b0ca69d4e0bc53ad07724d66565598b82cc6bbaf8456c3d9ff6eb092ab38ba6308c82717daefe30ee3a963a531e6006f550d72bdf9335fd81e98dc5ba2e694b34b4a5db8f13f015f9c0a0f7bca527e2e164635dd3503026b2060d6d1ee44f887d5c3debbab2189b285585801d85a5af8b0374e58bf839ef6eed1c6a912d57983ebd5292af99b15f1e4f08a5c8cce02b625e729f5a5a1130a1f78de0bc3da9860349f4825963906b01c49ed7fed54bee7ed8af418d109d6acd"}, + {"seed": 21, "len": 32, "xof": 16, "key": "d6a839c8611a7bd19c9791970aa61e9453a6ce5014f54c84af38377d711ce1d7", "ctx": "lambda-vm oracle review ctx 32/16", "hash": "1f8db1bfdd8dd100a84848f8824b888f", "keyed": "0bcb7116c7635074d6df75bf5d5c932f", "derive": "c3191d21509721b6885d7498e86005c7"}, + {"seed": 22, "len": 32, "xof": 32, "key": "6b8488c931ef0c80a7a15aa039a2a4da72e5fd97686f344c037c08c151a7ee29", "ctx": "lambda-vm oracle review ctx 32/32", "hash": "39eb5d3bf52d1c4855e9e7475cb795c9eda725f55f8e77cc5e97decb3a0bb2b7", "keyed": "cc5dfd72264e95db8a511bc890e2339bfe3c8330b368fbc5208b33f072a9e0a1", "derive": "659185a1756867f59590deba74d025854eb6b6d3d2a61ee8340ea9efea42cb09"}, + {"seed": 23, "len": 32, "xof": 64, "key": "ffae277d623842753e96b906f13377ec53f8218c863a7ffe52a5805ac20c9468", "ctx": "lambda-vm oracle review ctx 32/64", "hash": "ddf820aa0395b67040adf9de32bfa880bd7bd6a63776c5834309436e331523aa3bbab766ae85510377fce8259d0b5037bae8954b037f0faf9136d5858053f1af", "keyed": "108fb30d7a4ce089e673a206df2ba91600b992d5ba40f04fdbc0a42b99e6e1dfced81ddccb58738b92037a6903d2d669c9a8bb8880e45302f38824fc7aa3177d", "derive": "b14a08b72587ef6f8e95ad7c39c02ad3f635da206d80dae709fe6a215f0f1a5e6d897e6a0d6666a380bc4896dcc9eba9f711845f921c5a745eefcb16a88b9566"}, + {"seed": 24, "len": 32, "xof": 131, "key": "445637047e96984af1f65f8d683deeb28858305868dcd5da010644a281f75396", "ctx": "lambda-vm oracle review ctx 32/131", "hash": "04153ec8aae42ce4e967b24e4f0fcea47b0b5bc6ff27ce555d0e29ad1afb044681191753eb69bf3c246d04bb83f0c5fa6c2b169cc1f84188a72c52e28b86c953f9bcfc0428c49c4a9ef3ac2d6d9f372ba83f8fa4bbcf568b38c05d4e0baf9f8cc5a1b915b8f86f1eb8baf428b79bf4eb24eda5831a105e0cd7b9af8954d5342b4d259c", "keyed": "b78d7c2f73e8bb10ae2d6ba2af88034e071497f362978f2c6dfb41038c2ae7cc8466a2e599237145b1117907fc23c9f74b3f9d7cc81e8c75cf849d31c24d88e256feb79810ca0c90a07c066b95464a0ca3acf5848ad903883c4cbc31c033f6e71b0ea152433769feb322503af0700be9833e8b8101d4ab0bd7302880ff4df965e442c5", "derive": "33a9f9cb60dae21962da63ee1d6926ce0bf1292e6f03760396d5b292c295843b13ff19472fe964caddbe09559f9840d1609ad0a3be1ba5a4b6c0c4688585ecb1a399beef3d4bca758fd94f96e0aba1c86d815fb5ea72a7d03a29b7abc2459a77aaaf8db735dc80abade3181de8bd9ad2a0f4b7d155b49e8300666252337b4aa37c975f"}, + {"seed": 25, "len": 32, "xof": 200, "key": "6a021b8bc4caeed9bf9bdca81c38ab88786e351821c08d22314b361187bc7318", "ctx": "lambda-vm oracle review ctx 32/200", "hash": "e04c6f90e7c7c64dc4482711d6fc69dfd0311807c5de185abdc58bff4e5dc75c950ca3397b856cec6a773e31ea95470bf2f8d16072142c35104d90537e6644d191387ee0250d617def6ac698c20324cb1a7ad4ee0ca9a72a0be13478341383041ef3855c6871ca02f92c0a30ef98748200d1ff5c483f28068325d80880b37ec3050587fb66b3bd40cd84c701c163b0fd33a22a26c7fddf7833e7012f0d282613e263ffe5f93ba2997829ad0b6583e8973cd656e531ffc34418bb5c454ef1571a8d719ce4aa9dcbdc", "keyed": "3ee76706e52a25d0b11fda2008594cb7c0d108e0a30a799685e68a36fc4d576a872dae2c86182c06f744cb84088296f26397726455a9cd3076027851f94cb2db376daa3f2fb478f3df351487f0d55bb9b789f3481496894af5f7295e76c9144bd6db381d89aa984f0d0f1fee767c61899809979057129f8395ed90a6eaf53b910629dd38026013769f2a0761681449cbe68e03e7e87201bf02b68f1f89397bab16da8b38dbd50278618fdc4dae10946246fa6cca05264bd6f4333921e2fbeca60bba45240b2e248a", "derive": "f8ce88a2f6f80dacafd922474d734c038d7eb0e6bf14a82055d64f14343336c88bebf482471aa7d0b4119d6bbe762b4d9f6284ceb83acb215e575f14e0ec4331b1e5337f0b78f5aeb683830b8337df041c0270aea9d7afb7fc04875a0ec985e16a7b2a60473a3241a7aa404a713b86061d8f87d20135a29294ff9cf4137175064834410219d6d8cfa009f9f866d4560488eb5933925cd15a4461b25a9a2f002c02a39c2e3a7c0b0517ffca563c1398aafb02c7a786376321a38d09b8ae68491b5f2d1c43db26d2a9"}, + {"seed": 26, "len": 33, "xof": 16, "key": "90a4ae28a4be801aad16678441130895919436864549da4392404ce5abe7f28b", "ctx": "lambda-vm oracle review ctx 33/16", "hash": "ef56ebde8209e7f40467b2a34930f5d5", "keyed": "3cf2f08202f6d76e0ca02fdbf2cc3f19", "derive": "0fc74c6095da0ff7afbddb487f526fa5"}, + {"seed": 27, "len": 33, "xof": 32, "key": "b54f00030d216c7e1db895eecafc89352606995fd98cbbb88426abdea6c0e38f", "ctx": "lambda-vm oracle review ctx 33/32", "hash": "61472133b26ffd46ed8a0321fc278559d720a5ed6c43a55bd771bf08aacf247a", "keyed": "3df5ec7583584487ab47307fc118aef9bbcc697193ca78ce4d010a1537359e66", "derive": "346249306132497649f765a746576fec3c8640d5648f6c821e01c94d6635a0ed"}, + {"seed": 28, "len": 33, "xof": 64, "key": "dfdeb3bedd683ea5e7be357f352108909e305927738447cd1e3b12868f09c861", "ctx": "lambda-vm oracle review ctx 33/64", "hash": "9d2b24db2bc6fe675948d163f6d44d6575581aa479afe1c1e831044c1af8872a48a68959d7b6c82f8b8593a69af6270c73732fa96b5c53a83a5f3e73a0c7d239", "keyed": "350e2f86f87fae4ec59a64492eccd9da82d59a3e995b4aa5adcbbc66c1569fdde120c789a3a7a52f41fde3707ba3a06cb4a1d35af80ed299e9b3fccf538ea0a1", "derive": "568cbc5904963dafb7b00e79915b0b684b153b3a4a53d11067a7f01ef6b84676f75ca7844f50dd15ce9c60cac2e7652828df27ba9a671545b53ce885548f41e5"}, + {"seed": 29, "len": 33, "xof": 131, "key": "73092fc14a279f4086511f0fb033b375c241f8786cac69a76b9e7741e23ab604", "ctx": "lambda-vm oracle review ctx 33/131", "hash": "bb438da5ee6ee50a61e2c984008a4b816e327c95f6a47ac8c11d7ee537c0706eb77612e0f2dad149286b098f892043a62cd91c9fb3856556d7334e1a5de3ea36fd3df32cde9e7d68e3efcf38b4a0acecce095c698d464765526d39146e66a6f2382f72e0b2db791f43844de7975cd448d1183a9aa6c5346941a5059349e372dd7ea593", "keyed": "1adb706a90fcced156afccbacda27db9e57ff84b19b4f73cc72cdcfbe9221b01832e8ef79ef9829f9fd922eeb4584ae68d23d61d19516048748e3414a9d0c495fdf513523a881fad9f36703fc9d6eacd0e36aa3d79deaa64bc6ec476fc5364b70ccb188b1400f77ae4038cfe490cae20f58486f33a6c0e0adec5be45f037631ea24c08", "derive": "08bd3f3f80c8cc8d61c67f361948a703b3c648b00df9543c02e1d894a8006707a204e9605106f2ec362e4e5a230ae89267ff2708aa45d04be8f4e5be36b539c3138bc847f827671d46aa4321c5dfbeeab4ec376687bc91f9df331dae0a187ed31a88b733eb2176438d773a111158ae75561f32beda6cf7cd923dccd14350addc44f4f4"}, + {"seed": 30, "len": 33, "xof": 200, "key": "2a2c4ce3bb5f2815a142a9155a0092a92a965f8cb20c2100498799b76422b677", "ctx": "lambda-vm oracle review ctx 33/200", "hash": "dac08212ee797b2e713b25a596642be078022fbcd06859a4ceb0990e105f971581078d79719fce9ab8baecf7d44f7742c5e7faeacc67795455549120cf86d2599af82d19d76b4df7c7a9332eef0589a3ea71ff56f7d762adb57617854ed8340b59e5ce7716f61f7e16fef7db370a1ca355df79948ab0635bc33c8df93d72706344cae8a9d94cc17b35b869aa6bc28257e871f336ca931f68b5ca2594af9ad1b4518e3028422660db19d48a810cc0c837f1785e0d94091c847799c732a9b5a9effb41823c6f32030c", "keyed": "888a4c62b8c06353dbd9889296c692675f44f803a5465d1395109e0f8a07f2b2190490613894aa47ef30b3f545daaad007df969935d3bec5643320f3660ad959160382b47c1559f4cc5351ea29fcbbad62320644a1546343f7c1717418207b239abd8834f453ffa07b12c743f4d697dc8c8ef10eef74967c50a6d425afd387b1f13cc97f53774810547bfc3f8377a76a75fa7df2267e19d62dde7e5b84a6f64a6df693da3db93f2c20debb4763341e6e70a79cd290cfc5393336d4c18ae70f3cc11263eed615d07f", "derive": "4665c81a65f084fc050577c83edcc924438c5ae94efcf08833de53feb697ddbe43a6114cb825a7962455ac7834ea85add26679b75e80abe73b260e3db1f604cc570f918bf97eb9106d029033d92bcfd8bb800d160b9aeed608c94a098837cd09a8f04cb5c9a9820bb100740ae5f41023831a0fdfc40c7676b3112cb8865911211efb9f39c3f8abd15d6c255b694c94e151f922750acd1957b5ed83de0b8ddd28f685a14855174ceaca79094fbe6f76be6c61c9f7f28d582bee692c246f325cd16b7ec854f8583e79"}, + {"seed": 31, "len": 63, "xof": 16, "key": "bf567d39b3e6ef07817fe823e62a01cd76885b59b1ca71618f1145521868d4f0", "ctx": "lambda-vm oracle review ctx 63/16", "hash": "4262e019e820ef30e1e25fdad359c3ea", "keyed": "9f6b2487c3fd74c2a9d9be0e2581fe2d", "derive": "caa2126e0420650fee91fffaf7ede68e"}, + {"seed": 32, "len": 63, "xof": 32, "key": "fc4dd8ec37ce39f879d8e00a0c73114a45e91730dc70d90d18e5e654c0538431", "ctx": "lambda-vm oracle review ctx 63/32", "hash": "aaa39d28da9cb6b404bcb7d32a2920ba27738aef1c3483a6cae69684bff16889", "keyed": "7f999b8c7266cd4d41d379e431090084c2cb66cfc23b94b90de8aec7e08d04d0", "derive": "1d27ac79f3f877ffac23c9398d7c82514b41939ac6ed4b5eeb04c355a0a03802"}, + {"seed": 33, "len": 63, "xof": 64, "key": "228af733f9098e51b260f6f3273f85c2c90dfb84ef8e22d8a34b97c534a28a21", "ctx": "lambda-vm oracle review ctx 63/64", "hash": "d05537d45f4d3f83c2a99116e2cf7be2360a0b25ada3f5f5df23ea147ee2dbeb3839bdcdcfd838a0a8515e0648af60d51c422b7acbce9a9e6b88b1783b307fc0", "keyed": "7c5849505cf5500ff148af37164943ba45fd7d471f23e7b81d8cfdbb75ad5e560fe1e35b063ede7d7ee23ab2b82b3ac68495e05623b3ccb8b3497a5bd7cd11f3", "derive": "33d52e8cfa97f0fa1178a5517f8bfdf44b77ba41488613124d3061e601a1f09d39bab70e5a1fd7bf50d753f58d52773ec61a1969e103265b4a900a4fe9c8aa4e"}, + {"seed": 34, "len": 63, "xof": 131, "key": "2576e5cef6878d6ad1d871bd733ff3b19eb40c1ba2382108f942ca47a157dbf0", "ctx": "lambda-vm oracle review ctx 63/131", "hash": "c9e1b5beebbc4b4555f390e7671456ea4e99bd0c609cbc4879cd7993bbba429a1796e0c86ec72ab580ec6cb97624d1c4fbae8188d7e712b666e41f73258a6bff44ab4077ec5caca38a7433be48c486e99c165087a546594e84a04518d46d572f50fc452e62c5123889bd4082085cfa4dece385df7263294220fe2e284046529658fed2", "keyed": "61dc08eddf340491716507f5df1eb74b0864d8a736357041a057a556b5539a69ce82f9b91bace73c49b01d841edc68f1ab8dab66b8d0e87ab8366fba501f7a5c4ae280f7db9f459187b18cec8a753fb500de2b7f6e4a35f7475ee9b71d0f570407106db39e6831e609f03957fc1fcebc8799536498fbd1d713b2671474004cc5474594", "derive": "da1263ffa944e8b880d03b5e8e34ffc10a310abf855c028bd483542e51ef3fb27f50c9bc25a276bc993255b93c1132ba8fe8870e8a8326fab129ced339aa145930e1ff881082ca6b9abcf2dd11849b4fcc17e3d8e871b331f662b088f2d32682f96ee983e913867b78eadb514a5d57eb9a60574b6d5b5c4eb9ec42fee23de04b77b8f2"}, + {"seed": 35, "len": 63, "xof": 200, "key": "4bb395e425b26cc9923fcb8a4b7d3b4780c76f48520d1037b950aa020de21884", "ctx": "lambda-vm oracle review ctx 63/200", "hash": "7f7d40e208d339f811314b5eb5d1bac48af4124805f7a02c78240c0f967e67c29a41d9a649308da8c46487bea1bf6eb8e0f3a38eb7175e68cbd731bace4048adc87502e2708c7579a5d070c5c0c6fdf57164a8a9a8b7b1329882d0bc0208b5bda0717706e61f677ebf648fc890ecd8adc02df65793c0b4500c96695c832d473d97962410b3c4d4e9e4431af5990f22082d6aa4ec841067b90222972e63d8b387ac52e7bc31d5d7861cf4bde69a46ccc376550144c92fe0dc1b02d3355930a017bd1ad14b0690f0cc", "keyed": "1048cb6fbe2f95bd6d2dc387c01d9efb207f63a30adc9b34567e9b7c364380cdd90fddf1cfba6fb406bf7dfd24c4d84c17c562f20459a739218c839e31593d69b4676fba089efd4367b047c10b2f4543a24fd66a9913f4ef84cad9a299382ad63226d507a2f5fd97a3e54792d71bdb2b6ab407f6ccb2675b97e5f56773f81fd776014c88cf04d0a3d0adf2e715a8f6e94876da4dde49189cdad6bdd4a460df60869a03b5a2c27a2375b022293e7f847893874d6d1bc1d037fb812035a4394bc8a7dcd2fd67ee4afb", "derive": "ebb1db90953880b49ed3c1cc8af140b21342c04fef76b6ab6580f7ba7eaaf332d49f38d1ee789b89a5a179e977beb397ee57fc1735a98e5ed7dff73ade7df059c9f3212d358c1132a6db7f12348758c7c1190348f703193590a6495a313c9cd746cdf4a2b41228e4ef0e5e8562e7e31825a9565293a545bda5d536fb5060bef37025fa3f17998c237a01a22c430b6344358b3caf4e47d06c0c382be109b97ab283459bc6fc06c71309dc38f7bff007b78efa679eaf7d34ae4227f2348f408f6205409b2c3942fedb"}, + {"seed": 36, "len": 64, "xof": 16, "key": "dcd8315fa4d93ab7401b877d6faf3d91216ef5be067af7f88203136a6c930cd6", "ctx": "lambda-vm oracle review ctx 64/16", "hash": "d157ddbefa378438f4da2f56ba3999d7", "keyed": "7d9689c610ea2ee1adf08f5fbafda494", "derive": "5aab60d350237580fd2e72c7fc88e25a"}, + {"seed": 37, "len": 64, "xof": 32, "key": "70712f68d2f62b0ca8b89e939e02ea75ce4906752926f53e13e5123e565c2ab8", "ctx": "lambda-vm oracle review ctx 64/32", "hash": "2f9e3ee7adf6cdfe9c2eb4c5258a7a6407a7ca7c6097c31e9016b7a2d3041a6e", "keyed": "811d9602e043f9300dd08eef1b7d80d2636dce0beb1e3d3021375830073e779b", "derive": "779db5d851ab425e380f54d3e3b7dbd35e86163d5791b7a7e6596846d54fb981"}, + {"seed": 38, "len": 64, "xof": 64, "key": "0501a6428cd4abddb3ba9d7c7c7b7233b7de463f9b2bc76271f69c2587bae2f3", "ctx": "lambda-vm oracle review ctx 64/64", "hash": "927bc3149626165538fd524c3bbfc3b9dfc2569807604b16625f7f1a6284d1ef4c8f416a9ab9d7ba0a345a302a823d5b5cb1c8c6a5773d3d3ebfb894013b6800", "keyed": "dde8934e7634a0d90b5ee4875c1cdb9efbed89f6772920f6b79e355385ee69cd431d8038ecbc511073404d93af5e5a7e5458dc7c661f229d509bf012c4c04696", "derive": "7e5cbf68decda9fadec8f78802089a0b5174adff9e5dd810e048361ef1ff2584914c0e5c60d13e25751fde8a581879b4400b857cb9953e9210784cfea0919a63"}, + {"seed": 39, "len": 64, "xof": 131, "key": "999aef198891c1934233fe032a2b70c199be1eaf910d9a67882bab195bb665ce", "ctx": "lambda-vm oracle review ctx 64/131", "hash": "896f9dcadbeea0c1e49fe27f7d1174641cd6e5a9c281c0b953a4eda56e89b0fc4ba78af86bc387aaf7d8de8895fb03e759242a139971db04e2fd40b63ddcd4cffd009e5eaf22a8e39fb3c8d1bc12f05983f4a9a033e7b67fe505b89ab36af9b5ac95874685e6b6b27d78423187d00119aa2cdc3a273d1e1c08067be0452831de1e9608", "keyed": "0e51ec8678880a2633455762a587de426a4872976fd22d77d6e5911e97acd1a880178c3c23cf9d0bd160e0d892dba818d287da315646730c7da310809c43061c38e87f561025c098e716edff0196671d113fd61f7e85b5b6214be0221549619effe0aee19c32ed46c4a2fcb2a8642278ab8d06caa80ab65206cd974b3a2c31ba191859", "derive": "b2c4be9febf61f6cee8db15af3c3db73029ff205dbf16af4254b0a069c5a948ca7374ec85f27821bf9f6d144d1270c5d1e080c811844fb24c702d9f05e9804ff83d04e1decacd935d307cb5214cb1d2c26a177f48c6ae6f651ee87fa73977b70ae50c8035df33d208af5c2a0941c68a1c5461f5b8c39abee9b37a5ec8d15b61017e458"}, + {"seed": 40, "len": 64, "xof": 200, "key": "0fc85f0e2a4062e32fa0ff98397bb7d36232ca22dd0577918026b85492ca3a6f", "ctx": "lambda-vm oracle review ctx 64/200", "hash": "602a68f3084d758d683e9bb1e87f90cf0eb3f5fa9b1c72c67ae7ea73844265307fdd7bd060407314419dfeb50831db7271624dcd8a306666a73e2eb7dfc7e0d802dc0521fa05eb69203d791e90b1e4bea32887225cb5980e2631488f10f0d18a98f09a62030d4dee3f03f58a0f0151d3c533e731df86e4df2bafe9d395df1f7e43e1ecb51c6baf1115452d04c11cadd5d5fd467fdaba98bc04537ad265e45c8a52fea8831c8ebfdf37053576584270f4ceb12d4352e3a4ca386d0c94fae692f94bc264d956fdf6a0", "keyed": "ff0f40bef8b7b7229ae920abfca4a8c95648070075d8c960f1add315ecb9528df208bd76e4755128e09b8db80d5f3264ebfeda150910a1bb9597105681632a6eddf071300520c5260a2be70ea6840e2e9678ff236aab3548e151cb3d50cfe224d9d679c0c63ade61af9128d3ad2832e4370820d896486b3aa7a45edffb2a2a20b9f6d0390a3d0021c81be5a078261d05d3ff2c4b1fda0bfb20db84d70e3e1b3890861415a9f0be072e923e36a8f58e06003746e51cf12bc001ae7d7fd7afd24819c7aa4e7101fe28", "derive": "05dc3dedf0ff11444fc56c14533af0fea2487ec806b6fb375447d898f91d31df614676bac84d24a592f9e720be0f75b878894dd31d2cf91c55e98c029a4b966e3d8bd014d503040fefa66d032d47c4f6f0f49dc8ec3b417c75e2365bb1de95cc3ea68a933a15330ab3b9854d93715cf3761cd67769349c5e2e26f68510e01d7bd5145940aa2fe0fb6bdad0c4a0f3b2644387acaa6a09df2da5658686adb1530620d9fb3fa0f8cfa117993737d84c445c0aad5c3405efa569880c34f5f52a03c4edcba84e1307c607"}, + {"seed": 41, "len": 65, "xof": 16, "key": "3405ed72ee6e1e23b3056a704653ef50f87fe7b96d04fdb34006c9e4cfae0999", "ctx": "lambda-vm oracle review ctx 65/16", "hash": "dfb16d551fbe1c4aa129d94330963e3e", "keyed": "fab08e278a8ce8560dda049029890668", "derive": "b602c3e3951abc54f9cb839e0614a03b"}, + {"seed": 42, "len": 65, "xof": 32, "key": "5af3a92e08b2d165afa1ffaf4b4a718184038d58cc4f9ae4d82c26efd3f990c6", "ctx": "lambda-vm oracle review ctx 65/32", "hash": "4c7dfbf6b14dfd1b1b6cfb4d7cc5fddffffaed6a8b0d22fac9d1c2a23659d356", "keyed": "19edef3cb85a9dd2aa998b98d32f1894962d92f737a5d90b07ec6cf56c57631d", "derive": "f2900904342a7229e5a5291301b60d6e188d591338ce5a69393f0ef72ef608f8"}, + {"seed": 43, "len": 65, "xof": 64, "key": "8030ebe642164919b2ff6dd0813434a78d960adadd27bf31473939e8b852211b", "ctx": "lambda-vm oracle review ctx 65/64", "hash": "e1050d263b6e42425a3943383bb6decbc90f6d7efac5353c2d71bc7db0555783781da185db68be42db253d7db0e473adc4751ced78b31aade695fb6173743825", "keyed": "701bc44c78df866dac167e79d952882c02ed7360755cd86ba79900f49630eba449fea7cff119f08a77086143178fa25b866effd36b4ff7928bd9baa241b04bad", "derive": "f52acc57658d78b7b9b320788eb71fcc3af457c04e12c94068d7805529815ee65af87693a00b873b332f5e9b4084aedbd678245493d2a5d30a9361abfcd96ae3"}, + {"seed": 44, "len": 65, "xof": 131, "key": "ee528981c9463bb1736ed4617b4d514eb78d9e7589f2169a2e9c63de8ec916c0", "ctx": "lambda-vm oracle review ctx 65/131", "hash": "36077359619cc4e2676b8fa3026eb9c6794bdbc76d73209a22276f7ef6724faa9ca61c43e9851dc56752d630469146dc84791fbd82c8213854c08cc4fde6148b19486a004872a9b54f9677ef41f3efcf112d3dbf217e8684b6fb48e002b78292f01bae8e0bdd10411d5953dc17c9a2830d1570a9158e15f76d1b67e371078bd1adcf79", "keyed": "b76a2dde6eebd30fc4844e18e704a7d3a128ffa2b996aa89b8f8d8c57400c67c4bfdd7591266d43d359b9e89ba680b6ab6145bacd51d72c2869982fb6bf7c526b319c5b123dc6233f5664f54922ebddfd1d8d25841c3bf7a12ac4a044292ba069c318fa73d29ec98fb86a9e25b0f369a3b79c3742e12e4fff2f8672d38c8b37c145df2", "derive": "77c702e02e4ff332882554f0cd9310ee7be65596c76e1c77e50b7df84065547f71efce4cea00d8bed8bda6ba8d1740ded9ccc42fe1685c3c89f6624f0800295659d0fc690f4bfb0da9945790b9ca93a44bcd48286400ea9dfa3aa38ba0dc1dee2977a932056cd0ab02f96856a44919079c03359201fdccee8d31881923495e0f3fa163"}, + {"seed": 45, "len": 65, "xof": 200, "key": "83ebf4a74303c8137f44412c714708d00e639b7094a7810fedd9ab0c28165013", "ctx": "lambda-vm oracle review ctx 65/200", "hash": "a5e8ad40fb449a0ca7f50cdf35b8b5f192d84c646c48e1f3db70a8ad6f9de5743b26582ef3095962c2e70cde4205085562fa2003bbd4d6ff05d6e549bdf140cee596c179f3b4c83f06e1900fddac36630d16a89000010a2dffd35f70019fe3d35084e0171731907acab1ada9de117ac51a6bce94131931f4c1d6d2a16fadc000df322010c869c9e957f3d23798a08854abf3214792033766e0e6c01fe14d0061a397e264e4ba36e08e7262db6d20114816d1bd5c952e60b233bb83628da2161321d09ab7794445c0", "keyed": "ac1ccae4705f7f9247e0c6e92fbbaadb4c07dfd817b81a0c1ae6e6c3b8cf10a7281ff442a2991121b383029ad9fb0896965a77a4f8c0ba025eae95ed1acf98fcd59901212fae11e458be939b28ad7ab45eb0eb74e2c54b8fec25ed0c8dd2e4c5f785d98b07604e03e87f4936459f078d05c9ad9c01e1be625e61a982b6f01ff3398d47972624ec97cda2e421f73ecbc9f6c95feda44fd303c98065c1b848b1c623fcf9b2ddf7a94198894a9733731b5f9b33f93a5a502b39cb95d7a2c98eaf9de101f8b8d1e01706", "derive": "18b00d90ca81bd1c666c78c35bd781310aac74c7d982d6b30569bdfdfe157d2f2f41163cea7067b9d20572c946ac04344f8f680ce8b57f5c2f16e6a5e6b9579315283b0e661dbd1c873aa1061fc92c9e6c173540fad0a591d6886ef2953c9a6b5d2aef3eab4955afcc18e287b3adb4127b55febc507fa290746c8d5ccea4bec4ad278b3c06feea13725fb6e527d7e5956b65bb5743744117ae79d9a604055be0425cbd0007b127279a54aa09f5de423a72ef5b3d1421f5ffaeefb00a9adbdc338478b9d80086a385"}, + {"seed": 46, "len": 127, "xof": 16, "key": "3a7db0a1fefa0fe7771a95a8b9d8f21e7f7874fefd179b050e46904740340539", "ctx": "lambda-vm oracle review ctx 127/16", "hash": "4778a894166f5af5a67a7efde5af3adf", "keyed": "2dc7c28af3cfcee3cf578e4695d9b138", "derive": "f7912e7c9bd409ab95c01b5b7a068585"}, + {"seed": 47, "len": 127, "xof": 32, "key": "ce178a1a3b9de3ce2d077eef0e0e99be95106dfc3c0c7c89d99186078a9fd209", "ctx": "lambda-vm oracle review ctx 127/32", "hash": "cc328f5912ecc945f8cc29a43e6091d914392a54af859f09607d836b79a3e745", "keyed": "36a924463ef891bd94690c46fa412e395bf7bf812a9463645dd6135f08b116e8", "derive": "2b0da95bcadadbc0fb612055ed5370b016f354b620b1a5b7d108554bf1986cf4"}, + {"seed": 48, "len": 127, "xof": 64, "key": "7b57a2d7709314e0891179c6df8ed261124481c5a5619e735e48bf762831ab09", "ctx": "lambda-vm oracle review ctx 127/64", "hash": "acaab09bce3654c8398ebc7d7253b4b103d22133b8e0c880612b59ebc0575a1a786286c8b9278936372b029a79c963e10046142c1dbce65ec9048260cbcf6c5f", "keyed": "a095b8ac8eb13e496f2afa818bdd18a43bdda3959c2f2dd45f45322b6d0891a7c6ef9203ad21ebfb9da8463f13dd3f92f7bc461bfffe8fd9c12e03859e5cac1a", "derive": "a4712351b39281e204f97908eccc9326059fd4498f833ee9531bd083c567b1506dcc1170eed9ac0a3a88fc6fd4d8531bb756d0b2925ac515a68ef8a46da24bae"}, + {"seed": 49, "len": 127, "xof": 131, "key": "a103179229a58b5516244c74202acf3458ec9c505c675fcec68f583eb6d4610c", "ctx": "lambda-vm oracle review ctx 127/131", "hash": "997bab908b56bdf7ce69a0f1c6a4eeeecdc2d5516fd7fc18cf29a959fdc0e85000b8074e90d17c3fe63213476aa17a193b6f42e1a76db2ee05d5b511ca44d59fa84242d12aa698a21b63a7b96c899cbaec11c95214c0d55dfe5465c9f977c02ace60856fdcbbb35a7c591da0a1cdf5a9cc044c0d87f778acdb258fb21ca6617b51c29f", "keyed": "239e5db4ca6fe92c80f9680341715cbeafd6733cbf111248088215b72223553f47aa63c55e7a15bb8a9802984f53fbf43033e3c23b334fa16a06b59c18551ccca47b34c4f6fbf1c820942efd2d55cac5c3f9df826ebac493d70cb7dcbef7d4d2277a595461d7a8046c544ae779fc327efc6f3e56bc16ce190efbde419cc5b03fbe4d8f", "derive": "b86ba40d7ae3eccbe5a6d9e4533e65326a04355d58d803e97935b524b937358c9ba594a6053e38d043bc2e15396a77e4c1285000d8861244bafb12f66e740b9477c80f5d980da2da1cb40ee06fa3e2f639c8e7e37ce6e217161efaea2196d1d3d8d22f11716efa004cfa19595d44f78c084352c0b99062584a8d9e7eb29292f7cd7ebb"}, + {"seed": 50, "len": 127, "xof": 200, "key": "a480d2a7c82d1bcda32057f4f418bb56e9a6384fea983e669b9b9e75f065e6b8", "ctx": "lambda-vm oracle review ctx 127/200", "hash": "9cb06e7f440251995a8b91c1f92e5ff1047860df725c557372f26d4b14f80c4a7b5f294659a344c67b04affa0bf0a0872f64c7dfbf2c8336c88b09dbfab82250c389e98dc0c0a92ba3c850fdd7d14f2e0c1d3b1f8043d82aae6216a528dedafa75bbd034bf977dec016ca5b943b7fdb063fe1a4daaf7441b66ccd1180b87f3382e262ae1ea5e864424a123ff1c4886f98c8c219a26d2857a5724dedd58d483d3a7b11a436f15ff94e7d179516e6b900e0fc1978f53048e27b7d28ca5ccb6a0221abfa605fc3b1616", "keyed": "d7c39ed0632dc56e37158d34a01fd4983b4e3a80788e96887c7e4d98e27962689b3ed261b26b198b64919703ce60a32a620a67887fd956a03c9b07cb3d0a0bd1dcd42252b3ec4589573e70c5f7b28c78c9ba340b7984865a276ab331da59d0a7a5400593c4f1f55a9b3e6a0b3e7c49459254976ad5eb514171662d6eb7700a8332bc233252c9bb11f747e741df39ab0d3aba1202446e2bfda08f8ee69b77fc4d21a5ebe00096381bc15bb9508fc775e826d1d8fbdae063bb9780993e25f65c60c2dfb28bcf9c6f41", "derive": "e1124d9e220190870e3f796d4658bd050485df3008274f1449cd0d63b2ad056810a1f5b884141401aed964d6430d3ffd9d204d7e7b5fb0bf9c523cfccb1be6fe0910b9eadc21ca3dc8be146a96de93906937efef45a1b18c40f5a69524ecf651952f117fe8df66ce769e276736208172699b23d4db026d199c587ebd3b5cacd186ba696e968ab96ae32f4f5dbe52d15ecfd5fc6eb643b86058c1e1efd5df0576d1078126efd431f4e24474a762a013fae60bf874a1e75c29b7055a405b0c6dce05651fc7107c4e6f"}, + {"seed": 51, "len": 128, "xof": 16, "key": "ca2c9331b09bd8318cb472ef0ff2385f367fcf277359132fe18225b14e18293d", "ctx": "lambda-vm oracle review ctx 128/16", "hash": "7260b268b8f6338baff4c78c44faf52c", "keyed": "e5f4f67b8657119ba6aea3e641564caa", "derive": "9a9c46b5b7dd0b724aa8b0457b7ebc56"}, + {"seed": 52, "len": 128, "xof": 32, "key": "16dfa94a87ffe443e3dec0dc13f3bd6d19796c33de8e376ba772f551b9e59d8d", "ctx": "lambda-vm oracle review ctx 128/32", "hash": "4a63e95e25c4670d94ec1a1e01459041794e05b2e5a9bb1a7818935564290013", "keyed": "5973445a97d555a406ea562b257248f44a478e11d3dfca0b59c83c8123e60557", "derive": "6a43acd3d3798a352d137db9e0afdec0589c6d00b5b11d4dad5baaa4473f89f3"}, + {"seed": 53, "len": 128, "xof": 64, "key": "aa0ab7c74612cfe86fd6b0258a8253a5b40fb2c37e3280d3c99c9d641ad8046d", "ctx": "lambda-vm oracle review ctx 128/64", "hash": "94745003839a79dc8c227e95dfa24e6ed4de58c15b03360c8c16b4c3476c7a191a9234fa8180c5dc599f2664a94eb7d38253cbd97114942301d3c24ad0375f74", "keyed": "c01eab4bb9511b877e0a9b1e20c47d99a9ad9517297eaf140f5503a5becb4773b121527af1060e320e51ce5c60330f3f59096b08cf7794d828c3e12605d53f0f", "derive": "961479805b01dfe530e97214ea5ea55ff58b2b1a0b653ed7087e2ed83ab50ee39548af3dd3f50de83b067d2b97546bc41d956244e9e58a4ede6266dfad32950a"}, + {"seed": 54, "len": 128, "xof": 131, "key": "3f08fb1a9b5ee03886bce106e723704cf0c3c1258e196572e9bf0fa2d66f70ea", "ctx": "lambda-vm oracle review ctx 128/131", "hash": "1c018763f8d48fc81bbd5ab76e8e20b04159f50e19fda1406aedfcb712f9e018f9ddb4d9bffbbb29b38ef100700c68c8fd97cc14b510b77c746090620e4e1198db41b55e25ff6a08bba111c0287c61b5be00cac4803954d604ea293d16b983f53a3287ed43ed1f02b64debf46941a8273a3762cb81d54bca602dbd02dd525cbbef0ed4", "keyed": "3b46b8a4e038f114a6e4a0d5721cd4ae79cbf2dc28227ac353fa7f24f6c0248fc38f39da6fb574fb82e138c47f24824860136fe20023e754c9e603e20f2601b1ed7c29d52a9e555c681b663927e9f168fc423d077188224d49167f7c15631a317af5af52652ca91133285810e646a7efedde51452bcdbd889a296c821c77cfad74d625", "derive": "41618dc6fe81da9ba6d4a79f10381a6ac4a39959b1741082b18e9314fccf1e82f6a8ca667aeffb4bb6430fdab35f3e3df1ba578fee5debf55da9a1d347304feffe5139864ba1bf6a427adbecedc3f80b197f0d67ed92c2a8e986d52dbc0fc01aa7dc92f7e666e07e0cbb7ddcb45a7d8843d1ee417d97e3a409bce4304f36650050f572"}, + {"seed": 55, "len": 128, "xof": 200, "key": "d3329a66c41dfbc16d0b8e80fb27d046b6514604d87510ee726c05808310379f", "ctx": "lambda-vm oracle review ctx 128/200", "hash": "58bda839d3594b2568a91d58139ab02d39a296ffb3903c766ea08efb4c94d3f360c990b52efa7d0b3e5ea44fdc0c31563aeca3142f4cfb711f24739b2dd20383474d2c7999c39954ea4bd37860ab1903a35b42e826551e619111bae4d0a9b682676fe2ef9c12d1f55706ae1b070a59485045ade6ec1e7c335b1b4b98b5d7b73fb3f17c659996cfdc79054b02be7361b088bb2a27044041021949e37a147bb285142a218a1c6bae7c7a71a586822f1665be830c46e055ce443b3ded25793b8d151a1e0b0c3c031ae0", "keyed": "d3f6bdd9f8574460d01f25f2cc463b007cf28812493d05531ed305963789338306b85cb83986e0923fe12c1a30f79326255ba8406ded0c47cd6116f18e5d09f299fd7579f69223f91a546b535481a292948c832354dd4a823c2b458b5158f1c42a6fcfacc8ff5e35d047dd0a45daab8ea491e8e9622ce71ee4314213c0487c14dd7ef310f0461f67556243c5505dc7f73a5ae676b2ef017502048f2fb9ca4f84240bdccd955b6d2c9b50eb12dd5cac000d08b33d1874b910bd6bc808e25b75168476bb04ea76938d", "derive": "6450a7dec0869a1e3c8acfe39112f70f3d0b3e8c296f0c3096235b2917624fc98827f8cedff426ff6e189bd360d64d77fee2d293dc5f62b8b1a0ea90b73bfb40ffc5c139796e969bb589ac8afbfe8b8e501142b8e2d966d70e86c87de016ecf7815430225cfde36a706090c3e71439836ea38a481c6bca101ccf496de13b2ae4919b7d9d426c9d41560c598b624e45b366020a850c4e135f1c44e99d978f4d99d02a076c51080e6e8041552a9b9ea8ef6df548613e8d1115a40643c658a444698a75cb7a334481d6"}, + {"seed": 56, "len": 129, "xof": 16, "key": "8ed29fb36819743a8aafa7633b901089aed972c78f4e659dfa303fe3e7d5f763", "ctx": "lambda-vm oracle review ctx 129/16", "hash": "53933218cd3a7dc226dfda7657611642", "keyed": "f9806797d6ff511f653fc782afc27c5f", "derive": "bff8e1942bdc60787105dee8ecb3c163"}, + {"seed": 57, "len": 129, "xof": 32, "key": "b37d838c35afc413d4f4a0834775240e29b5cee2b4a4fd4cf076fafb3f075341", "ctx": "lambda-vm oracle review ctx 129/32", "hash": "788921dd78ce7a8af5f477677d8242ab2c1fec000dbea6e20356105594ef6977", "keyed": "6719b59a5e51d0a42ae683629e6d8b67c5c49a11cd69a78ddca2fba1af7767fe", "derive": "720c8d0efea80152a623dd2b67c8e707fbb93f2dfbdc6b2d39ddb1d649b61ec5"}, + {"seed": 58, "len": 129, "xof": 64, "key": "d9fd0cc1961360d4031cd4d5f1ec05cccb77565ef4b82eb4ae4d6f9a7883a731", "ctx": "lambda-vm oracle review ctx 129/64", "hash": "661f362fe1a4536f81abbb67e8c8505e9ecb0722aaa843a51f32613db21e4c16f165d3d237eecb99260e3ff88f8b43f3cacd0e323a9d0938e4ccbafb3212c4ff", "keyed": "b2b0f1978f87ad14a9d583d4cc5512192e15a3e9f425213a44895cd9ab3541023609832d8bd47a53d23d6a6ee091a860c8e5d3e8171c320c7d8374aa950b5a4d", "derive": "5dfa9b9f0ac8703982bd9f1dd20910601acb82b9d5a4cdcce3971b4fad21fcba9b5fe2ea7247dbc4f0ffb21f46daa8256ca0f04eed428f05f544ccd34b1754b6"}, + {"seed": 59, "len": 129, "xof": 131, "key": "ffa85eed76fc44128bcad9bcd7ae3713c2dc1206f3c61e0ce2774834da5e0617", "ctx": "lambda-vm oracle review ctx 129/131", "hash": "5a7a52e7bf5e79814bd861e2cb80e3af706036b544f8ed5e84a57b32e0acb6b641d009f9fa8d095ffdb0f00374c743c1c34edf3709f88cd33ced94e99887e3b1c709f0b835eb79c8726a36e1f032286d70b474843b87f6be8fb55584ef12b25adf776ecd544eef2794e72b77821ce05ba955d39be3ce18e791506e904051747883e6bd", "keyed": "98c57f6aa22ef67c09b40eb6c9f25133e3f439077d6f44669c5fc1562276affeb135ac5a030b438549803e05f07c384b1088cedd3849ae5dabcf2fce16fb7bac3065ba7b03de9b3c50c03508a29402463aa0876cff9604a71b8537856ddd8b49b0375e3e5582cd149293680909f40dafa59e7efa9ef0245ab3e810c1702e5edae2c261", "derive": "261ded465c98c65aa2623b43d3661f1dce5c48276fe0288ddfebd50967877716f60ec020080e6364f3c9577f2c0fc806b960eeb0ee61d96aa6e3340246e9a089f9bcc13d689454eb14a6994f5e6aeb2b5efd1d7b478557faf83a6b6da686b8a3f0aaae6639a66d948c3f76ca955b9fda44f09500e15cc75d98b999f2f72db52a63278f"}, + {"seed": 60, "len": 129, "xof": 200, "key": "285a6026fc81bdd9abc31cbbdea815f5ea45e1ec71d6617912542068e54634c1", "ctx": "lambda-vm oracle review ctx 129/200", "hash": "1121fbad05a0ed2ba8ff2498d45a9ea7e860384b021aeabdfed3c67f0eef7408ec6a15ec53723904e10c890e7323d1b509b3fdc0ba8fbaba2765eec1ab6db11f0feb360d1c92ee019b83a2f86aeec5554850ab924965ddd94d294cabe56b1db35efdada8d81e0b8832da33a47ec9ce135d96179ce3eeecfd71562187ceb9f5a550dd006d17f71fbd83826e1b1abbabb9719bd4a43ebfbf88572a9576d4e91fbb9da188de5a7cf2f2e2f3aae7ccb3eab767c49f82e127b172fafd76672179638768fb5fb06e946647", "keyed": "398d09d5b5cf7402631bc8eba3a4d4050e07a814376eec7edd52db6cee980fc104dcb8b8efc8a0132c2749f2565c78bc19db1163da064ea20a78c344c2219203fef359e4e5f9a40bbec9d401c14f041be0a034046c424d1c7754c25aa41f7ec749ec7f47e461a6c676b19fe4068f925a7d9792539d4e5acf92d16015ec05e2f4d889eddf55dfa5155098ccb903dc871e54fdbc09e83d546558db1ec6902c0cfda170f2fc2de829195766f65880fb8658d993301976f0e9f1799fbb5d27d38b344ca4d70b91b54b79", "derive": "1172b99ed6904a716737def55363442c70ee1ca85a08b6e62bcd0b1f15418b209babf8516c130a4732480acc4ca3d052c3d1644b71b7c6785d70083b6bbeb1945f4ff6179f9d80e177f44fd956e6ee4895807dfb70e7c8bbb8771bf3447f426c3e418af5bf5142d70090b1de34d2f1367aaf7f28ac844227bcf7633dc9c7f1c220164e9b70b00f1d26abb97c3e4f0746123788739224119e6071d4d7fa49892689c8e2463fb22ed28efc9207f7c64f2e768e172362e3382fa9ebeccab5f3d1a8abd7cb3c837c0a10"}, + {"seed": 61, "len": 512, "xof": 16, "key": "bc84ddc1f4c405a0d70a7a20b0a65b90abd9c1523a575929a31b51356a1406b1", "ctx": "lambda-vm oracle review ctx 512/16", "hash": "128659308e8a7103b89954328ff3dcc5", "keyed": "66ac5531db455adfae33d01f2424b2e3", "derive": "3714887158aff8b39e20460d1b1df7ce"}, + {"seed": 62, "len": 512, "xof": 32, "key": "74856534ef4035d35cf4cbb65aa0513428d5722c34950e422b79253741136300", "ctx": "lambda-vm oracle review ctx 512/32", "hash": "61e3770e80dffcb46dbd31d0f0cfe311cc746d2d051ce2fad4ce235b8249b6aa", "keyed": "9701c520da175568b55b68ebb9b9e38671c9354befc3751fdb9279cc6dde019d", "derive": "f81137c37343d28421733991ceba4f7a7a8445c58daf78a5f941c48935091771"}, + {"seed": 63, "len": 512, "xof": 64, "key": "08af96226b264d763b57413e3bf21605001c95eae5ea05b5c43da130647b664a", "ctx": "lambda-vm oracle review ctx 512/64", "hash": "ab098f1a5b5fa7e09c5681c5091a5c160ff00715c2824a4a2633401521ca46ab86239dbc65f0397dfda02711e457be739b40a4fd03bcf729503e57885a7bb1a1", "keyed": "b48711a18f51ffe38c41060057a47e0174cec29eaf16b10a0a6613cb23fb5b06e347e3ddbba4a2b6dde67587561452b2dd584c133bc5581062fe9f97b0ed98f3", "derive": "80ded338ae4f1d27505253e93dab1949bc51e8a2011ebfb24d99678057b6a18a8b0c7c12eebbc78bc7b343566b8fc7eb4be7705f7e975b35af72dffa911d343a"}, + {"seed": 64, "len": 512, "xof": 131, "key": "96f47b23ef2dd2dc947462660df783ddfbd7c45639e8d2107bdc5646657c5cf4", "ctx": "lambda-vm oracle review ctx 512/131", "hash": "c735b7a9e0afe293904be51a137cff72222202ca676435ae5b41efadcbc12653b85688a678e80d95f9109c10c689718a825311d8e99f5cc1f30bae70420a8e2ad9bb1b8fed710c789577500aed255249ba6ed21abd355de74dfee33d3648bcd439a79514b64416073c7db4b635b107a002ab71c9019de007996d51539ae9881759106f", "keyed": "f9d0d51d41f7a7916272ae75ab5330a4feeb92e927bb53e948ddf5a81d3cba6ebb757ccdb34719226cd4e425fad2f762cedacfdb57f65a63e9169cc5c35c46f02f37820e2d3087b3fd8a64bcd28d67df0ec9ce78a4d2caa02c0c80d43fb005a8b73e7f265c108c72a47baea9532698639dd9e12cc6275dd1b03c7bf687b3bbfd776f1f", "derive": "f5c3a9f44e179eaaeb3380f48f03fcc9a84ab03aa7db263b1580da1c6e3ebaf91daacf3cd305414ea5d90f2d368133acc4b579e6b84cade514e93df6ad20f9f39bfa10a796eda1cb142da2790e445294e3b12f3360d1f65bca819647c07234505d1fcbc63f56649047e4885bf1713f62c9a6b66f1e699d6f2324d0161507997e22c42e"}, + {"seed": 65, "len": 512, "xof": 200, "key": "bb310801144662e300b5f7e356d7d4fede0efe9a98aafcf4ff570ec3968023d8", "ctx": "lambda-vm oracle review ctx 512/200", "hash": "6fb76fa8d36be7d4f31b0bc7c952282740b91f430d12fdf645d2b5bd598da9c2fe8f616b84594763d2421979a7c0c31932952cbf669a5ca8e60ee3f3457c9e21041cedf59045c06e31625ce457205eabf5f7f6a9cddb34a5f79a7092c084915d509b7bc141d29630c9a520277a0bd5044cc3a2035b45464c928bc5e6361c9d2aa1be71be9f88406a2cb8a015df4ae3b0e44fbc709ed80d019598936df42e956251813351d4f7c420914363cab352f7e08e1747cbdfa7dee4c118daf1d6cc7269c34ee7ad2eed2b29", "keyed": "35a85673fd6202f1e550f2ee748e8d76d146134a5a2e7e503603d934fd884cddfb0b1b96046877118ce52cec5459f414606fb538cb75b3373e975566667a7088640cbd4b746faa3705236012ecc94b9a0de54783d679e87587ec9e7d9eeafc515ce532a88a68746fd615189d81f638d3ab2fa1d82a09c058d7a2fff2e91fffbc4a307f2836203f9897b52696448e041bc5628fd6063c43d480c4da2a8dcf16b2ef0cdbd039ebe96a08ed64f3470a026399a13d70990d6b569035521ef789471b904346448defaced", "derive": "06ac2355277dceb26a410851af772954e2e94f4ac8f2f43e55b06b0baa0ae50dfb33744efe8e6ea2aae1fe758c187707bc4975d381e67e173714267ae944d6504f8858e5c9f59a56db538ff002c7d0f305ad253951ace5e363ffc5652410e8ef74aa31cb92bdc3c370ab562dbd0e2094f1d687602ebcc0d3f9a911e40f1f917bb9937dc0d64c6f202a0f051ec1dc9bc9c96cae56fd5618ec9607ec7753a42b7e4e9f03ec73b4ac3264210c2873fd5caf468eb00dd8520e93f7b049dd2f05d99458d7a53126193407"}, + {"seed": 66, "len": 1000, "xof": 16, "key": "befa8b059d09f3a8dbc368577b4d33676a30b41578c029b9fdcf417ffe9a43a2", "ctx": "lambda-vm oracle review ctx 1000/16", "hash": "6f7175181e05840cf5bca9a1d59ddd39", "keyed": "31b3ca8354e46ac283cdfa7de0becca0", "derive": "3e44e34bf6578e1ff975f05cf77e3510"}, + {"seed": 67, "len": 1000, "xof": 32, "key": "e437cdb211ffa77321a331a3bd48759e26cd010375f1d68219457be4fd38b6f0", "ctx": "lambda-vm oracle review ctx 1000/32", "hash": "a7a6ab659e13f7f98a6c3500998ffc3a17416de7c9d5f3b3163cd123838e028e", "keyed": "ecb1d9e27fba7b612c9fb8a4f4715706b3b838a33c357279128fc49b9c454a40", "derive": "3b541d49dbaf94f167bca1d31223ba17009363e27fe1527b771392e0ec2e5f25"}, + {"seed": 68, "len": 1000, "xof": 64, "key": "75c4430c6cdcda06c095d3f03bdbda5b779e718476be00d654c73ac6330d70ea", "ctx": "lambda-vm oracle review ctx 1000/64", "hash": "89fa21877d363271797bdf327d4ba0f311139f593f4ffcaba00d13eabb9837203ae6f4ee806582664a8160a4a065346c6f923a6102e0f9606889040b16b6c36a", "keyed": "5b9f65641c5dc2438c736ea0207ea79f801385078ad64dd33afd2dcc8fbd101e90896b4c672eabefb1be15c3a2908e6d68a6b0d2789614aa75a1e771d3910800", "derive": "ef3972e6aa2c92126b3f8a982bfefb489d86031239d7d6da94c742d533016c602e5b053d60df3549775864d39350ec69c80b38178e0c297184cd014248c45fed"}, + {"seed": 69, "len": 1000, "xof": 131, "key": "0a5daf6694db1d9b7660c1b126fc81066d359c5b5142a13581a2f9a542d40a70", "ctx": "lambda-vm oracle review ctx 1000/131", "hash": "4629528981a3c2c0b64c229c8df8d050ff272caaf4a415032c625bb8e21538174b8d38c7e549836c968ed937cc98298f6deeec12ba4811da87545a4c831a965417181f7b5c59b5b60d1a2ab5c6c4e736949fa5e24c193a61025eb0bfa574fc360e0b237107b3922ea180fb81c5e694dbcbf10c5f541bcb085edcaa56119a3fc5cf3e5a", "keyed": "686683bd2c69e03888ff6a9249f7fd27a793f4573f129bb1aa2afce4d38d2e3118ef6ed035d090d68bc0cace642f168bbf4b1e14d382817cfeef8c51c87ae63c356094a40cbc2d3dd48a321ae5d988ea4c0594390e9277ab97358f76ab01790a841449d04340ec9e2bed929def2490f6d32fc831db678f01466f9692b7b435ce84255c", "derive": "7cedaf8af519c0efd6c7f7b7917b512cf33e3ee555262625041131f660ae4cbdfe96a4a81a3f14542197696cecd174efa1754894b8c30aac8b382b082f8252ea0060d11945ed3e3d6c94912dc4624ae3512551715b4e18d2d74ba41df6c633839f8b33c846561aa1ff08db9631f4ce0bfec8f7ebe94fe02282b72b31e32c373b4b3d59"}, + {"seed": 70, "len": 1000, "xof": 200, "key": "9eca31ee091f8bc6ff2a265069d32a2080cc6aadfae7b2f4473258faf88ea762", "ctx": "lambda-vm oracle review ctx 1000/200", "hash": "4c4c68cb630b73370f497003ed1fb20d23540096ef693cf499afb67ae5285bca71a17877c8c5fce6ed6bc9a78c42cbe77a978f3684ffb6179e619d3d9a1df86145546ade247622455b41af82eba6cd14fe268ed0d8a3e7d50675d796ac8464c42ad20edbee74e584441150b3b77ecb5cff422d1e684823f5b947aafc47c61400696302e36bbc05b367a09d229c153761fbae1604ccbc1e7d9cae9e7f0e938879405c2525459c5b98431c1eb70f8fffe3e909d5b5ea8027eac22fe790202255f1925d3274d7ddec2a", "keyed": "1abe83b666af57dc91ac342cbb0f1ba56490a53609cb4c38c3297f32cc31a919c9ed827bcbaadb32d861539f7920017aafe4e2613fa2c042bbe734da49d296e75bb5d0037f7daf91afc9ed8ec1eee2acf65fe9f12e6d2a43d0b809a6a67d77a28823189e03fd16ea2acfaece5c5b6439322d5d5456248826faef2feb18066953aacadd5391a37c9153fdd776f5dcfec1fdc831869a711a61ad54f677c10467df28362a1ea956a7b8c47fe50779e17700ed3065a9aa0a7b1b8e07c47156afc8d101b67316f199a427", "derive": "31453aadda0b9fc43f8f9e3163c11135dfd69c894e5924e9ae45c17e00466abba8c86a85f3b22a7f6ecdff2c2b8f599dc5dc8ab35b7436e70d922425b598ce2f17f6a10efbc0cbb7a1673b70d8417583a264647e1ccbdc8e219408986b448c7a14aed4e5191587a533eeb4e6fd8b493556ff5e1c28bf8994868674abeec5e1ac8c6eca2e04ada8f24d46db692784bb5e2dd336d5b24e28dca09d80a5c7888a2123cde8183f8ccaeae5225426e8932a1c726ca63e962d2236581c12abda25c6e214b45ea14bf32ea9"}, + {"seed": 71, "len": 1023, "xof": 16, "key": "33630b173362a69dbae2c07065866154f76f244e99f1315197f109eae4b74b56", "ctx": "lambda-vm oracle review ctx 1023/16", "hash": "21d7ad694e7edf37046214b4f872c6d3", "keyed": "9b73988949059085d68ecb8518945058", "derive": "816ad3aaaf4c9ff75502cc055e88f84a"}, + {"seed": 72, "len": 1023, "xof": 32, "key": "32552bfbc19d7b92b714b972065e0a182c3846351fdbf9f8da473a66d07bb089", "ctx": "lambda-vm oracle review ctx 1023/32", "hash": "7d4502d0e2d35b69bb2b6cd2f5ef61e38944736c200ed043d7b2ad52310e7be6", "keyed": "e63ba15cf67e4788f9aa504e42dbe78dfdf51de51aa5fd61a20bfea0c29cabe1", "derive": "6c90bdc9d0743dd9a18bcad1dff79d71c117b8c238fd54ca0af0b74648eefe0a"}, + {"seed": 73, "len": 1023, "xof": 64, "key": "58924bf7db31de95950c6df116b7c3d92d800d4d9194e144d568a96cda1f5404", "ctx": "lambda-vm oracle review ctx 1023/64", "hash": "4efb0d84bcec540b6313a0c6da16c19f6c548d825322430a54b3c1b15d60f17f3006c0199263f3183180a80c47752b2b890269524d0710254576dfa13fbd75f6", "keyed": "58ab643960e5e2fb588ae0192a44ee111dd1ad4ac3edbf46c4b56f69da3732cc08d70a89440a7cc3301f7aadb307a77378b569aadc756f45269eaa7250bd4e01", "derive": "b10359e0c1a9645b79fc3b3c49a4c0f5b8b4a33739e38572521b1908cc3c8b857349e672cc8d7f976b91ccb7bf37ed104cf6de35aa81700cc69123d311651dbd"}, + {"seed": 74, "len": 1023, "xof": 131, "key": "7ea3111b6bd050ee3b0f5804f3b26260e9be092dba7a3cedf86e31c2064c676c", "ctx": "lambda-vm oracle review ctx 1023/131", "hash": "392e9eb9433f4498cd9d9968ab58ba0f7a3a0e07e2187f0c5e35b655ed6e94a1318e6cbcc7fcd27f09a4d48eea14feb22add27d001c4ba2e331b5a8d37264fc034ad4fa8431284ebd1aae6f9b7cdf5e12862a46378b3bb2c94e8271e71979cdd4fe21eb3b582e83db6b50f712012cbcf904e021527d1835e06c99ccaef73a356fd72b1", "keyed": "94fdbd4d70e4266087dc015d7c404db05941db771b69b5958a8d563b2a59736c8df29b4f1462f39a868859446287e2a9f9193f616097f2799512d501078a6f2f16dc396e9a4aa7e42bf84b78d6a83fcbcecefad6cc7aa7e3d08638f4b69f4dc09ac3ff4087b95c90db8aee339dff663710f152c06626244c5ee58c75efcafd9b739f84", "derive": "30a5ba2a68b85a8a76ae8eb44b657782f4b845173d98169b9d79c3447b8e349489cc3141ea763b61d7e0cc825af7cfc9d3d3a78d86437b2e41531902bee1ddef2f5433032e5461840f6e89c691172eb486da91c1536ca59b4403baae9ec6eff03270b5cbc3b11d068b98a4614e3fe5e0354d10983df886cc00c4b98f1fb4eb2c7b9b9a"}, + {"seed": 75, "len": 1023, "xof": 200, "key": "a4dfc16ade4a493f07e112ebbd4a73ab4edfd7d7899761ccebfb8492a295b756", "ctx": "lambda-vm oracle review ctx 1023/200", "hash": "8478f370e33d3592d7991f12964bc4af23d387bb53aac03930f9ae194f4379196a814ec634008448bcde817ab14237fd0e6869649c3fdb74ec94ae5b1c0a434d0d5b3c60d38e53d07da811aebd7195f6c8bafa2a52460eb52ce7d64201c59dff2e764bc643f3579392f90ff9895fc5c573df0633f638b05936fd17fa84cde873e5b95a57e156073c2bc1ca48048f7078bb4520b81d7b1d4acf9558817e024c9a09872b3bf2260909b01ce4a21c594c91d579457749e0b0b801fd6c9d6b9e466a59d621f78d407ed8", "keyed": "17c56f3fe4fdc6d0e3df2cd517df23c1db7e100ec5aa8745b11007608c32c9a58a83573685159f9724b28e13330841964e626f9fcefd62a12314fb2a3f12ddc30d246beb5093926d199762853846e8e2cb050c44f6ca11cb4db8e6a69e0ee7e9c44b8a78c9ba43b6dde1a31ecaaad50d732eb22a7773ecae84098a5f8915562b5bf9ec1713e8e8a9d8d6daffe5e7b2daf3745a76ba1f2ef4f1d459f2e91103b89977e98e5192d10669d6351ed5779a9b44506fd5a707944179c2bb631c1dece54610f6981fe082ee", "derive": "bb14d0c8f0faefb4a6ce5cfacb240ea982419c862da6463a0feb08fec167edb8aabedd5abe2584b5b44508146e47399173927d08d0e05f2a442065c7d68d135d2bb4fa39c3eed3e5289074a695d4b7fadd8eb55f44aeb50195ff2424da1a9b0d34e7dcf73078d7a90414e1dbd603826596763990e46385fea060131d2597736ce4dbec61a6870d9e3b7332c5d9f5874fbede6b078646c6d0fb003a1f0435ad76aacaa47b804032622589fb474b8473e67794c42d99d5360c39ede03da8dcd427307bde1d74bfd73a"}, + {"seed": 76, "len": 1024, "xof": 16, "key": "122524e4b147ba890bcb4b36a1b39dc0588f4c21eb663bd20397e4b0cb2cde27", "ctx": "lambda-vm oracle review ctx 1024/16", "hash": "22c2bc21637d5a3a5c5772a2b41daf34", "keyed": "7914fffe0f698d80bc7ddab03e05c6fa", "derive": "076f9fc6b3a35312c5c79c687e9f1921"}, + {"seed": 77, "len": 1024, "xof": 32, "key": "a7be215c976dfd1555cb25917f28d872d4660e506ddac26586e1d1547ba4a5e0", "ctx": "lambda-vm oracle review ctx 1024/32", "hash": "9d05e80a80d43143baf493decaa5f031fb91f3ce6fd81bcc0270e9473ac94d6e", "keyed": "c3b7affcc71c6efe4fccb3278d0d41acf8085c81170a03470924311c8c6f6167", "derive": "52a053439dde9e4d68eac7ecf2e32e8a3baa17f6ec0e291fd0338718806a9c44"}, + {"seed": 78, "len": 1024, "xof": 64, "key": "5e72720477e1a7523601591e2adf103c8e5bc3c630a43315f36fb8a7cdf62e0f", "ctx": "lambda-vm oracle review ctx 1024/64", "hash": "d0abc07ee8290bc21e5fc11dcd7a2cbdd7ece80a5b916089bd023767d49e0a07e5ee1cc38e851dcf448494397f2374d5a0c370b6a70a2393a6ac06fd7ea6e145", "keyed": "f446907661aef0a2c702dc83f991902ae807f8c009fb16be9b93735456e4d41b0bb01a824d683c45854d11e2bce40bfe5cc82122fd9c465b06110fd966041fd7", "derive": "769c7cb81c0a7a115c8688f45902cb3573aec2266b8784f71443e657e777744a31a031cce9bf5d0252c5380b66dd9519b2c151632aa1a36c1e6790a09d97b094"}, + {"seed": 79, "len": 1024, "xof": 131, "key": "f20bbbcf5654a52a0d073e10a2c95a039f4c386ed0d8aae138eba6eb08cc443c", "ctx": "lambda-vm oracle review ctx 1024/131", "hash": "4e82ae97ec70084c61ae28f8f724ba3ceef17cfb6e2e89c4dd1f075a4d74f6743139e37de17ce3bdc7dfe370d212f46bb580cd659d445ce1a278781a2f83d4a06dac14aed49afa922506fdd0a59267cfc72ead8e7a7c0396428d9e941b6490908773bca2821de661219b6b51ebe33166d3eb293b95db50822e0b6f231f956bfbadd336", "keyed": "6cb89e82c21309efb425ed74278e6b7d8812d1f210b6ba85e7eecf2251a644d9670b8e05e721d60c7c9050ef68e722bc01cb59b1a3f91fe5395655e0805c1421ff3830033c234393af2a9abdae8b06827cfa9e01855e41f53fabe8cc47129cfa5a937530a337e207f86ab70523694c5071ffaf1aae614160b7feb18c271ac2d4c136a5", "derive": "0a9c479d7c63493957c01053e26c54b45ab2a302af25ce1ed7e820e82e4b787f897530604a257932d097a28518722fed68f570f6134c239c55f827141bef08930f41ef0c0e82b9acef19d4de5c661f101a09b8e8d80e47cdc189044b2151dea929234399774e12d44cf0c07390625cfd2cc17aa15eb24db2afad017904f3be4bf803de"}, + {"seed": 80, "len": 1024, "xof": 200, "key": "ff32ba83c1a39f53fb158d0e7b3adde55e6312af13b7143f67a106771f0ef5ec", "ctx": "lambda-vm oracle review ctx 1024/200", "hash": "4c7831baed8e7b961c5196ce58fe3b8954e90e77db26ca67cf7bdb66c4122a7efd33827be7372dbfe6c199374601eeb83c65ae54b3343b54d8fffc0b8e1a5af166e46ae44f5a140dd0d210f2e646099e91c4b4107ee5182028d32d66a534fa90dd9d45d4b54ee8fb482a9acdb0cf405711140e6944b619d4f185de6be35cc8e69157b876e7c6632c6dacce7ef3f591f1b45706a3d75744d6e6fcd8e193a52eaa273cba7afe60a52bb9c90772a3fb8e4004c51446cf31de17b7cac209e6033be62aef01cd86fe9e53", "keyed": "06fad5dca2d0c1ab104879bfcd86f10b9c2cd16a77cae797c40dd5584fdd65dd7003db899983836f9cbb8144c95d9dd5105fddeb5bd46559b6f89cd597e8a762de0a6814705813434c532380c9fdd6dd086c951a364e32a699f9dea809cade917f528f8ca390763079220b505be258c353f574ed30468a666055d60a7f8338137e57ca4d83b3e1dcbc86e30bb6a0ba69a9e33cee53bab754fc8c0ffcb4bd3592d9e1d6a274bb2d2da0e4bd8c32d34a3842fd5d3ce564d22ddb04bc35ff619f54250edce6c36201fe", "derive": "ceae53602dc8bda3ba5f70e37eac6c52e82380ed108241729c6def03ecb1c958b1dcfbc953368f85f1b95b96573bdc960bfcb1d23f86c7d4d05e05a79a9075707a833227e308173e06714ee3f6a24b2e620ff5daba96b8c63a3777ad6e50d3a6886465c194eb70406b261defad4dcc429813c4c46bae92b2d534561be169c5da0aa4e9b19ab67677cfa3262eab29ca8632484edb8f61bb6fcda32da33eb6bca12d5c5c595866a1e65695ad4f40a3a981e788b442fcafaa9f291e8ad4e7aa38119cff8cfddff5611c"}, + {"seed": 81, "len": 1025, "xof": 16, "key": "25de9eec69b24b07f16d278ec58ded50a93efa7241313709b81091cf0d732b97", "ctx": "lambda-vm oracle review ctx 1025/16", "hash": "eb7596ed1137e1cf9a7c4552bd016a96", "keyed": "36724602bbe0368b2123e429ed365a32", "derive": "b4f1243fc61d26dd1125b33a1c1b668f"}, + {"seed": 82, "len": 1025, "xof": 32, "key": "2838ed52fb997fd9e34ce40ab8e87831dead7c724b4254c8b59b6279a38b2c01", "ctx": "lambda-vm oracle review ctx 1025/32", "hash": "515d83974324f6b2fc3576abb7cb35d5a806079ac79b008183ab128bc687c3cf", "keyed": "17d066fd3a500a6a67ff4219bfb8607ea216f1c9d28373f287ed2c15632db630", "derive": "3485757d5c86ebf235ddbfa8d321602cb09e18dc428ba4170193c70b0f9c56cb"}, + {"seed": 83, "len": 1025, "xof": 64, "key": "4ee4408b9c7f1660100715784b4ce89905fc8f75be8ee48080dd392bfaf97a38", "ctx": "lambda-vm oracle review ctx 1025/64", "hash": "9ab339188c1e0f5503f7f464d22a46de9f6557bf26f6899f16cf6917ecd90c06823978130837cdd82a919b60a800eaaf284a6c910b8f1db578e5c2eb8885eb9f", "keyed": "c7cf6bddc2e55816047dbc1203a4d28ba617a5486b5a6771730edf6ce0d83863fbf07da183fead72816e6f986bd788136b341f97a1067139241acb9f11e736bc", "derive": "7e44d46c1f99a546c9352afd1bc02be2876b27664d1140c424e4cc244b16740dc62524e5a60330ab93d0faeb4fa3f2f6ae1a5e73b375b58fab7eaa9c723f4052"}, + {"seed": 84, "len": 1025, "xof": 131, "key": "9a751b6cc27fef7a0468d503107e27d8abd2f9c7779a39da42c6b0681f0c397d", "ctx": "lambda-vm oracle review ctx 1025/131", "hash": "67377651f2b0f5b916afb417f7fff4e3dece82aede4f89160be3a8956e41343e878a86f2352a5e7502583e1ad82173843897304d421f3e63849148580465a8533525c014f6421291cb028a1a84ac06e828e1d9ce378aaee0284e38e634f2cd9d74320302b4c1e0f165cc6cf90de17dfc9103afe60c8bb9ab81a062093a5d9df14ddd31", "keyed": "96c29d9fe588587498c961de7a23df82d564a8723ac5bc3e64d0bfe96b3fbb5a36bc482b4d9589b1421829b28d846e8df53729010627e3f855eda5e8fe3ca91dd83cc2b32d99e49fd511af21a0edf5eb56db0bed06584bf2c84a78ebfefcf12440af201ea7aef0c7f84068dabb3260ad3fa43eefd4b7015caa4bc55dbaec08916e9fb2", "derive": "ddecdfaf6e4b58a7e29d4508738406a4e5358e2492db8d5523d7ac7b4ec5b93f55d663fabe40000599cf1442159c864420d0ede939ec960b3c9182defd0d204542e9498a7bb769b828b181bcd75e36843db0480da80ee6b0fb698887c6c74c9991ea379564ca97b6e23ba329073fabd11dce75de6fc779f21670abaec2a74b7a8ba8e4"}, + {"seed": 85, "len": 1025, "xof": 200, "key": "2e9f9851066dc5a89e022a1b1fcb9dd3ea3ad37231117f01a9a30c393ea4d93d", "ctx": "lambda-vm oracle review ctx 1025/200", "hash": "561b9a7107a3a0ec41da537e6436d614de0928fd8c9f14e5e4db96469612c0461b5659031dc5b33f8f46b05c260b455cb4b400f1c16af79d18abb9c7ac134fca2d36a9024b31fe9ef4d3050628512ea0f945466b9c7d1ecb2a807d9f7e9c2611882747ed315e6e5236a11fd31ac672d36740ee41ac58db2330bc9f2efaffe5ee5ddf2253436013fe604435128ec434422b166e7b64a87c14eac7105e52e2c09c3fac5fdea6b3416ac44e29252c4dfcf4cb873488493a1aec0b9fa6ebe222744414c5982a6878d4da", "keyed": "8d7ad356afca0147cad9a7586babfb2e699d00d42c1dcf1342510b995beb3330ecb5d7f65a1170f325b0ab8a598fd3cc1d93e300a7124f032a5431255e931c56a0a90f67904d2996f01497d9eb3655dfeb7b55c44e3dcc3c41165d8c8608a51333a9474b6aa6c0c3622b5ee2261757381d12edd67939162ea1502ec17b94d9dea4ad33b42d8381591e1f91bd28f4f0b97f339c8d9e8ec07263c0e3bd12eba080fc4bde37413ecf9b7ba98ce52eb130185e219b826d8c747d132f58df61348f616fab68d72786be98", "derive": "0cc6758101c11c1e83655fe193ba15974da360aa10103233f08e46dd8e6dc9775ab0547d2ceaf644fff8b0601b04030793416915e97a0f0e5f0e4abf9a1d21e3d5b2ca252ec85be73fc26b22aa6f226eccddc2d7371498fa4ebe2e8f25f927fe57d6d4bcb7ae772146256a59d05e77a9e6e4710dd5f42605b0ca29122b9832b07e789c79806ec493a47313a088f17e30085f4aee3d90b7455a95f2024ebd00e096766724239a45c4a506ec3bc4ad44d1ea9a0e77921026537aa737350ae1a316c4df190b03e72b9a"}, + {"seed": 86, "len": 2048, "xof": 16, "key": "c37be63b7d15d6a397ff96ed27f207d7e29d9a22280c4c6e3147346f8b4c09a9", "ctx": "lambda-vm oracle review ctx 2048/16", "hash": "eccfe2bee6eca100dff3592bae156923", "keyed": "addf3e4da8e55eb85d9f0e38721f1f60", "derive": "be4ab18eeceb4e26cad91450546f2da5"}, + {"seed": 87, "len": 2048, "xof": 32, "key": "57a617ef49735dbededd8faa977a78443dcfa50bf196c96963bfb744972763fb", "ctx": "lambda-vm oracle review ctx 2048/32", "hash": "4749afc7faa42ccaf1222708f798c18a1e11146039f23da9f3b009486822f209", "keyed": "723b806ec4a871554612db8ce1c077f082793feb24af79da0da1a86d43d443db", "derive": "48e7b73f77f56d57dbda815fe5be0409bb668ddc0ef2c9774516ab27eca6b96b"}, + {"seed": 88, "len": 2048, "xof": 64, "key": "9c93f6150956b1b932501635387e36d88f1e9134fa430aa814181522c36d814b", "ctx": "lambda-vm oracle review ctx 2048/64", "hash": "e9e20cb49eab02a1b08dbccf5b09fd3b22cc43d84981865680eefefbae22584b3f53c0b3c9808b6d555e355e3b2fd4626dd5d00c4b65a781c1c7b5499969ac00", "keyed": "11ab1e09e164786a65ae13aecef2165c98fa89ef0567189b0dd8ab1ad6e78c4e2e5196272cdc58f02340a5f25dcad3bb2482c8e6b6e9b9c09fc181067046fbfd", "derive": "0ce71bf1f1bb53dc0c19f775da02b69d0bdd2d355f3c0d1a2fafb45551cafbefa6a15b3a1376a8435565449861c0341d37bc2c7cfc88958fb52b2555d7b707cc"}, + {"seed": 89, "len": 2048, "xof": 131, "key": "c23e6b9cb89f41a0c7d7e7bf22165dec740bac9f63895152802e43d3ac0553ed", "ctx": "lambda-vm oracle review ctx 2048/131", "hash": "dd91ebca42a55f4899cd0e25a0ec102c087d25c1b5c230ac1b5e8a074448720dfdb41736dffff0f5717f6c8e9a0dd6eda746cdc7f7634d4a69bbc82fad49f76ca38f7d58d8aa46728fd78231e9efc5d5249d0a412b2cf9b275c564d24c2fde7d5d968bfd3ea35d981e32d8f17ac66c4f0f29bc4dcd8082f324f1b1d6883ea2bfd5f6b6", "keyed": "457a806e76d666cd198569275734101273ea0c936e6b9380cf85ead6eead97a5f31a936c0bc6951aae3ca46178efe99f4d6ed98b83efb28ec5ec9d660e6b1048d8550b3001e29dfad2772f381b891577aa92e7f7d597411f537b3712bd5db8f7d1c5005699982fdba5bacd5ad66762f7d2b7fe2ec9a6dbc15a715fbf6e0f026100cdf0", "derive": "7819e47d406aeb41c0316d7db3f9d539ae763e768701334e313e938986369bb4297a90be5fbca72cb126074431248f0d570291c5906f71a01a479e66244d9cbfeddf966b83b53f578db9b5deb2568e62616142bef9edc4157c4e9e7fc91d3ad87f6c7340ab7dc493e3deaf6c4beee6278e100de4f90cb16e20c6ee62d5a7541ef9a5cd"}, + {"seed": 90, "len": 2048, "xof": 200, "key": "e8e0fe22f74b3bc9d2796d01e9a5461359ba22761ec3ae62fd250247808873e8", "ctx": "lambda-vm oracle review ctx 2048/200", "hash": "d1acdc026a5436d968dad88e6f04542b8d0df5846ee2a6ac387c44e9295e16df61c66897b623dd45a8033372bc77320274be1b5c38511ee24886b7d53e088f4e15ed3e140f5f99a829c4bfb70b70bdca42565e98bd38e785319eb5a4ee0bf5931a4bd6b8ee0b6e59eec5d9ea8bd792e426a1a56ea56e5545a8e8b65cad9ccf1d9678fd64c38f33879e17d1312359b2a4d314e8ef6ab2f1587e2ceb161bc4552df062416a58474ad3056defce3f2f6e7e7a175d6ecb012bceb91385930f8a47fa557640ce4276bb61", "keyed": "da61c6d28ac5f5f719ab444f8013ad9c56888df92dad2e939fc092438a0e83b17ac0824bea6c015330c7c3f2589a33e58bbb2ac5e14c5477c2b2024e698c47f52ede48d55563dadc46e783c06c24067aad1d8417cf7e2d8fec59444ea700cb6ded3dfb8379c73f39325c8a0ced75d62e995711542170784afba801ac92a14d95161fc109f220b6c120d4797155c3678ad0bbe529ce587250728b26fb64061a06a2ee67f8d1e3492b6a07d338d327ed709c42608bf2781279821cb1b21749203f6f6c9d04fff3a4ae", "derive": "2ed11775a07d4c1b517d354e5c89151b5336dab0588125e8f85990502ef6d85ef3309272cb31f4acb3a7fcf4942de8edc6f4329dbab4a1200d0facca24518edfe06f706e76a95b012e50a33e3f5ae3cdf41af17985317f19cccb7f753ea9bc57c30e0a7f1d0f7fd5234b5fcd2ed8a8ee1d60fd6b07af2b5ebfad1b5aef91c5f6be13d5f33981c9ef7f60a4ade34b9523a0d75bfb3a365e645a849caa98c12836dc0cc89fc2c4c322260a76c6ec499d39bb8ce8449eb3746a875356d5c27ac428513da2f9ae19424a"}, + {"seed": 91, "len": 4096, "xof": 16, "key": "0e8cbffd842407148829d0b971714b98068d3bc8afe1d04b5ebf2742245f5d1c", "ctx": "lambda-vm oracle review ctx 4096/16", "hash": "215da79fcb78be79dab730b78467a89d", "keyed": "17e84551f53d55feccf53cd8752232ab", "derive": "744b75aed4882da612ecabbeed555a24"}, + {"seed": 92, "len": 4096, "xof": 32, "key": "37d59cfec72fe64019e0139c588a3b2ba063e011e4b870cdb1ffe366582349af", "ctx": "lambda-vm oracle review ctx 4096/32", "hash": "2b2db219466f8183cd837cffb20505126a523114d0e2a69ddb425f9501933912", "keyed": "bd68a9f1b0d69d19bd1d0737004df16d3ff1ac827f80aef3804bb59f2d11961c", "derive": "1a485b0f2b1a6dcffa47e71159adceb8eec678d2576ee68d1dbde82c9a00d590"}, + {"seed": 93, "len": 4096, "xof": 64, "key": "cb00aa01b802dfc2eaa2256eed82a0208816629b4e905a9b3090c69e68274d03", "ctx": "lambda-vm oracle review ctx 4096/64", "hash": "0492d1ff2de887b4febea60dfa8ab27f9d322708de806629c2378282b3686518606927b089be53a118bde1205242973aa32eb3c32708cea83aecb0cb5ae0d08e", "keyed": "3939705bf049fecda37a7b3adfdfc277139ec2243353bccdafa11d7a841f662dd49379862bd294a7f3a04567bd9f9a5ba2785e7eff94fc16e5886eaf2439f4ab", "derive": "c71aa99c235917053a09e55bfda0ae9d5214f5faa6d54498f7ee690f6c6a2af0dae3578f9aef9ad1b4fb7af11656f97e95955af8d697b0570e41f18ef5c92e46"}, + {"seed": 94, "len": 4096, "xof": 131, "key": "8223c60b3ec3110cce7438b53f7223e9baa8c3abf2b43bffb5ce0df1b1118ca2", "ctx": "lambda-vm oracle review ctx 4096/131", "hash": "ad5d14700821d7ee7842e21815bde1f0f2c4374b57349b259b3d3de06337db474de14e8bf3d3315a5cec5cffd2a31c8a0a54150feda2b4f7dab19057a1e5d4ba13ddf1ad855968bf33cd77de4a46571f3b0c7869e5ec88ed34cbab8b4c18e2e3f64f736ca9cc06f145138de1ad586c2fbba5fd22cfd5d31c4fcbf8c3d71ab120ec552e", "keyed": "47765f1c959389a8a342f9050feb7d202d51f6c76e718dd9913289e00cac33d09e53dd6a9486f24fbe927dfa921c9e4060e5ec0587ab0eea545c998de6292fadc74ca5874629c9d2e0a85bd2d44dd0967bd6da74103f9b7062bbd3c0153b53f0342db20740e0e5065197a6c56362152649fd106662a2a3136f4bec163472633aedce57", "derive": "a966caccc348466a42226f6ed900ef7031c22eb751422ebc32c904e44dcbf5f5e3dd676a6cc16bbdfd6a87bd91d46c867bd850a864d2eb39be3f946a90b0ece4527838452b18d041b86b5e3b726905845f664afb0fed035d51dfdcb8e907b9a75ba216acd7f2182e34e5b47f65fc1977796f5a2132f4b6bcbc691bc9e7fae805c28287"}, + {"seed": 95, "len": 4096, "xof": 200, "key": "174e6662d2c05955c063523f5288eadb34328c4fa2b27165858897cae98b048d", "ctx": "lambda-vm oracle review ctx 4096/200", "hash": "05420df742c2a977945cd2090e7faf6d969e63a2c6238cc0770e40ac9a86669fe7ff09b94c0b2ceb41ae5ff5321864c1247f5bd42040ab4f02c49330807b307e30191bc6ae242805ff2512d417c847b12fb9d85555052ec43db978585716de1b6246b170d82cb92ac1fdef0b34818b4c06739152d25009f284eda20e493736fa55c688c9e6d6407c648dff4e72caad3b7f51aac4aec4c53b831b785006f0a1338212600b2ef65c3af11fcd729409bca76308a41da0639b6e23737ce70c86426fcc1beeed41559d7a", "keyed": "d9997ec1c3f46309869da8818b3e5f634a48adf8a2227dc3e2eeed9f9bd4b1842e442e1d444a2047e711de7c97bfd1b368f73c0157843e35ee4d46f4662d5eb259765ca7c4d50e57a7b93cbc7e2f18a085b083315782253a60049b5a1e5aafaa7f425ff81afc5f306ecd7306db3060d16f7df63e5dd81ea43fbee15f8c6d2198c1a001a576538bf22e489d1d812604c46e2dd6cfc405bb5241ac80b3dd5c0ef9e778daa460c3ad084bec4889c6a0009d200b765e131074781a0b247cde5700fc04d46a6fc1b019b6", "derive": "ec2762b33af35e89aa4c07c733054536582a31fb630bdd6fd8c25aed6ba6b36f1e66f577cc2458342d87c56474f9abeb6c81d2666fa1e9342d01abc12e35f1175395247bd1e654d74797bbd934424200de4f735307cc79fb0d640111559771911f0adb0ee081880b6a72c3281f6c2827453ee54c4cdd64fd419237353a1da8e6a08046c26b833407830a767febcc65e500e90195e13601285b4109d54c0d94eb55f15f3846b2b92dcc2e9ddc6d7f0232e50c151db38fd8495882764456e15893237c77345d5d95db"}, + {"seed": 96, "len": 4097, "xof": 16, "key": "69567c30361e6b67e55967d288477bf78c8e0af84fcda6abc9258567e858c90b", "ctx": "lambda-vm oracle review ctx 4097/16", "hash": "867aca04bbac9c1d3a07278138e9ad1f", "keyed": "38e03b39b3e55beb53bdd77ce350c832", "derive": "4753c2c7254782c32be47b7a410f4572"}, + {"seed": 97, "len": 4097, "xof": 32, "key": "8f930a77102fa62bb15ffc8cce6ff1314dcc31eb20f2d17050b09fc6602f520d", "ctx": "lambda-vm oracle review ctx 4097/32", "hash": "ed871b6a5ee95e25f03e981e5bec7758ec00523f4986852510db5a4162961c85", "keyed": "9d5efe1b892cbe1ae7ab8a04f0f56fd4882d0477a34a655b48218fae1c83165a", "derive": "3548c2cee59bbc4fdc4266169ddf3464dfdded393b0066bbe7a33490dd1107ad"}, + {"seed": 98, "len": 4097, "xof": 64, "key": "927ff7293c0dd8b741106fbb7479028a1fce6b6fefa6f6266296e989358c56e5", "ctx": "lambda-vm oracle review ctx 4097/64", "hash": "b6e61602cee007c2cc998ef402a2a50d02cc1b988dfbe7362139f06226f533e8f9556a1ec8b0ae749c7761da36cb5a97663eae8a13b13c7fd4ba39140ea12e1f", "keyed": "38390cedeb4636357acffbb91d85265c1d32f872c55e5e9e341e8f581df4e23e8823fa7cb5e239c857368bdd2c21984398d3662393910e2612f5c2ab1fe61780", "derive": "0f4b6becfee37bc9612d188bc1d80b5ff3f81396ada402e0daa5dc899e9e55c6cf2dc5cc759e77e0ffaa7eb26ec5785b1d589b9ddb14fab4963002670452194f"}, + {"seed": 99, "len": 4097, "xof": 131, "key": "b8bc393f6cd619e9b5748cc48c90ca3bdc39bf0ab4d2d43c502a939bdfbfbd56", "ctx": "lambda-vm oracle review ctx 4097/131", "hash": "c5e5938ad2e8304bf0d5bda578ddc94dfedc903f9d6cc4401b8b443ebb8cb6cf21ae9e8c453a5ea7d1a6e50c04995bbeb9bcedf458d0581d9c0ed2845d1c9bd9163da1e26478341d619b4aa441211ebad0344f44ad621d211e9b6295c9d74a121707802ec788a8f453c090cb6da6e95be787afa0b1c3efb8a82a0badb816919bf7be19", "keyed": "def95aa4c1ff157d04ff55cbd23310031603969f5b45decd791bb0ff5064643b6a31a69a16de293af9a7fe719c60cc5e3bba6c8bf4ebb04705da1ef8caba76635f84dffca60164493bb5beaec75224ec7a4b0818d4ad88ba3b8efdb70b52d10e66418757457e667b9e3239b3a5dbb8c6dd823dc93e0f3d0b7ee68736e010ecb271bafe", "derive": "68ae58a3f9156bf80acbc356843f6eff735adb807ef7fca62f034d02e85111f26b043baac74a36dda7531c8d24406dde16827c0e0dec91ec969d980e09ee2582b8e1b4f693efee008e9ff2f28fe8cab0b821b1a4dab4fb24813c09dccc3d195e3af888f9bb6e3487b53b3607913e9381e83e9c66784a7e1586b46e83681a9f8d2903e3"}, + {"seed": 100, "len": 4097, "xof": 200, "key": "4926ffd129f4d4da38a0464c3cd5ae104e540c96a195dd522573610ae621345a", "ctx": "lambda-vm oracle review ctx 4097/200", "hash": "b4b19f35b3eab71e25b86a0d12f234756d79531e32ea6b9ad70c9f275f949353ac8e6c18fbe4124bc16a572fdd3c1f89243eb10c5e0e86b4d344148513c3f802a8544d87bcc9aefa6a318602cce20fcc486fe7f265769cd4154a7b978f6e1d76946927f67af6967aa4906f74d09047fbdf203379d6f9eadff3612ac884b6839852a79716bc4bce6f7853325202b0fcf23573f41af4f8de558f9e62eb5268dc8712f7786194118f81694cdd73bc4d5153ae6c32fab2f35d5caf5f9c8d6ac0ee4ad0458f56c132efa4", "keyed": "ffdb15a8f97a8e8f27d1e6fa9f83b862bce80f164f894aa56a330e5366db864787fe52d1ec92af5ff062bb1f12564e8c004be44e59ae8734b131d609da13d6b915b1fb4c954afda7ad5ab0bcc086f203c03e05232d2be549ef0c73243418f249e31d68c2a022bd6a140402c444514213e8060df9d96d50ab7e10bc40e2dcae2506d4fa910e3639102967b8671489589b156068a28089dc5d278deaf7045ab64ff84c237245fd98f8951a5b17fe7de46a48eca62f6862cf986fa808b4ab8a3a0fbe3c65d49423574a", "derive": "c9bccf719c967863b07ccaaab3c44c0644af36192e6f7091680ffc5d2a36b4847082068e42de15377d55b4dd06ce66e60294168989306f12183933cb1e86289aadf85d7e09e3b6b71264be605b1d7774fdc4741c3b569362d2e158088430c1985a8eb41dde271c4d53eac15b4268794c2bfa46032d2f598e0d02f897b9767379b988466ecfbb7eecd9b9621c73ddc78e7017f63189aa06441594adf632bfd439ec66f98ab49d2591e0fdc906e7006ff057f149430f756ee06ab323506921f9552a475bf61d5ae158"}, + {"seed": 101, "len": 10000, "xof": 16, "key": "debf6bda8ae84c02678eb007feec1ae5037fe2fdd886c749bbd8ba07fa751ae1", "ctx": "lambda-vm oracle review ctx 10000/16", "hash": "298af4e1235a1a2cc4b1f7e6371e4296", "keyed": "985164b642278a61f7885477fcdd7d4a", "derive": "9edd4f3905cba501cf38174df53cfa57"}, + {"seed": 102, "len": 10000, "xof": 32, "key": "724ee3cb2af83fd3c7fe1479201facaef25de4d47ab374b04d51dc2a7dd2414c", "ctx": "lambda-vm oracle review ctx 10000/32", "hash": "8797cc68e0e8f52c95e0e1c805d3ed9b66ccee9b629f9c0ad014faaebd96877b", "keyed": "08bfaeb67847f9d8da30caf3daad2c4dd763013166dd954e33e001c5fcb38543", "derive": "195f9d3c9d30f5dec592cc0a81ed4d71c5bfe2b647ba8b2dbe48877e6b8ecc56"}, + {"seed": 103, "len": 10000, "xof": 64, "key": "07e8bda29e5416f49e9edd67d44e6a6eb3604796baa70a54c03d6db1b53a3246", "ctx": "lambda-vm oracle review ctx 10000/64", "hash": "691f34c88445969efaa89be227fd360393e3b9fd66075583c7214d3a22685894b5d5779035a9b3114de3e8c88c48d12f28165cf3b921f933e9758116f46b2cc6", "keyed": "8736e42e82ac72ad5296420d9b8ebdc1f7f16be14ee45f0f98c8bd608ad9dd878ed5a0794997ddf234b57a44eb56233bae99087f5de09bcab7b7256865fe77c6", "derive": "186867f6ee0a0a30cf90e931c687519c5c6015db1855bec7f687c0d14408f3f2f6170bb892a8104814730b5616eae4185abeb2184e4805c84304ff633871da89"}, + {"seed": 104, "len": 10000, "xof": 131, "key": "7cd1fd085fe8d82385a78adc0c2fb6967c37bceaf58c1ca36f75ccf94597297d", "ctx": "lambda-vm oracle review ctx 10000/131", "hash": "0a31dc0885cb9f01f608c472c927642bfc6da83faaeb3cbd08df693fa970e32fe63752f15ad7b44acc7aa84184f57f230256307e0e19d41bd7045b6e0acd2e0f1030d59fce33e6ec690eb1d61123b629f661af35f5b5289599a8c610e91954dfcfe2e10c51e989d2afac0f137db9cd9ed349bc9c268ecdb4de985890a780f9b4583491", "keyed": "789727759c1fa74d67fb10747ea2d6b63f880fa2c65384c7001edaa7a036c6a20e517f8722f41cba69eae3250ba124c6d5223382c7d7e715d12896091d4e06836afa08fb946a6e4095caed92fa1b32bf184170e17bd993664b243e25b4bad23b1fd502903ef89df669f42a33d3b8ecc0ae6e34c1e36d0fc30a205fca676b546bacdc8b", "derive": "6d23542e2401bdde33d6cfb6e1d5e711012e509a4d13075fc42b0f1f263c81807285351fe8fce9e755d45ebdaf1abec13e3031102472b775fde583672313104fbda0636b15485f925ce8a2a2bae14c4d7d9e4a595c1733c5e5017d9a54b67fa8238e43079b9ad609c0cc27c0bedf6bad841f8bf9f37cec048d6f99244df1e0c1e8bec2"}, + {"seed": 105, "len": 10000, "xof": 200, "key": "a20d1c6d3a8c8000132928d6d8de2d7a01b4352deb394230b46fe47a9d6d2319", "ctx": "lambda-vm oracle review ctx 10000/200", "hash": "de90c25b7c0cee636412502de2183d0f0ddac7238f059d540335550a92507c51181c3f525fe2072f17208a06e035a6b090c67184f792915181ff35b6986008ec6868104328fcc6f451da8478998ac6e310e4ce86f1a4356f839da5d21c44753a15486730c13028dd542d70ccbd220e32b4958e93845b6886937e7e1d2d720c3cb748876318bc9aecfcd7b7c2d66af44abe5757db8f0bc8eb697c1b6b4844d95569e295128700cfe2567def2ea7b13bcb53c315d020d9d9cf4abd77e4e242c16533fca443eb3aeeda", "keyed": "d5322c8131d3f7490f9b61b92ace6f127e372344af6acd440109be79ba3f3bbc0c9e7c68d251f1f6ba23b93d4f3c1e2bab8fbe4a6fabaabe17a27f12424117db0f4d24ca91ced9744601c9eea0083f0d0fa234af5ddd38bd316971f12570654c6871fb53edff0b2d2cc268a5898782b4e22c392d4b933e4f9120667caecb8c5f78bbc15ab143a97d13841dfebc3b7097569180a12b6d02350892625fab5bb723aa170c42da7f6213c005e76e82354ebcc2aa4074020000bb14a94035835748675a9ad1dac4d21abc", "derive": "49a119c6297ac0cfd9282cec209071d6b269b77b20cf1f32d700bb29d0174e65bd8149052f31efd0a6b1f9c902a6e8994ab821a84454877c412a7c8830219c8dd6dc0143accbe984687600e7e3b73f8c18929e98f434e8d151bf8ef849f3c48605d840a29065391ad342dcbd267e046d3558254f443eb539bf5878eb71721fed08922869cc7ab9d21218ca393dc300e970d0f7c0dcb15953639639c1ba29ffd6b411a297dd3288476d11c832cd326636d67ccacf258b48658877dc874594ee33163abf82196c02ce"}, + {"seed": 106, "len": 65536, "xof": 16, "key": "c7fcd83f19782a915cf9d246aec8c8ba5356c0236f5aab21745ca8706cd378e5", "ctx": "lambda-vm oracle review ctx 65536/16", "hash": "250e73c83ef8c2547bc66cbb62db7cf9", "keyed": "68a9254c8bca25432f39914236af4697", "derive": "c2610f8c15910689cc8b9071b3e618b6"}, + {"seed": 107, "len": 65536, "xof": 32, "key": "ed3888f753dca5e6740f4e89c20c13885e463030c76298c6c0cad9343166d9ea", "ctx": "lambda-vm oracle review ctx 65536/32", "hash": "d987ec830797a4b3c39f7b2da883106acde4dca8088a0c61c9ba75efb3db9f88", "keyed": "34b1c21894a95a245a74e0681ed5b38774e0b282adc477bdd1c58f3f5898f75b", "derive": "95b688c7d8a773cd6ed197bdfe9c3ddfcb3983c4b8a0aabaa36e1ffa6f21ad95"}, + {"seed": 108, "len": 65536, "xof": 64, "key": "5ca03baa8317d67cc39f60d104b35d1aef8869c74baccea4341bd825d246e351", "ctx": "lambda-vm oracle review ctx 65536/64", "hash": "32b03bba7802ca730ca05939ff1e1ffdec0a09af285130a4cdc148ef81d78eb98baa097a8ffad1e4b876f0bb59a18c7e2b45c8cb3813898e2403984f7a6c6071", "keyed": "3b0a32ca1166401b8d712f829dc16688095153d108912de3c8414e0a6750d6d0497d72fe25247c8d7e0e6e9c9023880cf0752310a7584748dc31891bfd9ef33c", "derive": "82899fe0ac33c8eaabc4b08dc0094cb450d9ccf0aebdda852eaab18adc4ad343d8d638e331496ef18fc23dfa48e3ce3cf77b3ad76403353d244b48e617b44da6"}, + {"seed": 109, "len": 65536, "xof": 131, "key": "f03938cf2f49ac7e99fdc4a957b6b3e99cc5288c4feacc5837526a08f35f5ad4", "ctx": "lambda-vm oracle review ctx 65536/131", "hash": "5ec02bd02ee4d382db8d10d0c4a2a014087558869944c60b97531eee05c2a5ee5fb8664fbf041bb64f09c5e997bd8a6db7dc18ecdf22285fe7a1ad368df8b0683e37c328cece89b0e90992bafc70f7987bb064338dc4cb025cf366aaee42c7e8fbcb9fe28a816b97c1766c7184bbdea668fe11db1f6aab31f189b28f9f4014a24b17b6", "keyed": "21c7d240682985db03161379efff7c36d04594c5a5b166b345a54a35f6fbb6c0c60a202e34a71ac97fb7f9888c86dfe66ef0781888245c332a45c7ff494385f6e7c950a1a561ab98dbb352ce4f9ea6fabb1684ac152ef00cb7d03f6811f7faf424e0dd4cb24de541a2a4762be4d2f76c22f752d18942a7c8961d0c99221da20a0b81a9", "derive": "f1c0de3ac81f9e3907a67c39b1a09949da0ff0561cfd84c6878cc7b969ee7cb89949bd0a4d217c7780f038cf7659a2e8e9f8472d9f414a3870e5adfecd48ecfe0187225bd5acffd898b25cbf7faeaa3afce321b2add22e62459e907f99843fd689ecc0e448c02b4a389db38c8fc52ca0f978e727077ec4dc58e5a6521ef07ad59a3c5a"}, + {"seed": 110, "len": 65536, "xof": 200, "key": "a7cbf3e166bbab1632153b7b7a9053d6b42f4d853b55e21867323ad5b55fcb0f", "ctx": "lambda-vm oracle review ctx 65536/200", "hash": "9d3608bf4225fd301cb08dbb27d22bdcd902d833ee5e8896e535d799c2ec10129c8aab47a6602262ad17b3589b66ee97047f29ab1606de811d00e3faa6bbfefc54afef9dbd61ca79675d3d583a819fae4e8adc0307da8e5e3202e2b25cc78af20e13d73a16656edb5ade79f77163d748813776916247f627452123b2aa46910a7fb7706460e8f72aa5bcb6f2685819960e6c85ce66ee560daf963c3afbe4a21e8fcf5750fcac93f62f13bd273d02082aa462f03497e97d0fca5a1b39c8a6fff4bd83aed05525c5e1", "keyed": "9b312955f025b12c33f0fb6d1a7724847b6bc5416bc8a7545b5b1c40e95047c479157113b57c0a4cc2d1acc5f16643f22285499f06f211c4ba208d9c2969e3a2bf21ccb3b1808ce309bb887eea150047a0c4e7ea981a57f7fbffbcf596ab616865fb3bdf428e77d1ec60e52a1f7f2f4796d13edfcc793ff7a5973793537793a902f315058d0c3f0a67968ff812657d46585c063165ff311c7ae19f218dde369cd5e89885ba08d2f4d5bb96c00ebc3e33dbda71fa8a873cbdd2c26b102aa49248fa4177f958fe5ce1", "derive": "8772e113081fe301b461adf0a1fbf105a199eee3d0c0f27836662a6ed7c62c01037ff56c18b298028babcb45aa1c7f13335f9fb8dc2f5931021be53ef21da4727b20927d3860eb17b3dd7ee51aeaae18aea989de6b897baa14547cb3e6c1bbec9bc63e66a078cac93815c4646e5690fb7ebbeabe010e44c9dd59eb2b5e6399b24c66f9d859390e170e94e45fe942d230788b9abce7392a3efe7d9652ca5e94cc63996cb47260dd9111864415368f206513609d744a807e7a2fd044b627d1ecc9b317bee514aaa7f4"}, + {"seed": 111, "len": 100000, "xof": 16, "key": "3b643c5a1b5e239a0799eaf8ff13931b82edb8fea42a47caa825b0244d4b7b00", "ctx": "lambda-vm oracle review ctx 100000/16", "hash": "dafc038f963a4fbb56a970d51433e9c7", "keyed": "ff1fe75b79c283887239712536a6f8b5", "derive": "4ca500d55e5ceec2fe5c02021403c4b5"}, + {"seed": 112, "len": 100000, "xof": 32, "key": "d394a71b890df7dfee174b95ed22722c143195fef201d138f2a8df894d16f144", "ctx": "lambda-vm oracle review ctx 100000/32", "hash": "58d738cba5b0b79e4d6f2035fb41acf271f1fed88b8e432a51958f43e827a212", "keyed": "6dd9792a242527b2d3013515dc5a7c75e36ddb6df6a6d41b373533f753c09454", "derive": "82f5c674b0ae22e41dd9424c64f66d4ff4ca3bc67ab20f0fcee1a97dfb6901f8"}, + {"seed": 113, "len": 100000, "xof": 64, "key": "f93f8bd6a30919f2178dd9860e65cc1da2d5a23e6dfda14cb14a2aa8e181732e", "ctx": "lambda-vm oracle review ctx 100000/64", "hash": "bf0c93ac003d8e92e511fe59d3309141fc1bc993c38d9a63737e4f508d68040f9422080ad44bb74dcb578d37bafb82ab021f758e98d347fe407576bce315f86d", "keyed": "90f3212d27f2a1919d209a19754424ff0b2704d7480d84ed882cb5fad926e7229d8d92a2b8daf1bc7b17e0333098265e7caf0eae401e0cbfcb898421a9245553", "derive": "7829eeda7bd48bb51b4b28147245a7bafb2df37e018029598098c99797cb174d2526858c341bf54a8d90eae6240572e0e590653c6486c5413bcbbdd390af9a25"}, + {"seed": 114, "len": 100000, "xof": 131, "key": "fcbd4502b4c333ae50c954a138f4b41913588e1d5e966ab5580f9d3c98d5bf28", "ctx": "lambda-vm oracle review ctx 100000/131", "hash": "4d1c5f62bf411a2b6f6711f4d9913c0669cc472b231099c41c7f9a45343ec421ac727a010b1e495a4311defea2baa88141a9dd60192a4d97a06f4818a527b4b65c322b4554b4a0c29fdeacd4fec1ba1afa2e0abec8b577c84f9417c376b2472272ac055878b0d42e5f9c4dd4939f27a4b46c0ef080c2e2abefe3fad3abb728c067cde5", "keyed": "bfa9973a8c02b8882e1450dd434a4b9d7072c877d8a2f94346fe06dbd5abd96d8fee4ed82aac7f3fec8addd2d7b197f030ea5322dc7d4217815a0a1f73987818ea935d09fa8f1513a8c18d87f9ba5f0f6a7fb57ab90c01c6aa8fa5aa59b304ae1240d0bcdd3e670786033ca4137ab6968a3491e2ebcddd638c633bbce0488b54eea92d", "derive": "a64ef45282602fb582847effffdddf218568cf916d38bab6cf711e8ec7569cb54d1ae624928ef8476942c2958c9d021e72ac012bd5c28cbbbfacf2239d968e783b310f23f6301eed08402d08078bda9da21a46c41c9728053e97b6d964f55aa3c8cfbb8698d6ef94dd2507ead85e454947d8bbea81159162c16c25134c0c92b519c0a7"}, + {"seed": 115, "len": 100000, "xof": 200, "key": "2268978ce6bb8db3b5133d660df23099096d6778ee1946a59d217165c34429a4", "ctx": "lambda-vm oracle review ctx 100000/200", "hash": "9d0a9882067f0f7e8adab6330d9e62498da7623daa31f39e984189b42830d444a4c7ad037fbe55db999626c6ccafb1737145a2ce28c63fa0380322bed696938395f8e18cf7c1ec1d032a81f044e4bd58a2f8b930b9f91fab466dcdbb97825ce61b696c1d9af7653380ca80c4ad3224338b2bb42d420c7d44d2d42e2beb9bd6e5d3f714df6dcd651c2ca655601c3ff2d50020df05b9f8c60617633f6f0648d4f57d1b98b9ff861dec6e0e3ee494729489d7b81f587fd0192fb7d93e25e28ca359967f6ce919f1ffa3", "keyed": "e6c5d4fb218f27ac862338bc0a6a272336d53ef38d415584fefe7989807ee222a683a3222031a9839dd5f02cd61687e9c6bd68792308dd86e43b3daf159defc21ca8f7663d4d8ce2c3c80539f9b9eae2656a4b299ca44d94381be56720599b799fef8e674e83f8114ac14f180deb55bdc0e3527fc2b69492540aa28dcf84f56ba9dcc89225122a0f7918b8892ee3afe40b24335ddb84a2f92796819602d8c88085cb77f45228cd7ed23d5b323c1bc7601fa85ceb1f9d57e541b834633ea7f5af70d52fcb3cc1fe4c", "derive": "1e6b77b3cfd3bb00000c04a2575b772c18720102877b8cc95379163219e67e03f3881cb7e78b893a11f19254cacf00302d68f8032999d7352f132341bb3a58eb002dfbc27dbd09694c82a8f34d006d1479508184954d36f58c821cfb7ae6965f4cce34037777213a433705a76390f33f6e5c80c4276dbdc1514a7e26437c5f490d5ec20b429016bb79d6eedf9b244694f6bd35cdc214fc27662b3e34bcc5ddee99c89fd4e4c443ad5f52958f64a4bc3514a4fb3b6bb4ec3e738836b2e03f0f096b98fdd9f91d1a43"} + ], + "known": {"empty": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", "abc": "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85"} +} diff --git a/thoughts/blake3/blake3-oracle/test_oracle.py b/thoughts/blake3/blake3-oracle/test_oracle.py new file mode 100644 index 000000000..24eccef98 --- /dev/null +++ b/thoughts/blake3/blake3-oracle/test_oracle.py @@ -0,0 +1,364 @@ +""" +Validation suite for the BLAKE3 compression-function oracle. + +External anchors (independent of `blake3_ref.py`): + 1. Official BLAKE3 `test_vectors.json` (authored by the BLAKE3 team). Covers + the whole-hash output in all three modes (hash / keyed_hash / derive_key) + for 35 input lengths up to 102400 bytes. Passing these exercises the + compression function under every flag combination and many counter values. + 2. The official `blake3` PyPI package (the reference Rust implementation), + differential-tested on randomised inputs of many lengths in all 3 modes. + 3. Plonky3's independent `blake3-air` compression (ported below from + others/Plonky3/blake3-air/src/generation.rs), differential-tested DIRECTLY + at the compression-function level (flags = 0) on random (h, m, t, block_len). + +The 6-round variant has no external vectors; we (a) show it differs from the +7-round function only in the round count and (b) emit 10 canonical vectors. + +Run: ./venv/bin/python test_oracle.py +""" + +import json +import os +import random +import sys + +import blake3_ref as ref + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# Test inputs in test_vectors.json follow a fixed pattern: byte i is (i % 251). +def pattern_input(n): + return bytes(i % 251 for i in range(n)) + + +# --------------------------------------------------------------------------- +# ANCHOR 1: official BLAKE3 test_vectors.json +# --------------------------------------------------------------------------- + +def test_official_vectors(): + path = os.path.join(HERE, "official_test_vectors.json") + data = json.load(open(path)) + key = data["key"].encode("utf-8") + assert len(key) == 32, f"expected 32-byte key, got {len(key)}" + context = data["context_string"] + + cases = data["cases"] + checked = 0 + for c in cases: + n = c["input_len"] + inp = pattern_input(n) + out_len = len(c["hash"]) // 2 # hex -> bytes (extended output length) + + got_hash = ref.blake3_hash(inp, out_len).hex() + assert got_hash == c["hash"], \ + f"[hash] len={n}: mismatch\n got={got_hash}\n exp={c['hash']}" + + got_keyed = ref.blake3_keyed_hash(key, inp, out_len).hex() + assert got_keyed == c["keyed_hash"], \ + f"[keyed] len={n}: mismatch\n got={got_keyed}\n exp={c['keyed_hash']}" + + got_dk = ref.blake3_derive_key(context, inp, out_len).hex() + assert got_dk == c["derive_key"], \ + f"[dkey] len={n}: mismatch\n got={got_dk}\n exp={c['derive_key']}" + + checked += 1 + return checked, len(cases), context + + +# --------------------------------------------------------------------------- +# ANCHOR 2: official `blake3` PyPI package (reference Rust impl) +# --------------------------------------------------------------------------- + +def test_pypi_blake3(): + try: + import blake3 as blake3_pkg + except ImportError: + return None # signal "unavailable" + + rng = random.Random(0xB3B3B3) + lengths = [0, 1, 2, 31, 32, 33, 63, 64, 65, 127, 128, 129, 512, 1000, 1023, + 1024, 1025, 2048, 4096, 4097, 10000, 65536, 100000] + n_checked = 0 + + # 2a. Default hash, default (32-byte) and extended output. + for n in lengths: + msg = bytes(rng.randrange(256) for _ in range(n)) + assert ref.blake3_hash(msg, 32) == blake3_pkg.blake3(msg).digest(), \ + f"pypi default hash mismatch at len={n}" + xof = rng.choice([16, 32, 64, 131, 200]) + assert ref.blake3_hash(msg, xof) == blake3_pkg.blake3(msg).digest(length=xof), \ + f"pypi XOF mismatch at len={n}, xof={xof}" + n_checked += 2 + + # 2b. Keyed hash. + for n in lengths: + key = bytes(rng.randrange(256) for _ in range(32)) + msg = bytes(rng.randrange(256) for _ in range(n)) + assert ref.blake3_keyed_hash(key, msg, 32) == \ + blake3_pkg.blake3(msg, key=key).digest(), f"pypi keyed mismatch at len={n}" + n_checked += 1 + + # 2c. Derive key. + for n in lengths: + ctx = f"lambda-vm blake3 oracle test context {n}" + material = bytes(rng.randrange(256) for _ in range(n)) + got = ref.blake3_derive_key(ctx, material, 32) + exp = blake3_pkg.blake3(material, derive_key_context=ctx).digest() + assert got == exp, f"pypi derive_key mismatch at len={n}" + n_checked += 1 + + return n_checked + + +# --------------------------------------------------------------------------- +# ANCHOR 3: Plonky3 blake3-air independent compression (flags = 0) +# +# Ported directly and independently from +# others/Plonky3/blake3-air/src/generation.rs +# (verifiable_half_round + generate_trace_row_for_round + feed-forward), which +# hardcodes flags = 0 and does exactly 7 rounds. This is a SECOND independent +# implementation of the compression function, checked at the compression level. +# --------------------------------------------------------------------------- + +# Plonky3 constants (constants.rs). IV stored as [lo16, hi16]. +_P3_IV = [ + (0x6A09 << 16) | 0xE667, (0xBB67 << 16) | 0xAE85, + (0x3C6E << 16) | 0xF372, (0xA54F << 16) | 0xF53A, + (0x510E << 16) | 0x527F, (0x9B05 << 16) | 0x688C, + (0x1F83 << 16) | 0xD9AB, (0x5BE0 << 16) | 0xCD19, +] +_P3_MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + + +def _p3_permute(m): + return [m[_P3_MSG_PERMUTATION[i]] for i in range(16)] + + +def _p3_rotr(x, n): + x &= ref.MASK32 + return ((x >> n) | (x << (32 - n))) & ref.MASK32 + + +def _p3_half_round(a, b, c, d, m, flag): + # verifiable_half_round(generation.rs:203) + rot1, rot2 = (8, 7) if flag else (16, 12) + a = (a + b) & ref.MASK32 + a = (a + m) & ref.MASK32 + d = _p3_rotr(d ^ a, rot1) + c = (c + d) & ref.MASK32 + b = _p3_rotr(b ^ c, rot2) + return a, b, c, d + + +def _p3_round(state, m): + # generate_trace_row_for_round(generation.rs:120), state is [row][col]. + for i in range(4): # columns, first half + state[0][i], state[1][i], state[2][i], state[3][i] = _p3_half_round( + state[0][i], state[1][i], state[2][i], state[3][i], m[2 * i], False) + for i in range(4): # columns, second half + state[0][i], state[1][i], state[2][i], state[3][i] = _p3_half_round( + state[0][i], state[1][i], state[2][i], state[3][i], m[2 * i + 1], True) + for i in range(4): # diagonals, first half + state[0][i], state[1][(i + 1) % 4], state[2][(i + 2) % 4], state[3][(i + 3) % 4] = \ + _p3_half_round(state[0][i], state[1][(i + 1) % 4], state[2][(i + 2) % 4], + state[3][(i + 3) % 4], m[8 + 2 * i], False) + for i in range(4): # diagonals, second half + state[0][i], state[1][(i + 1) % 4], state[2][(i + 2) % 4], state[3][(i + 3) % 4] = \ + _p3_half_round(state[0][i], state[1][(i + 1) % 4], state[2][(i + 2) % 4], + state[3][(i + 3) % 4], m[9 + 2 * i], True) + + +def plonky3_compress(chaining_value, block_words, counter, block_len): + """Independent Plonky3 blake3-air compression. flags is hardcoded 0 + (v[15]=0), matching generation.rs. Returns 16 output words.""" + cv = list(chaining_value) + m = list(block_words) + state = [ + [cv[0], cv[1], cv[2], cv[3]], + [cv[4], cv[5], cv[6], cv[7]], + [_P3_IV[0], _P3_IV[1], _P3_IV[2], _P3_IV[3]], + [counter & ref.MASK32, (counter >> 32) & ref.MASK32, block_len & ref.MASK32, 0], + ] + for r in range(7): + _p3_round(state, m) + if r < 6: + m = _p3_permute(m) + out = [0] * 16 + for i in range(4): + out[i] = state[0][i] ^ state[2][i] + out[4 + i] = state[1][i] ^ state[3][i] + out[8 + i] = state[2][i] ^ cv[i] + out[12 + i] = state[3][i] ^ cv[4 + i] + return out + + +def test_plonky3_differential(): + rng = random.Random(0x9110C43) + n = 20000 + for _ in range(n): + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + t = rng.randrange(1 << 64) + block_len = rng.randrange(0, 65) + mine = ref.compress(h, m, t, block_len, flags=0, rounds=7) + theirs = plonky3_compress(h, m, t, block_len) + assert mine == theirs, ( + f"Plonky3 differential mismatch\n h={h}\n m={m}\n t={t}\n " + f"block_len={block_len}\n mine={mine}\n theirs={theirs}") + return n + + +# --------------------------------------------------------------------------- +# Internal self-consistency (NOT an external anchor): compress_cv, feed-forward. +# --------------------------------------------------------------------------- + +def test_internal_consistency(): + rng = random.Random(7) + for _ in range(1000): + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + t = rng.randrange(1 << 64) + bl = rng.randrange(0, 65) + fl = rng.randrange(0, 128) + full = ref.compress(h, m, t, bl, fl) + assert len(full) == 16 + assert ref.compress_cv(h, m, t, bl, fl) == full[:8] + # feed-forward invariant: output[8:16] = v[8:16] ^ h ; recompute v to check. + return 1000 + + +# --------------------------------------------------------------------------- +# 6-ROUND VARIANT: derivation check + canonical vectors. +# --------------------------------------------------------------------------- + +def test_6round_derivation(): + """Confirm the 6-round variant equals 7-round with the loop bound changed, + and that it genuinely differs from the 7-round function.""" + rng = random.Random(0x6) + differ = 0 + for _ in range(2000): + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + t = rng.randrange(1 << 64) + bl = rng.randrange(0, 65) + fl = rng.randrange(0, 128) + v6a = ref.compress_6round(h, m, t, bl, fl) + v6b = ref.compress(h, m, t, bl, fl, rounds=6) + assert v6a == v6b, "compress_6round must equal compress(rounds=6)" + if ref.compress(h, m, t, bl, fl, rounds=7) != v6a: + differ += 1 + assert differ > 1990, "6-round and 7-round should differ on essentially all inputs" + return differ + + +def canonical_6round_vectors(): + """Deterministic canonical vectors for the 6-round variant (fixed seeds). + These become the variant's reference going forward (recorded in ORACLE.md).""" + vectors = [] + # 10 deterministic inputs derived from fixed seeds 0..9. + for seed in range(10): + rng = random.Random(seed) + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + t = rng.randrange(1 << 64) + bl = rng.randrange(0, 65) + fl = rng.randrange(0, 128) + out = ref.compress_6round(h, m, t, bl, fl) + vectors.append(dict(seed=seed, h=h, m=m, t=t, block_len=bl, flags=fl, out=out)) + return vectors + + +# --------------------------------------------------------------------------- + +def main(): + print("=" * 74) + print("BLAKE3 compression-function ORACLE — validation") + print("=" * 74) + + # Each anchor is independent: a missing fixture SKIPs that anchor only. It must + # never cascade — a FileNotFoundError here used to abort anchors 2 and 3 AND the + # canonical-vector emitter below, which silently blocked the z3 gate's positive + # controls on an unrelated download. + status = {} + + # Anchor 1. NOTE: the vector file ships regenerated from the official `blake3` + # Rust crate (see ../ground-truth/), not downloaded from upstream. Same official + # parameters and a non-circular reference, but not the published artifact — the + # label says so rather than claiming more than we have. + try: + checked, total, ctx = test_official_vectors() + print(f"[1] Official-parameter vectors : PASS ({checked}/{total} cases x 3 modes)") + print(f" modes: default hash, keyed hash, derive_key context={ctx!r}") + print(" source: regenerated from the official blake3 crate, not the published file") + status["official_vectors"] = "PASS" + except FileNotFoundError as e: + print(f"[1] Official-parameter vectors : SKIP (missing fixture: {os.path.basename(str(e.filename or e))})") + status["official_vectors"] = "SKIP" + + # Anchor 2 + n2 = test_pypi_blake3() + if n2 is None: + print("[2] Official `blake3` PyPI pkg : SKIP (package not importable)") + status["pypi"] = "SKIP" + else: + print(f"[2] Official `blake3` PyPI pkg : PASS ({n2} randomised differential checks, 3 modes)") + status["pypi"] = "PASS" + + # Anchor 3 + try: + n3 = test_plonky3_differential() + print(f"[3] Plonky3 blake3-air (direct): PASS ({n3} random compressions, flags=0)") + status["plonky3"] = "PASS" + except (FileNotFoundError, ImportError) as e: + print(f"[3] Plonky3 blake3-air (direct): SKIP ({e})") + status["plonky3"] = "SKIP" + + # Internal + ni = test_internal_consistency() + print(f"[.] Internal self-consistency : PASS ({ni} checks) [not an external anchor]") + + # 6-round + differ = test_6round_derivation() + print(f"[4] 6-round variant derivation : PASS (=compress(rounds=6); differs from 7r on {differ}/2000)") + + # The banner reports what actually ran. It previously printed "VALIDATED ... + # anchored on official test vectors + official PyPI package + Plonky3" + # unconditionally, including when anchors had SKIPped — the status dict was + # written and never read. Claiming an anchor you did not run is worse than + # running none. + passed = [k for k, v in status.items() if v == "PASS"] + skipped = [k for k, v in status.items() if v == "SKIP"] + label = { + "official_vectors": "official-parameter vectors", + "pypi": "official PyPI package", + "plonky3": "Plonky3 independent compression", + } + print("=" * 74) + if not passed: + print("VALIDATION STATUS: NOT VALIDATED (no external anchor ran)") + elif skipped: + print("VALIDATION STATUS: PARTIALLY VALIDATED") + else: + print("VALIDATION STATUS: VALIDATED") + print(f" 7-round reference: anchored on {', '.join(label[k] for k in passed) or 'nothing'}.") + if skipped: + print(f" NOT anchored on : {', '.join(label[k] for k in skipped)} (skipped this run).") + print(" 6-round variant : derivative anchor (loop-bound diff) + canonical vectors below.") + print("=" * 74) + + # Emit canonical 6-round vectors. + print("\nCANONICAL 6-ROUND VARIANT VECTORS (seeds 0..9):") + vecs = canonical_6round_vectors() + out_json = os.path.join(HERE, "canonical_6round_vectors.json") + json.dump(vecs, open(out_json, "w"), indent=2) + for v in vecs: + out_hex = "".join(f"{w:08x}" for w in v["out"]) + print(f" seed={v['seed']}: t={v['t']:#018x} block_len={v['block_len']:2d} " + f"flags={v['flags']:#04x} -> out[0]={v['out'][0]:#010x} out[15]={v['out'][15]:#010x}") + print(f" (full vectors written to {os.path.basename(out_json)})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/blake3/ground-truth/Cargo.toml b/thoughts/blake3/ground-truth/Cargo.toml new file mode 100644 index 000000000..01cc4bf46 --- /dev/null +++ b/thoughts/blake3/ground-truth/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] + +[package] +name = "gt" +version = "0.1.0" +edition = "2021" + +[dependencies] +blake3 = { version = "1.8.5", default-features = false, features = ["std", "pure"] } + +[profile.dev] +debug = false diff --git a/thoughts/blake3/ground-truth/src/bin/counter_probe.rs b/thoughts/blake3/ground-truth/src/bin/counter_probe.rs new file mode 100644 index 000000000..ac3dadff7 --- /dev/null +++ b/thoughts/blake3/ground-truth/src/bin/counter_probe.rs @@ -0,0 +1,32 @@ +// SCRATCH (audit, not committed): probe the XOF counter path of the official +// blake3 crate. For a fixed single-block input, the root output block at +// counter t is compress(key, block, t, block_len, flags|ROOT) — so seeking an +// OutputReader to byte position t*64 exercises the v[12]/v[13] counter split +// at arbitrary t, including t >= 2^32. +use blake3::Hasher; +use std::io::Write; + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{:02x}", x)).collect() +} + +fn main() { + // 64-byte single-block input, same pattern as the Python side. + let input: Vec = (0..64).map(|i| (i % 251) as u8).collect(); + let counters: Vec = vec![ + 0, 1, 2, 0xFFFF_FFFE, 0xFFFF_FFFF, 0x1_0000_0000, 0x1_0000_0001, + 0x100_0000_0000, // 2^40 + 0x8000_0000_0000, // 2^47 + ]; + let stdout = std::io::stdout(); + let mut w = std::io::BufWriter::new(stdout.lock()); + for &t in &counters { + let mut h = Hasher::new(); + h.update(&input); + let mut reader = h.finalize_xof(); + reader.set_position(t * 64); + let mut out = [0u8; 64]; + reader.fill(&mut out); + writeln!(w, "{} {}", t, hex(&out)).unwrap(); + } +} diff --git a/thoughts/blake3/ground-truth/src/main.rs b/thoughts/blake3/ground-truth/src/main.rs new file mode 100644 index 000000000..e79a10660 --- /dev/null +++ b/thoughts/blake3/ground-truth/src/main.rs @@ -0,0 +1,138 @@ +// Ground-truth BLAKE3 vector generator using the OFFICIAL blake3 crate (v1.8.5, +// pure-Rust feature, built offline from the local cargo registry). +// Emits JSON on stdout in the same shape as the upstream test_vectors.json, +// plus a randomised differential set. + +use blake3::Hasher; +use std::io::Write; + +const KEY: &[u8; 32] = b"whats the Elvish word for friend"; +const CONTEXT: &str = "BLAKE3 2019-12-27 16:29:52 test vectors context"; +const XOF_LEN: usize = 131; + +fn pattern_input(n: usize) -> Vec { + (0..n).map(|i| (i % 251) as u8).collect() +} + +fn hash_hex(input: &[u8], out_len: usize) -> String { + let mut h = Hasher::new(); + h.update(input); + let mut out = vec![0u8; out_len]; + h.finalize_xof().fill(&mut out); + hex(&out) +} + +fn keyed_hex(key: &[u8; 32], input: &[u8], out_len: usize) -> String { + let mut h = Hasher::new_keyed(key); + h.update(input); + let mut out = vec![0u8; out_len]; + h.finalize_xof().fill(&mut out); + hex(&out) +} + +fn derive_hex(ctx: &str, input: &[u8], out_len: usize) -> String { + let mut h = Hasher::new_derive_key(ctx); + h.update(input); + let mut out = vec![0u8; out_len]; + h.finalize_xof().fill(&mut out); + hex(&out) +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{:02x}", x)).collect() +} + +// xorshift64* — deterministic, self-contained RNG so the Python side can +// reproduce the exact same inputs without sharing any code. +struct Rng(u64); +impl Rng { + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545F4914F6CDD1D) + } + fn byte(&mut self) -> u8 { + (self.next_u64() >> 33) as u8 + } +} + +fn main() { + let lengths: Vec = vec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 63, 64, 65, 127, 128, 129, 1023, 1024, 1025, 2048, 2049, 3072, + 3073, 4096, 4097, 5120, 5121, 6144, 6145, 7168, 7169, 8192, 8193, 16384, 31744, 102400, + ]; + + let stdout = std::io::stdout(); + let mut w = std::io::BufWriter::new(stdout.lock()); + + writeln!(w, "{{").unwrap(); + writeln!(w, " \"key\": \"{}\",", String::from_utf8_lossy(KEY)).unwrap(); + writeln!(w, " \"context_string\": \"{}\",", CONTEXT).unwrap(); + writeln!(w, " \"cases\": [").unwrap(); + for (i, &n) in lengths.iter().enumerate() { + let inp = pattern_input(n); + writeln!(w, " {{").unwrap(); + writeln!(w, " \"input_len\": {},", n).unwrap(); + writeln!(w, " \"hash\": \"{}\",", hash_hex(&inp, XOF_LEN)).unwrap(); + writeln!(w, " \"keyed_hash\": \"{}\",", keyed_hex(KEY, &inp, XOF_LEN)).unwrap(); + writeln!(w, " \"derive_key\": \"{}\"", derive_hex(CONTEXT, &inp, XOF_LEN)).unwrap(); + writeln!(w, " }}{}", if i + 1 == lengths.len() { "" } else { "," }).unwrap(); + } + writeln!(w, " ],").unwrap(); + + // Randomised differential set. Inputs are generated from a self-contained + // xorshift64* stream that the Python side re-implements independently. + writeln!(w, " \"random\": [").unwrap(); + let rlens: Vec = vec![ + 0, 1, 2, 31, 32, 33, 63, 64, 65, 127, 128, 129, 512, 1000, 1023, 1024, 1025, 2048, 4096, + 4097, 10000, 65536, 100000, + ]; + let xofs: Vec = vec![16, 32, 64, 131, 200]; + let mut seedctr: u64 = 1; + let mut first = true; + for &n in &rlens { + for &xl in &xofs { + let seed = seedctr; + seedctr += 1; + let mut rng = Rng(seed); + let msg: Vec = (0..n).map(|_| rng.byte()).collect(); + let mut krng = Rng(seed ^ 0xDEADBEEF); + let mut key = [0u8; 32]; + for b in key.iter_mut() { + *b = krng.byte(); + } + let ctx = format!("lambda-vm oracle review ctx {}/{}", n, xl); + if !first { + writeln!(w, ",").unwrap(); + } + first = false; + write!( + w, + " {{\"seed\": {}, \"len\": {}, \"xof\": {}, \"key\": \"{}\", \"ctx\": \"{}\", \"hash\": \"{}\", \"keyed\": \"{}\", \"derive\": \"{}\"}}", + seed, + n, + xl, + hex(&key), + ctx, + hash_hex(&msg, xl), + keyed_hex(&key, &msg, xl), + derive_hex(&ctx, &msg, xl) + ) + .unwrap(); + } + } + writeln!(w, "\n ],").unwrap(); + + // A couple of well-known digests, for a human sanity check. + writeln!( + w, + " \"known\": {{\"empty\": \"{}\", \"abc\": \"{}\"}}", + hash_hex(b"", 32), + hash_hex(b"abc", 32) + ) + .unwrap(); + writeln!(w, "}}").unwrap(); +} diff --git a/thoughts/blake3/poseidon2-cost-study.md b/thoughts/blake3/poseidon2-cost-study.md new file mode 100644 index 000000000..887902f11 --- /dev/null +++ b/thoughts/blake3/poseidon2-cost-study.md @@ -0,0 +1,123 @@ +# Poseidon2 accelerator — cost study vs BLAKE3-6r and keccak (2026-08-05) + +Produced by a multi-agent study (three mining agents over the vendored +references in `others/` — Plonky3, zisk, stwo, openvm, SP1 old+new, risc0, +airbender, pil2-proofman — plus a synthesis agent applying this repo's cost +model). Model calibration: reproduces Plonky3's Goldilocks w8/SR=1 column +count exactly (180) and zisk's measured 490 cells/perm to 1.0%. + +Companion measured numbers (this branch, 32-core box, blowup 2): +keccak-f 72,672 table / 73,020 end-to-end; BLAKE3-6r 5,316 table / 7,337 +end-to-end per 2-to-1 merge; blake3 throughput 5,217 compressions/s at 2^17 +rows vs keccak 433 perms/s at 2^20 rows. + +--- + +Both calibrations land: my model reproduces Plonky3's Goldilocks w8/SR=1 figure **exactly** (180), and at zisk's degree budget it gives 495 against zisk's measured 490 — **1.0%**. That's a two-point validation of the whole cost model before applying it to our constraints. + +--- + +# Poseidon2 accelerator chip — cost study (final) + +**Headline: ≈ 651 cell-equiv table-only per 2-to-1 merge** (recommended in-place ABI; 753 under the brief's separate-output ABI). Against BLAKE3-6r's 5,316 that is **8.2× cheaper**; against keccak-f's 72,672, **112×**. End-to-end the advantage over BLAKE3 holds at roughly 5–9×, but the absolute win is small change next to what BLAKE3 already banked. + +## Calibration first — the model reproduces two independent mined numbers + +Before trusting it on our constraints, I ran the same model at other systems' degree budgets: + +| target | their budget | my model | mined | agreement | +|---|---|---:|---:|---:| +| Plonky3 Goldilocks w8, `SBOX_REGISTERS=1`, lookup-free | deg 3, ungated | `8 + core(8,REG=1)` = **180** | 180 | **exact** | +| zisk Goldilocks perm, no sbox registers, incl. memory plumbing | deg 7, ungated | `187 + 86 + 24 + 198` = **495** | 490 | **1.0%** | + +The zisk check is the valuable one: it exercises the core formula, the byte-level I/O apparatus *and* the LogUp aux rate simultaneously, and lands within 1%. It also isolates the one thing that makes our number bigger than everyone else's — the degree budget, nothing else. + +## (b) Our number, line by line + +**Width 8, truncated permutation — justified.** A digest is 4 Goldilocks elements (32 B), so a 2-to-1 merge absorbs 8 elements. Two shapes do that in *one* permutation: width 8 as a truncated permutation (`P(left‖right)[0..4]` — Plonky3's `TruncatedPermutation`, `others/Plonky3/symmetric/src/compression.rs:17`), or width 12 as a rate-8/capacity-4 sponge (Plonky2 style). I priced both on identical I/O: **width 8 = 651, width 12 = 779**. Width 8 wins by 16% and is what Plonky3/SP1 ship for merges. Parameters are forced: `RF = 8 (4+4)`, `RP = 22`, S-box `x⁷` — `others/Plonky3/goldilocks/src/poseidon2.rs:22,32,70-73`; x³ and x⁵ are not permutations since `p−1 = 2^32·3·5·17·257·65537` (`goldilocks/src/poseidon1.rs:41-44`). + +**4 committed cells per S-box — forced by μ-gating, and minimal.** Max degree 3 *including* ×μ means bodies are capped at degree 2. Chain: `a=x²`, `b=a·x=x³`, `c=b·b=x⁶`, then `post = M·(c·x)` absorbs the last multiply into the linear layer. All four constraints are degree 2 → 3 after ×μ. Four is provably minimal: from `{1}`, three degree-≤2 steps reach at most exponent 6. This is Plonky3's `SBOX_REGISTERS=3` — their width formula (`poseidon2-air/src/columns.rs:12-69`) is generic in REGISTERS, but `eval_sbox` (`air.rs:288-323`) only ships `(7,1)→deg 3`, so we are one rung past anything in the wild. + +Committing the S-box *output* (Plonky3's `post_sbox`, `air.rs:274-277`) rather than the post-linear element (SP1's `s0`) keeps the whole state at expression-degree 1 through all 22 internal rounds, so the 7 non-S-boxed elements ride free and **no boundary re-commit is needed** — SP1's choice would cost +8 cells here. + +``` +CORE (one row per permutation, fully unrolled) + full rounds 2 × 4 rounds × 8 elems × (3 registers + 1 post) = 256 + partial rounds 22 rounds × 1 elem × (3 registers + 1) = 88 + core = 344 cells + sends in the core = 0 + — field-native: no ByteAlu, no AreBytes, no lookups whatsoever + cross-check: 8 inputs + 344 = 352 = Plonky3 num_cols<8,7,3,4,22> ✓ + +CANONICITY (byte→field must be injective or the tree isn't binding: + x and x+p are distinct byte strings with the same field element) + per element: commit is_max, dinv; constrain + μ·(is_max + (H−(2³²−1))·dinv − 1) deg 3 + μ·(is_max·(H−(2³²−1))) deg 3 + μ·(is_max·L) deg 3 booleanity implied + 2 cells, 0 sends × 12 elements = 24 cells + +I/O APPARATUS (idiom copied from the shipped chip, prover/src/tables/blake3.rs:97-123 + columns and :747-1030 interactions; 2 bytes per AreBytes send, 4 IsHalfword per + dword pointer, pointer-arith carries are expression-form with no cells) + + A: 12 dwords (brief) A′: 8 dwords, in-place (SP1 ABI) + TIMESTAMP_0/1 2 2 + ADDR bytes 8 8 + PTR halfwords 48 32 + IN bytes 64 64 + OUT bytes 32 32 + OLD_OUT bytes 32 0 ← old = the input bytes + MU 1 1 + I/O columns 187 139 + + Ecall receive 1 1 + Memw register read 1 1 + Memw per dword 12 8 + IsHalfword 48 32 + AreBytes addr 4 4 + ByteAlu AND (align) 1 1 + AreBytes IN/OUT/OLD 64 48 + sends N 131 95 + +TOTAL + A : main 187+344+24 = 555 ; aux = 1.5×131 = 198 ; TOTAL 753 + A′: main 139+344+24 = 507 ; aux = 1.5× 95 = 144 ; TOTAL 651 ← recommended +``` + +Arithmetic machine-checked: `/private/tmp/claude-501/-Users-maurofab-workspace-lambda-vm/931cf0e4-cfb3-4d8a-b940-5360f4374a8b/scratchpad/pos2_cost.py`. + +**End-to-end plumbing — the weakest number here, and I won't pretend otherwise.** The two known marginals disagree about what a memory op costs: BLAKE3 is `7,337 − 5,316 = 2,021` over 23 chip Memw ops (**88/op**); keccak is `73,020 − 72,672 = 348` over 26 (25 lanes + register read, `prover/src/tables/keccak.rs:3-5`) — **13/op**. A 6.5× spread means "per Memw op" is the wrong model. The likely driver is guest-side marshalling: BLAKE3's ABI makes the guest lay out a fresh 176-byte region every call, while keccak operates in place on a resident 200-byte state (hypothesis, unverified). Poseidon2-A′ is in-place over a 96-byte region with 9 ops, i.e. structurally keccak-shaped, so I expect the low end — but I quote the full band: + +``` +A′ end-to-end = 651 + 9 ops × [13 … 88] = [768 … 1,443] central estimate ≈ 900 +A end-to-end = 753 + 13 ops × [13 … 88] = [922 … 1,897] +``` + +## (c) Comparison, per 2-to-1 merge (64 B in, 32 B out) + +| | table-only | end-to-end | vs keccak (e2e) | vs BLAKE3-6r (e2e) | +|---|---:|---:|---:|---:| +| keccak-f (measured) | 72,672 | 73,020 | 1× | 0.10× | +| BLAKE3-6r (measured) | 5,316 | 7,337 | 10.0× | 1× | +| **Poseidon2 A** (derived) | **753** | ~922–1,897 (est) | 39–79× | 3.9–8.0× | +| **Poseidon2 A′** (derived, recommended) | **651** | ~768–1,443 (est) | 51–95× | 5.1–9.6× | +| Poseidon2 B (deg-4 bodies) | 479 | ~596–1,271 (est) | 57–123× | 5.8–12.3× | +| Poseidon2 C (internal bus, no memory) | 358 | 358 | 204× | 20.5× | + +Note the shape change between the two columns: table-only, Poseidon2 looks 112× better than keccak; end-to-end that collapses toward ~50–95×, because keccak's plumbing is rounding error against its enormous table while Poseidon2's plumbing is comparable to its entire chip. + +## (d) Caveats + +1. **The chip is I/O-bound, not hash-bound.** Core 344 cells; syscall apparatus 283 cell-equiv (139 cols + 144 aux) even in the in-place variant. Every lever that removes memory crossing beats every lever inside the permutation: in-place ABI −14%, internal Merkle-parent bus −45% (358). +2. **Canonical `< p` input range checks are required and cheap.** 2 cells + 3 constraints per element, 0 sends, 24 cells for all 12. SP1 Hypercube ships exactly this check on both inputs *and* outputs (`others/hypercube-verifier/crates/core/machine/src/operations/sp1_field_word.rs:44-88`; `input_range_checkers[16]` + `hash_result_range_checkers[16]` at `syscall/precompiles/poseidon2/air.rs:66-70`) — so it isn't optional in practice. Separately: absorbing *arbitrary* byte strings rather than chip-produced digests needs 7-byte-per-element packing to stay injective, cutting sponge rate 32 B → 28 B. +3. **The degree budget is the one thing making us expensive, and it's ours alone.** Every mined design runs ungated bodies. Our ×μ factor doubles the core (344 vs 172 at deg-3 bodies) and quadruples it against zisk's deg-7 budget (344 vs 86). Good news: `logup_max_degree` already floors any table with committed pairs at 3 (`crypto/stark/src/lookup.rs:2287-2298`), so degree 3 is free. Going to 4 costs one composition part for that table alone — `composition_poly_degree_bound = trace_length·(max_degree−1)` (`lookup.rs:1078`), i.e. 3 parts instead of 2 — in exchange for −172 cells/row. That trade is plausibly a win and should be measured, not assumed. +4. **The verifier hash must switch, and that is the real bill.** All three Merkle backends are keccak (`crypto/stark/src/config.rs:10,19,23`). A Poseidon2 chip pays for nothing unless FRI/Merkle/FS move to Poseidon2 — which means a new GPU Merkle kernel (the keccak one is at `crypto/stark/src/gpu_lde.rs:861`) and a native-prover slowdown of roughly 5–10× per byte versus keccak (**order-of-magnitude, unmeasured**). BLAKE3 is the opposite trade: faster than keccak natively, so switching costs the prover nothing. This asymmetry appears nowhere in the cell count and is the single biggest difference between the two candidates. +5. **Keccak is not displaceable either way.** EVM/ethrex needs keccak256. Poseidon2 and BLAKE3-6r compete for the same internal-hash slot. +6. **Constraint-eval cost ≠ cell cost.** The 22 internal rounds carry non-S-boxed state as symbolic linear combinations — degree stays 1 (that's the point) but fan-out reaches ~30 terms by the last round, ~700 extra field mults per row. Fine, provided the IR stays a DAG. +7. **Always-on AIR tax.** `FIXED_TABLE_COUNT` +1. Per the EC regression (PR #871: +3 near-empty AIRs → +25% prove time), a real-block ABBA is mandatory regardless of how good the cell count looks. +8. **Uncertainty.** Core 344 is exact given the design and validated to 1% against zisk. I/O is exact given the shipped idiom. Table-only band: **620–700 for A′**. End-to-end is the soft number, band **768–1,443**, and it is directly measurable rather than arguable. + +## (e) Verdict + +Poseidon2 beats BLAKE3-6r here, and by a solid margin: **8.2× table-only (651 vs 5,316), 5–9× end-to-end.** The derivation is well-anchored — the same model reproduces Plonky3's Goldilocks figure exactly and zisk's to 1% — so I'd defend the number itself. What I would not defend is the conclusion that this justifies building it. Measured against keccak end-to-end, BLAKE3-6r already captures **91%** of the total addressable saving per merge (65,683 of 72,120 cell-equiv); Poseidon2 adds the remaining 9%. And Poseidon2 cannot go much lower as a syscall — roughly half its cost is the ecall/MEMW apparatus it shares with every other chip, so even a perfect permutation would only reach ~400. Meanwhile it uniquely imposes a native-prover hashing slowdown and a new GPU Merkle kernel that BLAKE3 does not, and the in-VM digests it produces are field elements crossing a byte-addressed memory, which is what the canonicity gadget and the 64 AreBytes sends are paying for. The decision should turn on one measurement nobody has taken: after the BLAKE3 switch, what share of a real recursion-verifier trace is still hashing? That is precisely the question the EC campaign skipped — a −61.9% win on 0.61% of the trace — and it is cheap to answer before committing to a chip. If Poseidon2 is pursued anyway, the leverage order is unambiguous and none of it lives in the permutation: in-place ABI (−14%), internal Merkle-parent bus (−45%), then relaxing the μ-gated degree cap (−23%). \ No newline at end of file diff --git a/thoughts/blake3/reference-impl/.gitignore b/thoughts/blake3/reference-impl/.gitignore new file mode 100644 index 000000000..28eddb5da --- /dev/null +++ b/thoughts/blake3/reference-impl/.gitignore @@ -0,0 +1,3 @@ +b3ref6 +b3ref7 +__pycache__/ diff --git a/thoughts/blake3/reference-impl/PARAMETERISATION.diff b/thoughts/blake3/reference-impl/PARAMETERISATION.diff new file mode 100644 index 000000000..b83cdb960 --- /dev/null +++ b/thoughts/blake3/reference-impl/PARAMETERISATION.diff @@ -0,0 +1,48 @@ +--- upstream/blake3_portable.c 2026-08-10 17:15:50 ++++ blake3_portable_paramrounds.c 2026-08-10 17:16:06 +@@ -1,6 +1,17 @@ + #include "blake3_impl.h" + #include + ++/* ---- LAMBDA VM PARAMETERISATION: round-count knob (added) ------------- ++ * 7 = standard BLAKE3 (bit-compatible with published vectors). ++ * 6 = the internal variant this repo's LFM chip prices (assumption A6R). ++ * --------------------------------------------------------------------- */ ++#ifndef BLAKE3_ROUNDS_PARAM ++#define BLAKE3_ROUNDS_PARAM 7 ++#endif ++#if BLAKE3_ROUNDS_PARAM > 7 || BLAKE3_ROUNDS_PARAM < 1 ++#error "BLAKE3_ROUNDS_PARAM must be in 1..7 (MSG_SCHEDULE has 7 rows)" ++#endif ++ + INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); + } +@@ -72,13 +83,20 @@ + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + +- round_fn(state, &block_words[0], 0); +- round_fn(state, &block_words[0], 1); +- round_fn(state, &block_words[0], 2); +- round_fn(state, &block_words[0], 3); +- round_fn(state, &block_words[0], 4); +- round_fn(state, &block_words[0], 5); +- round_fn(state, &block_words[0], 6); ++ /* ---- LAMBDA VM PARAMETERISATION (the ONLY edit to this file) ---------- ++ * Upstream unrolls exactly seven calls here: ++ * round_fn(state, &block_words[0], 0); ++ * ... (rounds 1..5) ... ++ * round_fn(state, &block_words[0], 6); ++ * They are replaced by a loop whose bound is BLAKE3_ROUNDS_PARAM, which ++ * defaults to 7. At the default the loop executes the identical seven calls ++ * in the identical order, so the parameterisation is inert by inspection -- ++ * and that is re-checked empirically against the official test vectors. ++ * MSG_SCHEDULE has exactly 7 rows, so the bound may not exceed 7. ++ * -------------------------------------------------------------------- */ ++ for (size_t r = 0; r < BLAKE3_ROUNDS_PARAM; r++) { ++ round_fn(state, &block_words[0], r); ++ } + } + + void blake3_compress_in_place_portable(uint32_t cv[8], diff --git a/thoughts/blake3/reference-impl/blake3_portable_paramrounds.c b/thoughts/blake3/reference-impl/blake3_portable_paramrounds.c new file mode 100644 index 000000000..32cff4421 --- /dev/null +++ b/thoughts/blake3/reference-impl/blake3_portable_paramrounds.c @@ -0,0 +1,178 @@ +#include "blake3_impl.h" +#include + +/* ---- LAMBDA VM PARAMETERISATION: round-count knob (added) ------------- + * 7 = standard BLAKE3 (bit-compatible with published vectors). + * 6 = the internal variant this repo's LFM chip prices (assumption A6R). + * --------------------------------------------------------------------- */ +#ifndef BLAKE3_ROUNDS_PARAM +#define BLAKE3_ROUNDS_PARAM 7 +#endif +#if BLAKE3_ROUNDS_PARAM > 7 || BLAKE3_ROUNDS_PARAM < 1 +#error "BLAKE3_ROUNDS_PARAM must be in 1..7 (MSG_SCHEDULE has 7 rows)" +#endif + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = IV[0]; + state[9] = IV[1]; + state[10] = IV[2]; + state[11] = IV[3]; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + /* ---- LAMBDA VM PARAMETERISATION (the ONLY edit to this file) ---------- + * Upstream unrolls exactly seven calls here: + * round_fn(state, &block_words[0], 0); + * ... (rounds 1..5) ... + * round_fn(state, &block_words[0], 6); + * They are replaced by a loop whose bound is BLAKE3_ROUNDS_PARAM, which + * defaults to 7. At the default the loop executes the identical seven calls + * in the identical order, so the parameterisation is inert by inspection -- + * and that is re-checked empirically against the official test vectors. + * MSG_SCHEDULE has exactly 7 rows, so the bound may not exceed 7. + * -------------------------------------------------------------------- */ + for (size_t r = 0; r < BLAKE3_ROUNDS_PARAM; r++) { + round_fn(state, &block_words[0], r); + } +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/thoughts/blake3/reference-impl/build.sh b/thoughts/blake3/reference-impl/build.sh new file mode 100755 index 000000000..d03229fb9 --- /dev/null +++ b/thoughts/blake3/reference-impl/build.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Build the two round-parameterised reference binaries. +# +# Only `blake3_portable_paramrounds.c` differs from upstream (see +# PARAMETERISATION.diff); blake3.c / blake3_dispatch.c / blake3_impl.h / +# blake3.h under upstream/ are verbatim BLAKE3 1.8.5. +# +# NEON is disabled and no x86 SIMD is available, so the dispatcher resolves +# every compression to the portable path -- i.e. to the parameterised file. +# That is what makes the round knob apply to the WHOLE tree hasher and not +# only to a directly-called compress. +# +# This is a ~1 second single-file C compile. It is not a cargo build. +set -e +cd "$(dirname "$0")" + +SRC="driver.c blake3_portable_paramrounds.c upstream/blake3.c upstream/blake3_dispatch.c" +COMMON="-O2 -Wall -Iupstream -DBLAKE3_USE_NEON=0 -DBLAKE3_NO_SSE2 -DBLAKE3_NO_SSE41 -DBLAKE3_NO_AVX2 -DBLAKE3_NO_AVX512" + +cc $COMMON -DBLAKE3_ROUNDS_PARAM=7 -o b3ref7 $SRC +cc $COMMON -DBLAKE3_ROUNDS_PARAM=6 -o b3ref6 $SRC + +echo "built: b3ref7 (standard BLAKE3) and b3ref6 (internal variant)" diff --git a/thoughts/blake3/reference-impl/check.py b/thoughts/blake3/reference-impl/check.py new file mode 100644 index 000000000..ab269ee4b --- /dev/null +++ b/thoughts/blake3/reference-impl/check.py @@ -0,0 +1,296 @@ +""" +SECOND-SOURCE validation of the 6-round BLAKE3 vectors. + +Source 1 is `thoughts/blake3/blake3-oracle/blake3_ref.py` -- an in-repo Python +oracle written from the spec, anchored on the official vectors at 7 rounds. +Source 2 is upstream BLAKE3's own portable C implementation (crate `blake3` +1.8.5, `c/blake3_portable.c`) with its round loop parameterised; see +PARAMETERISATION.diff for the entire edit. + +The two sources are independent in the ways that matter: + - different authors (the BLAKE3 team vs this repo) and different languages; + - different message-schedule CONSTRUCTION: the C indexes a precomputed + MSG_SCHEDULE[7][16] table, the Python/Rust iteratively apply a single + permutation between rounds. A bug in the iterative composition -- exactly + the kind of thing a single source cannot catch -- shows up here; + - the C drives the FULL tree hasher through the parameterised compression, + so its 7-round run is a direct external anchor rather than a borrowed one. + +Checks, in order: + [A] parameterised C at rounds=7 reproduces official_test_vectors.json + (35 cases x 3 modes) -- the parameterisation is inert. + [B] rounds=6 actually changes the function (negative control). + [C] MSG_SCHEDULE[r] == permute^r(identity) -- the two schedule + constructions denote the same thing. + [D] C at rounds=6 reproduces all ten CANONICAL_VECTORS byte for byte, + compared against BOTH canonical_6round_vectors.json AND the Rust + constants in prover/src/lfm/blake3.rs. + [E] randomised differential, C vs Python oracle, at rounds 7 and 6. + +Run: python3 check.py (after ./build.sh) +""" + +import json +import os +import random +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ORACLE_DIR = os.path.join(HERE, "..", "blake3-oracle") +REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..")) +# Where the canonical vectors live. This was `prover/src/lfm/blake3.rs` until +# P-a Stage 1 sank the primitive into `crypto`, so that the CUDA kernels, the +# commitment backends and the LFM chip could all be checked against ONE +# definition; `prover::lfm::blake3` is now a re-export and no longer holds the +# table. Checks [D] and [E] were silently dead between that move and 2026-08-15, +# because `parse_rust_vectors` failed with a bare `ValueError` from `str.index` +# rather than saying what had happened — see `read_rust_primitive`. +RUST_PRIMITIVE = os.path.join( + REPO_ROOT, "crypto", "crypto", "src", "hash", "blake3", "vectors.rs" +) + + +def read_rust_primitive(): + """The vectors file, or a diagnosis of where it went. + + A harness that dies on `ValueError: substring not found` when the code it + validates is refactored is a harness that gets deleted instead of fixed. If + this ever fires again, grep for `CANONICAL_VECTORS` and update the path + above — the parser itself is layout-independent and needs no change. + """ + if not os.path.exists(RUST_PRIMITIVE): + sys.exit( + f"check.py: {RUST_PRIMITIVE} does not exist.\n" + "The canonical vectors have moved again. Find them with\n" + " grep -rn 'pub const CANONICAL_VECTORS' --include='*.rs' .\n" + "and update RUST_PRIMITIVE at the top of this file." + ) + src = open(RUST_PRIMITIVE).read() + if "pub const CANONICAL_VECTORS" not in src: + sys.exit( + f"check.py: {RUST_PRIMITIVE} exists but no longer defines " + "CANONICAL_VECTORS.\nFind them with\n" + " grep -rn 'pub const CANONICAL_VECTORS' --include='*.rs' .\n" + "and update RUST_PRIMITIVE at the top of this file." + ) + return src + +sys.path.insert(0, ORACLE_DIR) +import blake3_ref as ref # noqa: E402 + +B3REF7 = os.path.join(HERE, "b3ref7") +B3REF6 = os.path.join(HERE, "b3ref6") + +FAILURES = [] + + +def check(name, cond, detail=""): + if cond: + print(f" PASS {name}") + else: + print(f" FAIL {name} {detail}") + FAILURES.append(name) + + +def pattern_input(n): + return bytes(i % 251 for i in range(n)) + + +def run(binary, *args, stdin=None): + r = subprocess.run([binary, *[str(a) for a in args]], input=stdin, + capture_output=True, text=True, check=True) + return r.stdout + + +# --------------------------------------------------------------------------- +# [A] the parameterisation is inert at rounds = 7 +# --------------------------------------------------------------------------- + +def check_official_vectors(): + data = json.load(open(os.path.join(ORACLE_DIR, "official_test_vectors.json"))) + key_hex = data["key"].encode("utf-8").hex() + context = data["context_string"] + cases = data["cases"] + + bad = [] + for c in cases: + n = c["input_len"] + out_len = len(c["hash"]) // 2 + got = run(B3REF7, "hash", n, out_len).strip() + if got != c["hash"]: + bad.append(("hash", n)) + got = run(B3REF7, "keyed", key_hex, n, out_len).strip() + if got != c["keyed_hash"]: + bad.append(("keyed", n)) + got = run(B3REF7, "derive", context, n, out_len).strip() + if got != c["derive_key"]: + bad.append(("derive_key", n)) + check(f"[A] parameterised C @ rounds=7 vs official vectors " + f"({len(cases)} cases x 3 modes)", not bad, str(bad[:5])) + + +# --------------------------------------------------------------------------- +# [B] rounds = 6 is genuinely a different function +# --------------------------------------------------------------------------- + +def check_six_differs(): + diffs = 0 + total = 0 + for n in (0, 1, 63, 64, 65, 1024, 1025, 4096): + total += 1 + if run(B3REF6, "hash", n, 32).strip() != run(B3REF7, "hash", n, 32).strip(): + diffs += 1 + check(f"[B] rounds=6 differs from rounds=7 on all {total} probe lengths", + diffs == total, f"only {diffs}/{total} differed") + + +# --------------------------------------------------------------------------- +# [C] the two message-schedule constructions denote the same thing +# --------------------------------------------------------------------------- + +def check_schedule_equivalence(): + """C uses a precomputed MSG_SCHEDULE table; Python composes one permutation + repeatedly. Confirm row r equals permute applied r times to the identity.""" + text = open(os.path.join(HERE, "upstream", "blake3_impl.h")).read() + blob = re.search(r"MSG_SCHEDULE\[7\]\[16\]\s*=\s*\{(.*?)\n\};", text, re.S).group(1) + rows = [[int(x) for x in re.findall(r"\d+", row)] + for row in blob.strip().split("\n") if "{" in row] + assert len(rows) == 7 and all(len(r) == 16 for r in rows), rows + + cur = list(range(16)) + ok = True + for r in range(7): + if rows[r] != cur: + ok = False + print(f" row {r}: table={rows[r]} composed={cur}") + cur = ref.permute(cur) + check("[C] MSG_SCHEDULE[r] == permute^r(identity) for r in 0..7", ok) + + check("[C] MSG_SCHEDULE[1] == the repo's BLAKE3_MSG_PERMUTATION", + rows[1] == ref.MSG_PERMUTATION, + f"{rows[1]} vs {ref.MSG_PERMUTATION}") + + +# --------------------------------------------------------------------------- +# [D] the ten canonical 6-round vectors, from the C, vs JSON and vs Rust +# --------------------------------------------------------------------------- + +def parse_rust_vectors(): + src = read_rust_primitive() + start = src.index("pub const CANONICAL_VECTORS") + blob = src[start:src.index("\n];", start)] + out = [] + for part in blob.split("Vector {")[1:]: + v = {} + for field in ("h", "m", "out"): + body = re.search(field + r":\s*\[(.*?)\]", part, re.S).group(1) + v[field] = [int(x, 0) for x in re.findall(r"0x[0-9A-Fa-f]+", body)] + v["t"] = int(re.search(r"\bt:\s*(0x[0-9A-Fa-f]+|\d+)", part).group(1), 0) + v["block_len"] = int(re.search(r"block_len:\s*(0x[0-9A-Fa-f]+|\d+)", part).group(1), 0) + v["flags"] = int(re.search(r"flags:\s*(0x[0-9A-Fa-f]+|\d+)", part).group(1), 0) + out.append(v) + return out + + +def encode_record(v): + words = [f"{w:08x}" for w in v["h"]] + [f"{w:08x}" for w in v["m"]] + return " ".join(words) + f" {v['t']:016x} {v['block_len']} {v['flags']}\n" + + +def check_canonical_vectors(): + js = json.load(open(os.path.join(ORACLE_DIR, "canonical_6round_vectors.json"))) + rust = parse_rust_vectors() + check("[D] Rust CANONICAL_VECTORS count == JSON count == 10", + len(rust) == len(js) == 10, f"{len(rust)} / {len(js)}") + + # The C driver takes block_len and flags as uint8_t; confirm lossless. + check("[D] all vector block_len/flags fit in u8 (driver is lossless here)", + all(v["block_len"] < 256 and v["flags"] < 256 for v in js)) + + # Inputs come from the JSON; OUTPUTS come from the C. + stdin = "".join(encode_record(v) for v in js) + lines = run(B3REF6, "compress", stdin=stdin).strip().split("\n") + check("[D] C emitted one output per vector", len(lines) == 10, str(len(lines))) + + c_out = [[int(line[8 * i:8 * i + 8], 16) for i in range(16)] for line in lines] + + bad_json = [i for i in range(10) if c_out[i] != js[i]["out"]] + check("[D] C @ rounds=6 == canonical_6round_vectors.json (all 10, 16 words)", + not bad_json, f"vectors {bad_json}") + + bad_rust = [i for i in range(10) if c_out[i] != rust[i]["out"]] + check("[D] C @ rounds=6 == Rust CANONICAL_VECTORS in " + "crypto/crypto/src/hash/blake3/vectors.rs", + not bad_rust, f"vectors {bad_rust}") + + # Inputs must match too, or the output agreement is about different things. + bad_in = [i for i in range(10) + if any(rust[i][f] != js[i][f] for f in ("h", "m", "t", "block_len", "flags"))] + check("[D] Rust vector INPUTS == JSON vector inputs", not bad_in, f"vectors {bad_in}") + + # Negative control: the same inputs at 7 rounds must NOT match. + lines7 = run(B3REF7, "compress", stdin=stdin).strip().split("\n") + c7 = [[int(line[8 * i:8 * i + 8], 16) for i in range(16)] for line in lines7] + check("[D] negative control: C @ rounds=7 matches none of the 10 vectors", + all(c7[i] != js[i]["out"] for i in range(10))) + + +# --------------------------------------------------------------------------- +# [E] randomised differential against the Python oracle +# --------------------------------------------------------------------------- + +def check_differential(n=5000): + rng = random.Random(0x5EC0D) + recs = [] + expect7 = [] + expect6 = [] + for _ in range(n): + v = { + "h": [rng.randrange(1 << 32) for _ in range(8)], + "m": [rng.randrange(1 << 32) for _ in range(16)], + "t": rng.randrange(1 << 64), + "block_len": rng.randrange(0, 65), + "flags": rng.randrange(0, 256), + } + recs.append(encode_record(v)) + expect7.append(ref.compress(v["h"], v["m"], v["t"], v["block_len"], + v["flags"], rounds=7)) + expect6.append(ref.compress(v["h"], v["m"], v["t"], v["block_len"], + v["flags"], rounds=6)) + + stdin = "".join(recs) + for binary, expect, label in ((B3REF7, expect7, 7), (B3REF6, expect6, 6)): + lines = run(binary, "compress", stdin=stdin).strip().split("\n") + got = [[int(ln[8 * i:8 * i + 8], 16) for i in range(16)] for ln in lines] + bad = [i for i in range(n) if got[i] != expect[i]] + check(f"[E] C vs Python oracle @ rounds={label} ({n} random compressions)", + len(got) == n and not bad, f"{len(bad)} mismatches, first={bad[:3]}") + + +def main(): + if not (os.path.exists(B3REF7) and os.path.exists(B3REF6)): + print("binaries missing -- run ./build.sh first") + return 2 + + print("=" * 74) + print("SECOND-SOURCE CHECK: upstream BLAKE3 C (round-parameterised)") + print("=" * 74) + check_official_vectors() + check_six_differs() + check_schedule_equivalence() + check_canonical_vectors() + check_differential() + print("=" * 74) + if FAILURES: + print(f"RESULT: {len(FAILURES)} FAILURE(S): {FAILURES}") + return 1 + print("RESULT: ALL GREEN -- two independent sources agree on the ten") + print(" 6-round vectors, and the 7-round anchor is external.") + print("=" * 74) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/blake3/reference-impl/driver.c b/thoughts/blake3/reference-impl/driver.c new file mode 100644 index 000000000..d6338508d --- /dev/null +++ b/thoughts/blake3/reference-impl/driver.c @@ -0,0 +1,188 @@ +/* Test driver for the round-parameterised upstream BLAKE3 C reference. + * + * This file is Lambda VM's; everything it links against in `upstream/` is + * upstream BLAKE3 (CC0 / Apache-2.0), unmodified except for the round-count + * loop in blake3_portable_paramrounds.c (see PARAMETERISATION.diff). + * + * Modes: + * hash default hashing mode + * hashhex default hashing mode, explicit msg + * keyed keyed_hash mode + * derive derive_key mode + * compress raw compression from stdin: + * one whitespace-separated record per line -- + * h[0..8] (8 hex u32) m[0..16] (16 hex u32) t (hex u64) + * block_len (dec) flags (dec) + * prints the 16-word output as 16 concatenated 8-hex-digit words. + * + * The `compress` mode calls blake3_compress_xof_portable directly, which is + * the 16-word (64-byte) output of the compression function `f` -- exactly the + * object CANONICAL_VECTORS pins. + * + * Input bytes for the hashing modes follow the official test-vector pattern: + * byte i is (i % 251). + */ +#include "upstream/blake3.h" +#include "upstream/blake3_impl.h" +#include +#include +#include + +static void fill_pattern(uint8_t *buf, size_t len) { + for (size_t i = 0; i < len; i++) { + buf[i] = (uint8_t)(i % 251); + } +} + +static void print_hex(const uint8_t *b, size_t n) { + for (size_t i = 0; i < n; i++) { + printf("%02x", b[i]); + } + printf("\n"); +} + +static int hex_to_bytes(const char *hex, uint8_t *out, size_t out_len) { + if (strlen(hex) != out_len * 2) { + return -1; + } + for (size_t i = 0; i < out_len; i++) { + unsigned v; + if (sscanf(hex + 2 * i, "%2x", &v) != 1) { + return -1; + } + out[i] = (uint8_t)v; + } + return 0; +} + +static int run_hasher(blake3_hasher *h, size_t input_len, size_t out_len) { + uint8_t *input = malloc(input_len ? input_len : 1); + uint8_t *out = malloc(out_len ? out_len : 1); + if (!input || !out) { + return 1; + } + fill_pattern(input, input_len); + blake3_hasher_update(h, input, input_len); + blake3_hasher_finalize(h, out, out_len); + print_hex(out, out_len); + free(input); + free(out); + return 0; +} + +static int mode_compress(void) { + uint32_t h[8], m[16]; + unsigned long long t; + unsigned block_len, flags; + char line[4096]; + + while (fgets(line, sizeof(line), stdin)) { + char *p = line; + int consumed; + int ok = 1; + + for (int i = 0; i < 8 && ok; i++) { + if (sscanf(p, " %x%n", &h[i], &consumed) != 1) { + ok = 0; + } + p += consumed; + } + for (int i = 0; i < 16 && ok; i++) { + if (sscanf(p, " %x%n", &m[i], &consumed) != 1) { + ok = 0; + } + p += consumed; + } + if (ok && sscanf(p, " %llx %u %u", &t, &block_len, &flags) != 3) { + ok = 0; + } + if (!ok) { + continue; /* blank or malformed line */ + } + + /* The compression function takes the message block as 64 little-endian + * bytes; serialise m[] the way BLAKE3 loads it (load32 is LE). */ + uint8_t block[BLAKE3_BLOCK_LEN]; + for (int i = 0; i < 16; i++) { + store32(&block[i * 4], m[i]); + } + + uint8_t out64[64]; + blake3_compress_xof_portable(h, block, (uint8_t)block_len, + (uint64_t)t, (uint8_t)flags, out64); + + for (int i = 0; i < 16; i++) { + printf("%08x", load32(&out64[i * 4])); + } + printf("\n"); + fflush(stdout); + } + return 0; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "usage: %s hash|keyed|derive|compress ...\n", argv[0]); + return 2; + } + + if (strcmp(argv[1], "compress") == 0) { + return mode_compress(); + } + + if (strcmp(argv[1], "hash") == 0 && argc == 4) { + blake3_hasher h; + blake3_hasher_init(&h); + return run_hasher(&h, strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10)); + } + + /* Whole-message hashing of an explicit byte string. This is what turns the + * socket spec's "compress(a,b) == BLAKE3(a || b || tag) truncated" claim into + * something executable against upstream code rather than against a compress + * call this repo assembled itself. */ + if (strcmp(argv[1], "hashhex") == 0 && argc == 4) { + size_t msg_len = strlen(argv[2]) / 2; + if (strlen(argv[2]) % 2 != 0) { + fprintf(stderr, "message hex must have even length\n"); + return 2; + } + uint8_t *msg = malloc(msg_len ? msg_len : 1); + if (!msg || hex_to_bytes(argv[2], msg, msg_len) != 0) { + fprintf(stderr, "bad message hex\n"); + return 2; + } + size_t out_len = strtoul(argv[3], NULL, 10); + uint8_t *out = malloc(out_len ? out_len : 1); + if (!out) { + return 1; + } + blake3_hasher h; + blake3_hasher_init(&h); + blake3_hasher_update(&h, msg, msg_len); + blake3_hasher_finalize(&h, out, out_len); + print_hex(out, out_len); + free(msg); + free(out); + return 0; + } + + if (strcmp(argv[1], "keyed") == 0 && argc == 5) { + uint8_t key[BLAKE3_KEY_LEN]; + if (hex_to_bytes(argv[2], key, BLAKE3_KEY_LEN) != 0) { + fprintf(stderr, "bad key hex (need %d bytes)\n", BLAKE3_KEY_LEN); + return 2; + } + blake3_hasher h; + blake3_hasher_init_keyed(&h, key); + return run_hasher(&h, strtoul(argv[3], NULL, 10), strtoul(argv[4], NULL, 10)); + } + + if (strcmp(argv[1], "derive") == 0 && argc == 5) { + blake3_hasher h; + blake3_hasher_init_derive_key(&h, argv[2]); + return run_hasher(&h, strtoul(argv[3], NULL, 10), strtoul(argv[4], NULL, 10)); + } + + fprintf(stderr, "bad arguments\n"); + return 2; +} diff --git a/thoughts/blake3/reference-impl/upstream/LICENSE_CC0 b/thoughts/blake3/reference-impl/upstream/LICENSE_CC0 new file mode 100644 index 000000000..0e259d42c --- /dev/null +++ b/thoughts/blake3/reference-impl/upstream/LICENSE_CC0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/thoughts/blake3/reference-impl/upstream/blake3.c b/thoughts/blake3/reference-impl/upstream/blake3.c new file mode 100644 index 000000000..00f91f444 --- /dev/null +++ b/thoughts/blake3/reference-impl/upstream/blake3.c @@ -0,0 +1,651 @@ +#include +#include +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" + +const char *blake3_version(void) { return BLAKE3_VERSION_STRING; } + +INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + self->buf_len = 0; + self->blocks_compressed = 0; + self->flags = flags; +} + +INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], + uint64_t chunk_counter) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = chunk_counter; + self->blocks_compressed = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + self->buf_len = 0; +} + +INLINE size_t chunk_state_len(const blake3_chunk_state *self) { + return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + + ((size_t)self->buf_len); +} + +INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, + const uint8_t *input, size_t input_len) { + size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); + if (take > input_len) { + take = input_len; + } + uint8_t *dest = self->buf + ((size_t)self->buf_len); + memcpy(dest, input, take); + self->buf_len += (uint8_t)take; + return take; +} + +INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { + if (self->blocks_compressed == 0) { + return CHUNK_START; + } else { + return 0; + } +} + +typedef struct { + uint32_t input_cv[8]; + uint64_t counter; + uint8_t block[BLAKE3_BLOCK_LEN]; + uint8_t block_len; + uint8_t flags; +} output_t; + +INLINE output_t make_output(const uint32_t input_cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + output_t ret; + memcpy(ret.input_cv, input_cv, 32); + memcpy(ret.block, block, BLAKE3_BLOCK_LEN); + ret.block_len = block_len; + ret.counter = counter; + ret.flags = flags; + return ret; +} + +// Chaining values within a given chunk (specifically the compress_in_place +// interface) are represented as words. This avoids unnecessary bytes<->words +// conversion overhead in the portable implementation. However, the hash_many +// interface handles both user input and parent node blocks, so it accepts +// bytes. For that reason, chaining values in the CV stack are represented as +// bytes. +INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { + uint32_t cv_words[8]; + memcpy(cv_words, self->input_cv, 32); + blake3_compress_in_place(cv_words, self->block, self->block_len, + self->counter, self->flags); + store_cv_words(cv, cv_words); +} + +INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, + size_t out_len) { + if (out_len == 0) { + return; + } + uint64_t output_block_counter = seek / 64; + size_t offset_within_block = seek % 64; + uint8_t wide_buf[64]; + if(offset_within_block) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + const size_t available_bytes = 64 - offset_within_block; + const size_t bytes = out_len > available_bytes ? available_bytes : out_len; + memcpy(out, wide_buf + offset_within_block, bytes); + out += bytes; + out_len -= bytes; + output_block_counter += 1; + } + if(out_len / 64) { + blake3_xof_many(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, out, out_len / 64); + } + output_block_counter += out_len / 64; + out += out_len & -64; + out_len -= out_len & -64; + if(out_len) { + blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); + memcpy(out, wide_buf, out_len); + } +} + +INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, + size_t input_len) { + if (self->buf_len > 0) { + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; + if (input_len > 0) { + blake3_compress_in_place( + self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + self->buf_len = 0; + memset(self->buf, 0, BLAKE3_BLOCK_LEN); + } + } + + while (input_len > BLAKE3_BLOCK_LEN) { + blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, + self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + input += BLAKE3_BLOCK_LEN; + input_len -= BLAKE3_BLOCK_LEN; + } + + chunk_state_fill_buf(self, input, input_len); +} + +INLINE output_t chunk_state_output(const blake3_chunk_state *self) { + uint8_t block_flags = + self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; + return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, + block_flags); +} + +INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], + const uint32_t key[8], uint8_t flags) { + return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); +} + +// Given some input larger than one chunk, return the number of bytes that +// should go in the left subtree. This is the largest power-of-2 number of +// chunks that leaves at least 1 byte for the right subtree. +INLINE size_t left_subtree_len(size_t input_len) { + // Subtract 1 to reserve at least one byte for the right side. input_len + // should always be greater than BLAKE3_CHUNK_LEN. + size_t full_chunks = (input_len - 1) / BLAKE3_CHUNK_LEN; + return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time +// on a single thread. Write out the chunk chaining values and return the +// number of chunks hashed. These chunks are never the root and never empty; +// those cases use a different codepath. +INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(0 < input_len); + assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); +#endif + + const uint8_t *chunks_array[MAX_SIMD_DEGREE]; + size_t input_position = 0; + size_t chunks_array_len = 0; + while (input_len - input_position >= BLAKE3_CHUNK_LEN) { + chunks_array[chunks_array_len] = &input[input_position]; + input_position += BLAKE3_CHUNK_LEN; + chunks_array_len += 1; + } + + blake3_hash_many(chunks_array, chunks_array_len, + BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, + true, flags, CHUNK_START, CHUNK_END, out); + + // Hash the remaining partial chunk, if there is one. Note that the empty + // chunk (meaning the empty message) is a different codepath. + if (input_len > input_position) { + uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, key, flags); + chunk_state.chunk_counter = counter; + chunk_state_update(&chunk_state, &input[input_position], + input_len - input_position); + output_t output = chunk_state_output(&chunk_state); + output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); + return chunks_array_len + 1; + } else { + return chunks_array_len; + } +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time +// on a single thread. Write out the parent chaining values and return the +// number of parents hashed. (If there's an odd input chaining value left over, +// return it as an additional output.) These parents are never the root and +// never empty; those cases use a different codepath. +INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, + size_t num_chaining_values, + const uint32_t key[8], uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(2 <= num_chaining_values); + assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); +#endif + + const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; + size_t parents_array_len = 0; + while (num_chaining_values - (2 * parents_array_len) >= 2) { + parents_array[parents_array_len] = + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; + parents_array_len += 1; + } + + blake3_hash_many(parents_array, parents_array_len, 1, key, + 0, // Parents always use counter 0. + false, flags | PARENT, + 0, // Parents have no start flags. + 0, // Parents have no end flags. + out); + + // If there's an odd child left over, it becomes an output. + if (num_chaining_values > 2 * parents_array_len) { + memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], + BLAKE3_OUT_LEN); + return parents_array_len + 1; + } else { + return parents_array_len; + } +} + +// The wide helper function returns (writes out) an array of chaining values +// and returns the length of that array. The number of chaining values returned +// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, +// if the input is shorter than that many chunks. The reason for maintaining a +// wide array of chaining values going back up the tree, is to allow the +// implementation to hash as many parents in parallel as possible. +// +// As a special case when the SIMD degree is 1, this function will still return +// at least 2 outputs. This guarantees that this function doesn't perform the +// root compression. (If it did, it would use the wrong flags, and also we +// wouldn't be able to implement extendable output.) Note that this function is +// not used when the whole input is only 1 chunk long; that's a different +// codepath. +// +// Why not just have the caller split the input on the first update(), instead +// of implementing this special rule? Because we don't want to limit SIMD or +// multi-threading parallelism for that update(). +size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb) { + // Note that the single chunk case does *not* bump the SIMD degree up to 2 + // when it is 1. If this implementation adds multi-threading in the future, + // this gives us the option of multi-threading even the 2-chunk case, which + // can help performance on smaller platforms. + if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { + return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, + out); + } + + // With more than simd_degree chunks, we need to recurse. Start by dividing + // the input into left and right subtrees. (Note that this is only optimal + // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree + // of 3 or something, we'll need a more complicated strategy.) + size_t left_input_len = left_subtree_len(input_len); + size_t right_input_len = input_len - left_input_len; + const uint8_t *right_input = &input[left_input_len]; + uint64_t right_chunk_counter = + chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); + + // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to + // account for the special case of returning 2 outputs when the SIMD degree + // is 1. + uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t degree = blake3_simd_degree(); + if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { + // The special case: We always use a degree of at least two, to make + // sure there are two outputs. Except, as noted above, at the chunk + // level, where we allow degree=1. (Note that the 1-chunk-input case is + // a different codepath.) + degree = 2; + } + uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; + + // Recurse! + size_t left_n = SIZE_MAX; + size_t right_n = SIZE_MAX; + +#if defined(BLAKE3_USE_TBB) + blake3_compress_subtree_wide_join_tbb( + key, flags, use_tbb, + // left-hand side + input, left_input_len, chunk_counter, cv_array, &left_n, + // right-hand side + right_input, right_input_len, right_chunk_counter, right_cvs, &right_n); +#else + left_n = blake3_compress_subtree_wide( + input, left_input_len, key, chunk_counter, flags, cv_array, use_tbb); + right_n = blake3_compress_subtree_wide(right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs, + use_tbb); +#endif // BLAKE3_USE_TBB + + // The special case again. If simd_degree=1, then we'll have left_n=1 and + // right_n=1. Rather than compressing them into a single output, return + // them directly, to make sure we always have at least two outputs. + if (left_n == 1) { + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); + return 2; + } + + // Otherwise, do one layer of parent node compression. + size_t num_chaining_values = left_n + right_n; + return compress_parents_parallel(cv_array, num_chaining_values, key, flags, + out); +} + +// Hash a subtree with compress_subtree_wide(), and then condense the resulting +// list of chaining values down to a single parent node. Don't compress that +// last parent node, however. Instead, return its message bytes (the +// concatenated chaining values of its children). This is necessary when the +// first call to update() supplies a complete subtree, because the topmost +// parent node of that subtree could end up being the root. It's also necessary +// for extended output in the general case. +// +// As with compress_subtree_wide(), this function is not used on inputs of 1 +// chunk or less. That's a different codepath. +INLINE void +compress_subtree_to_parent_node(const uint8_t *input, size_t input_len, + const uint32_t key[8], uint64_t chunk_counter, + uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN], + bool use_tbb) { +#if defined(BLAKE3_TESTING) + assert(input_len > BLAKE3_CHUNK_LEN); +#endif + + uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, + chunk_counter, flags, cv_array, use_tbb); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); + // The following loop never executes when MAX_SIMD_DEGREE_OR_2 is 2, because + // as we just asserted, num_cvs will always be <=2 in that case. But GCC + // (particularly GCC 8.5) can't tell that it never executes, and if NDEBUG is + // set then it emits incorrect warnings here. We tried a few different + // hacks to silence these, but in the end our hacks just produced different + // warnings (see https://github.com/BLAKE3-team/BLAKE3/pull/380). Out of + // desperation, we ifdef out this entire loop when we know it's not needed. +#if MAX_SIMD_DEGREE_OR_2 > 2 + // If MAX_SIMD_DEGREE_OR_2 is greater than 2 and there's enough input, + // compress_subtree_wide() returns more than 2 chaining values. Condense + // them into 2 by forming parent nodes repeatedly. + uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; + while (num_cvs > 2) { + num_cvs = + compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); + memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); + } +#endif + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); +} + +INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->key, key, BLAKE3_KEY_LEN); + chunk_state_init(&self->chunk, key, flags); + self->cv_stack_len = 0; +} + +void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } + +void blake3_hasher_init_keyed(blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]) { + uint32_t key_words[8]; + load_key_words(key, key_words); + hasher_init_base(self, key_words, KEYED_HASH); +} + +void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, + size_t context_len) { + blake3_hasher context_hasher; + hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); + blake3_hasher_update(&context_hasher, context, context_len); + uint8_t context_key[BLAKE3_KEY_LEN]; + blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); + uint32_t context_key_words[8]; + load_key_words(context_key, context_key_words); + hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); +} + +void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { + blake3_hasher_init_derive_key_raw(self, context, strlen(context)); +} + +// As described in hasher_push_cv() below, we do "lazy merging", delaying +// merges until right before the next CV is about to be added. This is +// different from the reference implementation. Another difference is that we +// aren't always merging 1 chunk at a time. Instead, each CV might represent +// any power-of-two number of chunks, as long as the smaller-above-larger stack +// order is maintained. Instead of the "count the trailing 0-bits" algorithm +// described in the spec, we use a "count the total number of 1-bits" variant +// that doesn't require us to retain the subtree size of the CV on top of the +// stack. The principle is the same: each CV that should remain in the stack is +// represented by a 1-bit in the total number of chunks (or bytes) so far. +INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { + size_t post_merge_stack_len = (size_t)popcnt(total_len); + while (self->cv_stack_len > post_merge_stack_len) { + uint8_t *parent_node = + &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; + output_t output = parent_output(parent_node, self->key, self->chunk.flags); + output_chaining_value(&output, parent_node); + self->cv_stack_len -= 1; + } +} + +// In reference_impl.rs, we merge the new CV with existing CVs from the stack +// before pushing it. We can do that because we know more input is coming, so +// we know none of the merges are root. +// +// This setting is different. We want to feed as much input as possible to +// compress_subtree_wide(), without setting aside anything for the chunk_state. +// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once +// as a single subtree, if at all possible. +// +// This leads to two problems: +// 1) This 64 KiB input might be the only call that ever gets made to update. +// In this case, the root node of the 64 KiB subtree would be the root node +// of the whole tree, and it would need to be ROOT finalized. We can't +// compress it until we know. +// 2) This 64 KiB input might complete a larger tree, whose root node is +// similarly going to be the root of the whole tree. For example, maybe +// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the +// node at the root of the 256 KiB subtree until we know how to finalize it. +// +// The second problem is solved with "lazy merging". That is, when we're about +// to add a CV to the stack, we don't merge it with anything first, as the +// reference impl does. Instead we do merges using the *previous* CV that was +// added, which is sitting on top of the stack, and we put the new CV +// (unmerged) on top of the stack afterwards. This guarantees that we never +// merge the root node until finalize(). +// +// Solving the first problem requires an additional tool, +// compress_subtree_to_parent_node(). That function always returns the top +// *two* chaining values of the subtree it's compressing. We then do lazy +// merging with each of them separately, so that the second CV will always +// remain unmerged. (That also helps us support extendable output when we're +// hashing an input all-at-once.) +INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], + uint64_t chunk_counter) { + hasher_merge_cv_stack(self, chunk_counter); + memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, + BLAKE3_OUT_LEN); + self->cv_stack_len += 1; +} + +INLINE void blake3_hasher_update_base(blake3_hasher *self, const void *input, + size_t input_len, bool use_tbb) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_update(&hasher, v.data(), v.size()); + if (input_len == 0) { + return; + } + + const uint8_t *input_bytes = (const uint8_t *)input; + + // If we have some partial chunk bytes in the internal chunk_state, we need + // to finish that chunk first. + if (chunk_state_len(&self->chunk) > 0) { + size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); + if (take > input_len) { + take = input_len; + } + chunk_state_update(&self->chunk, input_bytes, take); + input_bytes += take; + input_len -= take; + // If we've filled the current chunk and there's more coming, finalize this + // chunk and proceed. In this case we know it's not the root. + if (input_len > 0) { + output_t output = chunk_state_output(&self->chunk); + uint8_t chunk_cv[32]; + output_chaining_value(&output, chunk_cv); + hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); + chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); + } else { + return; + } + } + + // Now the chunk_state is clear, and we have more input. If there's more than + // a single chunk (so, definitely not the root chunk), hash the largest whole + // subtree we can, with the full benefits of SIMD (and maybe in the future, + // multi-threading) parallelism. Two restrictions: + // - The subtree has to be a power-of-2 number of chunks. Only subtrees along + // the right edge can be incomplete, and we don't know where the right edge + // is going to be until we get to finalize(). + // - The subtree must evenly divide the total number of chunks up until this + // point (if total is not 0). If the current incomplete subtree is only + // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have + // to complete the current subtree first. + // Because we might need to break up the input to form powers of 2, or to + // evenly divide what we already have, this part runs in a loop. + while (input_len > BLAKE3_CHUNK_LEN) { + size_t subtree_len = round_down_to_power_of_2(input_len); + uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; + // Shrink the subtree_len until it evenly divides the count so far. We know + // that subtree_len itself is a power of 2, so we can use a bitmasking + // trick instead of an actual remainder operation. (Note that if the caller + // consistently passes power-of-2 inputs of the same size, as is hopefully + // typical, this loop condition will always fail, and subtree_len will + // always be the full length of the input.) + // + // An aside: We don't have to shrink subtree_len quite this much. For + // example, if count_so_far is 1, we could pass 2 chunks to + // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still + // get the right answer in the end, and we might get to use 2-way SIMD + // parallelism. The problem with this optimization, is that it gets us + // stuck always hashing 2 chunks. The total number of chunks will remain + // odd, and we'll never graduate to higher degrees of parallelism. See + // https://github.com/BLAKE3-team/BLAKE3/issues/69. + while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { + subtree_len /= 2; + } + // The shrunken subtree_len might now be 1 chunk long. If so, hash that one + // chunk by itself. Otherwise, compress the subtree into a pair of CVs. + uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; + if (subtree_len <= BLAKE3_CHUNK_LEN) { + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, self->key, self->chunk.flags); + chunk_state.chunk_counter = self->chunk.chunk_counter; + chunk_state_update(&chunk_state, input_bytes, subtree_len); + output_t output = chunk_state_output(&chunk_state); + uint8_t cv[BLAKE3_OUT_LEN]; + output_chaining_value(&output, cv); + hasher_push_cv(self, cv, chunk_state.chunk_counter); + } else { + // This is the high-performance happy path, though getting here depends + // on the caller giving us a long enough input. + uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; + compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, + self->chunk.chunk_counter, + self->chunk.flags, cv_pair, use_tbb); + hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); + hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], + self->chunk.chunk_counter + (subtree_chunks / 2)); + } + self->chunk.chunk_counter += subtree_chunks; + input_bytes += subtree_len; + input_len -= subtree_len; + } + + // If there's any remaining input less than a full chunk, add it to the chunk + // state. In that case, also do a final merge loop to make sure the subtree + // stack doesn't contain any unmerged pairs. The remaining input means we + // know these merges are non-root. This merge loop isn't strictly necessary + // here, because hasher_push_chunk_cv already does its own merge loop, but it + // simplifies blake3_hasher_finalize below. + if (input_len > 0) { + chunk_state_update(&self->chunk, input_bytes, input_len); + hasher_merge_cv_stack(self, self->chunk.chunk_counter); + } +} + +void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = false; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} + +#if defined(BLAKE3_USE_TBB) +void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len) { + bool use_tbb = true; + blake3_hasher_update_base(self, input, input_len, use_tbb); +} +#endif // BLAKE3_USE_TBB + +void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, + size_t out_len) { + blake3_hasher_finalize_seek(self, 0, out, out_len); +} + +void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_finalize(&hasher, v.data(), v.size()); + if (out_len == 0) { + return; + } + + // If the subtree stack is empty, then the current chunk is the root. + if (self->cv_stack_len == 0) { + output_t output = chunk_state_output(&self->chunk); + output_root_bytes(&output, seek, out, out_len); + return; + } + // If there are any bytes in the chunk state, finalize that chunk and do a + // roll-up merge between that chunk hash and every subtree in the stack. In + // this case, the extra merge loop at the end of blake3_hasher_update + // guarantees that none of the subtrees in the stack need to be merged with + // each other first. Otherwise, if there are no bytes in the chunk state, + // then the top of the stack is a chunk hash, and we start the merge from + // that. + output_t output; + size_t cvs_remaining; + if (chunk_state_len(&self->chunk) > 0) { + cvs_remaining = self->cv_stack_len; + output = chunk_state_output(&self->chunk); + } else { + // There are always at least 2 CVs in the stack in this case. + cvs_remaining = self->cv_stack_len - 2; + output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, + self->chunk.flags); + } + while (cvs_remaining > 0) { + cvs_remaining -= 1; + uint8_t parent_block[BLAKE3_BLOCK_LEN]; + memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); + output_chaining_value(&output, &parent_block[32]); + output = parent_output(parent_block, self->key, self->chunk.flags); + } + output_root_bytes(&output, seek, out, out_len); +} + +void blake3_hasher_reset(blake3_hasher *self) { + chunk_state_reset(&self->chunk, self->key, 0); + self->cv_stack_len = 0; +} diff --git a/thoughts/blake3/reference-impl/upstream/blake3.h b/thoughts/blake3/reference-impl/upstream/blake3.h new file mode 100644 index 000000000..423154ff7 --- /dev/null +++ b/thoughts/blake3/reference-impl/upstream/blake3.h @@ -0,0 +1,86 @@ +#ifndef BLAKE3_H +#define BLAKE3_H + +#include +#include + +#if !defined(BLAKE3_API) +# if defined(_WIN32) || defined(__CYGWIN__) +# if defined(BLAKE3_DLL) +# if defined(BLAKE3_DLL_EXPORTS) +# define BLAKE3_API __declspec(dllexport) +# else +# define BLAKE3_API __declspec(dllimport) +# endif +# define BLAKE3_PRIVATE +# else +# define BLAKE3_API +# define BLAKE3_PRIVATE +# endif +# elif __GNUC__ >= 4 +# define BLAKE3_API __attribute__((visibility("default"))) +# define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) +# else +# define BLAKE3_API +# define BLAKE3_PRIVATE +# endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#define BLAKE3_VERSION_STRING "1.8.5" +#define BLAKE3_KEY_LEN 32 +#define BLAKE3_OUT_LEN 32 +#define BLAKE3_BLOCK_LEN 64 +#define BLAKE3_CHUNK_LEN 1024 +#define BLAKE3_MAX_DEPTH 54 + +// This struct is a private implementation detail. It has to be here because +// it's part of the blake3_hasher structure defined below. +typedef struct { + uint32_t cv[8]; + uint64_t chunk_counter; + uint8_t buf[BLAKE3_BLOCK_LEN]; + uint8_t buf_len; + uint8_t blocks_compressed; + uint8_t flags; +} blake3_chunk_state; + +typedef struct { + uint32_t key[8]; + blake3_chunk_state chunk; + uint8_t cv_stack_len; + // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, + // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk + // requires a 4th entry, rather than merging everything down to 1, because we + // don't know whether more input is coming. This is different from how the + // reference implementation does things. + uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; +} blake3_hasher; + +BLAKE3_API const char *blake3_version(void); +BLAKE3_API void blake3_hasher_init(blake3_hasher *self); +BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]); +BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); +BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, + size_t context_len); +BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, + size_t input_len); +#if defined(BLAKE3_USE_TBB) +BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, + size_t input_len); +#endif // BLAKE3_USE_TBB +BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, + size_t out_len); +BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len); +BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); + +#ifdef __cplusplus +} +#endif + +#endif /* BLAKE3_H */ diff --git a/thoughts/blake3/reference-impl/upstream/blake3_dispatch.c b/thoughts/blake3/reference-impl/upstream/blake3_dispatch.c new file mode 100644 index 000000000..14dfbbe0c --- /dev/null +++ b/thoughts/blake3/reference-impl/upstream/blake3_dispatch.c @@ -0,0 +1,332 @@ +#include +#include +#include + +#include "blake3_impl.h" + +#if defined(_MSC_VER) +#include +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#elif defined(__GNUC__) +#include +#else +#undef IS_X86 /* Unimplemented! */ +#endif +#endif + +#if !defined(BLAKE3_ATOMICS) +#if defined(__has_include) +#if __has_include() && !defined(_MSC_VER) +#define BLAKE3_ATOMICS 1 +#else +#define BLAKE3_ATOMICS 0 +#endif /* __has_include() && !defined(_MSC_VER) */ +#else +#define BLAKE3_ATOMICS 0 +#endif /* defined(__has_include) */ +#endif /* BLAKE3_ATOMICS */ + +#if BLAKE3_ATOMICS +#define ATOMIC_INT _Atomic int +#define ATOMIC_LOAD(x) x +#define ATOMIC_STORE(x, y) x = y +#elif defined(_MSC_VER) +#define ATOMIC_INT LONG +#define ATOMIC_LOAD(x) InterlockedOr(&x, 0) +#define ATOMIC_STORE(x, y) InterlockedExchange(&x, y) +#else +#define ATOMIC_INT int +#define ATOMIC_LOAD(x) x +#define ATOMIC_STORE(x, y) x = y +#endif + +#define MAYBE_UNUSED(x) (void)((x)) + +#if defined(IS_X86) +static uint64_t xgetbv(void) { +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t eax = 0, edx = 0; + __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); + return ((uint64_t)edx << 32) | eax; +#endif +} + +static void cpuid(uint32_t out[4], uint32_t id) { +#if defined(_MSC_VER) + __cpuid((int *)out, id); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__("movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#endif +} + +static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { +#if defined(_MSC_VER) + __cpuidex((int *)out, id, sid); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__("movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#endif +} + + +enum cpu_feature { + SSE2 = 1 << 0, + SSSE3 = 1 << 1, + SSE41 = 1 << 2, + AVX = 1 << 3, + AVX2 = 1 << 4, + AVX512F = 1 << 5, + AVX512VL = 1 << 6, + /* ... */ + UNDEFINED = 1 << 30 +}; + +#if !defined(BLAKE3_TESTING) +static /* Allow the variable to be controlled manually for testing */ +#endif + ATOMIC_INT g_cpu_features = UNDEFINED; + +#if !defined(BLAKE3_TESTING) +static +#endif + enum cpu_feature + get_cpu_features(void) { + + /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ + enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); + if (features != UNDEFINED) { + return features; + } else { +#if defined(IS_X86) + uint32_t regs[4] = {0}; + uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; + (void)edx; + features = 0; + cpuid(regs, 0); + const int max_id = *eax; + cpuid(regs, 1); +#if defined(__amd64__) || defined(_M_X64) + features |= SSE2; +#else + if (*edx & (1UL << 26)) + features |= SSE2; +#endif + if (*ecx & (1UL << 9)) + features |= SSSE3; + if (*ecx & (1UL << 19)) + features |= SSE41; + + if (*ecx & (1UL << 27)) { // OSXSAVE + const uint64_t mask = xgetbv(); + if ((mask & 6) == 6) { // SSE and AVX states + if (*ecx & (1UL << 28)) + features |= AVX; + if (max_id >= 7) { + cpuidex(regs, 7, 0); + if (*ebx & (1UL << 5)) + features |= AVX2; + if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm + if (*ebx & (1UL << 31)) + features |= AVX512VL; + if (*ebx & (1UL << 16)) + features |= AVX512F; + } + } + } + } + ATOMIC_STORE(g_cpu_features, features); + return features; +#else + /* How to detect NEON? */ + return 0; +#endif + } +} +#endif + +void blake3_compress_in_place(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); + return; + } +#endif +#endif + blake3_compress_in_place_portable(cv, block, block_len, counter, flags); +} + +void blake3_compress_xof(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); + return; + } +#endif +#endif + blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); +} + + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks) { + if (outblocks == 0) { + // The current assembly implementation always outputs at least 1 block. + return; + } +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) + if (features & AVX512VL) { + blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); + return; + } +#endif +#endif + for(size_t i = 0; i < outblocks; ++i) { + blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); + } +} + +void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { + blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_AVX2) + if (features & AVX2) { + blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); + return; + } +#endif +#endif + +#if BLAKE3_USE_NEON == 1 + blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, out); + return; +#endif + + blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); +} + +// The dynamically detected SIMD degree of the current platform. +size_t blake3_simd_degree(void) { +#if defined(IS_X86) + const enum cpu_feature features = get_cpu_features(); + MAYBE_UNUSED(features); +#if !defined(BLAKE3_NO_AVX512) + if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { + return 16; + } +#endif +#if !defined(BLAKE3_NO_AVX2) + if (features & AVX2) { + return 8; + } +#endif +#if !defined(BLAKE3_NO_SSE41) + if (features & SSE41) { + return 4; + } +#endif +#if !defined(BLAKE3_NO_SSE2) + if (features & SSE2) { + return 4; + } +#endif +#endif +#if BLAKE3_USE_NEON == 1 + return 4; +#endif + return 1; +} diff --git a/thoughts/blake3/reference-impl/upstream/blake3_impl.h b/thoughts/blake3/reference-impl/upstream/blake3_impl.h new file mode 100644 index 000000000..88e71e41e --- /dev/null +++ b/thoughts/blake3/reference-impl/upstream/blake3_impl.h @@ -0,0 +1,333 @@ +#ifndef BLAKE3_IMPL_H +#define BLAKE3_IMPL_H + +#include +#include +#include +#include +#include + +#include "blake3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// internal flags +enum blake3_flags { + CHUNK_START = 1 << 0, + CHUNK_END = 1 << 1, + PARENT = 1 << 2, + ROOT = 1 << 3, + KEYED_HASH = 1 << 4, + DERIVE_KEY_CONTEXT = 1 << 5, + DERIVE_KEY_MATERIAL = 1 << 6, +}; + +// This C implementation tries to support recent versions of GCC, Clang, and +// MSVC. +#if defined(_MSC_VER) +#define INLINE static __forceinline +#else +#define INLINE static inline __attribute__((always_inline)) +#endif + +#ifdef __cplusplus +#define NOEXCEPT noexcept +#else +#define NOEXCEPT +#endif + +#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC) +#define IS_X86 +#define IS_X86_64 +#endif + +#if defined(__i386__) || defined(_M_IX86) +#define IS_X86 +#define IS_X86_32 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#define IS_AARCH64 +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#endif +#endif + +#if !defined(BLAKE3_USE_NEON) + // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness + #if defined(IS_AARCH64) + #if defined(__ARM_BIG_ENDIAN) + #define BLAKE3_USE_NEON 0 + #else + #define BLAKE3_USE_NEON 1 + #endif + #else + #define BLAKE3_USE_NEON 0 + #endif +#endif + +#if defined(IS_X86) +#define MAX_SIMD_DEGREE 16 +#elif BLAKE3_USE_NEON == 1 +#define MAX_SIMD_DEGREE 4 +#else +#define MAX_SIMD_DEGREE 1 +#endif + +// There are some places where we want a static size that's equal to the +// MAX_SIMD_DEGREE, but also at least 2. +#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) + +static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, + 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, + 0x1F83D9ABUL, 0x5BE0CD19UL}; + +static const uint8_t MSG_SCHEDULE[7][16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, + {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, + {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, + {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, + {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, + {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, +}; + +/* Find index of the highest set bit */ +/* x is assumed to be nonzero. */ +static unsigned int highest_one(uint64_t x) { +#if defined(__GNUC__) || defined(__clang__) + return 63 ^ (unsigned int)__builtin_clzll(x); +#elif defined(_MSC_VER) && defined(IS_X86_64) + unsigned long index; + _BitScanReverse64(&index, x); + return index; +#elif defined(_MSC_VER) && defined(IS_X86_32) + if(x >> 32) { + unsigned long index; + _BitScanReverse(&index, (unsigned long)(x >> 32)); + return 32 + index; + } else { + unsigned long index; + _BitScanReverse(&index, (unsigned long)x); + return index; + } +#else + unsigned int c = 0; + if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if(x & 0x0000000000000002ULL) { c += 1; } + return c; +#endif +} + +// Count the number of 1 bits. +INLINE unsigned int popcnt(uint64_t x) { +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_popcountll(x); +#else + unsigned int count = 0; + while (x != 0) { + count += 1; + x &= x - 1; + } + return count; +#endif +} + +// Largest power of two less than or equal to x. As a special case, returns 1 +// when x is 0. +INLINE uint64_t round_down_to_power_of_2(uint64_t x) { + return 1ULL << highest_one(x | 1); +} + +INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } + +INLINE uint32_t counter_high(uint64_t counter) { + return (uint32_t)(counter >> 32); +} + +INLINE uint32_t load32(const void *src) { + const uint8_t *p = (const uint8_t *)src; + return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | + ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); +} + +INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], + uint32_t key_words[8]) { + key_words[0] = load32(&key[0 * 4]); + key_words[1] = load32(&key[1 * 4]); + key_words[2] = load32(&key[2 * 4]); + key_words[3] = load32(&key[3 * 4]); + key_words[4] = load32(&key[4 * 4]); + key_words[5] = load32(&key[5 * 4]); + key_words[6] = load32(&key[6 * 4]); + key_words[7] = load32(&key[7 * 4]); +} + +INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], + uint32_t block_words[16]) { + for (size_t i = 0; i < 16; i++) { + block_words[i] = load32(&block[i * 4]); + } +} + +INLINE void store32(void *dst, uint32_t w) { + uint8_t *p = (uint8_t *)dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +} + +INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { + store32(&bytes_out[0 * 4], cv_words[0]); + store32(&bytes_out[1 * 4], cv_words[1]); + store32(&bytes_out[2 * 4], cv_words[2]); + store32(&bytes_out[3 * 4], cv_words[3]); + store32(&bytes_out[4 * 4], cv_words[4]); + store32(&bytes_out[5 * 4], cv_words[5]); + store32(&bytes_out[6 * 4], cv_words[6]); + store32(&bytes_out[7 * 4], cv_words[7]); +} + +void blake3_compress_in_place(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]); + +void blake3_xof_many(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64], size_t outblocks); + +void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +size_t blake3_simd_degree(void); + +BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out, bool use_tbb); + +#if defined(BLAKE3_USE_TBB) +BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb( + // shared params + const uint32_t key[8], uint8_t flags, bool use_tbb, + // left-hand side params + const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, + uint8_t *l_cvs, size_t *l_n, + // right-hand side params + const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, + uint8_t *r_cvs, size_t *r_n) NOEXCEPT; +#endif + +// Declarations for implementation-specific functions. +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +#if defined(IS_X86) +#if !defined(BLAKE3_NO_SSE2) +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_SSE41) +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_AVX2) +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif +#if !defined(BLAKE3_NO_AVX512) +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +#if !defined(_WIN32) && !defined(__CYGWIN__) +void blake3_xof_many_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t* out, size_t outblocks); +#endif +#endif +#endif + +#if BLAKE3_USE_NEON == 1 +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* BLAKE3_IMPL_H */ diff --git a/thoughts/blake3/reference-impl/upstream/blake3_portable.c b/thoughts/blake3/reference-impl/upstream/blake3_portable.c new file mode 100644 index 000000000..062dd1b47 --- /dev/null +++ b/thoughts/blake3/reference-impl/upstream/blake3_portable.c @@ -0,0 +1,160 @@ +#include "blake3_impl.h" +#include + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = IV[0]; + state[9] = IV[1]; + state[10] = IV[2]; + state[11] = IV[3]; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + round_fn(state, &block_words[0], 0); + round_fn(state, &block_words[0], 1); + round_fn(state, &block_words[0], 2); + round_fn(state, &block_words[0], 3); + round_fn(state, &block_words[0], 4); + round_fn(state, &block_words[0], 5); + round_fn(state, &block_words[0], 6); +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/thoughts/blake3/socket-kats/SOCKET.md b/thoughts/blake3/socket-kats/SOCKET.md new file mode 100644 index 000000000..d3974c9b4 --- /dev/null +++ b/thoughts/blake3/socket-kats/SOCKET.md @@ -0,0 +1,296 @@ +# The LFM 2-to-1 BLAKE3 compress socket + +**Status:** specification + reference vectors. **No chip code exists for this.** +**Date:** 2026-08-10. **Phase:** 1 (reference layer), per `thoughts/shared/lfm-real-hash/PLAN.md` §2.5 and §5. +**Scope:** the `compress` socket only. The `permute` socket is *not* specified here — see §7. + +Claims about the tree are marked ✓ VERIFIED (read the code, cited `file:line`), +? INFERRED (derived, arithmetic shown) or ✗ OPEN. + +--- + +## 1. What this pins, and why `CANONICAL_VECTORS` is not enough + +`prover/src/lfm/blake3.rs:151` pins the compression function `f(h, m, t, +block_len, flags)` with ten vectors. ✓ VERIFIED. That is the *primitive*. + +It says nothing about how a two-input hash **calls** `f`. Between "we have a +correct `f`" and "we have a correct 2-to-1 compress" sit six independent +choices, every one of which is a way to be wrong while every existing test stays +green: + +1. where the two input digest cells land in the 16-word message `m`; +2. what the chaining value `h` is; +3. what the counter `t` is; +4. what `block_len` is; +5. what the `flags` byte is; +6. which 4 of the 16 output words become the digest. + +This document fixes all six, and `socket_kats.json` pins them with a vector +table plus one negative control per choice. + +## 2. The specification + +### 2.1 Byte-level form (normative) + +Let `a`, `b` be the two input digest cells, each four lanes, **each lane an +unsigned 32-bit value** (see obligation O1). Write `LE32(x)` for the four-byte +little-endian encoding of `x`. + +``` +msg = LE32(a0) ‖ LE32(a1) ‖ LE32(a2) ‖ LE32(a3) (16 bytes) + ‖ LE32(b0) ‖ LE32(b1) ‖ LE32(b2) ‖ LE32(b3) (16 bytes) + ‖ "LFMC" ( 4 bytes, domain tag) + ------------------ + 36 bytes + +digest_bytes = BLAKE3(msg)[0 .. 16] (truncate 256 → 128 bits) + +c_i = LE32⁻¹( digest_bytes[4i .. 4i+4] ) for i in 0..4 +``` + +`BLAKE3(·)` is the standard default hashing mode — the plain one-argument hash, +no key, no context. + +**This is the whole specification.** It is deliberately written as a call to a +library rather than as a compression-function invocation, because that is what +makes it externally checkable: at 7 rounds, `compress(a, b)` is *literally* +`blake3::hash(a ‖ b ‖ "LFMC")` truncated to 16 bytes. There is no oracle in the +chain, and no assumption. §5 records that this equality is already executed. + +### 2.2 Word-level form (what the chip proves) + +The 36-byte message is one BLAKE3 block, so the byte-level form is exactly one +compression. The chip proves this: + +| input to `f` | value | +|---|---| +| `h` (chaining value) | `BLAKE3_IV[0..8]` — all eight words | +| `m[0..4]` | `a[0..4]` | +| `m[4..8]` | `b[0..4]` | +| `m[8]` | **mode-selected on the built chip** — `MODE_C·TAG_LFMC + MODE_T·TAG_LFMT`; `0x434D464C` on a Merkle row (`MODE_C = 1`). See the note below. | +| `m[9..16]` | `0` | +| `t` (counter) | `0` | +| `block_len` | `36` | +| `flags` | `0x0B` = `CHUNK_START | CHUNK_END | ROOT` | + +Output: `c_i = f(...)[i]` for `i in 0..4` — the **low four** words of the +16-word output, i.e. the low half of the truncated chaining value. + +Everything in that table except `a` and `b` and `m[8]` is a compile-time +constant, and the socket costs the chip no extra columns beyond the compression +it already proves. + +> **⚠ UPDATED FOR B1 (2026-08-11) — the conclusion holds, the REASON changed.** +> `m[8]` was a compile-time constant when this document was written, and +> "constant" was why it was free. The built chip selects it from the row's +> domain: `MODE_C·TAG_LFMC + MODE_T·TAG_LFMT`, a linear form over two +> **preprocessed** columns (`WordRef::ModeSelected`, evaluated `Σ col·tag`), +> because the Fiat–Shamir transcript now runs on this same socket under +> `"LFMT"` (option B1 — see §2.4 and §7). It remains free (only ever an `add3` +> operand, read as a whole word, never byte-decomposed) and remains +> prover-unchosen — but now because the selectors are preprocessed, not because +> the value is constant. +> +> **If you transcribe this table into a model, transcribe the linear form.** A +> model carrying a constant where the chip has a linear form still reports PASS +> while checking something the chip does not do. + +`gen_socket_kats.py` computes §2.1 and §2.2 by separate routes and asserts they +agree, for every vector, at both round counts. That equality is the framing +check; if the chip is ever re-expressed, it is the property to re-run. + +### 2.3 Why the domain tag sits in the message + +Plan §5 recommends "option D": a 128-bit digest **plus domain separation**. The +obvious place for a domain tag is the `flags` byte, and that is what BLAKE3 +itself does for `PARENT` / `CHUNK_START` / `ROOT`. + +**We put it in the message instead, and that choice is load-bearing.** Any tag +in `flags` (or in `t`, or in `h`) makes the socket a *nonstandard* invocation of +`f` that no library computes — so the KATs could only ever come from our own +oracle, at 6 **and** at 7 rounds. Putting the tag in the message keeps the +socket a standard BLAKE3 hash of a domain-separated byte string, which is the +entire reason §2.1 can be a library call. The domain separation is just as real: +distinct tags give distinct 36-byte messages. + +Cost of the choice: the message is 36 bytes rather than 32, which is still one +block. Zero extra compressions, zero extra columns. ? INFERRED — `block_len` and +`m[8]` is not a column either — post-B1 it is a linear form over preprocessed +mode columns, which is still zero columns and zero sends (✓ CONFIRMED against +the built arm). + +### 2.4 Tag allocation + +| tag | bytes | u32 (LE) | use | +|---|---|---|---| +| `"LFMC"` | `4C 46 4D 43` | `0x434D464C` | **this socket** — 2-to-1 compress / Merkle parent | +| `"LFMT"` | `4C 46 4D 54` | `0x544D464C` | **transcript step** — the compress-chain Fiat–Shamir transcript (`thoughts/shared/lfm-real-hash/transcript-spec/TRANSCRIPT.md`) | +| `"LFMP"` | `4C 46 4D 50` | `0x504D464C` | ~~`permute` socket (§7)~~ — **RETIRED UNUSED**, see below | +| `"LFML"` | `4C 46 4D 4C` | `0x4C4D464C` | **LIVE** — the felt-input leaf mode (`MODE_L`). O5 is now enforced by the tag rather than by review: a leaf row is one with `MODE_L` set, and `MODE_L` selects `LFML`. Spec: `thoughts/shared/lfm-real-hash/leaf-spec/LEAF.md` | + +⚠ **§7's `permute`-socket sketch is superseded and will never be built.** The +user ratified option **B1** on 2026-08-11 +(`thoughts/shared/lfm-real-hash/permute-socket-options.md`): the Fiat–Shamir +sponge becomes a **compress-based chain** over *this* socket under the new +`"LFMT"` tag, for all hashers, and `MODE_P` stays pinned to 0 permanently. Read +§7 as a record of a rejected direction, not as a plan. + +`"LFMP"` is **retired rather than deleted**, and the distinction is +load-bearing: the value is now permanently unused, but removing the row would +let a future allocation reuse `0x504D464C` and silently create a domain nobody +analysed. + +A tag is never reused for a second purpose, for the same reason +`HasherKind::as_tag` never reuses a discriminant. ⚠ `as_tag` is **not yet +committed** — it is another agent's in-flight Phase 3 work in this worktree and +is absent from `HEAD` (✓ VERIFIED via `git show HEAD:prover/src/lfm/hash.rs`). +Cited by symbol, not by line, because its line numbers will move. + +## 3. Security consequence, stated plainly + +The digest is **128 bits**, so this socket offers **64-bit collision +resistance** by the birthday bound, not 128-bit. That is the honest consequence +of `HASH_DIGEST_FELTS = 4` (`hash.rs:21`) and of `word.rs:1-9`'s declared +"128-bit target", both ✓ VERIFIED — it is not introduced by BLAKE3 or by the +truncation window. + +**This is the question the plan (§5) puts to the user and it is not settled +here.** If the target is 128-bit *collision* resistance, the digest must be two +cells (256 bits) and the frozen 1-cell `LFM_HASH` output contract has to be +reopened. If the target is a 128-bit *security level* in the ordinary +preimage sense, this socket meets it. Nothing below depends on which answer +comes back; only the digest width does. + +Preimage resistance of the truncated digest is 128 bits. ? INFERRED — standard +for a truncated random oracle; it is not an assumption specific to this design. + +## 4. Obligations for the chip arm (Phase 2) + +**O1 — input lanes MUST be range-checked to 32 bits. This is a soundness +obligation, not hygiene.** ✓ VERIFIED that it bites: `edsl::merkle_walk` +(`edsl.rs:65-80`) feeds `compress` sibling cells that are **arena-hinted**, i.e. +prover-chosen — the doc comment says so outright: *"Sibling digests come as +(arena-hinted) cells; every hinted value ends up inside a `compress`, which is +what authenticates it."* A lane is a Goldilocks felt, so it ranges over +`[0, p)` with `p ≈ 2^64`. If the chip derives the four message bytes of a lane +by reduction mod 2^32 rather than by a checked decomposition, then lane values +`v` and `v + 2^32` produce the **same** message and hence the same digest — a +free collision, chosen by the prover, and therefore a forged Merkle path. The +host-side `LfmHasher` impl must likewise **reject** an out-of-range lane rather +than silently reduce, or the host and the chip disagree about what was proved +(plan §3.2, same failure mode on the input side). + +**O2 — the socket must be closed on its own output.** `c_i` is a `u32` by +construction, so a digest produced by this socket always satisfies O1. Only +*leaf* digests and prover-hinted siblings can violate it, which is exactly where +O1's check must sit. + +**O3 — `compress_iv()` does not participate.** The trait's default `compress` +injects `compress_iv()` into state lanes 8–11 (`hash.rs:35-43`, ✓ VERIFIED). +The BLAKE3 arm **overrides** `compress` entirely — the IV enters through `h`, +all eight words, not through the state. Overriding is explicitly sanctioned: +*"a real hash may override it, but the bus contract (2 cells in, 1 cell out) is +frozen"* (`hash.rs:25-26`). Two consequences: `compress_iv()` should return +`BLAKE3_IV[0..4]` as felts so it is meaningful if read, with a doc comment +saying it is not part of the compress framing; and the override must be wired +into `HasherKind::compress`'s explicit delegation, whose own doc comment already +warns that a candidate overriding `compress` must be honoured through that +dispatch. (Cited by symbol: that part of `hash.rs` is being edited concurrently +by the Phase 3 agent, so its line numbers are in motion. The trait definition and +its default `compress` at `hash.rs:19-44` are *not* in the edited region and are +✓ VERIFIED stable against `HEAD`.) + +**O4 — the byte order is the `keccak_host` convention, and it is already the +machine's.** One felt carries one `u32` as four little-endian bytes +(`keccak_host.rs:17-32`, ✓ VERIFIED: `FE::from(u64::from(u32::from_le_bytes(half)))`). +This socket reuses it unchanged. Note this is *not* `word::pack_digest` +(`word.rs:44-50`), which serialises each lane as eight bytes; the two are +different serialisations of a cell and must not be confused. + +## 5. The vectors + +`socket_kats.json`, generated by `gen_socket_kats.py`. + +- **10 vectors × 2 round counts.** Five structural inputs (zeros, unit `a`, unit + `b`, all-ones, a nibble ramp) and five from an explicit formula. All inputs are + written out in the JSON, so nothing depends on a random-number generator. +- **9 negative controls per vector**, one per framing degree of freedom: + `swap_a_b`, `tag_changed`, `tag_omitted`, `truncate_high_half`, `flags_parent`, + `block_len_64`, `counter_one`, `lanes_big_endian`, `other_round_count`. The + generator asserts each applicable control **changes** the digest, and + separately asserts every control is discriminated by at least one vector. + + Two controls are declared inapplicable on degenerate inputs rather than + skipped: `swap_a_b` when `a == b`, and `lanes_big_endian` when every lane is a + byte-palindrome (`0x00000000`, `0xFFFFFFFF`, `0x11111111`, …). That is a real + property of those inputs, not a workaround — three of the five structural + vectors cannot detect a byte-order error, which is precisely why the formula + vectors are in the table. +- **Three independent computations agree** on every vector: the in-repo Python + oracle at word level, upstream BLAKE3's C at word level, and upstream BLAKE3's + **whole tree hasher** over the 36-byte string at byte level. + +Worked example (`nibble_ramp`, rounds = 7): + +``` +a = 00000000 11111111 22222222 33333333 +b = 44444444 55555555 66666666 77777777 +msg = 00000000111111112222222233333333444444445555555566666666777777774c464d43 +BLAKE3(msg) = c03eaa1a295bdd663056a4e9ff74d261051f49096ec2345cde112bda36168bf4 +digest (16 bytes)= c03eaa1a295bdd663056a4e9ff74d261 +c = 1aaa3ec0 66dd5b29 e9a45630 61d274ff (the same 16 bytes as u32 lanes) +``` + +At rounds = 6 the same inputs give `c = 2ef9ed44 4b4ab3f5 6be64dc6 dabef7b1`. +No library computes that value and no published vector contains it — which is +the whole of the A6R argument, in one line. + +## 6. What is executed and what is deferred + +| claim | status | +|---|---| +| word-level and byte-level forms agree, both round counts, all 10 vectors | ✓ EXECUTED | +| Python oracle and upstream C agree on every socket vector | ✓ EXECUTED | +| at rounds = 7 the socket equals upstream BLAKE3's whole-hash output, truncated | ✓ EXECUTED (against upstream **C**, which passes the official vectors) | +| all 9 controls discriminate | ✓ EXECUTED | +| the same equality against the Rust **`blake3` crate** | ✗ DEFERRED to a build phase — needs cargo | +| the chip's `OUT` columns match these vectors | ✗ DEFERRED — no chip arm exists yet | + +The deferred crate check is a formality rather than a risk: the C that was +checked *is* upstream BLAKE3, and it reproduced the official test vectors in all +three modes. It should still be written, as a one-line `blake3::hash` assertion, +because it is the version of the check that survives this directory being +deleted. + +## 7. ~~✗ OPEN: the `permute` socket is not specified here~~ +## ⛔ SUPERSEDED — NO PERMUTE SOCKET WILL EVER BE BUILT (option B1, 2026-08-11) + +> The user ratified **option B1**: the Fiat–Shamir sponge becomes a +> **compress-based chain** over the socket this document specifies, under the +> new `"LFMT"` tag; `MODE_P` stays pinned to 0 permanently. Spec, reference, +> KATs and gate extension: +> `thoughts/shared/lfm-real-hash/transcript-spec/TRANSCRIPT.md`. +> **Everything below is a record of the rejected direction.** It is kept because +> the options paper's analysis cites it, not because anyone should build it. + +The brief asked for the 2-to-1 compress socket and that is what this document +covers. Flagging the gap explicitly, because it changes what Phase 5's E1 +milestone can claim: + +✓ VERIFIED — `edsl::merkle_walk` compresses (`edsl.rs:75`, `b.compress(...)`), +but `edsl::SpongeVar` **permutes** (`edsl.rs:31` and `edsl.rs:43`, +`b.permute(...)`). `FriToyV0`'s Fiat–Shamir sponge is therefore built on +`permute`, not on `compress`. Specifying this socket makes `merkle_walk`'s +authentication real; it does **not** on its own make the sponge real, so the +F3.4 disclosure is only half retired by it. + +The `permute` socket needs its own mapping decision and its own KATs: 12 felts +in, 12 felts out. The natural shape under the u32-lane restriction is one +compression — 12 lanes = 48 bytes fits one 64-byte block, per plan §3.2 option +(i) — taking `h = IV`, `m[0..12] = state`, `m[12] = "LFMP"`, `m[13..16] = 0`, +`t = 0`, `block_len = 52`, `flags = 0x0B`, and `out[0..12]` as the new state. +That is a **sketch, not a decision**: it is unreviewed, has no vectors, and the +security argument for a 12-word permutation built from a truncated compression +output is not the same argument as §3's. It should get the same treatment this +document gave `compress` before any code is written against it. diff --git a/thoughts/blake3/socket-kats/gen_socket_kats.py b/thoughts/blake3/socket-kats/gen_socket_kats.py new file mode 100644 index 000000000..6e347f523 --- /dev/null +++ b/thoughts/blake3/socket-kats/gen_socket_kats.py @@ -0,0 +1,315 @@ +""" +Reference vectors for the LFM 2-to-1 BLAKE3 compress socket (see SOCKET.md). + +This generates the KATs for the SOCKET FUNCTION -- not for the bare compression +function `f`, which `CANONICAL_VECTORS` already pins. The socket adds five +framing degrees of freedom on top of `f` (where a and b land in the message, +the counter, the block length, the flags, and the truncation window), and every +one of them is a fresh way to be wrong. + +Three computations must agree for each vector, and the script fails loudly if +they do not: + + W. WORD level, Python -- the in-repo oracle's `compress`, called with the + socket's (h, m, t, block_len, flags). + C. WORD level, C -- upstream BLAKE3's parameterised portable compress, + called with the same tuple (reference-impl/b3ref{6,7} `compress`). + B. BYTE level, C -- upstream BLAKE3's WHOLE TREE HASHER over the byte + string `a || b || tag`, truncated (reference-impl/b3ref{6,7} `hashhex`). + +W-vs-C is the two-source check. **B is the one that matters for the framing**: +it says the socket is not merely "some compression call" but exactly a standard +BLAKE3 hash of a domain-separated 36-byte string. At rounds=7 that makes the +socket reproducible with a one-line `blake3` crate call and no oracle anywhere +in the chain -- which is the §2.3/§7 argument for 7 rounds, made concrete. + +Run: python3 gen_socket_kats.py (after reference-impl/build.sh) +""" + +import json +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ORACLE_DIR = os.path.join(HERE, "..", "blake3-oracle") +REFIMPL_DIR = os.path.join(HERE, "..", "reference-impl") + +sys.path.insert(0, ORACLE_DIR) +import blake3_ref as ref # noqa: E402 + +MASK32 = 0xFFFFFFFF + +# --- the socket's frozen constants (SOCKET.md §2) --------------------------- + +# Domain tag for the 2-to-1 compress socket: the four ASCII bytes "LFMC", +# read as one little-endian u32 message word. +DOMAIN_TAG_BYTES = b"LFMC" +DOMAIN_TAG_WORD = int.from_bytes(DOMAIN_TAG_BYTES, "little") # 0x434D464C + +SOCKET_COUNTER = 0 +SOCKET_BLOCK_LEN = 36 # 8 digest words (32 bytes) + the 4-byte tag +SOCKET_FLAGS = ref.CHUNK_START | ref.CHUNK_END | ref.ROOT # 0x0B +DIGEST_LANES = 4 # truncate the 8-word chaining value to its low 4 words + +FAILURES = [] + + +def check(name, cond, detail=""): + if not cond: + print(f" FAIL {name} {detail}") + FAILURES.append(name) + return cond + + +# --- the socket, defined twice -------------------------------------------- + +def socket_message_words(a, b, tag=DOMAIN_TAG_WORD): + """The 16-word message block m the compression consumes.""" + return list(a) + list(b) + [tag] + [0] * 7 + + +def socket_message_bytes(a, b, tag_bytes=DOMAIN_TAG_BYTES): + """The byte string the whole-hash form consumes: a || b || tag.""" + out = b"".join(w.to_bytes(4, "little") for w in list(a) + list(b)) + return out + tag_bytes + + +def socket_compress_python(a, b, rounds): + out = ref.compress(list(ref.IV), socket_message_words(a, b), SOCKET_COUNTER, + SOCKET_BLOCK_LEN, SOCKET_FLAGS, rounds=rounds) + return out[:DIGEST_LANES] + + +def c_binary(rounds): + return os.path.join(REFIMPL_DIR, "b3ref7" if rounds == 7 else "b3ref6") + + +def c_compress_batch(records, rounds): + """records: list of (h, m, t, block_len, flags). Returns 16-word outputs.""" + lines = [] + for h, m, t, bl, fl in records: + words = [f"{w:08x}" for w in list(h) + list(m)] + lines.append(" ".join(words) + f" {t:016x} {bl} {fl}\n") + r = subprocess.run([c_binary(rounds), "compress"], input="".join(lines), + capture_output=True, text=True, check=True) + out = r.stdout.strip().split("\n") + assert len(out) == len(records), (len(out), len(records)) + return [[int(ln[8 * i:8 * i + 8], 16) for i in range(16)] for ln in out] + + +def c_hash_bytes(msg, out_len, rounds): + r = subprocess.run([c_binary(rounds), "hashhex", msg.hex(), str(out_len)], + capture_output=True, text=True, check=True) + return bytes.fromhex(r.stdout.strip()) + + +def digest_from_bytes(digest_bytes): + return [int.from_bytes(digest_bytes[4 * i:4 * i + 4], "little") + for i in range(DIGEST_LANES)] + + +# --- test inputs (explicit; every one is written into the JSON) ------------ + +def test_inputs(): + """Five structural cases then five formula cases. The formula is + a[i] = 0x9E3779B9*(16k+i+1) mod 2^32, b[i] = 0x9E3779B9*(16k+i+9) mod 2^32, + so any language can regenerate them; the JSON lists them explicitly anyway.""" + cases = [ + ("zeros", [0] * 4, [0] * 4), + ("a_one", [1, 0, 0, 0], [0] * 4), + ("b_one", [0] * 4, [1, 0, 0, 0]), + ("all_ones", [MASK32] * 4, [MASK32] * 4), + ("nibble_ramp", + [0x00000000, 0x11111111, 0x22222222, 0x33333333], + [0x44444444, 0x55555555, 0x66666666, 0x77777777]), + ] + for k in range(5): + a = [(0x9E3779B9 * (16 * k + i + 1)) & MASK32 for i in range(4)] + b = [(0x9E3779B9 * (16 * k + i + 9)) & MASK32 for i in range(4)] + cases.append((f"formula_{k}", a, b)) + return cases + + +# --- negative controls: one framing degree of freedom each ----------------- + +def byteswap32(w): + return int.from_bytes(w.to_bytes(4, "little"), "big") + + +def control_applicable(name, a, b): + """Whether a control can discriminate on THESE inputs. + + Two controls are no-ops on degenerate inputs and would otherwise look like + failures: swapping a and b when a == b, and re-packing lanes big-endian + when every lane is a byte-palindrome (0x00000000, 0xFFFFFFFF, 0x11111111, + ...). Those cases are declared inapplicable rather than quietly skipped, + and `main` separately asserts that every control is still discriminated by + at least one vector -- otherwise a degree of freedom would sit unpinned + behind a green run. + """ + if name == "swap_a_b": + return list(a) != list(b) + if name == "lanes_big_endian": + return any(byteswap32(w) != w for w in list(a) + list(b)) + return True + + +def negative_controls(a, b, rounds): + """Each entry perturbs exactly one framing choice and must change the + digest. A control that does NOT change it means that degree of freedom is + unpinned -- the vectors would accept a wrong implementation.""" + iv = list(ref.IV) + m = socket_message_words(a, b) + controls = {} + + # N1 operand order. + controls["swap_a_b"] = ref.compress( + iv, socket_message_words(b, a), SOCKET_COUNTER, SOCKET_BLOCK_LEN, + SOCKET_FLAGS, rounds=rounds)[:DIGEST_LANES] + + # N2 domain tag value ("LFMC" -> "LFMD"). + controls["tag_changed"] = ref.compress( + iv, socket_message_words(a, b, int.from_bytes(b"LFMD", "little")), + SOCKET_COUNTER, SOCKET_BLOCK_LEN, SOCKET_FLAGS, + rounds=rounds)[:DIGEST_LANES] + + # N3 tag omitted entirely (message is 32 bytes, m[8] = 0). + controls["tag_omitted"] = ref.compress( + iv, socket_message_words(a, b, 0), SOCKET_COUNTER, 32, SOCKET_FLAGS, + rounds=rounds)[:DIGEST_LANES] + + # N4 truncation window moved to the high half of the chaining value. + full = ref.compress(iv, m, SOCKET_COUNTER, SOCKET_BLOCK_LEN, SOCKET_FLAGS, + rounds=rounds) + controls["truncate_high_half"] = full[4:8] + + # N5 flags: PARENT instead of CHUNK_START|CHUNK_END|ROOT. + controls["flags_parent"] = ref.compress( + iv, m, SOCKET_COUNTER, SOCKET_BLOCK_LEN, ref.PARENT, + rounds=rounds)[:DIGEST_LANES] + + # N6 block_len declared 64 rather than the true 36. + controls["block_len_64"] = ref.compress( + iv, m, SOCKET_COUNTER, 64, SOCKET_FLAGS, rounds=rounds)[:DIGEST_LANES] + + # N7 counter nonzero. + controls["counter_one"] = ref.compress( + iv, m, 1, SOCKET_BLOCK_LEN, SOCKET_FLAGS, rounds=rounds)[:DIGEST_LANES] + + # N8 lanes packed big-endian instead of little-endian. + be = [byteswap32(w) for w in list(a) + list(b)] + controls["lanes_big_endian"] = ref.compress( + iv, be + [DOMAIN_TAG_WORD] + [0] * 7, SOCKET_COUNTER, SOCKET_BLOCK_LEN, + SOCKET_FLAGS, rounds=rounds)[:DIGEST_LANES] + + # N9 the other round count. + controls["other_round_count"] = socket_compress_python( + a, b, 6 if rounds == 7 else 7) + + return controls + + +def main(): + for r in (6, 7): + if not os.path.exists(c_binary(r)): + print(f"missing {c_binary(r)} -- run reference-impl/build.sh first") + return 2 + + doc = { + "socket": "LFM 2-to-1 BLAKE3 compress (see SOCKET.md)", + "digest_lanes": DIGEST_LANES, + "digest_bits": 32 * DIGEST_LANES, + "domain_tag_ascii": DOMAIN_TAG_BYTES.decode(), + "domain_tag_word": DOMAIN_TAG_WORD, + "counter": SOCKET_COUNTER, + "block_len": SOCKET_BLOCK_LEN, + "flags": SOCKET_FLAGS, + "flags_meaning": "CHUNK_START|CHUNK_END|ROOT", + "chaining_value_in": "BLAKE3 IV", + "message_layout": "m[0..4]=a, m[4..8]=b, m[8]=tag, m[9..16]=0", + "rounds": {}, + } + + cases = test_inputs() + discriminated = {} + print("=" * 74) + print("LFM 2-to-1 BLAKE3 compress socket -- reference vectors") + print("=" * 74) + + for rounds in (7, 6): + # Batch the word-level C calls. + records = [(list(ref.IV), socket_message_words(a, b), SOCKET_COUNTER, + SOCKET_BLOCK_LEN, SOCKET_FLAGS) for _, a, b in cases] + c_out = c_compress_batch(records, rounds) + + vectors = [] + for idx, (name, a, b) in enumerate(cases): + w = socket_compress_python(a, b, rounds) + c = c_out[idx][:DIGEST_LANES] + msg = socket_message_bytes(a, b) + digest32 = c_hash_bytes(msg, 32, rounds) + bl = digest_from_bytes(digest32) + + check(f"r{rounds} {name}: python word == C word", w == c, f"{w} vs {c}") + check(f"r{rounds} {name}: word form == BLAKE3(a||b||tag) truncated", + w == bl, f"{w} vs {bl}") + + ctrls = negative_controls(a, b, rounds) + inapplicable = [] + for cname, cval in ctrls.items(): + if not control_applicable(cname, a, b): + inapplicable.append(cname) + continue + if check(f"r{rounds} {name}: control '{cname}' changes the digest", + cval != w, f"control equals the canonical digest {w}"): + discriminated.setdefault(rounds, set()).add(cname) + + vectors.append({ + "name": name, + "a": list(a), + "b": list(b), + "message_bytes_hex": msg.hex(), + "digest": w, + "digest_hex": "".join(f"{x:08x}" for x in w), + "full_blake3_digest_hex": digest32.hex(), + "negative_controls": {k: v for k, v in ctrls.items()}, + "controls_inapplicable_here": inapplicable, + }) + + doc["rounds"][str(rounds)] = vectors + # Every control must be discriminated by at least one vector, or that + # framing degree of freedom is unpinned by this table. + all_controls = set(vectors[0]["negative_controls"].keys()) + missed = all_controls - discriminated.get(rounds, set()) + check(f"r{rounds}: every control is discriminated by >=1 vector", + not missed, f"never discriminated: {sorted(missed)}") + print(f" rounds={rounds}: {len(vectors)} vectors, " + f"{len(all_controls)} controls, all discriminated") + + # The headline cross-check, stated once more as an explicit assertion. + a, b = cases[4][1], cases[4][2] + seven = socket_compress_python(a, b, 7) + lib = digest_from_bytes(c_hash_bytes(socket_message_bytes(a, b), 32, 7)) + check("HEADLINE: at rounds=7 the socket IS truncated standard BLAKE3", + seven == lib) + + out_path = os.path.join(HERE, "socket_kats.json") + json.dump(doc, open(out_path, "w"), indent=2) + + print("=" * 74) + if FAILURES: + print(f"RESULT: {len(FAILURES)} FAILURE(S)") + for f in FAILURES[:10]: + print(" -", f) + return 1 + print("RESULT: ALL GREEN") + print(f" wrote {os.path.basename(out_path)}") + print(" At rounds=7 every vector equals blake3::hash(a||b||\"LFMC\")[0..16],") + print(" so the build phase can re-derive this table from the crate alone.") + print("=" * 74) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/blake3/socket-kats/socket_kats.json b/thoughts/blake3/socket-kats/socket_kats.json new file mode 100644 index 000000000..5de329f7b --- /dev/null +++ b/thoughts/blake3/socket-kats/socket_kats.json @@ -0,0 +1,1655 @@ +{ + "socket": "LFM 2-to-1 BLAKE3 compress (see SOCKET.md)", + "digest_lanes": 4, + "digest_bits": 128, + "domain_tag_ascii": "LFMC", + "domain_tag_word": 1129137740, + "counter": 0, + "block_len": 36, + "flags": 11, + "flags_meaning": "CHUNK_START|CHUNK_END|ROOT", + "chaining_value_in": "BLAKE3 IV", + "message_layout": "m[0..4]=a, m[4..8]=b, m[8]=tag, m[9..16]=0", + "rounds": { + "7": [ + { + "name": "zeros", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 2494038600, + 807496444, + 2349420159, + 3886468141 + ], + "digest_hex": "94a8024830216afc8c094e7fe7a6cc2d", + "full_blake3_digest_hex": "4802a894fc6a21307f4e098c2dcca6e7fc7d0fa72963ad16b7f2f5b3fe8ebf84", + "negative_controls": { + "swap_a_b": [ + 2494038600, + 807496444, + 2349420159, + 3886468141 + ], + "tag_changed": [ + 1380423299, + 284758052, + 2995705233, + 967770429 + ], + "tag_omitted": [ + 3246643754, + 1918081665, + 2401493466, + 600956609 + ], + "truncate_high_half": [ + 2802810364, + 380461865, + 3019240119, + 2227146494 + ], + "flags_parent": [ + 2751940035, + 3130605041, + 4142867304, + 3207282746 + ], + "block_len_64": [ + 3903141400, + 471027207, + 182976528, + 957593216 + ], + "counter_one": [ + 34211462, + 3403980658, + 3870432635, + 832700268 + ], + "lanes_big_endian": [ + 2494038600, + 807496444, + 2349420159, + 3886468141 + ], + "other_round_count": [ + 2809853715, + 2395900105, + 421057723, + 4135460974 + ] + }, + "controls_inapplicable_here": [ + "swap_a_b", + "lanes_big_endian" + ] + }, + { + "name": "a_one", + "a": [ + 1, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "01000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 3104074695, + 1974443198, + 2882972316, + 1734279477 + ], + "digest_hex": "b9046bc775af9cbeabd6aa9c675f0135", + "full_blake3_digest_hex": "c76b04b9be9caf759caad6ab35015f67f58236136aa594f04f37d4fd228effdd", + "negative_controls": { + "swap_a_b": [ + 2262001349, + 3860899954, + 4164161403, + 3498592193 + ], + "tag_changed": [ + 1916281729, + 3643551808, + 4246786223, + 1899024518 + ], + "tag_omitted": [ + 169795982, + 3716750026, + 229665778, + 2293597141 + ], + "truncate_high_half": [ + 322339573, + 4036273514, + 4258543439, + 3724512802 + ], + "flags_parent": [ + 145848328, + 2425291439, + 3761905320, + 4033610076 + ], + "block_len_64": [ + 158220690, + 38458862, + 2507042741, + 3368512297 + ], + "counter_one": [ + 882407196, + 611886641, + 815139260, + 363323504 + ], + "lanes_big_endian": [ + 2788372340, + 3245998644, + 3487891597, + 2434952568 + ], + "other_round_count": [ + 3558314982, + 1135936504, + 1880898970, + 561388701 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "b_one", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 1, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000010000000000000000000000000000004c464d43", + "digest": [ + 2262001349, + 3860899954, + 4164161403, + 3498592193 + ], + "digest_hex": "86d366c5e620a872f8340f7bd08847c1", + "full_blake3_digest_hex": "c566d38672a820e67b0f34f8c14788d04bdfd0fa1ab2d9631965cfb01294a0e1", + "negative_controls": { + "swap_a_b": [ + 3104074695, + 1974443198, + 2882972316, + 1734279477 + ], + "tag_changed": [ + 1623525419, + 2027791051, + 645660697, + 2862263606 + ], + "tag_omitted": [ + 4116228417, + 2803557933, + 1059784955, + 2589069092 + ], + "truncate_high_half": [ + 4207992651, + 1675211290, + 2966381849, + 3785397266 + ], + "flags_parent": [ + 3023217942, + 839028134, + 1763510021, + 1117934494 + ], + "block_len_64": [ + 717435685, + 450081066, + 952856761, + 3210949074 + ], + "counter_one": [ + 721270745, + 3523371657, + 96133770, + 1789110949 + ], + "lanes_big_endian": [ + 807383266, + 495744330, + 3547883087, + 469609088 + ], + "other_round_count": [ + 2449789754, + 3285089314, + 37558328, + 958692464 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "all_ones", + "a": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "b": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "message_bytes_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4c464d43", + "digest": [ + 512112064, + 2194416191, + 3337763018, + 1475985439 + ], + "digest_hex": "1e8635c082cc223fc6f238ca57f9c01f", + "full_blake3_digest_hex": "c035861e3f22cc82ca38f2c61fc0f95759f8100aa7ac004473813b9d103603a9", + "negative_controls": { + "swap_a_b": [ + 512112064, + 2194416191, + 3337763018, + 1475985439 + ], + "tag_changed": [ + 4246162828, + 2310173126, + 1791106969, + 1807798013 + ], + "tag_omitted": [ + 1626354843, + 2867904763, + 3796955409, + 4292745573 + ], + "truncate_high_half": [ + 168884313, + 1140894887, + 2637922675, + 2835559952 + ], + "flags_parent": [ + 2197539164, + 2836806235, + 3237076327, + 716093000 + ], + "block_len_64": [ + 3994438324, + 3770977968, + 923159964, + 3186013365 + ], + "counter_one": [ + 3179594704, + 2467983634, + 544141181, + 2332499776 + ], + "lanes_big_endian": [ + 512112064, + 2194416191, + 3337763018, + 1475985439 + ], + "other_round_count": [ + 481531477, + 1181580457, + 2833532478, + 1295194463 + ] + }, + "controls_inapplicable_here": [ + "swap_a_b", + "lanes_big_endian" + ] + }, + { + "name": "nibble_ramp", + "a": [ + 0, + 286331153, + 572662306, + 858993459 + ], + "b": [ + 1145324612, + 1431655765, + 1717986918, + 2004318071 + ], + "message_bytes_hex": "00000000111111112222222233333333444444445555555566666666777777774c464d43", + "digest": [ + 447364800, + 1725782825, + 3919861296, + 1641182463 + ], + "digest_hex": "1aaa3ec066dd5b29e9a4563061d274ff", + "full_blake3_digest_hex": "c03eaa1a295bdd663056a4e9ff74d261051f49096ec2345cde112bda36168bf4", + "negative_controls": { + "swap_a_b": [ + 2602287742, + 1091793620, + 3246283176, + 341235127 + ], + "tag_changed": [ + 2061842008, + 2579150968, + 3782433031, + 73447802 + ], + "tag_omitted": [ + 1151907280, + 1301059756, + 3107717143, + 2810812534 + ], + "truncate_high_half": [ + 155787013, + 1546961518, + 3660255710, + 4102755894 + ], + "flags_parent": [ + 927375832, + 955246354, + 3985860880, + 2792384868 + ], + "block_len_64": [ + 2889744962, + 2067974519, + 793298342, + 1312070483 + ], + "counter_one": [ + 1352679873, + 554860665, + 1539803498, + 4159849763 + ], + "lanes_big_endian": [ + 447364800, + 1725782825, + 3919861296, + 1641182463 + ], + "other_round_count": [ + 788131140, + 1263186933, + 1810255302, + 3669948337 + ] + }, + "controls_inapplicable_here": [ + "lanes_big_endian" + ] + }, + { + "name": "formula_0", + "a": [ + 2654435769, + 1013904242, + 3668340011, + 2027808484 + ], + "b": [ + 2415085441, + 774553914, + 3428989683, + 1788458156 + ], + "message_bytes_hex": "b979379e72f36e3c2b6da6dae4e6dd788147f38f3ac12a2ef33a62ccacb4996a4c464d43", + "digest": [ + 1352084339, + 1257553708, + 2471043581, + 979121606 + ], + "digest_hex": "50972b734af4bf2c934921fd3a5c35c6", + "full_blake3_digest_hex": "732b97502cbff44afd214993c6355c3ae28c2a8868f9ad4554aeaaa5e8d8dbce", + "negative_controls": { + "swap_a_b": [ + 1945415379, + 4233881026, + 3205850450, + 4225502794 + ], + "tag_changed": [ + 1294865967, + 1032901097, + 1011704452, + 3085477351 + ], + "tag_omitted": [ + 3078139872, + 12999010, + 1621284358, + 2487237378 + ], + "truncate_high_half": [ + 2284489954, + 1169029480, + 2779426388, + 3470514408 + ], + "flags_parent": [ + 2641397030, + 1190045648, + 2010263668, + 572684143 + ], + "block_len_64": [ + 2964721981, + 214229786, + 3613093280, + 3499581620 + ], + "counter_one": [ + 3113424487, + 3979443217, + 3196439285, + 1435039563 + ], + "lanes_big_endian": [ + 770521080, + 3808920109, + 2259917097, + 3234040021 + ], + "other_round_count": [ + 726891656, + 4201603114, + 1574841782, + 2067353180 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_1", + "a": [ + 2175735113, + 535203586, + 3189639355, + 1549107828 + ], + "b": [ + 1936384785, + 295853258, + 2950289027, + 1309757500 + ], + "message_bytes_hex": "4915af81028fe61fbb081ebe7482555c11e36a73ca5ca21183d6d9af3c50114e4c464d43", + "digest": [ + 132588438, + 2442761586, + 2176293190, + 1214338625 + ], + "digest_hex": "07e723969199957281b7994648615641", + "full_blake3_digest_hex": "9623e707729599914699b78141566148acaaa0be22af3bc572d4d70ecc760809", + "negative_controls": { + "swap_a_b": [ + 784664938, + 2292607292, + 1011932111, + 3939407295 + ], + "tag_changed": [ + 1870667933, + 2671355940, + 1641812716, + 2927333739 + ], + "tag_omitted": [ + 2468894809, + 3785140260, + 3811229258, + 234192523 + ], + "truncate_high_half": [ + 3198200492, + 3309023010, + 249025650, + 151549644 + ], + "flags_parent": [ + 71492567, + 2888704277, + 565558006, + 2931217710 + ], + "block_len_64": [ + 4125074361, + 314346409, + 3875765083, + 1721593340 + ], + "counter_one": [ + 3137348510, + 3273463828, + 199475780, + 2367118021 + ], + "lanes_big_endian": [ + 43718566, + 1786734403, + 1082101060, + 1581329095 + ], + "other_round_count": [ + 4027429195, + 1236283876, + 500926337, + 927998998 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_2", + "a": [ + 1697034457, + 56502930, + 2710938699, + 1070407172 + ], + "b": [ + 1457684129, + 4112119898, + 2471588371, + 831056844 + ], + "message_bytes_hex": "d9b02665922a5e034ba495a1041ecd3fa17ee2565af819f513725193cceb88314c464d43", + "digest": [ + 3416941098, + 205516775, + 837313723, + 1674917660 + ], + "digest_hex": "cbaa622a0c3fefe731e864bb63d5371c", + "full_blake3_digest_hex": "2a62aacbe7ef3f0cbb64e8311c37d563f4a286c7daca47453fbf704d69e008e9", + "negative_controls": { + "swap_a_b": [ + 196055030, + 2554370323, + 3619258027, + 2740973158 + ], + "tag_changed": [ + 444198939, + 1371325364, + 3350880783, + 2994263437 + ], + "tag_omitted": [ + 2935443957, + 1421721941, + 2106819797, + 2095796501 + ], + "truncate_high_half": [ + 3347489524, + 1162332890, + 1299234623, + 3909673065 + ], + "flags_parent": [ + 2124899262, + 690393886, + 12792001, + 1005978492 + ], + "block_len_64": [ + 1507204277, + 3462901758, + 1940019576, + 1431825061 + ], + "counter_one": [ + 3279102718, + 143731077, + 3940202508, + 1563588901 + ], + "lanes_big_endian": [ + 1952789763, + 3281899451, + 245929336, + 797892551 + ], + "other_round_count": [ + 2611692713, + 2586154225, + 3722034585, + 4051179927 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_3", + "a": [ + 1218333801, + 3872769570, + 2232238043, + 591706516 + ], + "b": [ + 978983473, + 3633419242, + 1992887715, + 352356188 + ], + "message_bytes_hex": "694c9e4822c6d5e6db3f0d8594b94423311a5a3aea9391d8a30dc9765c8700154c464d43", + "digest": [ + 2906955538, + 732030644, + 603925077, + 4269925458 + ], + "digest_hex": "ad449f122ba1e6b423ff2a55fe81e452", + "full_blake3_digest_hex": "129f44adb4e6a12b552aff2352e481fe23586932b0c1403a80db974efc2577c6", + "negative_controls": { + "swap_a_b": [ + 64496243, + 2475530138, + 2974881064, + 253548690 + ], + "tag_changed": [ + 2271702164, + 1363059701, + 2327392930, + 403741379 + ], + "tag_omitted": [ + 4153517033, + 3953988304, + 4013544589, + 3676570025 + ], + "truncate_high_half": [ + 845764643, + 977322416, + 1318574976, + 3329697276 + ], + "flags_parent": [ + 3992715041, + 584964376, + 951636984, + 2488715381 + ], + "block_len_64": [ + 267090884, + 2577717240, + 3104375143, + 3638326480 + ], + "counter_one": [ + 1600424835, + 187457470, + 1895800034, + 4240360861 + ], + "lanes_big_endian": [ + 4248123175, + 196848378, + 74484507, + 924057161 + ], + "other_round_count": [ + 3285322819, + 1763914470, + 2042156769, + 3152015512 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_4", + "a": [ + 739633145, + 3394068914, + 1753537387, + 113005860 + ], + "b": [ + 500282817, + 3154718586, + 1514187059, + 4168622828 + ], + "message_bytes_hex": "f9e7152cb2614dca6bdb84682455bc06c1b5d11d7a2f09bc33a9405aec2278f84c464d43", + "digest": [ + 350076932, + 3861935018, + 145291395, + 3827942314 + ], + "digest_hex": "14ddc004e63073aa08a8f883e429c3aa", + "full_blake3_digest_hex": "04c0dd14aa7330e683f8a808aac329e44b6cbe0286d893d0a4442e6d603e97d3", + "negative_controls": { + "swap_a_b": [ + 2106642346, + 269327270, + 3907080289, + 1615491014 + ], + "tag_changed": [ + 1715528356, + 3503327777, + 2947820293, + 261431188 + ], + "tag_omitted": [ + 3654800449, + 4121041585, + 3685919015, + 3747735280 + ], + "truncate_high_half": [ + 46033995, + 3499350150, + 1831748772, + 3549904480 + ], + "flags_parent": [ + 2955091651, + 2548381447, + 296019950, + 818342147 + ], + "block_len_64": [ + 3084476732, + 3706341510, + 1958100940, + 974907637 + ], + "counter_one": [ + 3227835229, + 2269625861, + 536534159, + 3526011742 + ], + "lanes_big_endian": [ + 3137720933, + 909148841, + 1074048064, + 1686216241 + ], + "other_round_count": [ + 3930423792, + 2783591612, + 2006900685, + 3295138076 + ] + }, + "controls_inapplicable_here": [] + } + ], + "6": [ + { + "name": "zeros", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 2809853715, + 2395900105, + 421057723, + 4135460974 + ], + "digest_hex": "a77af7138ece88c91918d4bbf67e206e", + "full_blake3_digest_hex": "13f77aa7c988ce8ebbd418196e207ef6a3b9cefd6055504eb6de0f527873cc75", + "negative_controls": { + "swap_a_b": [ + 2809853715, + 2395900105, + 421057723, + 4135460974 + ], + "tag_changed": [ + 4093095823, + 4263061468, + 268330994, + 2625113450 + ], + "tag_omitted": [ + 3048415149, + 1475892664, + 1644902263, + 1911052230 + ], + "truncate_high_half": [ + 4258183587, + 1313887584, + 1376771766, + 1976333176 + ], + "flags_parent": [ + 341972669, + 3029577580, + 3331485068, + 3799596966 + ], + "block_len_64": [ + 745558093, + 847667665, + 1887832086, + 1953676804 + ], + "counter_one": [ + 2494159201, + 2176068449, + 2989284609, + 2427558519 + ], + "lanes_big_endian": [ + 2809853715, + 2395900105, + 421057723, + 4135460974 + ], + "other_round_count": [ + 2494038600, + 807496444, + 2349420159, + 3886468141 + ] + }, + "controls_inapplicable_here": [ + "swap_a_b", + "lanes_big_endian" + ] + }, + { + "name": "a_one", + "a": [ + 1, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "01000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 3558314982, + 1135936504, + 1880898970, + 561388701 + ], + "digest_hex": "d41793e643b503f8701c3d9a21761c9d", + "full_blake3_digest_hex": "e69317d4f803b5439a3d1c709d1c762185faa4d14a4cdfb0a1b577eda9d21518", + "negative_controls": { + "swap_a_b": [ + 2449789754, + 3285089314, + 37558328, + 958692464 + ], + "tag_changed": [ + 3622704346, + 1006447733, + 1471928985, + 2679991511 + ], + "tag_omitted": [ + 44512210, + 2202275017, + 3058762027, + 2963133862 + ], + "truncate_high_half": [ + 3517250181, + 2967424074, + 3984045473, + 404083369 + ], + "flags_parent": [ + 1328012960, + 1441905265, + 1566563243, + 2259864765 + ], + "block_len_64": [ + 3456350827, + 2465378020, + 186883414, + 1308251957 + ], + "counter_one": [ + 93437251, + 1279073643, + 3668904414, + 1584500791 + ], + "lanes_big_endian": [ + 968514830, + 3275024172, + 3366063996, + 1833349798 + ], + "other_round_count": [ + 3104074695, + 1974443198, + 2882972316, + 1734279477 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "b_one", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 1, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000010000000000000000000000000000004c464d43", + "digest": [ + 2449789754, + 3285089314, + 37558328, + 958692464 + ], + "digest_hex": "9204d33ac3ce7c22023d183839247c70", + "full_blake3_digest_hex": "3ad30492227ccec338183d02707c2439ad24af458d05428457fe5921792060f5", + "negative_controls": { + "swap_a_b": [ + 3558314982, + 1135936504, + 1880898970, + 561388701 + ], + "tag_changed": [ + 2298662015, + 301921525, + 3622891912, + 261141954 + ], + "tag_omitted": [ + 3334490412, + 785885940, + 762612797, + 3735107846 + ], + "truncate_high_half": [ + 1169106093, + 2218919309, + 559545943, + 4116717689 + ], + "flags_parent": [ + 3192145441, + 4255669687, + 2673121513, + 3488853830 + ], + "block_len_64": [ + 353166171, + 4055598462, + 3109386211, + 1771168700 + ], + "counter_one": [ + 1279418886, + 3747766226, + 1287549188, + 943360911 + ], + "lanes_big_endian": [ + 363086977, + 3234472873, + 2775555316, + 1032517280 + ], + "other_round_count": [ + 2262001349, + 3860899954, + 4164161403, + 3498592193 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "all_ones", + "a": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "b": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "message_bytes_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4c464d43", + "digest": [ + 481531477, + 1181580457, + 2833532478, + 1295194463 + ], + "digest_hex": "1cb39655466d7ca9a8e4463e4d33195f", + "full_blake3_digest_hex": "5596b31ca97c6d463e46e4a85f19334d7f6c9c4d7425904412031d12ec8b3bc1", + "negative_controls": { + "swap_a_b": [ + 481531477, + 1181580457, + 2833532478, + 1295194463 + ], + "tag_changed": [ + 2216560796, + 2174319362, + 194620432, + 1566427869 + ], + "tag_omitted": [ + 1621915756, + 1110253212, + 3230662765, + 2725076756 + ], + "truncate_high_half": [ + 1302097023, + 1150297460, + 303891218, + 3241905132 + ], + "flags_parent": [ + 1186841718, + 1963879649, + 3721767533, + 3500910472 + ], + "block_len_64": [ + 2207239952, + 2959686046, + 2781780849, + 1147654384 + ], + "counter_one": [ + 222281700, + 3982136183, + 733001904, + 2856569378 + ], + "lanes_big_endian": [ + 481531477, + 1181580457, + 2833532478, + 1295194463 + ], + "other_round_count": [ + 512112064, + 2194416191, + 3337763018, + 1475985439 + ] + }, + "controls_inapplicable_here": [ + "swap_a_b", + "lanes_big_endian" + ] + }, + { + "name": "nibble_ramp", + "a": [ + 0, + 286331153, + 572662306, + 858993459 + ], + "b": [ + 1145324612, + 1431655765, + 1717986918, + 2004318071 + ], + "message_bytes_hex": "00000000111111112222222233333333444444445555555566666666777777774c464d43", + "digest": [ + 788131140, + 1263186933, + 1810255302, + 3669948337 + ], + "digest_hex": "2ef9ed444b4ab3f56be64dc6dabef7b1", + "full_blake3_digest_hex": "44edf92ef5b34a4bc64de66bb1f7beda7fcc336c120bacd6f6abfed12df8d48e", + "negative_controls": { + "swap_a_b": [ + 235231114, + 1417868809, + 612301685, + 460014826 + ], + "tag_changed": [ + 597796601, + 3081433456, + 3193325980, + 3163013601 + ], + "tag_omitted": [ + 2859869993, + 416987530, + 2322331240, + 2564012397 + ], + "truncate_high_half": [ + 1815334015, + 3601599250, + 3523128310, + 2396321837 + ], + "flags_parent": [ + 2467882701, + 2267974464, + 3469229345, + 495033823 + ], + "block_len_64": [ + 14549805, + 2526977084, + 2722498859, + 1100477395 + ], + "counter_one": [ + 2043336428, + 1493534113, + 513715177, + 4060124582 + ], + "lanes_big_endian": [ + 788131140, + 1263186933, + 1810255302, + 3669948337 + ], + "other_round_count": [ + 447364800, + 1725782825, + 3919861296, + 1641182463 + ] + }, + "controls_inapplicable_here": [ + "lanes_big_endian" + ] + }, + { + "name": "formula_0", + "a": [ + 2654435769, + 1013904242, + 3668340011, + 2027808484 + ], + "b": [ + 2415085441, + 774553914, + 3428989683, + 1788458156 + ], + "message_bytes_hex": "b979379e72f36e3c2b6da6dae4e6dd788147f38f3ac12a2ef33a62ccacb4996a4c464d43", + "digest": [ + 726891656, + 4201603114, + 1574841782, + 2067353180 + ], + "digest_hex": "2b537c88fa6f602a5dde2db67b394e5c", + "full_blake3_digest_hex": "887c532b2a606ffab62dde5d5c4e397bcdaeb853fa024dc45e2d7e4ed0c3b16a", + "negative_controls": { + "swap_a_b": [ + 2287757126, + 2522288221, + 4204507564, + 1510858295 + ], + "tag_changed": [ + 1411093247, + 3277560855, + 2299820010, + 4220809803 + ], + "tag_omitted": [ + 1630871002, + 701634033, + 3668128791, + 2020914271 + ], + "truncate_high_half": [ + 1404612301, + 3293381370, + 1316891998, + 1790034896 + ], + "flags_parent": [ + 1783774852, + 1531870232, + 1670761061, + 912951135 + ], + "block_len_64": [ + 4137240477, + 2307323988, + 1234004509, + 3293559429 + ], + "counter_one": [ + 170937440, + 842443068, + 1967688488, + 3097257515 + ], + "lanes_big_endian": [ + 133464992, + 2020637449, + 2772550485, + 303181699 + ], + "other_round_count": [ + 1352084339, + 1257553708, + 2471043581, + 979121606 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_1", + "a": [ + 2175735113, + 535203586, + 3189639355, + 1549107828 + ], + "b": [ + 1936384785, + 295853258, + 2950289027, + 1309757500 + ], + "message_bytes_hex": "4915af81028fe61fbb081ebe7482555c11e36a73ca5ca21183d6d9af3c50114e4c464d43", + "digest": [ + 4027429195, + 1236283876, + 500926337, + 927998998 + ], + "digest_hex": "f00db14b49b031e41ddb878137502416", + "full_blake3_digest_hex": "4bb10df0e431b0498187db1d16245037d4c6c5332c1c2b954b8688d4693c3ce0", + "negative_controls": { + "swap_a_b": [ + 1061534104, + 3762300316, + 3860312183, + 3218150697 + ], + "tag_changed": [ + 1575757937, + 2953684583, + 433194671, + 2827605630 + ], + "tag_omitted": [ + 3522593650, + 2450106593, + 1465843341, + 1493459227 + ], + "truncate_high_half": [ + 868599508, + 2502630444, + 3565717067, + 3762044009 + ], + "flags_parent": [ + 3724301157, + 756354888, + 3094893203, + 3765926751 + ], + "block_len_64": [ + 4257426834, + 746029597, + 1224128594, + 2827557058 + ], + "counter_one": [ + 3357268267, + 696461381, + 3251232782, + 3588835807 + ], + "lanes_big_endian": [ + 3689163316, + 2265303062, + 2400643333, + 2523490381 + ], + "other_round_count": [ + 132588438, + 2442761586, + 2176293190, + 1214338625 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_2", + "a": [ + 1697034457, + 56502930, + 2710938699, + 1070407172 + ], + "b": [ + 1457684129, + 4112119898, + 2471588371, + 831056844 + ], + "message_bytes_hex": "d9b02665922a5e034ba495a1041ecd3fa17ee2565af819f513725193cceb88314c464d43", + "digest": [ + 2611692713, + 2586154225, + 3722034585, + 4051179927 + ], + "digest_hex": "9bab44a99a2594f1ddd9bd99f1781997", + "full_blake3_digest_hex": "a944ab9bf194259a99bdd9dd971978f1e7d3b29dbafbb19744d6703b4ea1d754", + "negative_controls": { + "swap_a_b": [ + 3305415968, + 3486437672, + 772494840, + 3778948088 + ], + "tag_changed": [ + 3549645016, + 642745547, + 951922256, + 2481436659 + ], + "tag_omitted": [ + 3144682363, + 3316019969, + 1507385536, + 3674770217 + ], + "truncate_high_half": [ + 2645742567, + 2545023930, + 997250628, + 1423417678 + ], + "flags_parent": [ + 920766747, + 7894494, + 4106111544, + 254850763 + ], + "block_len_64": [ + 1246838896, + 4080312985, + 3870129443, + 157857948 + ], + "counter_one": [ + 1679366029, + 2395353215, + 2098738463, + 3973155908 + ], + "lanes_big_endian": [ + 3443814819, + 1045146291, + 3770179767, + 3855527815 + ], + "other_round_count": [ + 3416941098, + 205516775, + 837313723, + 1674917660 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_3", + "a": [ + 1218333801, + 3872769570, + 2232238043, + 591706516 + ], + "b": [ + 978983473, + 3633419242, + 1992887715, + 352356188 + ], + "message_bytes_hex": "694c9e4822c6d5e6db3f0d8594b94423311a5a3aea9391d8a30dc9765c8700154c464d43", + "digest": [ + 3285322819, + 1763914470, + 2042156769, + 3152015512 + ], + "digest_hex": "c3d20c43692332e679b8d6e1bbdff098", + "full_blake3_digest_hex": "430cd2c3e6322369e1d6b87998f0dfbbb64224c81369e3f1c605929c54ad12f2", + "negative_controls": { + "swap_a_b": [ + 1894497042, + 14191344, + 1924318259, + 737778920 + ], + "tag_changed": [ + 3270827201, + 483846566, + 529175613, + 4072304835 + ], + "tag_omitted": [ + 100009501, + 4048472805, + 2176299551, + 4036661407 + ], + "truncate_high_half": [ + 3357819574, + 4058212627, + 2626815430, + 4061310292 + ], + "flags_parent": [ + 3522347107, + 1612075529, + 2397114859, + 726720979 + ], + "block_len_64": [ + 718521719, + 2132472759, + 810498, + 538747873 + ], + "counter_one": [ + 1527925611, + 2342327938, + 4029544076, + 3095335299 + ], + "lanes_big_endian": [ + 316622236, + 4054949641, + 3797035992, + 713045086 + ], + "other_round_count": [ + 2906955538, + 732030644, + 603925077, + 4269925458 + ] + }, + "controls_inapplicable_here": [] + }, + { + "name": "formula_4", + "a": [ + 739633145, + 3394068914, + 1753537387, + 113005860 + ], + "b": [ + 500282817, + 3154718586, + 1514187059, + 4168622828 + ], + "message_bytes_hex": "f9e7152cb2614dca6bdb84682455bc06c1b5d11d7a2f09bc33a9405aec2278f84c464d43", + "digest": [ + 3930423792, + 2783591612, + 2006900685, + 3295138076 + ], + "digest_hex": "ea4581f0a5ea3cbc779edfcdc467d11c", + "full_blake3_digest_hex": "f08145eabc3ceaa5cddf9e771cd167c4beb7838fb3b42e33d4fdd97c9ed809b2", + "negative_controls": { + "swap_a_b": [ + 2910938883, + 1984189902, + 983406695, + 2776337861 + ], + "tag_changed": [ + 3241307008, + 3697093809, + 996298134, + 91673458 + ], + "tag_omitted": [ + 888505490, + 3797579733, + 3762181083, + 597697236 + ], + "truncate_high_half": [ + 2407774142, + 858698931, + 2094661076, + 2986989726 + ], + "flags_parent": [ + 2554185738, + 2606785956, + 3516944530, + 3839186671 + ], + "block_len_64": [ + 3163081460, + 1161555519, + 2564356656, + 1671790406 + ], + "counter_one": [ + 2200533718, + 3875724195, + 852857365, + 932216256 + ], + "lanes_big_endian": [ + 2200313026, + 1190982824, + 2107398321, + 3711166871 + ], + "other_round_count": [ + 350076932, + 3861935018, + 145291395, + 3827942314 + ] + }, + "controls_inapplicable_here": [] + } + ] + } +} \ No newline at end of file diff --git a/thoughts/shared/HANDOFF-2026-08-12.md b/thoughts/shared/HANDOFF-2026-08-12.md new file mode 100644 index 000000000..e25ab2381 --- /dev/null +++ b/thoughts/shared/HANDOFF-2026-08-12.md @@ -0,0 +1,126 @@ +# Session handoff — 2026-08-12 + +Resume point after a long session. Read top-to-bottom. User preferences: **no AI attribution** +in any commit/PR (global rule); **benches run on the remote server** (hand the command, never +run locally); user wants me to **orchestrate/delegate and keep context lean**; user is the repo +owner (MauroToscano/MauroFab) — pushing to their branches/PRs is authorized; "we'll do more +reviews at the end" — reviews are deferred, don't over-gate. + +Repo: `yetanotherco/lambda_vm`. Main working dir `/Users/maurofab/workspace/lambda_vm` (on `main`). + +--- + +## 1. DONE THIS SESSION — the headline + +### ★ The BLAKE3 real-hash campaign is COMPLETE and now MERGED WITH main. PR #930. +Branch **`blake3-real-hash`** (worktree `/Users/maurofab/workspace/lambda_vm-blake3-impl`), pushed, +**PR #930 (draft, base main, MERGEABLE)**. Current tip **`fe7314b3`** (docs) on top of merge commit +**`253504d5`** (2-parent: pre-merge tip `ed1b7785` + main tip `58160b6f`). + +**F3.4 is retired** — the LFM machine's role-2 hash is real BLAKE3 across every domain (Merkle +parents `LFMC`, FRI leaves `LFML` felt-input, Fiat–Shamir transcript `LFMT` compress-chain), each +tagged/prover-unchosen/z3-gated. Chip gate PASS 86/86. Both registered programs (TrivialV0, +FriToyV0) prove+verify under BLAKE3. Full campaign detail in memory `[[lfm-real-hash-blake3]]` and +`thoughts/shared/lfm-real-hash/` (PLAN, phase reports, A6R-signoff, gate-oracle/{ORACLE,CHIP-GATE}.md, +transcript-spec/, leaf-spec/). + +### ★ The main merge — what "up to date with main" actually required +Bringing the branch (the whole unmerged LFM feature line) up to date was NOT just conflicts — main +had *evolved* two things the branch's forks depend on: +1. **Constraint-IR device redesign.** Main changed operands from node-index to OPK-tagged slots and + made `lower()` lossy. The branch's build-time artifact feature was reconciled via **approach A**: + the artifact owns a node-index POD `ArtifactNode` and re-derives the device blob through main's + own `lower()`. Round-trip 11/11. +2. **KECCAK_RND dropped 120 θ/ρ HWSL sends/round** (→ inline μ-gated identities). The LFM machine's + **forked** receiver-side collector `prover/src/lfm/keccak_adapter.rs::bitwise_ops_for` still + emitted them → LogUp bus imbalance broke **all 20 keccak machine/fri/join tests**. Fix = sync the + fork (drop the 120, count `1148→1028`). + +**The #2 root cause was found by a user-requested 4-agent debate** (2 defend / 1 attack / 1 judge) — +records in `thoughts/shared/lfm-real-hash/merge-plan/` (FIX-PLAN.md, debate-defender-{A,B}.md, +debate-attacker.md, JUDGE-VERDICT.md; also main-ir-spec.md, artifact-feature-map.md, reconcile-report.md). +**Critical lesson: my ORIGINAL diagnosis was wrong and would have been a SOUNDNESS REGRESSION** — +patching the verifier's `expected_public_balance` to "make the numbers agree" would have folded an +unmatched-bus residual into the recursion machine's only cross-table check, blinding it, with every +negative control staying green. The attacker found the real cause; the judge verified it link-by-link +and cleared the correct, fail-closed fix. **Run an adversary before touching soundness code.** + +Also: 5 conflicts resolved (lookup.rs Arc+precaptured, continuation.rs, 3 IR test files), HINT +coverage added, and test-expectations updated to main's semantics (preprocessed_tags → verify-time +rejection via main's precomputed-tree cache [probe-confirmed verify=false]; HINT design count 418 / +KECCAK_RND 14016→12998; epoch budget 63393→62375 attributed **entirely** to KECCAK_RND's HWSL→inline +swap [gate G3]; private-page follows main's OFFSET soundness fix). + +**Validation (all green, zero regressions):** lfm:: **306/19** (the 19 = pre-existing fibonacci.elf +fixtures, identical to pre-merge baseline); stark #909 opening_width/aux_opening_width **15/15**; +artifact round-trip 11/11; make fmt+lint clean. Pristine pre-merge tip tagged+pushed +**`blake3-campaign-preMerge`** (=ed1b7785). + +--- + +## 2. THE ACTIVE NEXT THING — GPU for LFM recursion (user is providing a machine "soon") + +User asked to explore GPU-in-recursion. Deliverable: **`thoughts/shared/gpu-recursion/EXPLORATION.md`** +(read §0). ⚠ **My first relay of this to the user was STALE — corrected after the trace-height census. +Use the corrected headline:** + +- The old "LFM gets ~zero GPU (preprocessed exclusion)" premise is **obsolete** — main's #863 + (split-tree GPU path for preprocessed tables) + #875 (device-resident rounds 2-4) already fixed most + of it; neither is preprocessed-gated. +- **THE REAL LEVER (verified):** the GPU LDE gate counts **ROWS ONLY** — `lde_size = n × blowup`, no + column term (`crypto/stark/src/gpu_lde.rs`, 3 sites; default threshold `1<<19`). LFM chips are + short+wide, so the gate **admits tall-small BITWISE (2^20 rows, 4.8% of the wrap's cells) and + REFUSES huge KECCAK_RND (2^17 rows × 1480 cols, ~84% of the wrap)**. KECCAK_RND is ALSO the one + NON-preprocessed chip (`airs.rs:68`), so putting it on GPU has no preprocessed complication. +- **Headline experiment, ZERO code:** `LAMBDA_VM_GPU_LDE_THRESHOLD=262144` (2^18) lets KECCAK_RND's LDE + clear the gate → the 84%-of-cost chip goes fully device-resident. **This is Stage 0 on the box.** +- Secondary lever: the D2H skip (`&& !is_preprocessed`), now ≳1.2 GiB (LFM_BALU at 2^21 also clears). +- ⚠ Reasoning about recursion from the toy registered programs (trivial/sponge) is a CATEGORY ERROR + (there BITWISE is 97-99%; in the real wrap it's 4.8%). **The box plan targets `lfm::wrap_tests` + (`#[ignore]d`, run with `--ignored`), NOT the toy programs.** + +Memory `[[lfm-real-hash-blake3]]` GPU line is updated. Full staged plan + box commands in EXPLORATION.md. + +--- + +## 3. OPEN ITEMS / NEXT STEPS +- **GPU box work** (when the machine arrives): run EXPLORATION.md Stage 0 first (the one-env-var + falsification test on `lfm::wrap_tests --ignored`). It either confirms the row-only-gate story or + corrects it before any code. +- **Clean-history curation** of the merge: it's currently ONE comprehensive merge commit (253504d5). + User said we'd curate later. If splitting for the real landing, the natural seam is + reconciliation-code vs test-expectation-updates (mixed in constraint_tests.rs / constraint_artifact_tests.rs + → needs `git add -p`). +- **origin/main advanced** slightly past 58160b6f since the merge; PR #930 is still MERGEABLE. A future + re-merge picks up newer main. +- **`blake3-real-hash-mainmerge`** (worktree `/Users/maurofab/workspace/lambda_vm-blake3-merge`) is a + throwaway branch now equal to the merge — safe to `git worktree remove` + delete the branch. +- **Merge precondition already satisfied by the merge:** #909's opening-width pin is now in the + branch's ancestry (came in with main), so the standing "rebase past #909 + re-run M-controls" + precondition is discharged; the M-controls all pass under the merged verifier. +- Older parked items (from the 2026-08-10 handoff, may be stale): PR #915 rebase/re-wrap; PR #912 + (keccak sponge, merged); PR #923 (keccak FV baseline). Check their state before acting. + +--- + +## 4. WORKTREES / BRANCHES / TAGS +- `/Users/maurofab/workspace/lambda_vm` — main (`528a8411`+). The debate/GPU docs also live here + untracked, but are now ALSO committed on the branch (fe7314b3) so they're safe. +- `/Users/maurofab/workspace/lambda_vm-blake3-impl` — **`blake3-real-hash` @ fe7314b3** (THE branch, PR #930). +- `/Users/maurofab/workspace/lambda_vm-blake3-merge` — `blake3-real-hash-mainmerge` @ 253504d5 (throwaway). +- `/Users/maurofab/workspace/lambda_vm-pr915` — `pr915`. +- `/Users/maurofab/workspace/lambda_vm-sponge` — `keccak-sponge-spec-verify` (PR #912). +- `/Users/maurofab/workspace/lambda_vm-lfm-gpu` — `lfm-gpu-experiments`. +- Tags: **`blake3-campaign-preMerge`** (=ed1b7785, pushed) — the recoverable pristine campaign tip. + +--- + +## 5. KEY LESSONS BANKED THIS SESSION +- **Adversarial debate before soundness edits.** The 4-agent debate caught that my merge fix would + blind the recursion verifier's cross-table check. Both defenders (honest) + the attacker converged; + the judge verified. This is the pattern to reuse for any soundness-adjacent change. +- **"main IR remains, the branch fork adapts"** — the reconciliation direction for every part of the + merge (artifact device-IR, keccak HWSL). Main's production code is authoritative. +- **Don't reason about recursion from toy programs** (the GPU BITWISE-vs-KECCAK_RND category error). +- **A stale relay is a real error** — re-brief when a delegated finding is revised; verify a + correction in code before relaying it (I relayed a wrong GPU headline, then corrected it). diff --git a/thoughts/shared/block-compression/BLAKE3-COST-MODEL.md b/thoughts/shared/block-compression/BLAKE3-COST-MODEL.md new file mode 100644 index 000000000..3896fbc84 --- /dev/null +++ b/thoughts/shared/block-compression/BLAKE3-COST-MODEL.md @@ -0,0 +1,125 @@ +# BLAKE3 in-circuit cost model — handover notes for the crypto team's split work + +Verified against `blake3-real-hash` (2026-08-13); every figure code-cited in the full +report (session record). Context: the team owns the algebraic/hash split; this is what +the machine-side measurements say about where the cost lives and what moves it. + +## The numbers (cells per compression = main + 3·⌈interactions/2⌉) + +| | standalone LFM_BLAKE3 (probe) | **LFM_HASH socket arm (what runs)** | #903 syscall chip | +|---|---|---|---| +| main cols @6r | 3,056 | **2,964** | 3,219 | +| interactions @6r | 1,259 | 1,190 | 1,397 | +| **cells/compression @6r** | 4,946 | **4,749** | 5,316 | +| cells/compression @7r | 5,714 | 5,517 | n/a | + +One row per compression, fully unrolled (48 G-blocks side by side). ⚠ Stale docs: +`blake3_probe.rs:392` + `phase2-report.md` say 4,741/5,509 — off by 8 (the option-C +canonicity block); the pinned tests carry the correct 4,749/5,517. + +## Where the cost lives + +- **The G core is 97.0%** of the socket's bill: 48 G × (60 main + 24 interactions) = + 96 base-equiv cells per G. +- **The representation is 8-bit limbs, and byte cells are 92.3% of main.** There is no + separate range-check family — bytes are bound by the XOR lookup itself (operands and + output), with explicit AreBytes sends only where no XOR consumes a value. +- What's already FREE and should stay free in any redesign: rotr16/rotr8 (pure byte + relabel — the payoff of byte limbs), the message permutation (index bookkeeping, no + copies), v-state columns (aliased, zero dedicated), add2 carries (an expression), + the m[8] tag (a linear form over preprocessed selectors). +- The socket contract's overhead is negligible: the LFM_HASH bus = 6 of 1,190 + interactions (0.5%); 16 of 28 shared columns are dead weight (0.54% of main). + +## What moves the number (the design question for the split circuit) + +1. **Reshaping does NOT**: cells/compression is invariant under packing (round-per-row + re-adds state/message carry columns; aspect ratio changes, the bill doesn't). +2. **Floor of the CURRENT primitive (byte-pair XOR table): ≈ 3,981 (−16%)** — via a + ternary add3 carry (degree-4, only legal on an ungated dedicated chip) and halfword + shift witnesses (needs a u16 range table). +3. **The real lever: the XOR primitive.** 64.8% of main and 64.5% of interactions are + byte-XOR-forced. A **16-bit limb design over a 16-bit XOR lookup** lands ≈ **2,200 + cells/compression (−54%)** — the one representation change that matters, and it is a + table-cost conversation (2^32-entry XOR table vs today's byte-pair table), i.e. + exactly the kind of trade a dedicated split-out hash circuit can make and the + general machine cannot. + +## Prior art for the split glue: SP1 v6's deferred shards (surveyed, code-verified) + +How SP1 moves precompile work (incl. keccak) into separate shards and binds them soundly +— the direct prior art for any split design: + +- **SP1 does not SOLVE the cross-proof-challenge problem — it SIDESTEPS it.** Their + LogUp challenges are per-shard, sampled after that shard's commitment; the cross-shard + bus is a **challenge-free, group-homomorphic multiset accumulator** (hash each message + to an EC point, negate sends, sum) — not a Schwartz–Zippel fingerprint, so nothing is + adaptively choosable and shard proving ORDER is irrelevant. That property is what any + port must preserve. Load-bearing caveat: the verifier's chip-cluster whitelist is what + stops a shard from simply omitting the accumulator chip. +- **The glue is NOT a cross-shard LogUp** (that's a dead enum in v6). Every boundary- + crossing event (syscall dispatch, memory init/finalize) is emitted twice — send in one + shard, receive in the other — hashed to a septic-extension elliptic-curve point + (domain-separated by kind, negated on send), summed per-shard into a public + `global_cumulative_sum`, and the whole-proof check is one equation: Σ over shards + + the program's memory-image digest = the fixed identity point. Aggregation re-sums it + in-circuit; the root asserts 14 felt equalities. +- **Measured glue cost:** the Global chip is 241 columns, of which the in-circuit + **Poseidon2 hash-to-curve is 74%**; per deferred keccak permutation the glue is + ~12.8k cells ≈ **17% of the precompile shard**. Precompiles that stay in-shard + ("retained": sha256, poseidon2, bn254/bls fp, u256) pay zero glue. +- **★ The design tension for OUR split:** SP1's cross-shard binding itself rests on + Poseidon2 (the hash-to-curve). Under the no-algebraic-hash posture that motivates our + whole blake3 direction, copying this glue re-imports the assumption we're escaping. + A blake-consistent split needs either (a) a different accumulator (e.g. an EC digest + with a non-algebraic hash-to-curve — costs more in-circuit), (b) LogUp-style + cross-proof accounting with shared challenges (the commit-all-then-prove barrier our + epoch-local FS design deliberately avoids — see streaming-proving-vs-zisk), or (c) + accepting Poseidon2 in the GLUE only, with the assumption scoped and documented. + This is the first decision the split design must make. +- Also transferable: SP1's per-syscall shard-sizing (row thresholds per cost table), + the "shard is transparent to the state chain" trick (non-execution shards pin + timestamp/pc so contiguity passes through), and the verifier's cluster allowlist + (a shard cannot omit the glue chip). + +## The backwards target: required cells/compression per fleet budget + +Block 25368371, arity-4 tower, throughput anchor MEASURED (481M cells / 7.17s on one +5090). "Residue" = the machine's felt-marshalling arithmetic around the hash calls — +tracks felts absorbed, untouched by ANY hash lever (calibrated at 439 cells/felt from +the measured chip census). `--` = the residue ALONE exceeds the budget: no hash chip, +however cheap, fits at that fleet size. Fleet budgets: 4/8/16/32/64 GPUs = 3.22/6.44/ +12.89/25.78/51.56 B cells/block. Reproduction: bench_cache/hash_split_2026-08-13/handover.py. + +Best configuration family (2^23 epochs, batching ON): + +| inner preset | RATE | comps/block | residue | required @64 GPU | +|---|---|---|---|---| +| blowup2/219q | 4 | 19.0M | 35.4 B | 849 | +| blowup2/219q | 8 | 10.2M | 35.4 B | 1,579 | +| blowup4/110q | 4 | 14.0M | 26.2 B | 1,816 | +| blowup4/110q | 8 | 7.5M | 26.2 B | 3,392 | + +All 4-32 GPU cells are `--` in every configuration measured. Verified candidates to +compare against: socket 4,749 @6r / **5,517 @7r (the DEFAULT build)**; standalone chip +4,946/5,714; #903 accelerator 5,316; plausible floor within the byte-decomposed family +~4,060. + +**The one-line conclusion for the table: per-compression cost is not the binding term.** +The required figures only leave `--` once batching lands AND epochs are large, and even +then they sit below every buildable candidate — what binds is the residue, i.e. the +machine's own arithmetic around the hash, which responds only to reducing FELTS +ABSORBED (narrower inner tables, fewer queries, single-row leaves — ROWS_PER_LEAF=2 +verified at commitment.rs:42, halving it halves the dominant term) and to row-shape +(D12: verification cost ∝ width, invariant proving cost — tower node 104→69 GiB @RATE 8). + +## Boundary conditions from the campaign (decided/measured) + +- Commitment digests stay **256-bit** (128-bit truncation = 64-bit collision bound, + below the 128-bit security floor — R2). +- **6 rounds** is the chosen variant (Mauro), −16% vs 7r; A6R ratification formally owed. +- The circuit's WIDTH is paid twice in recursion: the tower re-absorbs the hash chip's + own trace at every layer (57% of a tower node's leaf bill today) — column count + matters more than row count for the split circuit's downstream cost. +- RATE-4 leaf construction (in implementation) doubles felts-per-compression on the + absorption side — orthogonal to per-compression cost; both compose. diff --git a/thoughts/shared/block-compression/CENSUS.md b/thoughts/shared/block-compression/CENSUS.md new file mode 100644 index 000000000..c55bdfe0e --- /dev/null +++ b/thoughts/shared/block-compression/CENSUS.md @@ -0,0 +1,981 @@ +# Stage A — CENSUS FIT MAP + +**Result: Gate A FAILS at every point.** No `(epoch size, preset)` in the sweep fits +93 GiB (box) or 110 GiB (rigs) under keccak-inner. The cheapest real-block point +overshoots by 13×; the worst by 78×. + +Measured 2026-08-12 on the rented GPU box (79.161.122.162, 32 cores / 93 GiB / RTX +5090), repo `/root/lambda_vm` @ `blake3-real-hash`. + +**★ All artifacts are checkpointed OFF the rented box** (it will not outlive the +campaign) at `~/workspace/lambda_vm_bench_cache/lfm_census_2026-08-12/`: + +| file | what | +|---|---| +| `census_harness.diff` | the 374-line harness — **re-derive the reviewed commit from this** | +| `census_results.md` | all points + verdict tables as produced | +| `census_logs/` | 28 raw per-point logs, incl. full leg-shape dumps | +| `project.py`, `fitmap.py`, `final.py` | analytic chip model + Gate-A projections | +| `tower.py` | the Gate-D1 tower model | +| `census_run.sh`, `sweep.sh` | runners — `census_run.sh ` reproduces any point | + +On the box itself (while it lives): the same files under `/root/`. + +**Inner workload:** `ethrex.elf` (sha `133816f0`) + real mainnet block **25368371** +(`ethrex_mainnet_25368371.bin`, 1,110,156 B, sha `61eba49b`). Every ethrex point +censuses **epoch 0** of that block. Per-epoch profiles vary across a block, so a +later epoch would move the trace-length profile somewhat — but not the verdict, +which fails by more than an order of magnitude. + +--- + +## 1. The fit map + +Projected peak = census cells × 33.7 bytes/cell (`MEASURED_BYTES_PER_CELL`, the +slice-0 anchor: 481,327,124 cells → 15.1 GiB RSS). + +| epoch | sub-proofs | preset | total keccak perms | KECCAK_RND chunks | cells | projected peak | fits 93 GiB | fits 110 GiB | +|---|---|---|---|---|---|---|---|---| +| 2^4 fixture | 25 | blowup2/219q | 311,214 | 15 | 22.3B+ | 701 GiB | NO (7.5×) | NO | +| 2^4 fixture | 25 | blowup4/110q | 164,120+ | 8 | 12.7B+ | 399 GiB | NO (4.3×) | NO | +| **2^20** | 28 | blowup2/219q | 920,273 | 43 | 72.0B | 2,261 GiB | NO (24×) | NO | +| **2^20** | 28 | blowup4/110q | 490,998 | 23 | 38.2B | **1,199 GiB** | **NO (13×)** | NO | +| **2^21** | 32 | blowup2/219q | 1,194,441 | 55 | 93.3B | 2,929 GiB | NO (31×) | NO | +| **2^21** | 32 | blowup4/110q | 637,057 | 30 | 49.5B | 1,554 GiB | NO (17×) | NO | +| **2^22** | 43 | blowup2/219q | 1,797,233 | 83 | 140.8B | 4,421 GiB | NO (48×) | NO | +| **2^22** | 43 | blowup4/110q | 960,063 | 44 | 74.8B | 2,348 GiB | NO (25×) | NO | +| **2^23** | 64 | blowup2/219q | 2,895,737 | 133 | 230.6B | 7,242 GiB | NO (78×) | NO | +| **2^23** | 64 | blowup4/110q | 1,546,363 | 71 | 122.5B | 3,848 GiB | NO (41×) | NO | + +✓ MEASURED per point: sub-proof count, per-chip log-heights (full leg-shape dumps in +the logs), closed-form leg permutations, spine permutations and spine instruction mix. +? PROJECTED: cells at the real query counts (analytic chip model, §3) and peak RSS +(the 33.7 B/cell coefficient, §5). + +### Epoch trace-length profiles (log2), measured + +| epoch | profile | +|---|---| +| 2^4 fixture | `[2 ×15, 3, 4 ×4, 5 ×3, 7, 20]` | +| 2^20 | `[2 ×10, 4, 5, 7, 17 ×4, 18 ×2, 19 ×4, 20 ×4, 21]` | +| 2^21 | `[2 ×8, 5, 7, 10 ×2, 12, 17, 18 ×2, 19 ×8, 20 ×7, 22]` | +| 2^22 | `[2 ×7, 5, 7, 11, 13 ×2, 14, 17, 18 ×2, 19 ×12, 20 ×14, 22]` | +| 2^23 | `[2 ×7, 5, 7, 11, 14, 15, 16, 19 ×25, 20 ×25, 22]` | + +--- + +## 2. Two results that need no extrapolation + +### 2a. The 219-query program cannot even be BUILT + +The assembled verifier at blowup2/219q on the **16-cycle fibonacci fixture** — the +smallest epoch that exists, 25 sub-proofs — was **OOM-killed (signal 9) at 89.1 GiB +during EMISSION**, after 3 m 56 s, before any proving began. + +Emission is strictly cheaper than proving, so this alone settles the gate: the +smallest possible epoch at the cheapest secure preset does not fit on the box even +as a program in memory. + +### 2b. Eight of the required 219 queries already exceed the box + +Fully emitted censuses of a **real 2^21 block epoch at blowup2**, query count reduced +via `LFM_CENSUS_QUERIES` (never a security claim — the count travels with every number): + +| queries | cells | chunks | projected peak | fits 93 GiB | +|---|---|---|---|---| +| 4 | 1,965,702,420 | 2 | 61.7 GiB | YES | +| 8 | 3,706,469,652 | 3 | 116.4 GiB | **NO** | +| 16 | — | — | 225.7 GiB | **NO** | + +Spine/leg split at the q=4 point: spine 1,725,376 instr / 2,667 perms / 9,010 words; +legs 4,948,202 instr / 21,736 perms / 75,804 words — i.e. **5,434 leg permutations per +query**, closed-form CHECKED against `epoch_verify::query_permutations`. + +--- + +## 3. The scaling law + +**Linear in query count.** Measured ×1.89 then ×1.94 across a 4× span (61.7 → 116.4 → +225.7 GiB). The shortfall below an exact ×2 is fixed-height chips (BITWISE at 2^20, +KECCAK_RC, LFM_RANGE) plus power-of-two row padding, both of which dilute as the point +grows — so linearity is the right extrapolation and it is mildly conservative. + +**Roughly linear in sub-proof count**, which grows 28 → 32 → 43 → 64 across 2^20…2^23 +as VM tables split at `max_rows`. + +**Logarithmic in epoch size per table** (Merkle depth and committed FRI layers). + +Net: the epoch-size lever is **weak** — 2^23 → 2^20 buys only 3.2× — and it multiplies +the number of wraps the tower must then aggregate, so it partly pays itself back. + +**One chip is the bill.** `KECCAK_RND` is **92.5% of cells** at the real 2^21 point +(87.6% at the fixture; the share rises with size). One full chunk is 2^19 rows × +(1480 main + 3×516 aux) = 3,028 base-field-equivalent cells/row = 1.588B cells = +12.7 GB raw trace = **49.8 GiB projected**. Therefore: + +> The 93 GiB budget holds ~**40.8k permutations** — 1.9 chunks — in total. +> The sweep needs 491k…2.90M. + +Equivalently, per point, the query count that *would* fit 93 GiB: 2^20 → 9.7 queries; +2^21 → 7.5; 2^22 → 5.0; 2^23 → 3.1. Against a required 219. + +### Where the permutations go (2^21/blowup2 leg dump) + +Two distinct cost centres, which matter because they respond to different levers: + +- **Wide tables → leaf absorption.** Leg idx 4 is the inner proof's own `KECCAK_RND` + sub-proof: only 2^2 rows, but 1480 main + 516 aux columns, so one query's leaf costs + ~359 permutations. 79,935 perms at 219 queries from a 4-row table. Independent of + epoch size. +- **Deep tables → Merkle + FRI paths.** Leg idx 31 at 2^22 rows / depth 22 / 14 FRI + layers costs 288 perms per query almost entirely in path steps. Grows with epoch size + and with table count. + +--- + +## 4. Levers, and the headline conclusion + +From the cheapest real point, **2^20 / blowup4 / 110q = 1,199 GiB (13× over)**: + +| lever | result | still over 93 GiB | +|---|---|---| +| baseline | 1,199 GiB | 13× | +| + inner hash blake3-6r (4.06×, the plan's own hash matrix 11.17B → 2.75B) | **295 GiB** | **3.2×** | +| + a further 2× from anywhere | 148 GiB | 1.6× | +| + a further 4× from anywhere | 74 GiB | fits | + +> **The inner-hash switch is NECESSARY BUT NOT SUFFICIENT.** + +The plan anticipated that a keccak-inner failure would promote the inner-hash switch +"from optimization to prerequisite". The measurement says something stronger: after the +switch, the best point in the whole sweep is still **3.2× over** the box. + +### Coefficient-free floor — this is not an artifact of the 33.7 B/cell anchor + +Counting only the raw committed trace at 8 bytes per felt — zero LDE, zero Merkle trees, +zero quotient, zero allocator overhead: + +| epoch | preset | raw trace | vs 93 GiB | after blake3-6r | +|---|---|---|---|---| +| 2^20 | blowup4/110q | 285 GiB | 3.1× | 70 GiB (fits) | +| 2^21 | blowup2/219q | 695 GiB | 7.5× | 171 GiB (1.8×) | +| 2^23 | blowup2/219q | 1,718 GiB | 18.5× | 423 GiB (4.6×) | + +No prover, however efficient, holds the 2^20/blowup4 trace in 93 GiB today. The +blake3 switch brings the *floor* under the budget at exactly one point — which is +what makes the residual ~3.2× a question about prover residency rather than about +the census. + +### The remaining ~3.2× — candidate + +**Bounded-residency proving.** Peak is currently the *sum* over 23–133 `KECCAK_RND` +chunks because `airs.air_trace_pairs(&mut traces)` hands every trace to a single +`multi_prove` call. One resident chunk at a time would be ~50 GiB regardless of chunk +count. There is a `disk-spill` feature and a `StorageMode` enum already in the tree +(`StorageMode::Ram` is passed explicitly at the epoch-prove sites), which may already +be most of this. + +> **★ ANSWERED — this paragraph was written before the read; see +> `residency-seam-audit.md` and Part 2 §1.** Two things above are now known to be +> wrong: `disk-spill`/`StorageMode` do **not** already provide most of this (the +> LDE is never spilled and the path is unreachable from LFM), and "~50 GiB +> regardless of chunk count" needs the TRACE streamed as well as the LDE dropped — +> LDE-only bounding lands at 309-819 GiB. Bounded residency is a real refactor with +> named seams (S1-S7), not a flag. The `airs.air_trace_pairs` observation stands. + +✗ **UNCERTAIN at the time of writing — I had not read `multi_prove`'s residency +behaviour.** That read was the cheapest next step and is now done; it determined +that the hash switch alone does **not** close the gate at 2^20/blowup4. + +Adjacent parked item: `max_rows` / `KECCAK_RND_MAX_CHUNK_ROWS` tuning changes the chunk +*count* but not total rows, so it does **not** move peak unless residency is bounded. + +--- + +## 5. Method, and what the numbers cannot see + +**The analytic chip model** (`project.py`) reproduces `lfm_chip_census` from shapes +alone: per-chip padded rows = `next_pow2(instruction count)`, `KECCAK_RND` rows from the +chunk policy (21,845 perms per 2^19-row chunk), fixed heights for `BITWISE` (2^20), +`KECCAK_RC` (32), `LFM_RANGE` (2^16). It was validated against **three independent +measured censuses** — the fixture at 1 query, and the real 2^21 epoch at 4 and 8 +queries — reproducing `main`, `aux`, total cells and chunk count **exactly** in every +case. Leg instruction cost per query came from differencing the emitted q=8 and q=4 +censuses (the spine cancels exactly); spine permutations and mixes are measured +directly at the real query counts via `LFM_CENSUS_SPINE_ONLY`. + +**The 33.7 B/cell coefficient is the one soft link.** It rests on a single anchor +(slice 0, one `KECCAK_RND` chunk). Extrapolating it assumes peak RSS stays roughly +linear in total cells across 23–133 chunks, which holds only while all traces are +simultaneously resident — the same assumption the bounded-residency lever above would +break. This is why §4's coefficient-free floor is stated: the verdict does not depend +on the coefficient. + +**Not covered by either side of the cells count:** preprocessed columns, the composition +polynomial's own commitment, LDEs and Merkle trees. The recursion ratio quoted by the +harness (4.0× its own trace cells at the real 2^21/q=4 point) is trace-to-trace only. + +--- + +## 6. Traps discovered + +1. **2^23/blowup4 inner prove dies on a 32 GiB 5090.** `CUDA_ERROR_OUT_OF_MEMORY` + (`[gpu] resident aux LDE failed (rows=524288 cols=10 blowup=4)`), then it **panics + instead of falling back**: `crypto/stark/src/prover.rs:1637` — *"R2 composition fell + back to the host evaluator, but the trace is device-only (empty)"* — and + `prover.rs:2321` for R4 DEEP, surfacing as `prover.rs:701` "a scoped thread panicked". + Same class as issue **#927** (uncovered cliff asserts). Reproduced twice. + `LAMBDA_VM_GPU_LDE_THRESHOLD=999999999` does **NOT** avoid it. The point only + completed with `LAMBDA_VM_DISABLE_DEVICE_ONLY=1 LAMBDA_VM_DISABLE_GPU_COMPOSITION=1 + LAMBDA_VM_NO_GPU_LOGUP=1` (57.5 s on CPU, 45 GiB RSS). **This will bite Stage B/C**, + which must prove real 2^23 epochs. +2. **Emission is the first wall, not proving** — 89 GiB merely to *build* the 219q + program. Any future census must budget for the emitter, and `LFM_CENSUS_SKIP_EMIT` / + `LFM_CENSUS_SPINE_ONLY` exist for exactly this. +3. **The July-built `ethrex.elf` (sha `133816f0`) executes fine** against the box's + August `blake3-real-hash` tree with the real block input — no guest rebuild needed, + and the ELF-drift risk flagged in the brief did not materialise. +4. `pgrep -f` / `pkill -f` on this box match the invoking ssh command's own argv — I + self-killed a running point that way. Use a bracketed pattern (`lambda_vm_prov[e]r`). +5. The fibonacci fixture ELF was copied aside to `/root/fibonacci.elf.SAFE` before any + work; no make target that would overwrite it was run. + +--- + +## 7. Harness change (for the reviewed commit later) + +Two files, +260 lines, all test-only. + +**`prover/src/lfm/epoch_tests.rs`** — `real_epoch_with` now reads three environment +overrides, defaulting **byte-for-byte to the existing fibonacci fixture path** so no +existing test moves: + +- `LFM_CENSUS_ELF` — inner guest ELF path (default: `proof_fixture::read_inner_elf()`) +- `LFM_CENSUS_INPUT` — private input path (default: empty) +- `LFM_CENSUS_EPOCH_LOG2` — epoch size (default: `FIXTURE_EPOCH_LOG2` = 4) + +The private input is threaded into `Executor::new`, `build_initial_image_paged` and +`Traces::from_image_and_logs` (position 6 of that call was `&[]` and *is* the private +input — easy to miss). Everything else already mirrors `continuation::prove_epoch` +faithfully (no PAGE configs, L2G bookend, REGISTER preprocessed with FINI), so the +generalisation is genuinely minimal. Plus one `eprintln!` reporting inner prove time +and sub-proof count. + +**`prover/src/lfm/wrap_tests.rs`** — one new `#[ignore]`d test, +`the_census_fit_map_point`, driven by: + +- `LFM_CENSUS_PRESET` — `min|blowup2|blowup4|blowup8` +- `LFM_CENSUS_QUERIES` — query-count override for the linearity calibration +- `LFM_CENSUS_SKIP_EMIT` — stop after the leg-shape dump (what made the big points measurable) +- `LFM_CENSUS_SPINE_ONLY` — emit the spine alone + +It prints the per-leg shape table (`log2_trace_length`, LDE, main/aux width, Merkle +depth, groups, committed FRI layers, `query_permutations`), then — when emitting — the +full chip census, the spine/leg split with the closed form **asserted** equal to the +emitted leg permutations, the recursion ratio, and the fit verdict against 93/110 GiB. + +The runner (`/root/census_run.sh`) wraps every point in `timeout` + `/usr/bin/time -v` +and appends headlines to `/root/census_results.md` as they land. + +--- + +# Part 2 — Residency, the emission wall, and the tower node + +Follow-up to Part 1's Gate-A failure, answering the three questions its verdict +raised. Read-only analysis in `/Users/maurofab/workspace/lambda_vm-blake3-impl` +(branch `blake3-real-hash`); no builds. + +> ### Companion documents — read these alongside +> +> Residency and emission were each analysed **twice, independently**. The +> standalone audits are the primary records; §1 and §2 below are the second read, +> and where the two differed the audits won (corrections are marked in place): +> +> | topic | primary record | this document | +> |---|---|---| +> | bounded-residency proving (P-b) | **`residency-seam-audit.md`** — S1-S7 seams, the `17.37·N + 30.2·k GiB` peak model, the 309-819 / 48-56 GiB ladder, coefficient correction, confidence ledger | §1 below (second read) | +> | emission memory (P-c) | **`emitter-memory-audit.md`** — the row-intermediate + scope-held `read_counts` mechanism, the four wins, streaming seams, "emission is not the last wall" | §2 below (second read — **its mechanism was wrong**, see the correction box) | +> | tower node / Gate D1 | §3 below (only record) | — | +> +> Both audits agree with §1-§2 on **every verdict**; the differences are in +> accounting and in which allocation dominates. `PLAN.md` §A cites all three. + +## 1. RESIDENCY — verdict **(b) moderate refactor, one named seam** + +### The code answers it directly + +`crypto/stark/src/prover.rs:265-274`, the `Lde` struct's own doc comment: + +> *"Memory trade-off, asymmetric since the per-table scheduler fused aux build, +> aux commit and rounds 2-4 into one task:* +> - *main: produced by the Round 1 main commit, **which is a phase-wide barrier, +> so all N tables' main LDEs are live at once** (O(N x main_cols x lde_size)).* +> - *aux: produced and consumed inside the same fused task, so **at most +> `table_parallelism()` of them coexist** (O(k x aux_cols x lde_size))."* + +So: **all N main traces AND main LDEs are simultaneously resident; aux is already +bounded to k.** ✓ VERIFIED. Note the `debug-checks` caveat in the same comment — +there the fused task is split and aux becomes all-N-live too. + +`air_trace_pairs` hands `multi_prove` a vector of `(air, &mut trace, &publics)`, +so every trace must be materialized *before* the call: there is no per-table +streaming at the entry point either. + +### This makes Part 1's projection ~2x CONSERVATIVE + +Part 1 charged all N tables for aux cells as well as main. Correcting for the +bounded aux (k = `table_parallelism()` = cores x 2/3 = **21** on the 32-core box, +`prover.rs:588-605`, overridable by the `TABLE_PARALLELISM` env var at `:591`): + +| point | N | main-side (xN) | aux-side (xk=21) | corrected | Part 1 said | +|---|---|---|---|---|---| +| 2^20/blowup4 | 23 | 666 GiB | 635 GiB | **1,300 GiB** | 1,199 GiB | +| 2^21/blowup2 | 55 | 956 GiB | 381 GiB | **1,337 GiB** | 2,929 GiB | +| 2^22/blowup2 | 83 | 1,442 GiB | 381 GiB | **1,823 GiB** | 4,421 GiB | +| 2^23/blowup2 | 133 | 2,311 GiB | 381 GiB | **2,692 GiB** | 7,242 GiB | + +**Gate A's verdict is unchanged** — every point still fails by 14x-29x — but the +margin at the large points is roughly half what Part 1 reported. (At 2^20 the two +agree closely because N < k there, so nothing was over-charged.) + +### Where peak accretes + +| buffer | scope | one 2^19-row KECCAK_RND chunk, blowup 2 | +|---|---|---| +| main trace | **all N** (input to `multi_prove`) | 5.8 GiB | +| main LDE | **all N** (Round-1 phase-wide barrier) | 11.6 GiB | +| main tree | all N (small) | 0.03 GiB | +| aux trace | k concurrent | 6.0 GiB | +| aux LDE | k concurrent | 12.1 GiB | +| composition / DEEP / FRI | inside the fused per-table task, k concurrent | — | + +**The binding constraint is the main LDEs**: 532 GiB (2^20/blowup4) to 1,538 GiB +(2^23/blowup2) on their own. + +### What `disk-spill` / `StorageMode` bound today + +`StorageMode` is `{Ram (default), Disk}` (`crypto/stark/src/storage_mode.rs:4-8`). +The feature is opt-in — `prover/Cargo.toml:8` has `default = ["parallel"]`, spill +at `:17` — and **our Stage-A builds used `--features cuda` only, so spill was +compiled out and every Part-1 measurement ran in `Ram`.** + +What it actually spills: `prover.rs:3113-3118`, *"Spill main traces to mmap before +Round 1 LDE"* — the main **traces**, via `spill_to_disk()`, and only under +`StorageMode::Disk`. It does **not** spill the main **LDEs**, which are the +binding buffer. So spilling bounds the input side, not the peak. + +### The seam, and the achievable floor + +Cumulative, at 2^21/blowup2 (N=55): + +| change | peak | reachable today? | +|---|---|---| +| today (N main traces + LDEs, k=21 aux) | 1,335 GiB | — | +| `TABLE_PARALLELISM=1` | 972 GiB | flag exists (`prover.rs:591`) — but see correction | +| + `disk-spill` on main traces | 654 GiB | **NO — see correction** | +| + **main-LDE re-derivation at query time** | **35 GiB — FITS** | does not exist | + +> ### ⚠ CORRECTION — superseded by `residency-seam-audit.md` (2026-08-12) +> +> Full detail, including the S1-S7 seam list and the confidence ledger, is in +> **`residency-seam-audit.md`** (§4 "Verdict" and §4 "The seams" / "The floor"). +> The "exists" column above was **too generous**, and the ladder should not be +> quoted as a set of free levers: +> +> - **`disk-spill` is UNREACHABLE from the LFM path.** The feature is off in our +> builds *and* the LFM prove call site pins RAM — `lfm/proof.rs:140` calls +> `Prover::multi_prove` directly, and the test path is literally +> `test_utils::multi_prove_ram` (`test_utils.rs:134-142`). ✓ VERIFIED. Wiring +> spill through to LFM is therefore **part of P-b**, not a precondition of it. +> - **`TABLE_PARALLELISM` bounds only the aux / rounds-2-4 transients**, which is +> the k-term — it does nothing to the O(N) main-LDE term that binds. +> - **The 33.7 B/cell coefficient is ~2.1x high for the KECCAK_RND shape.** The +> audit's direct peak model for that family is **17.37·N + 30.2·k GiB**, which +> reads the Gate-A band as ~560-3,200 GiB rather than the figures in Part 1. +> **No verdict moves** — every point still fails by a wide margin — but the +> Part-1 absolute numbers are upper bounds, not estimates. +> - Bounding only the LDE lands at **309-819 GiB**; the flat floor additionally +> needs the TRACE streamed. That is available because chunks are pure functions +> of their `round_ops` slice (no cross-chunk logic), giving **~48-56 GiB flat +> regardless of N, at ? +40-60% wall time**. Seams S1-S7 are named in +> `residency-seam-audit.md` §4. +> +> What survives from my reading: the `Lde` doc comment (`prover.rs:265-274`) +> establishing that **all N main LDEs are live while aux is k-bounded**, and the +> soundness argument below. + +**Verdict (b).** No flag bounds residency today. The change that turns O(N) into +O(1) is dropping each table's main LDE after its root is committed and re-deriving +it when Round 4 needs openings — plus streaming the trace for the flat floor. + +**Why this is a refactor and not a protocol change:** the Round-1 barrier itself is +required by soundness — every main root must be in the transcript before the shared +LogUp challenges are sampled (`prover.rs:3216`). But only the **roots** are needed +for that; retaining the **LDEs** is a performance choice. The seam is the `Lde` +struct (`prover.rs:275-287`) and the `main_ldes: Vec<(Vec>, +usize)>` accumulator at `prover.rs:3145`, which is what makes retention O(N). + +**Cost:** one extra LDE + tree pass per table, i.e. roughly 2x prover time on the +hash chips, traded for O(1) memory. + +⚠ **This corrects my earlier "one chunk resident at a time ≈ 50 GiB flat" claim** +as recorded in PLAN.md §A and the campaign memory. The ~35-50 GiB figure is a +*target reachable only with the re-derivation change*; it is not what flags buy +today. + +--- + +## 2. EMISSION WALL — verdict **(b), with two nearly-free wins first** + +The 219q emission was OOM-killed at 89.1 GiB. No architectural change is implicated +— but the dominant allocations are **not** the ones I first named, so read the +correction box before the arithmetic. + +`Addr` is `pub struct Addr(pub u64)` — 8 bytes (`instr.rs:20`). The largest `Instr` +variant is `Unpack { input: Addr, outs: [Addr;4], mults: [u64;4] }` = 72 bytes of +payload, so `size_of::()` is **80 bytes** with tag and alignment +(? INFERRED — reasoned from the field types; not measured, since builds were out +of scope). `KeccakF(Box)` is boxed, which is what keeps the enum +this small. + +At 219 queries the program is ~272M instructions (measured leg slope 1,237,050 +instr/query + measured spine): + +> ### ⚠ CORRECTION — my attribution below was WRONG; see `emitter-memory-audit.md` +> +> The primary record for this section is **`emitter-memory-audit.md`** (§0 measured +> type sizes, §1 "What dominates", §4 the verdict and the four wins). Read it +> instead of the arithmetic below. +> +> I attributed the 89 GiB peak to a **`Vec` doubling spike**. That was an +> inference from the enum size, never a reading of the emitter's allocation path, +> and `emitter-memory-audit.md` §1, which did read it, found otherwise: +> +> - **The instruction stream is only ~24% of the peak** (271M x 80 B = 21.7 GB). +> - The dominant term is the **per-instruction `Vec>` row intermediate** +> (~47 GB, with ~80% capacity waste — 10-wide rows landing at capacity 18). +> ✓ VERIFIED it exists: `compiler.rs:38`, `fn from_rows(width: usize, rows: Vec>)`. +> - Plus a **drained-but-unshrunk `read_counts` HashMap** (~18.3 GB) held by scope +> through the peak. ✓ VERIFIED: `compiler.rs:140` binds it `mut`, `take(...)` +> drains it entry by entry (`:164-201`) and `:208` asserts it empty — but a +> `HashMap`'s allocation does not shrink on removal, so it sits at full capacity +> across the subsequent column-group emission. +> +> **The correct cheap wins are therefore not mine but these:** +> 1. `drop(read_counts)` before `emit_column_groups` — **-18.8 GB, one line**. +> 2. A flat-append `ColumnGroupBuilder` replacing the row-of-`Vec`s — **-27 GB, +> ~50 lines**, zero semantic change (`program_id` commits over matrices, so the +> result is bit-identical). +> +> Together: peak **~99-102 GB -> ~53-56 GB**. +> +> My two suggestions (`Vec::with_capacity`, dense `read_counts`) are not wrong as +> micro-optimisations, but they target the ~24% term and would not have moved the +> wall. **Task #29 tracks `emitter-memory-audit.md` §4's list, not mine.** +> +> ⚠ And emission is **not** the last wall even once streamed: `execute` still wants +> ~21 GB of memory plus ~10 GB of records, and `LFM_BALU` pads to 2^28 rows at +> 219q. The P-b prover streaming stays load-bearing. + +The superseded arithmetic is left below for the record. + +| term | size | +|---|---| +| final `Vec` (272M x 80 B) | 20.3 GiB | +| ~~`Vec` doubling spike~~ (SUPERSEDED — not the mechanism) | ~~60.0 GiB~~ | +| `read_counts: HashMap` (`builder.rs:101`), ~1 entry per instruction | 8.5 GiB | + +`Addr` is `pub struct Addr(pub u64)` — 8 bytes (`instr.rs:20`); the largest `Instr` +variant is `Unpack` at 72 bytes of payload, so `size_of::()` is ~80 bytes +(? INFERRED, never measured — `emitter-memory-audit.md` §0 measured the type +sizes directly and its 21.7 GB for this term agrees). + +Whether full per-leg streaming is possible hinges on the machine being +straight-line; `emitter-memory-audit.md` §3 names the seams (builder `instrs` field; `compile` merging +into the builder; the executor needing a 10-way merge by destination — the one new +algorithm). I did not verify the straight-line property end to end. ✗ UNCERTAIN. + +--- + +## 3. TOWER NODE PROJECTION — Gate D1: **FAILS** (1.3x-4.9x over) + +The plan expects "census says the 1-proof verifier fits comfortably (expected: +yes - 14 tables vs ~25-31, blake3 legs vs keccak)". **Falsified**, though by far +less than Gate A. + +Node's own options blowup 2; legs recompute BLAKE3 per `COMMIT.md` §1.4; 6-round +chip; projected peak = cells x 33.7 B/cell. + +| node | inner LFM proof | 110q | 219q | +|---|---|---|---| +| D1 (verify 1 proof) | fixture wrap (exists today) | **124 GiB** (1.3x) | 247 GiB (2.7x) | +| D1 | real 2^21 wrap, D0-consistent | 227 GiB (2.4x) | 452 GiB (4.9x) | +| D2 (aggregate 2) | fixture wrap | 248 GiB (2.7x) | 493 GiB (5.3x) | +| D2 | real 2^21 wrap | 454 GiB (4.9x) | 904 GiB (9.7x) | + +keccak control (D1 / fixture / 219q): 1,274 GiB. So blake3 buys **5.2x** here - +better than the plan's 4.06x aggregate. + +### Model validation — exact on four measured legs + +The cost model reproduces MEASURED per-query permutation counts **exactly** for +four structurally different real sub-proofs: + +| leg | shape | model | measured | +|---|---|---|---| +| leg 4 | 2^2 rows, 1480+516 cols (wide+shallow) | 365 | 365 | +| leg 31 | 2^22 rows, 9+3 cols, 14 FRI layers (narrow+deep) | 288 | 288 | +| leg 3 | 2^2 rows, 511+67 cols | 92 | 92 | +| leg 22 | 2^20 rows, 10+4 cols, 12 FRI layers | 239 | 239 | + +This confirms the leaf/Merkle/FRI decomposition, the group counts, the FRI layer +counts, and `num_parts = 2`. + +### The D1 lever is the LEAF RATE, not the hash + +blake3's advantage is **highly non-uniform**: + +- **Merkle parents: 14.7x cheaper.** One invocation either way - keccak + `KECCAK_RND` costs 24 rows x (1480 main + 3x516 aux) = 72,672 cells per + permutation; the blake3 chip is 1 row x (3056 + 3x630) = 4,946 cells per + compression (`blake3_chip.rs:162,224`, `airs.rs:246` for the + `interactions.div_ceil(2)` aux rule). +- **Leaf absorption: only 1.73x cheaper.** keccak absorbs 17 felts per + permutation; `COMMIT.md` §1.4 and `LEAF.md` §1.4 give blake3 **2 felts per + compression** (a 4-felt `LFML` row plus one `LFMC` fold). blake3 needs 8.5x + more invocations, nearly cancelling its per-invocation edge. + +And leaf absorption is **69.8%** of the D1 node's per-query bill (Merkle 10.7%, +FRI 19.5%), concentrated in the two wide chips: `KECCAK_RND` 3,229 +compressions/query + `LFM_KECCAK` 1,164 = 65% of everything. + +Raising the `LFML` rate is therefore the dominant D1 lever - and it is still an +**open spec decision** (`COMMIT.md` is DRAFT, S1 is the gating item), so it is +cheap now and expensive later: + +| node | inner | today | rate x2 | rate x4 | +|---|---|---|---|---| +| D1 | fixture, 110q | 124 | **81 FITS** | **59 FITS** | +| D1 | real, 110q | 227 | 148 | 108 | +| D2 | fixture, 110q | 248 | 161 | 118 | + +? INFERRED headroom: blake3's compression block is 64 B; a 4-felt `LFML` row uses +36 B of it (`LEAF.md` §1.3: 8 lanes x 4 B + a 4-byte tag), so ~7 felts fit one +block. **I have not checked what the chip's constraint layout can support** - +this is a question for the spec owner, not a claim that it is free. + +### Why the plan's expectation was directionally right but short + +| what is verified | tables | perms/query (keccak) | +|---|---|---| +| RV64 epoch 2^21 (Gate A inner) | 32 | 5,434 (MEASURED) | +| LFM fixture wrap (D1 inner) | 14 | 2,384 | +| LFM real-2^21 wrap (D1 inner) | 15 | 3,306 | + +D1's inner IS 2.3x cheaper per query than Gate A's, because the LFM proof has far +fewer DEEP tables (where Merkle depth and FRI dominate). But fewer tables does not +mean cheap: the 14 LFM chips are far WIDER (`KECCAK_RND` 1480+516) than RV64 +tables (mostly <50 columns), and **leaf cost is set by width, not height**. + +### The tower does not get cheaper as it climbs + +An upper-layer node whose inner has every chip at its 4-row floor still costs +**97 GiB at 110q**, against 124 GiB for the base layer - only 22% less. The 14 +chip WIDTHS are fixed by the machine, so only the Merkle/FRI 30% shrinks with +height. The plan's "per-layer cost is the D1 census number x 2" is therefore +sound. For N=36 base wraps: 6 layers, 38 aggregation nodes; peak is PER NODE +(sequential), so the binding constraint is the largest single node. + +### Two build-config traps + +1. **`blake3-6round` is OFF by default.** `prover/Cargo.toml:22` declares it; + `blake3.rs:82-85` cfg-gates `BLAKE3_ROUNDS` to `BLAKE3_STANDARD_ROUNDS` (7) + unless the feature is on. The chip is `8 x rounds` G-blocks wide, so the + default 7-round chip is 3,552 columns against 3,072 - **+16% on every tower + number** (D1/fixture/110q: 124 -> 144 GiB). The plan's hash matrix says + "blake3-6r", so the campaign intends the feature ON; it must be named + explicitly in the build. +2. **Second-order feedback:** under D0 the machine's own hash chip becomes BLAKE3 + at 3,056 main columns, which is **wider than `KECCAK_RND`'s 1,480**. The LFM + proof's widest table therefore gets wider and each tower layer pays more to + re-hash its leaf: D1/real/219q moves 381 -> 452 GiB (+19%). The hash switch is + still right, but it is not monotonically cheaper at every layer. + +### Sensitivity — the FAIL verdict is robust + +| variation | D1 fixture 110q | vs base | +|---|---|---| +| baseline (aux 630, num_parts 2, non-hash 6.5%) | 124 GiB | -- | +| blake3 aux 500 | 114 | 0.92x | +| blake3 aux 750 | 133 | 1.07x | +| num_parts 1 / 4 | 123 / 125 | 1.00x / 1.01x | +| non-hash chips 3% / 15% | 119 / 136 | 0.96x / 1.10x | + +Every variation stays above 93 GiB. The one soft input is the blake3 aux width: +`bus_interactions()` is built with `Vec::with_capacity(1_259)` +(`blake3_chip.rs:913`) and **no test asserts the final count**, so 630 is +? INFERRED; +-20% moves the result only +-8%. + +--- + +# Part 3 — spill ladder, measured + +Measured 2026-08-13 on the rented 5090 box (32 cores, 60.45 GiB RAM, 64 GB +overlay disk, 71 GiB of swap present), branch `blake3-real-hash`, wiring commit +`c5ffadf3`. Every number below is ✓ MEASURED unless marked otherwise; raw +`.meta`/`.samples`/`.timev`/`.stdout` per rung are in +`~/workspace/lambda_vm_bench_cache/lfm_spill_2026-08-13/`. + +This part answers the question PLAN.md's P-b fallback line poses — *"existing +flags + spill wiring first; streaming only if the numbers still demand it"* — +by wiring the flags and running them. + +## 0. What had to be built before anything could be measured + +`disk-spill` was unreachable from the wrap on two counts, and one of them was +not in the seam audit's list: + +1. `lfm/proof.rs:140-145` passed `Default::default()` = `Ram`. Now it calls + `auto_storage::decide_lfm()`, which honours `FORCE_DISK_SPILL`. There is + deliberately no estimate: `decide` keys off the RV64 executor's + `TableLengths`, and the wrap has no analogue — its table set is program + shape, and `KECCAK_RND`'s column profile was never calibrated into that + model. +2. **The fixture path does not run.** `real_epoch_with` passes an empty private + input to a fibonacci guest that reads its iteration count *from* private + input, so the guest halts inside the first epoch and every test asserting an + INTERMEDIATE epoch fails. This is the known "19 failing `lfm::` tests" drift, + and it is a fixture bug, not a prover bug. `LFM_CENSUS_INPUT` (a file holding + the input; unset = today's behaviour exactly) is what makes the wrap + harness runnable; the ladder ran with an 8-byte `n = 1000`. + +`LFM_WRAP_QUERIES` raises the blowup-8 wrap's inner query count above 1. +`make lint` already covers `lambda-vm-prover/disk-spill` (Makefile:660), so no +lint-matrix change was needed, and all four of its clippy passes are clean. +✓ VERIFIED + +**Default behaviour is unchanged**, checked against the oracle rather than +argued: with every knob unset the `lfm::` suite is **307 passed / 19 failed** — +the known baseline exactly. (All 19 are the fixture-input failures described +above; `machine_tests::continuation_fixture_generates_two_epochs` is the one +that names the cause outright.) ✓ MEASURED + +## 1. What the ladder lever actually moves + +The wrap spends **2,228 permutations of spine + ~1,565 per inner query**, and a +`KECCAK_RND` chunk holds 21,845 (`chunking.rs:40`, 2^19 rows). So: + +- q = 1 … 12 → **one** chunk, growing in height to its 2^19 cap; +- q = 13 → the **second** chunk appears (the first point where peak is a sum + over chunks at all); +- q = 20 would be the first point with two *full* 2^19 chunks. + +That is the whole reachable range: the box refuses the proof long before the +third chunk. The real wrap needs **N = 23 … 133**. + +## 2. The ladder + +`rung` = storage mode + `TABLE_PARALLELISM`. "anon" is `RssAnon`, "file" is +`RssFile` (both sampled at 2 Hz from `/proc//status`); "peak RSS" is +`/usr/bin/time -v`. "spill vol" is the filesystem high-water mark — spill files +are `tempfile()`s, unlinked at creation, so they are invisible to `du` and only +show up as filesystem usage. + +| q | perms | chunks | rung | peak RSS | peak anon | peak file | spill vol | wall | prove | result | +|---|---|---|---|---|---|---|---|---|---|---| +| 1 | 3793 | 1 | ram_def | 16.64 | — | — | 0.00 | 24s | 14.4s | ok | +| 1 | 3793 | 1 | ram_tp1 | 14.63 | — | — | 0.00 | 27s | 17.6s | ok | +| 1 | 3793 | 1 | spill_tp1 | 14.23 | — | — | 4.82 | 29s | 18.6s | ok | +| 2 | 5358 | 1 | ram_def | 17.14 | 16.69 | 0.01 | 0.00 | 25s | 15.0s | ok | +| 2 | 5358 | 1 | ram_tp1 | 15.35 | 15.35 | 0.01 | 0.00 | 28s | 18.3s | ok | +| 2 | 5358 | 1 | spill_tp1 | 14.52 | 12.39 | 3.65 | 4.86 | 29s | 19.1s | ok | +| 3 | 6923 | 1 | ram_def | 22.50 | 22.50 | 0.01 | 0.00 | 40s | 28.9s | ok | +| 3 | 6923 | 1 | ram_tp1 | 23.50 | 23.49 | 0.01 | 0.00 | 43s | 32.2s | ok | +| 3 | 6923 | 1 | spill_tp1 | 23.58 | 19.99 | 6.62 | 7.82 | 45s | 34.3s | ok | +| 4 | 8488 | 1 | ram_def | 31.33 | 30.42 | 0.01 | 0.00 | 47s | 32.1s | ok | +| 4 | 8488 | 1 | ram_tp1 | 27.28 | 27.27 | 0.01 | 0.00 | 53s | 37.6s | ok | +| 4 | 8488 | 1 | spill_tp1 | 26.37 | 22.27 | 7.13 | 9.35 | 56s | 40.1s | ok | +| 6 | 11643 | 1 | ram_def | 41.53 | 41.52 | 0.01 | 0.00 | 76s | 59.8s | ok | +| 6 | 11643 | 1 | ram_tp1 | 43.67 | 43.67 | 0.01 | 0.00 | 86s | 69.7s | ok | +| 6 | 11643 | 1 | spill_tp1 | 44.54 | 37.51 | 13.08 | 15.31 | 89s | 72.8s | ok | +| 8 | 14773 | 1 | ram_tp1 | 51.04 | 51.04 | 0.01 | 0.00 | 104s | 77.5s | ok | +| 8 | 14773 | 1 | spill_tp1 | 49.97 | 41.91 | 14.11 | 18.30 | 111s | 84.5s | ok | +| 12 | 21058 | 1 | spill_tp1 | 51.02 | 42.94 | 14.13 | 18.43 | 112s | 83.3s | ok | +| 13 | 22648 | **2** | ram_tp1 | 55.35 | 55.35 | 0.01 | 0.00 | 114s | 84.2s | ok | +| 13 | 22648 | **2** | spill_tp1 | 52.05 | 43.94 | 14.17 | 18.86 | 117s | 87.8s | ok | +| 14 | 24213 | 2 | ram_tp1 | 56.62 | 56.62 | 0.01 | 0.00 | 119s | 88.1s | ok, **swapped 71 MB** | +| 14 | 24213 | 2 | spill_tp1 | 53.00 | 44.88 | 14.17 | 19.22 | 121s | 91.1s | ok, no swap | +| 16 | 27343 | 2 | ram_tp1 | 57.41 | 56.71 | 0.01 | 0.00 | 74s | — | **OOM-KILLED (SIGKILL)** | +| 16 | 27343 | 2 | spill_tp1 | 57.38 | 49.63 | 13.57 | 24.41 | 144s | 111.9s | **ok** | + +All GiB. Every `ok` row proved *and* verified *and* passed the three +falsifications (the harness asserts all of them; a rung that only proved would +have failed the test). + +Sampler note: the q=1 anon/file cells are blank because the 2 Hz sampler was +still latching onto the wrong pid on those three runs; their `time -v` peak RSS +is unaffected. Fixed from q=2 onward — where the sampler and `time -v` agree to +within the 0.5 s sampling gap. + +## 3. Paired, at equal q, both at `TABLE_PARALLELISM=1` + +| q | anon Ram → spill | Δ anon | RSS Ram → spill | Δ RSS | wall Ram → spill | Δ wall | +|---|---|---|---|---|---|---| +| 2 | 15.35 → 12.39 | **−19.3%** | 15.35 → 14.52 | −5.4% | 28 → 29s | +3.3% | +| 3 | 23.49 → 19.99 | −14.9% | 23.50 → 23.58 | +0.3% | 43 → 45s | +4.5% | +| 4 | 27.27 → 22.27 | −18.3% | 27.28 → 26.37 | −3.3% | 53 → 56s | +4.3% | +| 6 | 43.67 → 37.51 | −14.1% | 43.67 → 44.54 | +2.0% | 86 → 89s | +3.7% | +| 8 | 51.04 → 41.91 | −17.9% | 51.04 → 49.97 | −2.1% | 104 → 111s | +6.7% | +| 13 | 55.35 → 43.94 | −20.6% | 55.35 → 52.05 | −6.0% | 114 → 117s | +3.0% | +| 14 | 56.62 → 44.88 | **−20.7%** | 56.62 → 53.00 | −6.4% | 119 → 121s | +2.3% | + +**Spill takes 14–21% off the anonymous working set for 2–7% of wall time.** +Mauro's recollection that spill "used to be quite efficient" is confirmed on the +time axis — this is a cheap mechanism, and it is now reachable. + +**But peak RSS barely moves (0 to −6%), and that gap is the whole story.** What +spill does is *convert* anonymous pages into file-backed ones: at q=8 it wrote +18.30 GiB to disk, dropped anon by 9.13 GiB, and grew `RssFile` from 0.01 to +14.11 GiB. The bytes are still resident — they are just **evictable** now. + +That is exactly why the ceiling moves and the peak does not. At q=16 the two +rungs peak at the *same* RSS (57.41 vs 57.38 GiB) and one dies: + +- Ram: 56.71 GiB of it is anonymous → nothing to reclaim → SIGKILL. +- spill: only 49.63 GiB is anonymous, 13.57 GiB is reclaimable page cache → + the kernel reclaims and the proof finishes. + +**Peak RSS is the wrong metric for this question. Peak anon is the right one.** + +## 4. Largest point each rung completes, on a 60.45 GiB box + +| rung | largest q that fits | chunks | first failure | +|---|---|---|---| +| Ram, `TABLE_PARALLELISM` default (21) | **q = 6** (41.53 GiB) | 1 | q=8 **OOM-killed** at 57.27 GiB | +| Ram, `TABLE_PARALLELISM=1` | **q = 14** (56.62 GiB, and it already had to swap 71 MB) | 2 | q=16 **OOM-killed** at 57.41 GiB | +| spill, `TABLE_PARALLELISM=1` | **q = 16** (57.38 GiB, no swap) | 2 | q=20 does not fit | + +q=20 (the first point with two *full* 2^19 chunks) reached **52.34 GiB +anonymous with 0.99 GiB of swap in use** and its anon still climbing, having +made ~7 minutes' progress against the 144 s that q=16 took. It was terminated by +the operator rather than burning the 50-minute timeout, so it is recorded as +`rc=143` (SIGTERM), **not** as an OOM kill and not as a completed run. Read it +as "did not fit": it was already paging on a box where every rung that fit used +no swap at all, and the two rungs that were killed outright had reached the same +place. ✗ NOT PROVEN that it would have failed — it was not run to conclusion. + +Two levers, and they are **not** the same size: + +- **`TABLE_PARALLELISM=1` buys q = 6 → 14.** This is the big one, and it is + invisible in the peak-RSS column at small q — there it looks like noise (−12% + at q=1/2/4, but *+*0.3–5% at q=3/6) for ~13% wall. At the ceiling it is + decisive: at q=8 the default k=21 was OOM-killed at 57.27 GiB while k=1 + finished the same point at 51.04 GiB. The k-term the `Lde` doc bounds is small + until it isn't. +- **Spill buys q = 14 → 16 on top of that.** In the units that matter that is + +2 inner queries, +3,130 permutations, and **zero additional chunks** — both + ceilings sit at N = 2. + +✓ MEASURED. The lesson for anyone quoting the paired table in §3: judge these +levers by where the rung breaks, not by the peak-RSS delta at a comfortable +point. The two disagree in both directions. + +## 5. Build-side spill for `LfmTraces.keccak_rnd` — measured NOT needed + +The brief asked whether the eager chunk-vector build (`trace.rs:162-167`, all +chunks materialised before `multi_prove`) needs its own spill-at-build. It does +not, and the RSS timeline says so directly rather than by argument: + +| run | span | peak occurs at | peak | +|---|---|---|---| +| q8 ram_tp1 | 104s | **t+87s (84% in)** | 51.0 GiB | +| q8 spill_tp1 | 111s | t+63s (57% in) | 50.0 GiB RSS / 41.9 anon | + +The inner epoch builds in 4.9s and `lfm_prove` runs 77.5s; the trace build is +the small early plateau (~8–11 GiB in the sampled timeline), and the peak is +4–5× higher and lands deep inside `multi_prove` — *after* the existing pre-R1 +spill point at `prover.rs:3113-3122`. Arithmetically it could not be otherwise: +a main LDE is `blowup` × its trace, so Σ traces is at most half of Σ main LDEs +at blowup 2. Spilling at build time would move a number that is not the peak. + +✓ MEASURED. The seam audit's S6 (lazy per-index trace generation) is still +worth what it claims — but as part of streaming, not as a spill target. + +> ### ⚠ CORRECTION — true at N = 2, and it does NOT extrapolate (2026-08-13) +> +> This section's verdict is sound for the range it was measured in and wrong as +> a general statement. The whole ladder above ran at **N = 1 or 2** chunks, +> where the trace build really is a small early plateau. At **N = 15** the trace +> build IS the peak and `multi_prove` is never reached at all. +> +> ✓ MEASURED on a 60 GiB / 32-core box: the **real-block** wrap (block +> 25368371, epoch 0 at 2^16, inner blowup4 / **110 queries** — the secure +> preset) is **OOM-killed at 56.91 GiB anon after 2m30s, BEFORE proving +> starts**. Spill volume 0.00 GiB, `RssFile` peak 0.01 GiB, disk untouched +> (60.11 GiB still free at the minimum). Emission SUCCEEDS first and prints its +> full census (26,197,950,740 base-field-equivalent cells), so the emitter is +> not the wall either. 15 chunks × 5.78 GiB of main trace = **87 GiB** in +> `build_traces_with_hasher` (`prover/src/lfm/trace.rs:162-167`) before +> `multi_prove` is called. +> +> The arithmetic argument above — "a main LDE is `blowup` × its trace, so +> Σ traces is at most half of Σ main LDEs" — is correct and beside the point: +> it compares two things that are only both alive if the prove is reached. Σ +> traces is what has to be resident *to call* `multi_prove`, so at large N it +> binds first no matter what the LDE side costs. +> +> Consequence: **S3 Phase A+B bound residency INSIDE `multi_prove` and are +> never reached at production query counts on a <128 GiB box.** S6 (lazy +> per-index chunk traces) is the enabler for the 64-128 GiB class, not an +> optimisation. On a 258 GiB box the eager build fits and S6 is not required. +> +> Build-side spill does not rescue the 110q rung either: 87 GiB of trace +> against 61 GiB of disk. + +## 5a. The two knobs — epoch size is WEAK, query count is STRONG + +> ### ⚠ CORRECTION to the campaign's climb strategy (2026-08-13) +> +> Part 1 §3 measured "logarithmic in epoch size per table" and concluded the +> epoch-size lever is **weak** (2^23 → 2^20 buys only 3.2×). That is right, and +> the operational consequence was never drawn: **epoch size is not the knob that +> decides whether a wrap fits.** The chunk count is, and it is +> +> ``` +> chunks = (spine_perms + per_query_perms × queries) / 21,845 +> ``` +> +> (`chunking.rs:40`). Per-query cost is dominated by **leaf absorption, which is +> set by table WIDTH** — Part 1 §3 says so itself ("Independent of epoch size"). +> ✓ MEASURED: **2,946.0 perms/query at 2^16** against the census's **5,434 at +> 2^21/blowup2** — a 32× change in epoch size moves per-query cost by 1.8×, +> while the query count moves the chunk count **linearly**. +> +> So a climb that walks epoch size looking for a fitting point at 110 queries +> finds nothing at any size, and a climb that walks the query count finds the +> boundary immediately. Measured boundary at 2^16 on a 36 GiB laptop under +> `RecomputeLde` + `TABLE_PARALLELISM=1`: q=4 → 1 chunk (19.76 GiB), q=8 → 2 +> (21.43), q=12 → 2 (22.87, completes), q=16 → 3 (killed). + +## 6. Spill volume against the disk + +| q | spill volume | of 61 GiB free | +|---|---|---| +| 1 | 4.82 GiB | 8% | +| 8 | 18.30 GiB | 30% | +| 13 | 18.86 GiB | 31% | +| 16 | 24.41 GiB | 40% | + +Volume tracks the trace+tree bytes, not the LDE, exactly as §3 of the seam audit +predicts. No run came close to the 62 GiB disk, and `posix_fallocate` +(`mmap_util.rs:47-70`) reserves blocks up front, so a full disk would surface as +a `ProvingError::DiskSpill` rather than a mid-write SIGBUS. **The disk is not +the binding constraint at any point this box can prove** — but note the ratio: +at q=16 the spill volume is 40% of the disk while buying 14% of RAM. Scaled to +the N=23 wrap the volume, not the disk headroom, is what would run out first. + +One trap worth recording: `/tmp` is on the overlay filesystem on this box, so +spill files really do land on disk. On a systemd-default distro `/tmp` is tmpfs +and **spill would be a no-op** — anonymous pages moved to RAM-backed files. +`mmap_util.rs:53-55` says so in its own comment; set `TMPDIR` to a disk-backed +path before trusting any spill measurement. + +## 7. GPU interaction — documented, one build, three runs + +Built `--features cuda,disk-spill` on the same box (RTX 5090, 32,607 MiB VRAM). +Mauro's caveat that spill is "not compatible with GPU" is **half right, and the +other half matters**: + +| run | result | +|---|---| +| q=1, spill + `TP=1`, cuda | **ok.** 16.10 GiB RSS / 11.59 anon / 4.00 file, **4.26 GiB spilled**, 24.0s | +| q=13, spill + `TP=1`, cuda | **panic** — `prover.rs:1657` | +| q=13, **Ram** + `TP=1`, cuda (control) | **panic** — `prover.rs:1657`, *identical* | + +The panic is +`"R2 composition fell back to the host evaluator, but the trace is device-only +(empty)"` — the #927-class uncovered cliff assert, fired with VRAM at +**32,086 of 32,607 MiB**. **The control settles the attribution: it is a +pre-existing VRAM-pressure failure, not a spill bug.** ✓ MEASURED. The same +q=13 point proves fine on the non-cuda build in every rung of §2. + +What is genuinely GPU-specific about spill: + +- **Aux Merkle trees are not spilled when a GPU aux commit succeeds.** Both cuda + aux arms `return Ok(...)` at `prover.rs:3442` and `:3480`, before the + `spill_tree(&mut tree, storage_mode, "aux Merkle tree")` at `:3524` that the + CPU fallback reaches. ✓ VERIFIED by reading. It shows up in the volume: at + q=1 the cuda build spilled **4.26 GiB against the CPU build's 4.82 GiB**, on + a byte-identical proof. +- **`StorageMode::Disk` disables the precomputed-tree cache** — + `prover.rs:1151-1157` sets `cache_ok = storage_mode != StorageMode::Disk`, so + spilled runs lose cross-prove reuse of preprocessed trees. The wrap has 11 + chips carrying preprocessed instruction column groups, so this is a real + (unmeasured, ? INFERRED) wall-time cost on repeated proves. +- Host RSS is *higher* under cuda at the same point (16.10 vs 14.23 GiB at q=1), + and wall is lower (24.0 vs 28.6s). + +**Net:** spill and cuda compose without corrupting anything — the q=1 cuda spill +run proved and verified — but on this GPU the wrap hits the VRAM cliff at +q=13 regardless of storage mode, so the CPU build remains the honest instrument +for residency work, exactly as briefed. + +## 8. Verdict — does spill suffice? + +**No. Keep it, but it is not the fix.** + +What was bought, measured end to end: **q = 6 → 14 from `TABLE_PARALLELISM=1`, +then 14 → 16 from spill.** Two levers, both now reachable from the wrap, both +cheap (spill costs 2–7% wall). What is needed: the real wrap has **N = 23 to +133** `KECCAK_RND` chunks. This ladder died at **N = 2**. + +### Calibrating the seam audit against measurement + +The marginal cost of a chunk falls straight out of the spill rung: q=12 (one +full 2^19 chunk) peaks at 42.94 GiB anon, q=16 (that chunk plus a 2^18 one) at +49.63 — **6.69 GiB for half a chunk, so ≈ 13.4 GiB per full 2^19 chunk with +spill on.** The seam audit's model says **17.37 GiB/chunk** persistent without +spill. The two agree to within 23%, and 23% is precisely the anon reduction §3 +measures. **The audit's coefficient survives contact with a real wrap.** That is +the most reusable thing in Part 3. + +Extrapolating on the measured marginal (? INFERRED — the non-chunk base also +grows with query count, so treat these as a floor): + +| point | N | spill + `TP=1` | vs a 124 GiB rig | +|---|---|---|---| +| 2^20 / blowup4 | 23 | ~337 GiB | 2.7× over | +| 2^23 / blowup2 | 133 | ~1,810 GiB | 15× over | + +### The specific question: post-blake3 (÷~4), on 124 GiB rigs? + +**Not a yes.** ÷4 on the keccak family takes the cheapest point to N ≈ 6, i.e. +~110 GiB by the marginal above — which lands *on* the 124 GiB line, not safely +under it, and that estimate ignores Part 2 §3's own trap: under BLAKE3 the +machine's `LFM_HASH` chip becomes **3,056 columns, wider than `KECCAK_RND`'s +1,480**, so the non-chunk base grows at the same time the chunk count shrinks. +A single geometry choice decides it either way. ? INFERRED. + +And the flags are now **spent**: both are on in that estimate. There is no third +flag. + +### So the recommendation stands, with one change + +**Main-LDE re-derivation (seam S3) remains required**, and the measurement +sharpens why: spill removes the *trace*, and the trace is the cheap half. At +q=16 spill wrote 24.41 GiB to disk to take 7.08 GiB off anon, because a main LDE +is `blowup` × its trace and **never** spills (`LDETraceTable` has no mmap field, +`trace.rs:316-343`). The buffer that binds is the one no flag touches. + +The change from PLAN.md's framing: spill is no longer "unreachable, therefore +unknown". It is reachable, it is cheap, it is worth keeping wired — it buys a +rung for free and it will multiply whatever structural fix lands. It simply +cannot be the structural fix, and now that is measured rather than modelled. + +### One more thing the ladder found + +`real_epoch_with` cannot run the fixture at all without `LFM_CENSUS_INPUT` (§0). +The 19 failing `lfm::` tests are **a fixture-input bug, not prover drift** — +the fibonacci guest reads its iteration count from a private input the fixture +never supplies. Worth fixing properly at the fixture rather than carrying as +known-red. diff --git a/thoughts/shared/block-compression/D0-DESIGN.md b/thoughts/shared/block-compression/D0-DESIGN.md new file mode 100644 index 000000000..6055a63f3 --- /dev/null +++ b/thoughts/shared/block-compression/D0-DESIGN.md @@ -0,0 +1,378 @@ +# D0 — LFM proof under the machine's native BLAKE3 scheme + +**Design record.** Scoping pass, read-only; no builds run. +**Ground:** worktree `/Users/maurofab/workspace/lambda_vm-blake3-impl`, branch +`blake3-real-hash` @ `2a8552f2`. **Date:** 2026-08-12. + +**Decision this implements:** the LFM machine's own proof (`lfm_prove`, +`prover/src/lfm/proof.rs`) moves from a standard STARK `MultiProof` over +`DefaultTranscript` + keccak256 Merkle commitments to the machine's **native** +scheme under `HasherKind::Blake3` — LFMC Merkle parents, LFML leaves, LFMT +compress-chain Fiat–Shamir (ratified option B, form B1). Purpose: a future LFM +program verifying an LFM proof recomputes the proof's hashes with the machine's +cheap blake3 chips instead of the hosted keccak family. + +Claims are ✓ VERIFIED (read the code, cited) / ? INFERRED / ✗ UNVERIFIED. + +> **Provenance note.** Two delegated sweeps (registry/pinned-digest; GPU +> hash-dependence) stalled without returning to the author. Their load-bearing +> claims were re-derived independently against the source; every citation below +> was read directly. + +--- + +## 0. Verdict + +The switch is tractable and **does not require a new proof format**. Three +structural facts make it so, and three hazards make the ordering +non-negotiable. + +**Why it is tractable.** The transcript is already injectable; the Merkle +backend is pinned in exactly one file with everything beneath it already +generic; and `word::pack_digest` already defines an `LfmWord` → `[u8;32]` +embedding, so the rkyv wire format never moves. + +**Why ordering matters.** Three separate places will *silently* stamp a Blake3 +label on keccak-derived data. All three type-check. None fails loudly. They +must be guarded **before** any Blake3 commitment path exists, not after. + +**The one genuine blocker.** `LFML` hashes exactly four felts. Production LFM +AIRs have arbitrary column counts, and no ratified spec covers a wide leaf. +That is a spec task, not a coding task, and it comes first. + +--- + +## 1. Host-side precedent (Q1) + +**There is a reusable host-side LFM-native commitment layer** — not test-local +helpers. It lives in `pub mod fixture` (`prover/src/lfm/mod.rs:33`), i.e. it is +production-visible, not `#[cfg(test)]`. + +✓ VERIFIED, `prover/src/lfm/fixture.rs`: + +| Role | Function | Location and shape | +|---|---|---| +| **LFMT B1 transcript** | `HostSponge` | `:60-136` — state = one cell (`:61,:80`); `absorb` = `hasher.transcript(state,c)` (`:101-103`); `absorb2` (`:105-108`); `absorb_felts` = leaf-encode then absorb (`:113-116`); `squeeze_cell` outputs *then* advances with `SQ(i)` (`:119-125`); `squeeze_operand` (`:87-94`); `squeeze_ext` = lanes 0–2 (`:127-130`); `squeeze_index(n)` = low `n` bits of lane 0 (`:132-135`) | +| **LFML leaves + LFMC parent** | `host_leaf_hash_pair` | `:145-147` — `hasher.compress(&hasher.leaf(c0), &hasher.leaf(c1))` | +| **LFMC Merkle parents** | `HostTree` | `:157-190` — parents `:170`, `root` `:177`, `open` `:182-189` | + +All three are `HasherKind`-parameterised, so they already run under Test / +Poseidon / Blake3. The primitives underneath sit in `blake3_socket.rs`: +`TAG_LFMC:227`, `TAG_LFML:241`, `TAG_LFMT:253`, +`socket_digest_rounds_tagged:294`, `transcript_digest:330`, `leaf_digest:414`, +`lanes_of:431`, `word_of:443`, `Blake3Permutation` impl `:456-520`. + +The **guest** side already exists in LFM-native form: `edsl::leaf_hash_pair` +(`:167-171`) and `edsl::merkle_walk` (`:177-190`) operate on one-cell digests, +distinct from the keccak twins (`keccak_merkle_walk:267`, +`KeccakDigest = [Cell;2]:196`). A tower verifier therefore walks **half as many +cells per Merkle level** as the keccak path — the recursion-tower payoff is +already built. + +**The limitation, stated by the file itself** (`fixture.rs:9-12`): this is *not* +the production proof format — "`crypto/stark` hardcodes keccak at its Merkle +layer; the measured 26-site migration seam is deliberately not touched here." +That seam has roughly doubled: **58** references to the four backend aliases +inside `crypto/stark/src` (`commitment.rs`, `config.rs`, `prover.rs`, +`verifier.rs`, `gpu_lde.rs`, `fri/mod.rs`, `tests/commitment_tests.rs`). + +`HostSponge`/`HostTree` are the right **reference**, not the right +**implementation**: they assume fixed 4-column rows, two-row leaves, fixed +depth. + +--- + +## 2. Prove-path genericity (Q2) + +**Injectable today — the transcript.** +`Prover::multi_prove(… transcript: &mut (impl IsStarkTranscript + Clone + Send) …)` +— `prover.rs:3032-3044`; mirrored at `verifier.rs:1219-1231`. ✓ VERIFIED +`prover.rs` never names `DefaultTranscript` (grep: zero hits). A B1 impl slots +in with no signature change. + +**Hardwired today — Merkle backend, FRI layer commitment, node type.** +`crypto/stark/src/config.rs:10-24` pins all three backends to keccak; +`Commitment = [u8;32]` at `:16-17`. `IsStarkProver` (`prover.rs:807-814`) +carries **no** hash parameter, and its default method bodies name +`BatchedMerkleTree` concretely (`:823, :879, :898`). + +**But everything beneath the alias layer is already generic** — this is what +makes the migration a parameterization rather than a rewrite: + +- `IsMerkleTreeBackend` with associated `Node`/`Data` — `crypto/crypto/src/merkle_tree/traits.rs:8-29` +- `Proof` — `merkle_tree/proof.rs:22-24` +- `verify_merkle_path_from_leaf_hash` / `verify_merkle_path` — `proof.rs:31-38, 57-65`, already turbofished at `verifier.rs:587, 671, 723` +- the keccak backends are themselves generic over `D: Digest, const NUM_BYTES` — `field_element_vector.rs:98-132, 135-203` + +### Does the 128-bit `LfmWord` force a distinct proof format? + +**No — and the tower-leg recomputation story survives.** + +`pack_digest` (`word.rs:44-50`) maps an `LfmWord` to `[u8;32]` as four canonical +LE u64 lanes; `unpack_digest` is the inverse (`:53-61`). A Blake3 digest cell +has all four lanes `< 2^32` (`word_of`, `blake3_socket.rs:443`), so it embeds +with 16 bytes of zero padding. + +Keeping `Node = [u8;32]` leaves `StarkProof`'s commitment fields +(`proof/stark.rs:47, 88, 89, 91, 94, 102, 106`) and the rkyv derives +(`:30-37, 52-59, 73-80, 128-135`) **byte-identical** — no wire-format bump, no +disturbance to the in-place rkyv verify path. Padding costs proof size only: +parent hashing operates on **cells**, never on the padded bytes, so the guest +recomputes exactly `LFMC(cell_l, cell_r)` with no padding in the preimage. The +tower leg is unaffected. + +⚠ **Two riders on that choice.** + +1. `unpack_digest` reduces mod p and does not bound lanes (`word.rs:52-61`), so + many distinct 32-byte strings decode to one node — **node malleability**. + Decode strictly via `lanes_of` (`blake3_socket.rs:431-438`), which rejects + rather than reduces. +2. `Node = [u8;32]` is precisely what makes a Blake3 backend type-check against + a keccak GPU kernel — see hazard **H3** (§4). + +### Recommended shape + +? INFERRED — design proposal, not compile-verified. A `StarkHash` config trait +carrying the three backends plus the node type; add it as a generic parameter +on `IsStarkProver`/`IsStarkVerifier` and on a `GenericProver`/`GenericVerifier`; +keep `pub type Prover = GenericProver`. Every +existing RV64 call site resolves unchanged, including the bare +`Prover::multi_prove` at `proof.rs:140`. + +A defaulted parameter on the *trait* will not work — `H` would be uninferable at +the call site; it has to ride on the concrete type via the alias. + +--- + +## 3. Transcript mapping (Q3) + +`IsTranscript` has five methods (`is_transcript.rs:7-26`); `IsStarkTranscript` +adds `sample_z_ood*` (`:28-90`). Call census ✓ VERIFIED by grep +(prover / verifier): `append_bytes` 4/8, `append_field_element` 2/4, +`sample_field_element` 3/5, `sample_u64` 1/1, `state` 1/1, `sample_z_ood*` 1/1. + +| `IsTranscript` method | B1 op | Verdict | +|---|---|---| +| `append_field_element` | `absorb_felts` (`fixture.rs:113-116`) | ✓ direct | +| `sample_u64(bound)` | `squeeze_bits(n)` (`fixture.rs:132-135`) | ✓ **exact** | +| `sample_field_element` | `squeeze_ext` (`fixture.rs:127-130`) | ✓ shape, ⚠ semantics | +| `append_bytes(&[u8])` | — | ⚠ B1 has no byte-level absorb | +| `state() -> [u8;32]` | — | ✗ **no equivalent** | +| `sample_z_ood*` | default body over `sample_field_element` | ✓ inherited | + +`sample_u64` maps **exactly**, which is worth stating because it looks like it +should not. The only call is `sample_u64(domain_size >> 1)` with `domain_size` a +power of two (`prover.rs:2132-2134`, `verifier.rs:138-140`). For a power-of-two +bound, `upper_bound.wrapping_neg() % upper_bound == 0`, so the rejection loop at +`default_transcript.rs:136-145` accepts the first candidate and returns its low +`log2` bits — precisely `squeeze_bits`. + +### Four items with no B1 equivalent + +1. **Grinding / PoW — the hard gap.** `transcript.state()` seeds + `grinding::generate_nonce` (`prover.rs:2093`) and `is_valid_nonce` + (`verifier.rs:1668`); both hash Keccak256 unconditionally + (`grinding.rs:1, 72, 87`). It is **live**: `MIN_PROOF_OPTIONS` sets + `grinding_factor: 1` (`prover/src/recursion.rs:39-45`). A tower guest + recomputing keccak PoW defeats the purpose of the switch. + **Recommend `grinding_factor: 0` for LFM proofs and scoping grinding out + explicitly**, rather than re-specifying PoW over the LFM hash. Note + `options.rs:114` asserts `security_bits > grinding_factor` — confirm 0 is + admissible before relying on it. +2. **Rejection sampling is loop-shaped; the machine cannot hold it.** + `sample_field_element` for E calls the base sampler three times + (`extensions_goldilocks.rs:575-581`), each an unbounded `loop` + (`goldilocks.rs:548-555`). The eDSL fully unrolls — "nothing loop-shaped + reaches the machine" (TRANSCRIPT.md §1.1, citing `edsl.rs:1-4`). The B1 impl + must use lane-direct `squeeze_ext`, whose u32 lanes are canonical by + construction, and must **not** reuse `sample_field_element_from`. +3. ⚠ **Challenge entropy drops to 96 bits.** A squeezed cell is four u32 lanes + = 128 bits (`word_of`, `blake3_socket.rs:443`); `squeeze_ext` takes lanes + 0–2. `DefaultTranscript` yields three near-full Goldilocks coordinates + (~192 bits). TRANSCRIPT.md §4.1 analyses the 128-bit state and its ~64-bit + collision bound but **does not** analyse per-challenge entropy at production + query counts. Needs a decision before it is a security claim. +4. **`append_bytes` needs an encoding.** `absorb_lfm_statement` + (`statement.rs:79-89`) feeds raw byte strings — tags, `program_id`, LE + integers. B1 absorbs cells. A padding-and-length-bound byte→cell convention + must be specified, not improvised. + +Also: TRANSCRIPT.md §4.2(b) warns the maximum squeeze run **is** `NUM_QUERIES`; +at the `Blowup2` preset's 219 queries the run is ~219 (≈7 bits loss). Within the +doc's own `k < 2^16` guidance, but record it rather than assume it. + +--- + +## 4. The three silent-mislabel hazards + +These are why the ordering is non-negotiable. All three type-check; none fails +loudly. + +### H1 — artifacts stamp Blake3 on keccak roots + +✓ VERIFIED `build_artifacts_with_hasher` (`registry.rs:128-171`) computes every +root via `commit_group(g, options)` (`:150`) plus +`keccak_rc::preprocessed_commitment` (`:155`) and +`bitwise::preprocessed_commitment` (`:157`) — all keccak (`commit.rs:56` → +`commit_columns:21-46` → `commit_bit_reversed`, `commitment.rs:140-155`). It +then stamps `hasher` into `program_id` and the returned `LfmArtifacts` +(`:163-170`). + +Under `HasherKind::Blake3` today you get artifacts *naming* Blake3 whose roots +were built with keccak. Harmless while the commitment hash is not part of the +claim — **silently wrong the instant a Blake3 commitment path exists.** + +The doc comment at `registry.rs:112-116` actively asserts the currently-true +reasoning ("its preprocessed width is the same under every candidate … so no +commitment moves with it"). That sentence becomes **false** and must move in the +same commit as the guard. + +### H2 — the cross-hasher test sweep does not cover Blake3 + +✓ VERIFIED `machine_tests.rs:2427-2430`: +`const ALL_HASHERS: [HasherKind; 2] = [Test, Poseidon]` — while `HasherKind` has +**three** variants (`hash.rs:196-212`). Its own doc at `:2424-2426` claims +exhaustiveness: *"Every `HasherKind` there is. Not derived — a new candidate must +be added here by hand, which is the point."* Blake3 was added and this was not. + +The two tests it drives — the digest-binding test (`:2456`) and the cross-hasher +reject test (`:2503`) — are exactly the ones that would catch hasher confusion, +and the Blake3 arm currently sits outside both. + +### H3 — the GPU tree path takes a backend parameter it does not honour + +✓ VERIFIED `try_expand_leaf_and_tree_row_major_keep` +(`gpu_lde.rs:680-695`) is bounded `B: IsMerkleTreeBackend` but +its body unconditionally calls the keccak device kernel +`math_cuda::lde::coset_lde_row_major_with_merkle_tree_keep` (`:720`). `B` is a +**type-level label only**. + +A Blake3 backend satisfying `Node = [u8;32]` — exactly what §2 recommends — +would compile silently and yield **keccak trees typed as Blake3**. The §2 +node-type recommendation is what opens this, so it must ship with the guard. +Same shape applies to `try_expand_split_trees_row_major_keep` (`:779`), +`try_expand_leaf_and_tree_ext3_row_major_keep` (`:863`), `..._keep_dev` +(`:1554`). + +--- + +## 5. GPU staging + +✓ VERIFIED **no BLAKE3 exists anywhere in CUDA** — `grep -ril blake3 +crypto/math-cuda/` returns nothing. Kernel sources: `arith.cu, barycentric.cu, +constraint_interp.cu, deep.cu, ext3.cuh, fri.cu, goldilocks.cuh, inverse.cu, +keccak.cu, logup.cu, ntt.cu`. Only `keccak.cu` is a hash. + +**Survives (hash-agnostic):** LDE/NTT (`ntt.cu`), constraint composition +(`constraint_interp.cu`), barycentric (`barycentric.cu`), DEEP (`deep.cu`), FRI +fold arithmetic (`fri.cu`), LogUp (`logup.cu`), inverse/arith. + +**Dies (all `keccak.cu`):** `keccak256_leaves_base_batched:152`, +`..._base_row_pair_batched:196`, `..._ext3_batched:237`, +`keccak_comp_poly_leaves_ext3:277`, `keccak_fri_leaves_ext3:326`, +`keccak_merkle_level:394`, `keccak_merkle_tail:408`, +`keccak256_leaves_base_row_major_row_pair:473`, `..._range:511`. Rust wrappers +in `crypto/math-cuda/src/merkle.rs`: `keccak_leaves_base:33`, +`keccak_leaves_ext3:83`, `build_merkle_tree_on_device:316`, +`build_comp_poly_tree_from_slabs_dev:494`, +`build_comp_poly_tree_from_evals_ext3_keep:544`, +`build_fri_layer_tree_from_evals_ext3:564`. Note tree *building* is on-device +too, not only leaf hashing. + +### The tree-less R1 entry the short-term staging needs + +✓ VERIFIED `try_expand_columns_batched(columns, blowup_factor, weights) +-> Option<()>` at `gpu_lde.rs:430-434`. It expands columns in place, takes **no +backend parameter** and builds **no Merkle tree** — GPU does the LDE, host does +leaves and tree. This is the correct entry for the accept-CPU-trees phase, and +it is immune to H3 by construction. + +### `device_only_gate` must be forced false + +⚠ `device_only_gate` (`gpu_lde.rs:189-212`) is entirely hash-agnostic — field +tower, thresholds, `!is_preprocessed`, contiguous offsets, uniform zerofier. +That is the hazard, not the relief: it would still evaluate **true** under +blake3, but device-only residency drops the host trace, and the module doc +(`:175-180`) says a violated precondition hits a `host_trace_empty` **hard +abort**. CPU leaf hashing needs the LDE on the host. + +**Follow-up (not step 1):** blake3 leaf + `merkle_level`/`merkle_tail` kernels. +`merkle_gather_paths:433` (`gather_merkle_paths_dev`, `merkle.rs:358`) is +already hash-agnostic and reusable once a device tree exists again. + +--- + +## 6. Change list (Q6) + +Each step independently verifiable. `cargo test --release` throughout — proving +tests crawl otherwise. + +| # | Change | Test oracle | +|---|---|---| +| **0** | **Guards first, before any Blake3 commitment exists.** H1: make `build_artifacts_with_hasher` reject or assert when `hasher` disagrees with the commitment hash actually used, and correct `registry.rs:112-116`. H2: add `Blake3` to `ALL_HASHERS` (`machine_tests.rs:2427`). H3: remove the unused `B` parameter from the GPU tree entries, or bind it to a keccak-only marker so a Blake3 backend cannot be passed | Existing suite green; H2's two tests (`:2456`, `:2503`) now exercise the Blake3 arm and must pass unchanged | +| **1** | **Spec, no code.** `commit-spec/COMMIT.md` covering the three things no ratified doc covers: wide-leaf construction, byte→cell absorb encoding, node embedding + strict decode | Python KATs in the style of `leaf_kats.py` / `transcript_kats.py`, written before any Rust — the discipline LEAF.md and TRANSCRIPT.md both followed | +| **2** | `StarkHash` config trait + `GenericProver`/`GenericVerifier`, keccak instance only. Pure refactor | Full existing suite + a cross-version verify (the king gate) | +| **3** | `LfmBlake3` leaf + pair backends implementing `IsMerkleTreeBackend`, `Node = [u8;32]` packed, LFML leaves / LFMC parents, strict decode via `lanes_of` | Step-1 KATs + host parity against `HostTree` (`fixture.rs:157-190`) | +| **4** | B1 `IsStarkTranscript` impl (`LfmTranscript`); `grinding_factor: 0` for LFM options | Op-for-op parity against `HostSponge` + transcript KATs | +| **5** | Generalize `absorb_lfm_statement` (`statement.rs:74`) and `replay_transcript_phase_a_view` (`prover/src/lib.rs:989-992`) from `&mut DefaultTranscript` to `&mut impl IsTranscript` | Compile-level; existing `lfm::` suite unchanged | +| **6** | Wire `lfm_prove` / `verify_against` (`proof.rs:133, 207`) | Prove+verify round trip, `TrivialV0` then `FriToyV0` under Blake3 | +| **7** | Registry: hasher-aware `resolve` key; **add** Blake3 rows, never flip the default | `compute_lfm_registry` + drift test; Test rows must still resolve and verify | +| **8** | GPU staging: route LFM prove to `try_expand_columns_batched` (`gpu_lde.rs:430`), force `device_only_gate` false under the blake3 config | `crypto/math-cuda/tests/{keccak_leaves,merkle_root_parity,fri_layer_tree}.rs` still green for keccak | +| **9** | *Follow-up:* blake3 device kernels, restore device-resident trees | Parity tests mirrored for blake3 | + +### On the registry (step 7) + +✓ VERIFIED all six entries (`registry.rs:194, 278, 362, 446, 530, 614`) are +`blowup_factor: 2`, `hasher: HasherKind::Test`, with inline literal roots and +program_ids; **no Blake3 entry exists**. + +`resolve` (`:176-186`) keys on `(kind, blowup_factor)` **only** — so a Blake3 +row cannot coexist with a Test row until the key includes hasher. Regeneration +is `cargo run --bin compute_lfm_registry --release` (`:5`) under the standing +policy at `:6-8`: *"a drift failure is investigated, never re-blessed to silence +the test."* + +**Add rows; never flip the default.** That preserves the Test entries as the +honest control the campaign depends on. Every root moves (H1's cause), so this +is a strictly larger re-bless than Phase 3's, which moved six program_ids but +**no root**. + +Keep `lfm_program_id` (`statement.rs:50-68`) on keccak for now — it is a +host/consumer artifact the tower guest does not recompute, and `statement.rs:6-7` +already reserves `_V2` for the ecosystem migration. + +--- + +## 7. Soundness register + +| # | Point | Pinned by | +|---|---|---| +| **S1 ★** | **Wide-leaf / opening-width binding.** LEAF.md §1.4 specifies only the toy shape (2 rows × 4 cols → 2 LFML + 1 LFMC). Production AIRs vary in width, and the leaf hash streams `evaluations ‖ evaluations_sym` **with no length prefix or separator** — `verifier.rs:204-206` states this verbatim; `:207-213` records it *was* exploitable (a prover could pick columns after challenges they must precede) and is closed today by an explicit I3 width check, not by the hash. Any LFML chain must bind width itself or preserve that check | **Nothing.** Same class as the standing `main↔aux` open item | +| **S2** | Node malleability on decode — use `lanes_of` (`blake3_socket.rs:431-438`), not `unpack_digest` (`word.rs:52-61`) | Nothing; add a KAT | +| **S3** | Extension packing — aux openings are `FieldElement` (`verifier.rs:666`), 3 base felts each, against a 4-felt LFML cell | Nothing | +| **S4** | Transcript domain separation — and the explicit correction that constraint idx 4 is *not* what makes selectors one-hot | TRANSCRIPT.md §2, §3.3; controls M5/M6/M8 | +| **S5** | Leaf domain separation — O5 retired, enforced by the tag | LEAF.md §4; M9/M10 | +| **S6** | Tree arity/padding — `HostTree::build` asserts power-of-two leaves (`fixture.rs:164`) and pads nothing; `build_from_hashed_leaves` off that shape unchecked | Nothing | +| **S7** | Grinding — scope out explicitly (§3, item 1) | Nothing | +| **S8** | 96-bit challenge entropy; squeeze runs scale with query count | TRANSCRIPT.md §4.1/§4.2 cover state collision and run length, **not** per-challenge entropy | + +S1 is the gating item. S2, S3, S6 and S7 are all "pinned by nothing" and belong +in step 1's spec. + +--- + +## 8. Scope + +PLAN.md §6.2 classifies this rung as **E2** and says it "requires the +*production* RV64 prover's Merkle and Fiat–Shamir hash to be BLAKE3 … Out of +LFM's control and far larger than everything above combined." + +The parameterization in step 2 is precisely what makes an **LFM-only E2** +possible while the RV64 path keeps keccak untouched. That is the central +architectural claim of this design and the reason the work is tractable at all. + +It is nonetheless materially larger than **P5** as tracked in ORCHESTRATION.md +("prove+verify a wrap under BLAKE3 (swap TestPermutation)"), which concerns the +machine's *chips* (role 2). This concerns the machine's *own commitments* +(role 1 for LFM). They are different axes and should be tracked as separate +rungs, not folded together. diff --git a/thoughts/shared/block-compression/HASH-SPLIT-PLAN.md b/thoughts/shared/block-compression/HASH-SPLIT-PLAN.md new file mode 100644 index 000000000..e29b42e17 --- /dev/null +++ b/thoughts/shared/block-compression/HASH-SPLIT-PLAN.md @@ -0,0 +1,680 @@ +# HASH-SPLIT-PLAN — the fleet endgame: split proving + a specialized blake3 circuit + +**Mandate (Mauro, 2026-08-13):** *"450 [GPUs] is a lot, the current sota is 4-8 gpus. We may +need the split proving with some specialized blake circuit."* + +**Status: SCOPING / Round-0 projection. Read-only; no code touched, nothing built.** +This joins SOLUTION-ARRAY.md as the fleet-endgame track. Its Round 0 is arithmetic and is +complete in this document; every number is reproducible from +`~/workspace/lambda_vm_bench_cache/lfm_census_2026-08-12/` plus the two scripts named in §7. + +--- + +## 0. Verdict — read this before scheduling anything + +**1. The 450 figure reconciles, and the model reproduces it without tuning.** Today's +configuration (2^21 epochs, blowup2/219q, leaf rate 4, no batching, 2-ary tower, hosted +socket) projects **1,267 GPUs**; the partially-optimised region Mauro is quoting from — +rate-8 absorption or partial batching — lands at 400-750. DERIVED, §1.4. So the fleet +question is real and the arithmetic behind it is sound. + +**2. ★ Neither the split nor a specialized circuit gets to 4-8 GPUs.** At the best +configuration the model can reach (2^23 epochs, blowup4/110q, rate 8, 4-ary tower, batched +FRI+MMCS, plus every hash-circuit lever in this document), **a hash chip that cost literally +zero still leaves 23 GPUs of work**. The residue — the LFM machine's own arithmetic +marshalling felts into and out of the hash chip — is 18.3 B cells per block and no hash +lever touches it. §2.3. The required cells-per-compression at 8 GPUs is *negative*: the +budget is exhausted before the first compression is priced. **This survives a 10× swing in +the residue coefficient** (§1.4): even at the optimistic end, 8 GPUs needs 891 +cells/compression against a ✓ VERIFIED chip of 4,946 and a plausible floor of ~4,060. +**32 GPUs is the reachable endgame; 16 needs everything to go right; 4-8 is not in this +design space.** + +**3. And there is no floor worth chasing under the current chip. ✓ VERIFIED from source.** +`blake3_chip.rs` is **one row per compression** (`:781`), **3,056 main** = 112 input bytes + +48 G-blocks × 60 cells + 64 output bytes (`:162, :224`), **1,259 bus interactions → 630 ext +aux** (`:911-913`), of which **1,248 are BITWISE byte-lookups** (`:990-991`). So +**4,946 cells/compression is fully verified**, including the aux width the campaign has been +carrying as inferred. The design is already tight: only the two non-byte-aligned rotations +cost anything (`ROT_SHIFT_R`, `:126-128`), and a tighter encoding of the 14 words per +G-block plausibly reaches **≈4,060** — **a 1.1-1.2× chip, not a 10× one.** §1.3. +**The brief's premise that a specialized blake3 circuit is the lever should be retired.** + +**4. ★ The finding that pays for this document is about SHAPE, not about splitting.** +Verification cost is proportional to a table's **WIDTH** and only logarithmic in its rows +(leaf absorption = `Σ_groups 2·cols·kind / RATE`; Merkle depth = `log2_lde − 1`). Proving +cost is `rows × width` — invariant under reshaping. So laying the blake3 AIR out +**narrow-and-tall (one G-call per row, ~100 columns × 48× the rows) costs the same to +prove and 1.5× less to verify**: +- the D1 tower node drops **157 → 103 GiB** at RATE 4 and **104 → 69 GiB** at RATE 8 (§3.2) + — the best tower-node figure the campaign has, within 8% of the 64 GiB production target, + and *independent of the split*; +- it is also what makes a split viable at all: at a 2^18-compression shard the + 1-compression-per-row layout needs ρ > 2.50 to break even (we have ρ ≈ 1.7, so it + **loses**), while the G-per-row layout breaks even at ρ = 1.17 and **wins**. §2.4. + +**This reprices D9.** RATE and row-shape are the same lever seen twice, and row-shape is +the cheaper half: it needs no bus-arity change, no `num_input_cells` change, and none of +the lane-map hazards the RATE=5 refute pass found. + +**5. What gets from 1,267 to 78 GPUs is scheduling, not silicon; the hash circuit takes it +from 78 to 51.** Ladder (§1.4): batching 1.7×, RATE 4→8 1.5×, tower arity 2→4 1.3×, +**epoch 2^21→2^23 3.7×**, inner blowup4/110q 1.4× — 16.2× cumulative, and the biggest rung +is a *scheduling* decision already unlocked by MMCS-PLAN §1.2 ("epoch size becomes nearly +free after batching") that costs no build. Then the hash-circuit rungs (packed AIR + the two +shape changes) add 1.54×. **Do the scheduling first: it is free, and it is 10× the size of +the thing the brief asked about.** + +**6. Cadence and latency are different problems and the brief conflates them.** "4-8 GPUs +keeping up with 12 s blocks" is a *throughput* requirement: fleet size = total work ÷ +(12 s × per-GPU throughput), and pipelining across blocks satisfies it. *Latency* — a +block proof available within one slot — is a separate requirement that the current shape +misses by 14-45×: one base wrap is 28-65 s on one GPU and the tower adds 2-3 layers of +69-162 s each. **Intra-proof distribution (§2.1 route d) is the only lever that attacks +latency, and it is also the only route with zero soundness surface.** §3. + +**7. One number decides the split's value and it is free to measure.** ρ — the ratio of +total wrap cells to hash-chip cells — is **MEASURED at 1.081 under keccak** +(`census_logs/ethrex_e21_b2_q4.log`: hash chips 92.5% of cells) and **DERIVED at ~1.7-1.9 +under blake3**, because blake3 shrinks the hash term ~4× while the marshalling residue +tracks *felts absorbed*, which is hash-independent. The whole split case turns on whether +the blake3-native emitter (`edsl::leaf_hash_pair` over one-cell digests) is as expensive +per felt as the keccak emitter's byte-level sponge packing. **Census one wrap under the +blake3 emitter and read off the hash-chip share.** That is the minimal experiment (§5.1), +it needs no proving, and it is decisive in both directions. + +**8. The obstacle to a *true* split is a real open problem, not an engineering gap.** Our +LogUp challenge is sampled only after every main root is in the transcript +(`crypto/stark/src/prover.rs:2447`), so a bus cannot cross proof boundaries. SP1's answer is +a **challenge-free** binding — each interaction is hashed to a septic-curve point, sends +minus receives must sum to the identity, asserted in the recursion layer +(`global_interaction.rs:33-45`, `complete.rs:147`) — ✓ VERIFIED. **But it costs a Poseidon2 +permutation per interaction.** That is fine for offloading a keccak-f from a RISC-V shard +and self-defeating for offloading the compression function the glue is made of. Routes +(a)-(c) all need a cheap challenge-free binding at hash rate, and none is known. §2.1. + +**Open decisions for Mauro: D10, D11, D12 — §6.** + +--- + +## 1. THE BACKWARDS TARGET + +### 1.1 Anchors + +| quantity | value | provenance | +|---|---|---| +| GPU throughput | **67.13 M base-field-equivalent cells/s** | 481,327,124 cells (`EXPLORATION.md:186`) ÷ 7.17 s ABBA mean (`BOX-RESULTS.md:53,57`), one RTX 5090, `LAMBDA_VM_GPU_LDE_THRESHOLD=262144`, verify green ✓ MEASURED | +| slot | 12 s | Ethereum | +| bytes per cell (host RSS) | 33.7 | `wrap_tests.rs` `MEASURED_BYTES_PER_CELL` ✓ MEASURED | +| hosted socket | **4,946** cells/compression | 3,056 main + 3×630 aux — ✓ **VERIFIED both**, `blake3_chip.rs:162,224` (main) and `:911-913` (1,259 interactions → 630 ext aux). MMCS-PLAN §5 and `tower.py:15` carried the aux as `? INFERRED`; it is now confirmed. | +| #903 standalone chip | **5,316** cells/compression | 3,219 main / 1,397 sends → 3,219 + 3×699; PA-PLAN.md:541 quoting commit `35038501` ✓ VERIFIED from the commit message, ✗ UNVERIFIED against the branch source | +| epochs per block | 2^20→72, 2^21→36, 2^22→18, 2^23→9 | block 25368371, 74.8M cycles, PLAN.md:8 ✓ MEASURED | + +**Budget** = G × 12 s × 67.13 M cells: + +| GPUs | 4 | 8 | 16 | 32 | 64 | +|---|---|---|---|---|---| +| cells/block | **3.22 B** | **6.44 B** | **12.89 B** | **25.78 B** | **51.56 B** | + +⚠ **The throughput assumption and what breaks it.** 67.13 M cells/s is measured on a +481 M-cell wrap whose dominant chip (KECCAK_RND, 88.1% of main cells) is device-resident +at 12.7 GiB peak VRAM on a 32 GiB card. Extrapolating it to a 2-4 B-cell wrap assumes: +1. **VRAM.** Linear scaling puts a 1.9 B-cell wrap at ~50 GiB VRAM — over a 5090. The + measured configuration therefore *cannot* hold; S3 host-recompute or device-recompute + (SOLUTION-ARRAY B/C+) must run, and the seam audit prices that at **+40-60% wall**. + Applying that penalty multiplies every fleet number below by ~1.5. +2. **Host RAM.** 33.7 B/cell puts one config-F base wrap at **59 GiB** — inside the 64 GiB + target, which is the first time in this campaign a wrap has fitted. §3.1. +3. **Fixed costs amortise the other way.** 7.17 s on 481 M cells implies ~26.8 KB of memory + traffic per cell against the 5090's 1.79 TB/s — i.e. the measured point is *not* + bandwidth-bound, so a larger proof may run faster per cell. Direction unknown; this is + the main reason to treat the anchor as ±40% rather than ±10%. +4. **The blake3 chip is lookup-heavy** (~57% of its cells are LogUp aux), and LogUp aux + generation is batch-inversion-bound, not NTT-bound. A blake3-dominated wrap may have a + materially different cells/s than a KECCAK_RND-dominated one. **Unmeasured.** + +### 1.2 Compressions per block + +Per-query leg cost from the calibrated model (`mmcs_project.py`, validated to the unit +against the measured census at four points — MMCS-PLAN §1.0), RATE-parameterised, × queries +× N epochs. `rate` = felts of message the socket absorbs per compression invocation; the +byte-optimal value is 8 (two 4-felt cells = one 64-byte BLAKE3 block). + +**Base layer (blake3 inner, after P-a), compressions per BLOCK:** + +| epoch | N | batching | RATE 4 | RATE 5 | RATE 8 | +|---|---|---|---|---|---| +| 2^20 | 72 | off | 59,891,040 | 54,085,680 | 43,797,600 | +| 2^20 | 72 | **ON** | 33,580,800 | 27,632,880 | **18,501,120** | +| 2^21 | 36 | off | 68,559,264 | 62,654,148 | 51,687,504 | +| 2^21 | 36 | **ON** | 34,374,240 | 28,295,676 | **18,961,020** | +| 2^23 | 9 | off | 34,352,559 | 32,612,166 | 28,636,659 | +| 2^23 | 9 | **ON** | 10,245,258 | 8,410,257 | **5,593,698** | + +DERIVED from the calibrated model. Two readings: **batching is worth 1.8× (2^20) to 5.1× +(2^23)** on compression count, and **RATE 4→8 is worth a flat 1.8×** everywhere. + +**Tower**, N leaves aggregated k-ary: total proof-verifications = N + ⌈N/k⌉ + ⌈N/k²⌉ + … +Node cost = one LFM-proof verify at 110q, native LFML/LFMC (`tower.py` construction). + +| epoch | arity | nodes | depth | verifications | tower comps (rate 8, batched) | as % of base | +|---|---|---|---|---|---|---| +| 2^21 | 2 | 38 | 6 | 73 | 28,281,660 | 149% | +| 2^21 | 4 | 13 | 3 | 48 | 18,596,160 | 98% | +| 2^21 | 8 | 6 | 2 | 41 | 15,884,220 | 84% | +| 2^23 | 2 | 11 | 4 | 19 | 7,360,980 | 132% | +| 2^23 | 4 | 4 | 2 | 12 | 4,649,040 | 83% | +| 2^23 | 8 | 3 | 2 | 11 | 4,261,620 | 76% | + +**★ The tower is not a rounding error — it is 76-149% of the base layer.** Every campaign +number quoted so far has been per-wrap; the block costs roughly *twice* the base layer. +Arity 2→4 removes a third of it and halves the depth; 4→8 adds little (§3.3). + +⚠ This holds node cost constant across layers, which is only legitimate under D1's +**static-shape premise** ("14 fixed tables, known log-heights, wrap options fixed → the +program shape is static per (K, options)", PLAN.md:184-190). If a node's own proof is +larger than what it verifies, upper layers cost more and the tower diverges. **Gate D1 must +demonstrate the fixed point, not just the single node.** ✗ UNVERIFIED. + +### 1.3 What a compression costs, and the floor + +**★ The existing chip is already within ~10% of its floor. ✓ VERIFIED from source** +(`lambda_vm-blake3-impl@blake3-real-hash`, `prover/src/lfm/blake3_chip.rs`): + +| fact | value | citation | +|---|---|---| +| **one row per compression** | — | `:781` *"One row per compression; padding rows are ALL ZERO."* | +| G-instances | `NUM_G = BLAKE3_ROUNDS * 8` = **48** at 6r | `:101-102` | +| per-G columns | `G_SIZE = 60` — *"56 bytes + 4 carry bits"* | `cols::G_SIZE`, `:158` | +| input bytes | `4 × IN_U32` = 112 (`h[32] | m[64] | t_lo|t_hi|len|flags[16]`) | `:104, :156-157` | +| output bytes | `4 × OUT_U32` = 64 | `:106, :161` | +| **main columns** | 112 + 48×60 + 64 = **3,056** = `NUM_COLUMNS(3072) − PREP_WIDTH(16)` | `:162, :224` | +| **bus interactions** | **1,259** → aux = ⌈1259/2⌉ = **630 ext** | `:911-913` `Vec::with_capacity(1_259)` | +| of which BITWISE lookups | **1,248 per compression** | `:990-991` `ops.len() * 1_248` | +| per-G operations | 2 add3 + 2 add2 (= **6 additions**), **4 XOR**, **2 rotations** | `:485-488` | +| rotations | only 12 and 7 cost anything: *"rotr12 = rotl20 = rotl16∘rotl4; rotr7 = rotl25 = rotl16∘rotl9"*; 16 and 8 are byte permutations, free | `ROT_SHIFT_R`, `:126-128` | + +> **4,946 = 3,056 main + 3×630 aux, ✓ FULLY VERIFIED** — including the aux width, which +> `tower.py:15` and MMCS-PLAN §5 both carried as `? INFERRED`. + +**What a floor would have to beat.** 2,880 of the 3,056 main columns are the 48 G-blocks; +the other 176 are I/O. Within a G-block, 56 bytes = 14 byte-decomposed 32-bit words (4 add +results + 4 XOR results + rotation split parts) plus 4 carry bits. A tighter encoding might +carry 10-12 words instead of 14 → `G_SIZE` ≈ 44-52 → main ≈ 2,300-2,700, with the lookup +count falling proportionally. **Realistic floor ≈ 4,200-4,700 cells/compression — a 5-15% +improvement, not a factor.** + +| candidate | main | aux (ext) | cells/compression | +|---|---|---|---| +| hosted socket today | 3,056 | 630 | **4,946** ✓ VERIFIED | +| #903 standalone chip | 3,219 | 699 | **5,316** (✓ from commit `35038501`'s message) | +| plausible floor, same byte-decomposed family | ~2,500 | ~520 | **≈4,060** DERIVED | + +**The chip is not where the win is.** 38% of a compression is LogUp aux paying for 1,248 +BITWISE byte-lookups; going materially below means trading those lookups for algebraic +constraints, which costs main columns roughly 1:1. **A specialized blake3 circuit is worth +~1.1×, not 10× — and the brief's premise that it is the lever should be retired.** + +### 1.4 The target table, and the ladder + +**Cost model.** The flat-ρ framing in the brief is not adequate, because the residue does +not scale with compressions — it scales with **felts absorbed**, which is a property of the +proof being verified and is invariant under every hash-chip lever. Two-term model: + +``` +cells_per_wrap = compressions × c_chip + felts_absorbed × 439 + 26.5 M (fixed-height tables) +``` + +✓ MEASURED calibration, `census_logs/ethrex_e21_b2_q4.log` chip census (2^21/blowup2/q=4): +total 1,965,702,420 cells; hash chips (KECCAK_RND + LFM_KECCAK) 1,818,755,072 = **92.5%**; +fixed-height tables (BITWISE 2^20 + LFM_RANGE + KECCAK_RC) 26,476,672; q-scaling arithmetic +120,470,676. Against 4 × 50,870 = 203,480 felts absorbed → 592 cells/felt, corrected to +**439** for the spine's 25.9% instruction share (which does not scale with q). +**ρ = 1.081 at this point** — ✓ MEASURED, and the single most load-bearing input here. + +**★ The ladder** (each rung cumulative; GPUs = cells ÷ (67.13 M × 12 s)): + +| step | comps/block | B cells | **GPUs** | cum | hash share | +|---|---|---|---|---|---| +| **A** today: 2^21 b2/219q, rate 4, no batching, arity 2, hosted socket | 138,942,214 | 1,020.6 | **1,267** | 1.0× | 67% | +| **B** + batched FRI + MMCS | 87,934,340 | 599.7 | **744** | 1.7× | 73% | +| **C** + leaf RATE 4→8 | 47,242,680 | 398.4 | **495** | 2.6× | 59% | +| **D** + tower arity 2→4 | 37,557,180 | 316.0 | **392** | 3.2× | 59% | +| **E** + epoch 2^21→2^23 (N 36→9) | 10,242,738 | 86.0 | **107** | 11.9× | 59% | +| **F** + inner blowup4/110q | 7,476,480 | 63.2 | **78** | 16.2× | 59% | +| **G** + packed AIR at 4,400 cells/compression (12% headroom, §1.3) | 7,476,480 | 59.1 | **73** | 17.4× | 56% | +| **I** + G-per-row shape on the tower's LFM_HASH (§3.2) | 5,926,800 | 46.8 | **58** | 21.8× | 56% | +| **J** + G-per-row shape on the inner blake3 chip too (§3.3) | 5,174,400 | 41.0 | **51** | 24.8× | 55% | +| **H** + a *free* hash chip — the residue-only floor | 5,174,400 | 18.3 | **23** | 55.1× | 0% | + +Rungs A-F are configuration and scheduling; **G-J are the hash-circuit work this document +was commissioned about, and together they are worth 1.54×** (78 → 51) — **of which the chip +itself is 1.07× and the two shape changes are 1.44×.** The shape rungs reduce *both* terms +— fewer compressions to host *and* fewer felts absorbed — which is why they dominate the +per-compression rung that costs far more to build. + +**Required cells/compression at config J** (5,174,400 compressions/block): + +| GPUs | budget | required, **pessimistic residue** (18.3 B) | **optimistic residue** (1.83 B) | +|---|---|---|---| +| 4 | 3.22 B | impossible | 269 | +| 8 | 6.44 B | impossible | 891 | +| 16 | 12.89 B | impossible | 2,138 | +| 32 | 25.78 B | **1,446** | **4,629** | +| 64 | 51.56 B | 6,428 | 9,225 | + +Against the §1.3 floor (**4,946 today, ≈4,060 plausible best**): **32 GPUs is reachable if +the residue is small — the required 4,629 is 6% under today's chip and comfortably inside +the plausible floor. 16 needs 2,138, which is 2× below anything this chip family can +reach. 8 needs 891 and 4 needs 269 — 4.6× and 15× below the floor. Those are not in this +design space, and no amount of blake3-circuit work puts them there.** + +The brief's requested {batching} × {RATE} × {GPU count} grid is in §7's script output; it is +not reproduced in full here because **every cell at ≤32 GPUs reads "impossible" once the +residue is priced**, and the grid computed against hash cells alone (which is what a flat-ρ +model does) is misleading in exactly the direction that would authorise the wrong build. + +**★ Robustness.** 439 cells/felt is the pessimistic end — it is calibrated on the *keccak* +emitter's byte-level sponge packing, and the blake3-native emitter over one-cell digests +should be much leaner (§5.1). The "optimistic residue" column above is that coefficient +divided by 10. **The verdict at 4-8 GPUs survives the full 10× swing**; what the swing +changes is *which lever matters next* (residue vs chip), and whether 32 needs a new chip at +all. That is exactly why §5.1 is the first thing to run. + +--- + +## 2. THE SPLIT DESIGN SPACE + +### 2.1 Four routes, and what each actually is + +**(d) — DISTRIBUTED PROVING of a table that is already separate.** *Named first because it +is the cheapest and is not what the brief assumed.* Our proofs are already multi-table: an +epoch proof has 28-64 sub-proofs, an LFM proof 14, each with its own commitment, all bound +by a **shared LogUp challenge sampled after every main root is in the transcript** — +✓ VERIFIED **`crypto/stark/src/prover.rs:2447-2448`**, "Round 1, Phase A: Commit all main +traces … All main trace commitments must be in the transcript before sampling LogUp +challenges." (MMCS-PLAN §1.0 cites this as `prover.rs:3213-3238`; line numbers have drifted +since, the constraint now sits at 2447.) So the hash chip is *already* a separate table on a +shared, already-sound bus. "Splitting hash out" can mean nothing more than **proving that +table on a different GPU**. +- *Soundness:* **unchanged — it is the same proof.** No new protocol surface at all. +- *What is new:* a distributed prover with a small number of synchronisation barriers + (commit mains → gather roots → advance transcript → broadcast challenge → commit aux → + …). The seams are already named: residency-seam-audit.md S1-S7 (`multi_prove` takes a + per-index producer; `LfmTraces` goes lazy). +- *What it buys:* **latency only.** Total work is identical, so the fleet number in §1.4 + does not move. It is the only route that attacks §3's 14-45× latency miss. +- *Effort:* **M** (orchestration + the S1-S7 refactor, which S3 has already started). + +**(a) — SP1-style deferred/precompile shards with a global EC-digest accumulator.** +✓ VERIFIED in `others/sp1`: +- Each global interaction's 8-word payload is **hashed to a point on a septic-extension + curve** — `SepticCurve::::lift_x(new_values)` — with the interaction `kind` folded into + the top byte of word 0 (`crates/core/machine/src/operations/global_interaction.rs:33-45`). + A **send is the point; a receive is its negation** (`:41-44`). +- The per-shard accumulation is the **elliptic-curve sum** of those points + (`operations/global_accumulation.rs`, `global/mod.rs:208` `global_cumulative_sum`). +- The reconciliation is **not in the per-shard verifier — it is in the recursion layer**: + `crates/recursion/circuit/src/machine/complete.rs:147` + `builder.assert_digest_zero_v2(is_complete, *global_cumulative_sum)`, seeded from the vk's + `initial_global_cumulative_sum` (`machine/core.rs:136-140`) and observed into the + challenger (`:149-150`). + +> **★ The architectural answer to the problem route (b) runs into: an EC digest needs NO +> shared challenge.** Soundness rests on the hardness of finding a non-trivial zero-sum +> combination of hash-to-curve points, not on a Fiat-Shamir challenge sampled after +> commitments. *That* is why it composes across independently-proved shards, and it is +> exactly what our LogUp bus cannot do. + +- ⚠ *And the cost is structurally wrong for our use case.* Each global interaction carries + `x_coordinate: SepticBlock` (7) + `y_coordinate` (7) + **a full `Poseidon2Operation` + permutation** + `offset` + `y6_byte_decomp[4]` + (`global_interaction.rs:24-30`). **The glue spends a hash permutation per interaction.** + At one interaction per blake3 compression, offloading hashing would cost a hash per hash — + the mechanism is priced for offloading *expensive* precompiles (a keccak-f, a 256-bit EC + op) from a cheap RISC-V shard, not for offloading the compression function that the glue + itself is built from. +- *Effort:* **L**, and the cost analysis above argues it is the wrong tool. Adversarial + debate mandatory if it is ever scheduled. +- ⚠ A fuller survey (per-shard row limits / `SplitOpts`, the deferred-proof digest path, + exact `Poseidon2Cols` width) was commissioned and had not returned; the figures above are + the load-bearing ones and are ✓ VERIFIED, the rest of route (a) remains ? INFERRED. + +**(b) — CROSS-PROOF LogUp with a joint transcript.** The wrap emits a fingerprint send per +compression; the hash shard emits the matching receive; both partial sums are public and +must cancel. +- *The obstacle is the challenge, not the bus.* The shard's aux trace needs the LogUp + challenge γ, and γ must be bound to the wrap's commitments too, or a malicious shard + prover chooses its list after seeing γ. That forces **both provers to interleave**: + commit mains → joint γ → commit auxes. This is a single Fiat-Shamir transcript spanning + two proofs — i.e. **route (d) with the two halves relabelled as separate proofs**, and it + keeps (d)'s synchronisation barrier while adding a new wire format. +- *The escape route is (a)'s trick, not more transcript engineering.* A **challenge-free** + accumulator — SP1's hash-to-curve digest — removes the interleaving requirement entirely, + which is precisely why SP1 chose it. But it re-imports a hash permutation per interaction + (§2.1a), so for *hash* offload it is self-defeating. **There is no known cheap, + challenge-free binding for a bus whose payload rate equals the hash rate.** That is the + real obstacle to routes (a)-(c), and it is a genuine open problem, not an engineering gap. +- *Effort:* **M-L**, and it is strictly worse than (d) unless the shard genuinely needs to + be a standalone verifiable object. + +**(c) — RECURSION-WITHIN-RECURSION.** The specialized blake3 STARK proves a batch of +compressions and publishes a commitment to its (input, output) list; the WRAP verifies that +proof instead of hosting the rows. +- *The binding still needs (b).* The wrap must check that the compressions it consumed are + the ones in the shard's list. Merkle-opening each one costs more than hashing it; the only + cheap check is a fingerprint under a shared challenge — which is (b). **So (c) = (b) + an + imported verify cost.** Its only advantage is that the shard proof is a standalone object + (schedulable, cacheable, re-usable across blocks for repeated inputs). +- *Effort:* **L.** + +### 2.2 When (c) wins over hosting — the break-even, honestly + +Hosting H compressions costs `H · c_host · ρ`. Splitting costs `H · c_ded · ρ_shard + +V · c_host · ρ`, where V is the compressions the wrap spends *verifying* the shard proof. +At `ρ_shard = 1` and `c_ded = c_host`: + +> **break-even ρ = 1 / (1 − V/H)** + +V is a native (LFM-hash) leg walk over the shard's single table at 110 q, batched. DERIVED: + +| shard holds | AIR layout | main | rows | V (comps) | V/H | break-even ρ | +|---|---|---|---|---|---|---| +| 2^18 compressions | 1 compression/row | 3,056 | 2^18 | 157,080 | 0.599 | **2.50** | +| 2^18 | 1 round/row (6r) | 509 | 2^20 | 48,730 | 0.186 | **1.23** | +| 2^18 | 1 G-call/row (48) | 63 | 2^23 | 37,070 | 0.141 | **1.17** | +| 2^20 compressions | 1 compression/row | 3,056 | 2^20 | 162,030 | 0.155 | **1.18** | +| 2^20 | 1 G-call/row (48) | 63 | 2^25 | 43,120 | 0.041 | **1.04** | + +(rate 8 throughout; the rate-4 rows are in §7's output and are ~1.2× worse.) + +**Read it against ρ ≈ 1.7-1.9 (blake3, §1.4) and against the memory ceiling.** A 2^20- +compression shard at 1 compression/row is 5.19 B cells = **163 GiB host** — unprovable. The +shard sizes that fit 64 GiB are ~2^18, and *at 2^18 the wide layout loses* (needs ρ > 2.50, +we have ~1.8). **So the split is only viable in the narrow-and-tall layout.** That is not a +tuning preference; it is the condition of the design. + +### 2.3 What the split actually buys, bounded + +The split's work win is `ρ / ρ_split × c_host / c_ded`, and both factors are smaller than +they look: +- **ρ ≈ 1.7-1.9** (DERIVED) — so removing *all* residue is ≤1.9×. But the split does not + remove all of it: the guest still computes the leaf bytes it is absorbing and still has to + present them to a bus. Only the socket's memory plumbing (LFM_LANES / LFM_HINT / address + arithmetic) goes away. **ρ_split ∈ (1.0, 1.9), unmeasured, plausibly 1.2-1.4.** +- **c_host/c_ded ≤ 1.22×** (§1.3, now ✓ VERIFIED rather than derived). + +> **Split ceiling ≈ 1.2-1.6× on total work**, and it is the *last* rung of a 25× ladder. +> Rungs G-H in §1.4 bracket the whole hash-circuit family: 78 → 51 GPUs with every lever +> landed, 78 → 23 with a chip that costs nothing at all. + +⚠ And note what the split does **not** do: it does not reduce the compression *count*, only +the cost of each one, so it cannot substitute for any rung A-F. It also re-imports a verify +cost (§2.2) and, unlike route (d), it buys no latency — the shards are parallel, but so are +the tables in route (d), for free. + +### 2.4 The shape lever, restated as the actual recommendation + +Everything above points at one cheap change that is **not** a split: + +> **Lay the blake3 AIR out narrow-and-tall.** Cells are conserved (`rows × width`), so +> proving cost is unchanged. Verification cost falls ~1.5× at the tower and makes the split +> break even where it currently does not. + +Costs to weigh (✗ UNVERIFIED, needs the chip read): a G-per-row layout must carry the +16-word state and the message schedule in every row, so the width floor is ~100 columns, +not 3,056/48 = 64; and it adds row-transition constraints plus a round/G selector. If the +realistic width is 100-120 rather than 64, §3.2's tower win drops from 1.54× to ~1.45× +— still the largest single lever on the tower. + +--- + +## 3. PARALLELISM STRUCTURE + +### 3.1 Cadence ≠ latency + +- **Cadence** (one block proof per 12 s): fleet = total work ÷ per-GPU throughput. + Pipelining across blocks satisfies it; no intra-proof parallelism required. **This is + what §1 answers, and it is what the "4-8 GPUs" target means.** +- **Latency** (a block proof within one slot): needs intra-proof parallelism *and* a shallow + tower. Config F, hosted socket: + +| stage | cells | 1-GPU latency | host RSS | +|---|---|---|---| +| one base wrap (2^23, blowup4/110q, rate 8, batched) | 1.86-4.37 B | **28-65 s** | 59-137 GiB | +| one 4-ary tower node | 4.65-10.88 B | **69-162 s** | 146-342 GiB | +| tower depth (2^23, arity 4) | — | 2 layers | — | +| **critical path, perfect fan-out** | — | **166-389 s = 14-32 slots** | — | + +**The 12 s cadence binds on total work; the 12 s *latency* binds on per-wrap and per-node +latency, not on tower depth.** Depth contributes 2 of the ~5 stage-times at arity 4. Even +an infinitely wide fleet cannot produce a block proof in 12 s without splitting a *single* +wrap across GPUs — which is exactly route (d). + +**Shard latency under intra-wrap distribution** (config F, floor chip): + +| shards | shard latency | shard host RSS | +|---|---|---| +| 1 | 27.8 s | 59 GiB | +| 4 | 6.9 s | 15 GiB | +| 16 | 1.7 s | 4 GiB | +| 64 | 0.4 s | 1 GiB | + +Route (d) at 4-16 shards puts a wrap inside a slot and each shard inside a 5090's VRAM. +**This is the parallelism structure the fleet endgame needs, and it is the route with no +soundness surface.** + +### 3.2 The arity trade, with the D1 node model + +N = 9 (2^23 epochs), two-term model, batched. A node verifying k proofs costs k × one +proof-verify; **one** proof-verify is 733,700 comps / 157 GiB as D0+D9 are specified today, +258,280 comps / 69 GiB with RATE 8 + G-per-row. + +| arity | nodes | depth | verifications | node host RSS (spec'd / recommended) | tower comps (recommended) | +|---|---|---|---|---|---| +| 2 | 11 | 4 | 19 | 315 / **138 GiB** | 4,907,320 | +| **4** | **4** | **2** | **12** | 629 / **277 GiB** | **3,099,360** | +| 8 | 3 | 2 | 11 | 1,258 / 554 GiB | 2,841,080 | +| 16 | 1 | 1 | 9 | 2,517 / 1,108 GiB | 2,324,520 | + +**Arity 4 is the knee on work: 2→4 buys 1.58× and halves depth; 4→8 buys 8% more and +*doubles* the node.** + +⚠ **But read the node column: at every arity the aggregating node is far over 64 GiB, and +the shape lever does not fix that.** Gate D1's ~81 GiB (PLAN.md:176) and this document's +69 GiB are both **one-proof-verify** figures; the smallest node that actually *aggregates* +is arity 2 at **138 GiB**, 2.2× the target. **The tower node — not the base wrap — is the +campaign's binding memory constraint, and the only lever that touches it is residency +(S3/S6), not any hash lever.** That is a finding for the tower track, and it argues for +running S3's seams on tower nodes from the start rather than treating them as a base-layer +concern. + +**Row shape at the D1 node** — width moved, **rows scaled to compensate so cells are +conserved**, priced under §1.4's two-term model (batched, 110 q): + +| LFM_HASH layout | main | aux | rows | node @ RATE 4 | node @ RATE 8 | +|---|---|---|---|---|---| +| 1 compression / row (**D0 as specified**) | 2,964 | 630 | 4 | **157 GiB** | **104 GiB** | +| 1 round / row (6r) | 494 | 105 | 32 | 110 GiB (1.43×) | 74 GiB (1.40×) | +| **1 G-call / row (48)** | 100 | 20 | 256 | **103 GiB (1.53×)** | **69 GiB (1.50×)** | +| (keccak-era LFM_HASH, the unreachable bound) | 28 | 3 | 4 | 102 GiB (1.55×) | 68 GiB (1.51×) | + +The row-shape change recovers **96% of the gap back to the keccak-era node**, and it holds +at 1.50× even after scaling rows (a G-per-row LFM_HASH is 256 rows, so Merkle depth grows +from 2 to 8 — six extra parent compressions per group per query, against ~1,200 leaf +compressions saved). + +⚠ **This does not reproduce MMCS-PLAN §1.4's 122 GiB at RATE 4 / D0 width — it gives 157 +GiB, 29% worse.** The difference is the residue model, and it is the same disagreement as +§1.4: `tower.py` and MMCS-PLAN price non-hash cells at a flat **6.5%** of the node, while the +measured chip census says the residue tracks *felts absorbed* and therefore does **not** +shrink when the hash term does. At the D0 width the two agree to within a few percent; at +RATE 8 with a narrow chip the flat model says 43 GiB and the two-term model says 69 GiB. +**§5.1's census resolves which is right, and the answer moves Gate D1 by 1.6×.** Until then +the tower numbers circulating in the campaign should be read as the optimistic end. + +RATE 8 + G-per-row lands at **69 GiB — within 8% of the 64 GiB production target**, against +**157 GiB** as D0 and D9 are specified today. Nothing else in the campaign gets a tower node +that close. + +### 3.3 Where the inner chip's shape matters (less) + +The same lever applied to the *inner* proof's blake3 chip is weaker, because a 2^23 epoch +has 64 sub-proofs and the hash chip is one of them: + +| inner chip layout | felts/query | comps/query | block GPUs | +|---|---|---|---| +| keccak KECCAK_RND (1,480/516) — what the leg model prices today | 21,614 | 2,856 | 78 | +| blake3, 1 compression/row (3,056/630) | 25,450 | 3,365 | **84** | +| blake3, 1 round/row | 17,230 | 2,273 | 73 | +| blake3, 1 G-call/row | 15,878 | 2,092 | **71** | + +⚠ **A correction the campaign should absorb: P-a makes the wrap's leaf absorption *worse*, +by 8%.** The leg sets in `mmcs_project.py` carry the keccak-era `(1480, 516)` inner hash +chip; after P-a that leg is the blake3 chip at `(3056, 630)`, and leaf absorption is +proportional to width. Every post-P-a number in MMCS-PLAN and in §1 of this document is +optimistic by ~8% for this reason. It does not change any verdict; it should be fixed in the +leg data before the next projection round. + +--- + +## 4. SEQUENCING, AND WHAT THIS OBSOLETES + +### 4.1 Against the live tracks + +| track | verdict under the split analysis | +|---|---| +| **P-a** (inner → blake3-6r) | **Unaffected, still first.** It is the ÷4 on the hash term at every layer. But it *widens* the inner hash chip 1,480→3,056, costing 8% back on the wrap's leaf absorption (§3.3) — worth knowing, not worth re-sequencing. | +| **D0 step 3-4** (LFM proof commits blake3) | **Unaffected, still required.** The tower legs recompute the LFM proof's own trees. | +| **Batching** (FRI + MMCS) | **Confirmed, and it is a prerequisite for everything here.** Every number in §1.2-1.4 assumes it. It is rung B (1.7×) and it is what makes rung E (epoch size, 3.7×) *possible* — MMCS-PLAN §1.2's "epoch size becomes nearly free". **Not obsoleted; promoted.** | +| **S3 / S6** (residency) | **Unaffected as a fit lever, and route (d) subsumes its seams.** S1-S7 are the same seams a distributed prover needs. Building (d) on top of S3 is nearly free; building (d) without S3 is not possible. | +| **D9 / RATE** | **★ Repriced, and the question changes.** RATE 4→8 is rung C, worth 1.5× — larger than the split. But **row-shape is the cheaper half of the same lever** (1.5× at the node, no bus-arity change, none of the RATE=5 lane-map hazards). D9 should be re-framed as "RATE *and* row shape", and row shape should go first. | +| **Tower arity** | New: **take 4**, not 2 (rung D, 1.3×, halves depth, §3.2). | +| **Epoch size** | New and largest: **take 2^23, not 2^21** (rung E, 3.7×). Pure scheduling. Census 2^24/2^25 before assuming it continues. | + +### 4.2 Does a specialized hash circuit reduce pressure on D9 and batching, or multiply it? + +**It multiplies both, and neither substitutes for it.** +- **On batching:** batching moves the wrap into the leaf-absorption-dominated regime (74-77% + of the bill after batching, MMCS-PLAN §1.1). Leaf absorption is `Σ cols / RATE` — + precisely what RATE and row-shape attack. So after batching, hash-chip levers are worth + *more*, not less. +- **On D9:** identical logic at the tower (94% leaf after batching). MMCS-PLAN §1.4 already + says "batching magnifies D9 rather than substituting for it"; the split does the same. +- **But the pressure that matters most has moved off all three.** At rung F the residue is + 41% of the block and rising as the hash levers land (rung H: a free chip still needs 33 + GPUs). **The next campaign question after this one is the emitter's cells-per-absorbed-felt, + not the hash chip.** + +### 4.3 Build order + +| # | item | effort | worth | gate | +|---|---|---|---|---| +| 0 | **Measure ρ under the blake3 emitter** (§5.1) | **S**, zero proving | — | decides the order of everything below | +| 1 | Census 2^24 / 2^25 (§5.2) | **S**, zero proving | — | finds where the epoch lever turns over | +| 2 | Take epoch 2^23 + tower arity 4 + inner blowup4/110q | **S** (config) | **7.4×** | census confirms | +| 3 | Batched FRI + MMCS (already scoped, MMCS-PLAN §2) | M | 1.7× | ≥2× $/wrap at LARGE | +| 4 | RATE 4→8 (D9) | M | 1.5× | the existing D9 gate | +| 5 | **Row-shape the blake3 AIR narrow-and-tall** (tower, then inner) | **M** | **1.44×** | D1 one-proof node ≤ 70 GiB at RATE 8 | +| 6 | **Route (d): distributed proving of one wrap across GPUs** | **M** | latency only | a wrap inside a slot; no soundness surface | +| 7 | Packed blake3 AIR at the floor (~3,000 cells/compression) | L | 1.3× | measured on the shape A/B harness | +| 8 | Route (a)/(b)/(c) true split | **L** | ≤1.4× | only if step 0 says ρ_split ≥ 1.5, step 5 landed, **and** the challenge-free-binding problem in §2.1(b) has an answer cheaper than a hash per interaction | + +**Steps 0-6 are worth ~19× and carry no new soundness surface. Steps 7-8 are worth ~1.4× +each and step 8 carries all the soundness risk in this document.** That ordering — and in +particular putting the row-shape change (step 5, M) ahead of both the packed AIR (L) and the +split (L) — is the document's main recommendation. + +--- + +## 5. THE MINIMAL EXPERIMENTS + +### 5.1 ★ Round 0 (decisive, free, no proving): measure ρ under the blake3 emitter + +The census harness already dumps the per-chip cell table +(`census_logs/ethrex_e21_b2_q4.log` shows the exact format). Run the same census point with +the blake3-native emitter and read off two numbers: **hash-chip share** and **q-scaling +arithmetic cells ÷ felts absorbed**. + +- **If cells/felt ≈ 439** (the keccak-emitter value): ρ_blake3 ≈ 1.9, the residue is 41% of + the block, and **the split is worth at most 1.9× while step 0-4 are worth 16×** — do the + split last or not at all. +- **If cells/felt ≈ 50-100** (a felt-native emitter over one-cell digests should be far + leaner than byte-level sponge packing): ρ_blake3 ≈ 1.1-1.2, the residue nearly vanishes, + **and the hash chip becomes ~90% of the block again** — at which point the split and the + packed AIR become the dominant levers and should be promoted above everything except + batching. + +**This single number flips the plan's order.** It is a census, not a prove. + +### 5.2 Round 0b (free): census 2^24 and 2^25 + +Rung E (2^21→2^23) is the largest in the ladder and the model says the direction continues. +But sub-proof count grows ~1.5× per doubling (28/32/43/64 measured at 2^20…2^23) and batched +leaf absorption is proportional to *total columns across all sub-proofs*, so returns damp. +The census is closed-form and free. **Find where the epoch-size lever turns over before +building anything.** + +### 5.3 Round 1 (M, after step 1): the shape A/B + +Prove one wrap with the blake3 AIR at 1 compression/row and at 1 G-call/row. Predictions to +falsify: **proving cells within 3%** (cells are conserved), **D1 node census 1.5× cheaper**, +**verify time unchanged**. If proving cells move more than 10%, the row-transition +constraints cost more than this model allows and §2.4/§3.2 must be re-derived. + +--- + +## 6. OPEN DECISIONS — need Mauro + +- **D10 — cadence or latency?** "4-8 GPUs keeping up with 12 s blocks" is a throughput + target (§3.1) and is satisfied by pipelining. If a block proof is also required *within* a + slot, route (d) becomes mandatory and moves to the front of §4.3. **These are different + builds; the plan cannot pick.** +- **D11 — is 4-8 GPUs a requirement or an aspiration?** The honest projection is **78 GPUs** + at the best schedulable configuration, **51 with every hash-circuit lever in this document + landed**, and **23 even with a free hash chip**. Reaching 8 needs ~5× more that no + identified lever supplies. If it is a requirement, the answer is not in + this design space — it is in reducing *felts absorbed* (fewer/narrower inner tables, + higher blowup / fewer queries, or **single-row leaves**: leaf absorption is + `ROWS_PER_LEAF · cols · kind` and ✓ VERIFIED `crypto/stark/src/commitment.rs:42` + `pub const ROWS_PER_LEAF: usize = 2` — dropping to 1 halves the dominant term at the cost + of one Merkle level, and is a wire-format change nobody has priced). That is a different + document. +- **D12 — D9 re-framing.** Row shape is a second, cheaper half of the RATE lever with none of + the RATE=5 lane-map hazards (§4.1). Should D9 be re-opened as "RATE and row shape", with + row shape scheduled first? **This changes what task #35 builds.** + +--- + +## 7. REPRODUCTION + +Scripts checkpointed at **`~/workspace/lambda_vm_bench_cache/hash_split_2026-08-13/`** +(out of tree, per the lean-PR rule). Run them from that directory with the calibrated model +directory on `sys.path` (each script inserts it): +`hashsplit.py` (anchors, compression counts, target grid, fleet inverse), +`hashsplit2.py` (blowup-4 derivation, break-even sweep, latency), `residue.py` +(ρ calibration from the measured chip census), `final2.py` (ladder A-H), +`ladder2.py` (ladder F-J, the shape rungs), `shape2.py` (tower node vs row shape, rows +scaled), `arity.py` (the arity table), `innerwidth.py`. All import the calibrated model at +`~/workspace/lambda_vm_bench_cache/lfm_census_2026-08-12/` (`mmcs_project.py`, `project.py`, +`tower.py`) and reproduce MMCS-PLAN §1.1's blake3 column exactly at RATE 8, which is the +regression check that the RATE parameterisation did not perturb the calibration. + +## 8. Confidence ledger + +| claim | mark | +|---|---| +| 67.13 M cells/s; 481,327,124 cells; 7.17 s | ✓ MEASURED (`EXPLORATION.md:186`, `BOX-RESULTS.md:53,57`) | +| ρ = 1.081 under keccak; hash chips 92.5% of cells | ✓ MEASURED (`census_logs/ethrex_e21_b2_q4.log`) | +| 439 residue cells per absorbed felt | DERIVED from that measurement (spine-corrected) | +| ρ ≈ 1.7-1.9 under blake3 | DERIVED — **the number §5.1 exists to replace** | +| compression counts per block | DERIVED from the calibrated model (unit-exact at 4 measured points) | +| hosted socket 4,946 cells/compression | ✓ **VERIFIED both terms** — `blake3_chip.rs:162,224` (3,056 main) and `:911-913` (1,259 interactions → 630 ext aux). Upgrades `tower.py`'s own `? INFERRED` caveat. | +| plausible floor ≈ 4,060 | DERIVED from the verified G-block encoding | +| break-even ρ table; row-shape 1.5× at the node | DERIVED from the calibrated model | +| tower is 76-149% of the base layer | DERIVED; rests on D1's static-shape premise ✗ UNVERIFIED | +| ladder rungs A-J | DERIVED | +| SP1 binds shards with a **challenge-free septic-curve digest**, reconciled in the recursion layer, at the cost of a Poseidon2 permutation per interaction | ✓ VERIFIED (`global_interaction.rs:24-45`, `complete.rs:147`, `machine/core.rs:136-150`) | +| SP1 `SplitOpts` / deferred-proof-digest detail | ? INFERRED — survey commissioned, not returned | +| blake3 chip: one row/compression, 48 G-blocks × 60 cells, 1,248 BITWISE lookups | ✓ VERIFIED `blake3_chip.rs:101,158,781,990-991` | +| #903 chip 5,316 | ✓ VERIFIED from commit `35038501`'s message; ✗ UNVERIFIED against source | diff --git a/thoughts/shared/block-compression/MMCS-PLAN.md b/thoughts/shared/block-compression/MMCS-PLAN.md new file mode 100644 index 000000000..37946d0b2 --- /dev/null +++ b/thoughts/shared/block-compression/MMCS-PLAN.md @@ -0,0 +1,1228 @@ +# MMCS-PLAN — batched commitments for the Lambda VM recursion campaign + +**Status: SCOPING, implementation-ready. Read-only analysis; no code touched.** + +Mauro green-lit batched-MMCS work ("we were delaying it until we had evidence we +needed it but we were expecting to do batched mmcs"). This document supplies the +numbers, the rebase-vs-reimplement verdict, the design, and the sequencing +against S3 and P-a. + +--- + +## 0. Verdict — read this before scheduling anything + +**1. Batching projects as the largest single lever in the campaign, larger than +the inner-hash switch.** At the real 2^21/blowup2 point it takes the wrap's leg +cost from 5,434 to 1,264 hash invocations per query — **4.30×**, against P-a's +measured 4.06×. At 2^23 it is **9.06×**. It is the only lever whose value +*grows* with epoch size, and it composes with P-a (combined 11.1× at 2^21, +20.8× at 2^23). DERIVED from the calibrated model; the model's validation is +§1.0 and its reconciliation against the one relevant measurement is §1.3. + +**2. The bill is not where the brief assumed, and that changes the design.** +Today's wrap spends **54–66% of its leg permutations on FRI** and 24–25% on +Merkle walks; **leaf absorption is only 9–22%**. Batching the *trees* alone +(MMCS-only) buys 1.28–1.33×. Batching the *FRI* alone buys 1.96–2.80×. The +MMCS is the smaller half of the win at the inner layer — do both, and if +anything sequence FRI first. + +**3. After batching, epoch size stops mattering for the wrap's memory.** All +three geometries land within 121–135 GiB (Part-2-corrected baselines, P-a +composed). The per-table O(N_tables) terms are what made 2^23 six times more +expensive than 2^20 in Gate A's sweep; remove them and the residual is leaf +absorption, which is set by table *widths* and barely moves. **Strategic +consequence: the campaign can use LARGE +epochs — fewer wraps for the tower to aggregate — at no memory cost.** That +reverses PLAN.md's "smaller epochs shrink each wrap but grow the total" trade. + +**4. In the TOWER the answer is the opposite and much weaker: batching buys +1.31×** (D1 fixture node 122 → 93 GiB at the D0 blake3-socket width). Leaf +absorption is 72% of a tower node and batching does not touch payload. The +tower's lever remains the LFML **RATE** (D9), and batching *magnifies* D9's +value rather than substituting for it. + +**5. #768 is bigger than the campaign's notes said, and both facts matter.** It +is not "batched FRI with a digests-only MMCS" — it is a complete wired +implementation: `fri/mmcs.rs` (+1,015, 7 tests), `fri/batched.rs` (+499), +prover integration (+2,053), soundness tests (+237). **Its MMCS layout is +line-for-line the construction this model prices**, arrived at independently — +the strongest structural validation available (§1.3). But it is 25 commits +behind, `CONFLICTING`, and predates StarkHash (879bdc0f), #877's rewrite of the +very R1 loop it changes, #863/#875/#914 and #909. **Verdict: port +`fri/mmcs.rs` + `fri/batched.rs` + the soundness tests; reimplement the +integration on StarkHash.** §2. + +**5b. ⚠ ONE NUMBER DOES NOT RECONCILE AND IT GATES QUOTING §1.** The −57% keccak +figure predicts −76.7% under this model with #768's MMCS wired. And that figure +is **not** the PR's CI number — ✓ VERIFIED the only CI-posted result is +**+3.61% cycles / −573 keccak at ONE query** at an intermediate commit; −57% +comes from Mauro's own sims in `pr768-batched-fri-state.md`. So step one is +pinning which measurement is being compared, not re-measuring. **Until then, +treat §1.1's ratios as upper bounds on end-to-end reduction.** The leg +arithmetic is exact and validated four ways, and for the *wrap* the legs are a +MEASURED 99.6% of permutations, so the wrap-side numbers stand. Item M-11, S. + +**5c. Two of my own claims were falsified by reading the branch, and both are +recorded in place rather than quietly fixed** (§2.1 correction box, §2.4): I had +`StarkHash` on main when it lives only on `blake3-real-hash`, and I called +#768's terminal-poly gap "not reproduced" after grepping the non-batched +verifier instead of `batched.rs`, where it plainly is. The second one has a +design consequence worth keeping: **#768's width binding is implemented and +tested; the residual gap is that the transcript absorbs heights but not widths** +(`batched.rs:196`), which §3.4's addendum now folds in as requirement M3. + +**6. The July caveat "hash choice gates the batching decision" is retired.** +Batching wins by 3.6–9.1× under keccak and 2.4–5.1× under blake3. The hash +changes the *size* of the win, never its sign, and never the design. §4.4. + +--- + +## 1. PROJECTION + +### 1.0 Method, and why the numbers are trustworthy + +The per-query cost function is the campaign's own closed form, re-derived from +source and then validated against measurement. + +✓ VERIFIED the closed form in the code — `epoch_verify.rs:552-559`: + +``` +per_query = leaf_permutations(shape) // Σ over groups + + groups * shape.sub.merkle_depth // one parent per level per group + + shape.fri.permutations_per_query() +``` + +with `leaf_permutations` = `Σ_groups num_blocks(leaf_bytes)` (`:413-419`), +`leaf_bytes = ROWS_PER_LEAF · num_columns · (24 if ext else 8)` +(`sub_proof.rs:88-90`), `merkle_depth = log2_lde − 1` (`sub_proof.rs:160-166`), +and `permutations_per_query = num_committed + path_steps_per_query` with +`layer_path_len(i) = n − i − 2` (`fri.rs:133-144, :146-155`). + +✓ MEASURED validation — the model reproduces the census harness's own +`query_permutations` **exactly, to the unit**, on all four real-block points: + +| point | sub-proofs | model | measured | +|---|---|---|---| +| 2^20/blowup2/219q | 28 | 4,185 /query | 4,185 | +| 2^20/blowup4/110q | 28 | 4,434 /query | 4,434 | +| 2^21/blowup2/219q | 32 | 5,434 /query | 5,434 | +| 2^23/blowup2/219q | 64 | 13,196 /query | 13,196 | + +Leg shapes are the MEASURED dumps in +`~/workspace/lambda_vm_bench_cache/lfm_census_2026-08-12/census_logs/ethrex_e2*_skip.log`. +Tooling: `mmcs_project.py` / `mmcs2.py` / `mmcs3.py`, parked beside `project.py` +and `tower.py` in the same directory. Run `python3 mmcs2.py` to reproduce every +number in this section. + +**The batched cost function**, same primitives, Plonky3 MMCS semantics +(mixed-height, tallest matrix sets the depth, shorter matrices injected at the +level whose subtree height matches): + +``` +digest = H(rows of the tallest matrices) # level 0 +for l in 1..=D: + digest = compress(digest, sibling) + if matrices inject at level l: + digest = compress(digest, H(their rows)) +``` + +The total absorbed payload over the path equals `Σ_matrices row_pair_bytes` — +**batching does not reduce the payload, only the framing and the walk.** That is +the brief's premise and the model honours it: at 2^21 the leaf term moves 967 → +932 (−3.6%), which is only the saved per-group padding block. + +**One MMCS per commitment ROUND, not one overall.** Fiat–Shamir requires every +main root in the transcript before the shared LogUp challenge — ✓ VERIFIED +`prover.rs:3213-3238` ("All main trace commitments must be in the transcript +before sampling LogUp challenges … the one ordering Fiat-Shamir requires"), with +the verifier mirror absorbing the same roots at `verifier.rs:1288-1316` +immediately before "Round 1, Phase B: Sample shared LogUp challenges" +(`:1319-1322`). So main / aux / composition-parts cannot share a tree. +Preprocessed is committed at setup. Four trees, not one. That is the "~1-3 trees" +of the brief, made precise. + +All projections below are **DERIVED-from-calibrated-model** unless a cell is +marked MEASURED. + +### 1.1 (a) BATCHED INNER — what the WRAP pays + +Per-query hash invocations in the wrap's legs, and the four corners of the +design space: + +**keccak inner (today's RV64 commitment hash)** + +| point | trees | today | FRI-only | MMCS-only | BOTH | BOTH ratio | +|---|---|---|---|---|---|---| +| 2^20/blowup4 | 88 → 4 | 4,434 | 2,236 | 3,426 | **1,228** | **3.61×** | +| 2^20/blowup2 | 88 → 4 | 4,185 | 2,135 | 3,261 | **1,211** | **3.46×** | +| 2^21/blowup2 | 100 → 4 | 5,434 | 2,517 | 4,181 | **1,264** | **4.30×** | +| 2^23/blowup2 | 196 → 4 | 13,196 | 4,713 | 9,940 | **1,457** | **9.06×** | + +**blake3 inner (after P-a)** + +| point | today | FRI-only | MMCS-only | BOTH | BOTH ratio | +|---|---|---|---|---|---| +| 2^20/blowup4 | 5,530 | 3,332 | 4,534 | **2,336** | **2.37×** | +| 2^21/blowup2 | 6,556 | 3,639 | 5,322 | **2,405** | **2.73×** | +| 2^23/blowup2 | 14,529 | 6,046 | 11,321 | **2,838** | **5.12×** | + +**The split, which is the finding.** Where the per-query bill goes, today vs +batched (keccak inner): + +| point | today leaf / merkle / FRI | batched leaf / merkle / FRI | +|---|---|---| +| 2^20/blowup4 | 21.1% / 24.6% / **54.3%** | 73.9% / 9.2% / 16.9% | +| 2^21/blowup2 | 17.8% / 24.5% / **57.7%** | 73.7% / 9.1% / 17.2% | +| 2^23/blowup2 | 9.1% / 24.9% / **65.9%** | 77.2% / 7.9% / 14.9% | + +Per-table FRI dominates because every one of 28–64 sub-proofs runs its own FRI +instance down to `fri_final_poly_log_degree = 7`, and a deep table's FRI is +expensive: leg 31 of the 2^21 epoch (2^22 rows, 14 layers) costs 288 +permutations per query of which **217 are FRI** — the census called it "almost +entirely in path steps" (CENSUS Part 1 §3) and this decomposes that remark. +Batched FRI replaces `Σ_t fri_t` with one instance over the largest domain: +3,134 → 217 per query at 2^21. + +The walks collapse as promised (1,333 → 115 at 2^21, a 4-tree walk of depth 22 +plus injection compressions). The leaf does not (967 → 932). **After batching, +leaf absorption becomes 74–77% of the wrap's bill** — i.e. batching moves the +inner layer into the same regime the tower is already in, where the only +remaining lever is the leaf rate. + +**Peak projection, full census** (leg permutations + MEASURED spine → KECCAK_RND +chunks → cells → 33.7 B/cell). Non-`KECCAK_RND` chips are held constant, which +is conservative (their instruction counts shrink too): + +| point | variant | leg perms | chunks N | cells | projected peak | +|---|---|---|---|---|---| +| 2^20/blowup4/110q | today | 487,740 | 23 | 35.7B | 1,122 GiB | +| | #768 FRI-only | 245,960 | 12 | 18.3B | 573 GiB | +| | **BOTH** | **135,080** | **7** | **10.3B** | **324 GiB** | +| 2^21/blowup2/219q | today | 1,190,046 | 55 | 87.3B | 2,742 GiB | +| | #768 FRI-only | 551,223 | 26 | 40.5B | 1,271 GiB | +| | **BOTH** | **276,816** | **13** | **20.6B** | **648 GiB** | +| 2^23/blowup2/219q | today | 2,889,924 | 133 | 211.1B | 6,630 GiB | +| | #768 FRI-only | 1,032,147 | 48 | 76.2B | 2,393 GiB | +| | **BOTH** | **319,083** | **15** | **23.8B** | **748 GiB** | + +Spine permutations are MEASURED (`*_spine.log`): 3,258 / 4,395 / 8,417. They are +0.3–0.7% of the total and are held constant; batching in fact shrinks them +slightly (4 roots to absorb instead of 100). + +**Chunk count is the number that matters for S3.** N = 55 → 13 at 2^21, N = 133 +→ 15 at 2^23. The S3 residency model is `17.37·N + 30.2·k GiB` (CENSUS Part 2 +§1, calibrated to a measured 13.4 GiB/chunk marginal, Part 3 §8) — batching +attacks the N term directly, which is exactly the term S3's Phase C exists to +flatten. §4.2 works the interaction. + +### 1.2 Composition with P-a, and the fit + +Batching factor is DERIVED here; the hash factor is the campaign's **MEASURED** +hash matrix (epoch-verify 11.17B cells keccak → 2.75B blake3-6r = 4.06×). They +are applied to both published baselines — Part 1's `33.7 B/cell` projection and +Part 2 §1's aux-corrected numbers, which the census itself calls upper bounds: + +| point | baseline (P1 / P2-corrected) | + batching | + P-a only | **+ BOTH** | +|---|---|---|---|---| +| 2^20/blowup4/110q | 1,199 / 1,300 | 332 / 360 | 295 / 320 | **125 / 135** | +| 2^21/blowup2/219q | 2,929 / 1,337 | 681 / 311 | 721 / 329 | **265 / 121** | +| 2^23/blowup2/219q | 7,242 / 2,692 | 800 / 297 | 1,784 / 663 | **348 / 130** | + +Two things to read off this table: + +- **The combined lever is 9.6× (2^20) to 20.8× (2^23)**, against Gate A's + required 13–29×. It does not close the gate on its own at every point, but it + is the first lever that gets within a factor of ~1.3 of the 93 GiB box and + *inside* the 124 GiB rigs at two of three points, before S3 contributes + anything. +- **Epoch size becomes nearly free.** 121 / 130 / 135 GiB across 2^20 → 2^23. + PLAN.md's framing — "the epoch-size lever is weak, 2^23 → 2^20 buys only 3.2×, + and it multiplies the number of wraps the tower must aggregate" — is + *reversed* by batching: the lever's remaining value is ~1.1×, so the campaign + should take the LARGEST epoch that proves, minimising N wraps and therefore + tower layers. That is a scheduling decision worth surfacing to Mauro + independently of when MMCS lands. + +⚠ These are wrap **work** numbers. Whether the work fits in RAM is the S3 +residency question, which is orthogonal and multiplies (§4.2). + +### 1.3 #768 — what it validates, and one number that does NOT reconcile + +**★ The construction is independently confirmed.** #768's `fri/mmcs.rs` is a +mixed-height row-pair MMCS, and its documented layout is the *same* construction +this model prices, arrived at independently. ✓ VERIFIED, quoting its module doc +(`crypto/stark/src/fri/mmcs.rs:1-56` on `origin/feat/batched-fri-per-epoch`): + +> *"A matrix of `log_height h` is injected at layer index `i = h_max - h` … +> Base layer node `k`: `layer0[k] = H( CONCAT_{m : h_m == h_max} (row_m(2k) || +> row_m(2k+1)) )` … Climb: `parent = C(layer_i[2j], layer_i[2j+1])`. If any +> matrix has `h_m == inject_h`, then `layer_{i+1}[j] = C( parent, H( CONCAT_{m : +> h_m == inject_h} (row_m(2j) || row_m(2j+1)) ) )` … For query `iota`, matrix +> `m` is opened at leaf `k_m = iota >> (h_max - h_m)`."* + +That is `batched_tree_cost` line for line — same injection level, same +concatenation, same `C(parent, H(injected))` two-compression step, same +index truncation. **The model's semantics are not a guess.** It also +retroactively justifies §3.4's recommendation to express injection as one extra +compression rather than a new step type: #768 reached the same shape. + +**⚠ But the magnitude does NOT reconcile, and I am recording that as open rather +than explaining it away.** Memory `recursive-verifier-batched-fri` records +**−36.9% cycles / −57% keccak** at real query counts. + +⚠ **First, that number's provenance, which is weaker than it looked.** ✓ VERIFIED +the only CI-posted figure on the PR is **+3.61% cycles / −573 keccak calls at ONE +query**, dated 2026-07-17 at intermediate commit `0880cff6` — not at the head, +and not at real query counts. The −57% / −36.9% figures come from **Mauro's own +sims recorded in `pr768-batched-fri-state.md`**, not from the PR. So the target +this model is being reconciled against is itself a simulation whose geometry and +denominator are not stated in the PR. **Pinning which number is being compared +is the first half of M-11**, and it may dissolve the discrepancy without any +re-measurement. + +I initially inferred from `pr768-memfix-mmcs-digest-only` +("digests-only MMCS by design") that #768 batched FRI only, which would put the +measurement at the FRI-only corner — where the model predicts −53.7% at 2^21, +within 3.3 points. **That inference is FALSIFIED by the branch.** ✓ VERIFIED all +three round-MMCS instances are built, absorbed and opened: +`prover.rs:612-614` (`main_mmcs` / `aux_mmcs` / `comp_mmcs`), `:2666-2667` +(built, root appended to transcript), `:4413-4415` (`open_batch` per query). +The "not yet wired into the prover/verifier" note at `mmcs.rs:10-12` is **stale** +— it was written at Task 1 and the branch went on to wire it. + +So the measurement should sit at the BOTH corner, where the model predicts +**−76.7%** at 2^21, not −57%. + +| point | model FRI-only | model BOTH | measured (#768) | +|---|---|---|---| +| 2^20/blowup4/110q | −49.6% | −72.3% | | +| 2^21/blowup2/219q | −53.7% | **−76.7%** | **−57%** | +| 2^23/blowup2/219q | −64.3% | −89.0% | | + +**Three candidate explanations, none yet checked:** + +1. **Different denominator.** The #768 number is the *whole guest verifier's* + keccak count; this model prices the *legs* only. The guest also absorbs the + transcript, samples challenges, and checks grinding, and none of that + shrinks. For BOTH to read −57% the fixed remainder would have to be ~26% of + the guest's keccak. In the wrap the spine is 0.4% — but the RV64 guest is a + different program, so this is plausible and is the explanation I would bet on. +2. **Different geometry.** The ratio is strongly geometry-dependent (−71% to + −89% across the sweep). A measurement on a small or toy epoch would land + lower. +3. **Measured mid-branch**, after the FRI work and before the MMCS wiring — + 45 commits, and the stale module note shows the branch was built in tasks. + +**This must be resolved before any projection in §1.1–1.2 is quoted as a +schedule input.** It is cheap to resolve: re-read the #768 bench record for its +denominator and geometry, or re-run it. Until then, treat §1.1's ratios as +**upper bounds on the achievable end-to-end reduction** — the *leg* arithmetic +is exact and validated four ways, but the fraction of the wrap that legs +constitute is measured (99.6%) only for the wrap, not for the guest verifier +#768 measured. + +**What survives regardless:** the leg model reproduces the measured census +exactly at four points (§1.0), the construction matches a real implementation +(above), and the *wrap's* leg share is measured at 99.6%, so for **the wrap** — +which is what this campaign's memory problem is about — the ratios stand. + +### 1.4 (b) BATCHED LFM MACHINE — the TOWER node + +Same model, `native = true`: the LFM proof's own commitments under the +COMMIT.md §1.2 LFML/LFMC construction at the adopted **RATE = 4** +(`compressions = ceil(2·num_cols·kind / 4)`, §1.4), parents 1 compression. + +**Baseline reconciliation first.** `tower.py` as published prices the OLD rate +(`2·ceil(felts/4)`) and gives D1/fixture/110q = 124 GiB; PLAN.md's RATE=4 +headline of ≈81 GiB is that number with the leaf term halved. This model at +RATE=4 with the same `LFM_HASH=(28,3)` width gives **78 GiB** — 4% from the +published 81, and the per-query split (leaf 71.7% / merkle 10.9% / FRI 17.4%) +matches CENSUS Part 2 §3's independently derived 69.8% / 10.7% / 19.5%. ✓ The +tower model is anchored — and note this is a *second* model agreeing with the +first, not a measurement: the tower has never been censused on hardware, unlike +the inner layer where §1.0's four points are real. + +| inner | LFM_HASH width | q | today | FRI-only | MMCS-only | **BOTH** | ratio | +|---|---|---|---|---|---|---|---| +| FIXTURE wrap | (28,3) | 110 | 78 GiB | 60 | 66 | **49 GiB** | 1.58× | +| FIXTURE wrap | (2964,630) D0 | 110 | 122 GiB | 104 | 110 | **93 GiB** | 1.31× | +| REAL 2^21 wrap | (28,3) | 110 | 116 GiB | 91 | 103 | **78 GiB** | 1.49× | +| REAL 2^21 wrap | (2964,630) D0 | 110 | 160 GiB | 135 | 147 | **122 GiB** | 1.31× | +| FIXTURE wrap | (2964,630) D0 | 219 | 242 GiB | 208 | 220 | **185 GiB** | 1.31× | + +Per-query split, D1 fixture at the D0 width: today leaf 4,773 (72%) / merkle 727 +(11%) / FRI 1,156 (17%) → batched leaf 4,769 (94%) / merkle 120 / FRI 208. + +**Why the tower gains so much less than the inner layer.** A tower node verifies +14–15 sub-proofs, not 28–64, and those chips are *wide* (`LFM_HASH` 2,964 main +at 6r, `KECCAK_RND` 1,480, `LFM_KECCAK` 736) rather than deep. Leaf absorption +is already 72% of the bill and batching leaves it untouched. **MMCS does not +rescue the tower; the RATE does.** + +**Batching magnifies D9 rather than substituting for it** (§ mmcs3.py G): + +| | per-table trees | batched MMCS | batching buys | +|---|---|---|---| +| RATE = 4 (adopted) | 122 GiB | 93 GiB | 1.31× | +| RATE = 5 (D9 open) | 105 GiB | 76 GiB | 1.38× | +| RATE = 8 (hypothetical) | 77 GiB | 50 GiB | 1.55× | + +After batching, leaf is 94% of the node, so the rate scales 94% of the cost +linearly instead of 72%. **If D9 goes to Mauro, this table belongs in the +question**: RATE=5 is worth 17 GiB unbatched and 17 GiB batched, but it is the +*only* remaining lever once MMCS lands. + +**★ A discrepancy found in CENSUS Part 2 §3, flagged for its owner.** That +section states the D0 second-order feedback (the machine's own hash chip +becoming BLAKE3 and therefore the widest table) is **+19%**, "D1/real/219q moves +381 → 452 GiB". Re-running the published `tower.py` with +`WIDTH['LFM_HASH'] = (3056, 630)` gives **559 GiB, i.e. +47%** — and this +model's independent D0-width variant shows +38% to +56% depending on the point. +? UNRESOLVED which input produced 452. It does not change any verdict here (the ++19% figure is the optimistic one, so the tower baseline is *worse* than +published and the case for every tower lever is stronger), but the tower numbers +circulating in the campaign should be re-derived before they gate a decision. + +--- + +## 2. REBASE vs REIMPLEMENT + +**Verdict: SALVAGE THE PRIMITIVES, REIMPLEMENT THE INTEGRATION.** Do not rebase +the branch. But `fri/mmcs.rs` is far more reusable than the campaign's notes +suggested and should be the starting point, not a reference. + +### 2.0 What is actually on the branch — ✓ VERIFIED + +`gh pr view 768`: head `feat/batched-fri-per-epoch`, base `main`, **state OPEN, +mergeable CONFLICTING**, opened 2026-07-02, **+5,097 / −1,099 across 25 files**. + +Merge base `3ea4f916` (2026-07-17, "verify continuation proofs in place via +rkyv (#845)"). **25 commits behind main; 45 commits on the branch.** + +`git diff --stat 3ea4f916..origin/feat/batched-fri-per-epoch`, the load-bearing +rows: + +| file | Δ | what | +|---|---|---| +| `crypto/stark/src/fri/mmcs.rs` | **+1,015 NEW** | `MixedMmcs`, `BorrowedMatrix`, `MixedOpening` — mixed-height row-pair MMCS, 7 tests, layout documented as "the single source of truth" | +| `crypto/stark/src/fri/batched.rs` | **+499 NEW** | `combine_by_height` — mixes DEEP codewords by FRI height with `alpha^i`, the batched-FRI core | +| `crypto/stark/src/prover.rs` | **+2,053** | the integration: `main_mmcs`/`aux_mmcs`/`comp_mmcs` (`:612-614`), built + absorbed (`:2666-2667`), `open_batch` per query (`:4413-4415`) | +| `crypto/stark/src/verifier.rs` | +764 | batched verify, `fri::terminal::FriFoldLayout`, terminal-codeword reconstruction (`:386-400, :436-474`) | +| `crypto/stark/src/proof/stark.rs` | +96 | **wire format changes** — `MixedOpening` enters the proof type | +| `prover/src/continuation.rs` | −810 net | rewritten | +| `crypto/stark/src/tests/bus_tests/batched_soundness_tests.rs` | **+237 NEW** | soundness oracles for the batched path | + +So the campaign's shorthand was wrong twice: it is **not** "batched FRI with a +digests-only MMCS", it is a **complete, wired, tested batched-commitment +implementation** — three round-MMCS instances plus batched FRI, exactly the +design this document scopes. + +### 2.1 Why it still cannot be rebased + +The merge base is 2026-07-17 and **every structural change this design depends +on landed after it** — ✓ VERIFIED by `git log 3ea4f916..origin/main`: + +| main-side commit | why it collides | +|---|---| +| `7644043b` **#877 per-table scheduler with VRAM admission for `multi_prove`** | rewrote the R1 commit loop into `run_admitted` — the exact region the branch changes most. It also **deleted `plan_table_chunks`, which #768 still calls** (`prover.rs:4683` on the branch) | +| `5749a956` #875 device-resident rounds 2-4 + fused NTT | rewrote the fused per-table task | +| `d83b4d9e` #863 halve GPU continuation proving | commitment production on device | +| `6949ceb9` **#909 pin each trace-opening column width to the AIR, not just their sum** | the opening-width soundness fix — lands directly on the batched leaf's binding surface | +| `d898a423` #914 VRAM pressure / R2 corruption race | same prover regions | + +Plus S3's `ResidencyMode` rewrite of the same R1 loop, **in flight this week**. + +> ### ⚠ CORRECTION — `StarkHash` is NOT on main +> +> An earlier revision of this section listed `879bdc0f` (StarkHash, D0 step 2) +> as a main-side commit. **It is not.** ✓ VERIFIED three ways: +> `git grep -c StarkHash origin/main` → zero matches; +> `git branch -a --contains 879bdc0f` → **`origin/blake3-real-hash` only**; and +> it does not appear in the 25 commits of `git log 3ea4f916..origin/main`. The +> `config.rs:55-192` I read is the **campaign branch's** file, not main's. +> +> **Consequence, and it is a scheduling fact not a nitpick:** `StarkHash` is a +> D0 artifact living on `blake3-real-hash` alongside P-a and S3. So "reimplement +> on `StarkHash`" means **building on the campaign branch**, and the eventual +> main merge is a separate, later problem that this plan does not cost. It also +> means M-1 is not independent of D0 — it inherits D0's merge risk. Anyone +> scheduling M-1 against main will not find the trait. +> +> The rebase argument is unaffected: the branch predates `StarkHash` either way, +> and the four genuinely-on-main commits above are sufficient on their own. + +⚠ Correction to the brief's list: **#823 (`a8648320`) and #826 (`18f3b8f2`) +predate the merge base** and are already in the branch's history; they are not +sources of conflict, and any note saying #768 "must rebase over #826+#823" is +stale. #863, #875, #877, #909 and #914 are the real collisions. + +`gh` reports the PR as **draft, CONFLICTING, zero reviews, empty body**, against +a 25-commit gap that includes two rewrites of the file the branch changes most. +The branch was never rebased — 13 of its 45 commits are `Merge branch 'main'`. + +**Two further reasons a merge is the wrong instrument**, both ✓ VERIFIED: + +- **The batched lane has ZERO CUDA.** `feature = "cuda"` appears 0 times in + `mmcs.rs`, 0 in `batched.rs`, and 0 across the branch's + `batched_table_deep_codeword..batched_round_4` region. Meanwhile 4 of the 5 + main-side `prover.rs` commits since the merge base are GPU work + (#863/#875/#877/#914, +1,913 lines in that one file). A merged batched path + would be CPU-only on a prover whose recent history is entirely GPU. +- **A merge would silently delete main's #845 zero-copy verify layer.** + `git merge-tree` conflicts in only two files, but in the resulting tree + `EpochProofView`, `ContinuationProofView`, `verify_continuation_view`, + `access_recursion_archive` and `verify_l2g_commitment_binding_view` all have + count **zero** — dropped during an on-branch conflict resolution and never + re-touched. The branch documents the debt itself + (`prover/src/lib.rs`: *"TODO(batched-fri): port the view machinery to + `BatchedMultiProof` to restore fully in-place verification"*), priced in the + campaign notes at **≈ +136M guest cycles**. This is the most dangerous + property of a rebase: it is a silent deletion that no conflict marker shows. + +### 2.2 What to salvage, and it is a lot + +The branch's value is concentrated in the two files that do **not** depend on +`multi_prove`'s structure: + +- **`fri/mmcs.rs` (1,015 lines, 7 tests).** A standalone primitive taking + matrices as `(row_major_bit_reversed_lde, log_height, width)`. Its only + coupling to main's churn is the concrete keccak backend, which is precisely + what `StarkHash::Mmcs` (§2.3) parameterizes. **Port this file, do not + rewrite it** — and keep its module doc, which is the clearest statement of the + layout contract anywhere in the campaign and should become the normative + reference §3.4's spec delta cites. +- **`fri/batched.rs`'s `combine_by_height` (499 lines).** Mixes codewords by + height with `alpha^i` and returns one combined codeword per height. That is + M-3's core and it is written. +- **`batched_soundness_tests.rs` (237 lines).** Oracles for a path that does not + exist on main yet — the most expensive thing to write from scratch and the + cheapest to port. + +⚠ **One salvage caveat, and it is the streaming requirement from §3.3.** +`combine_by_height` takes `inputs: &[(Vec>, usize)]` — *all* +codewords materialised at once. That is the `O(N)` shape §3.3 warns against. The +same question must be asked of `MixedMmcs::commit`'s matrix input. **Check both +before porting**; if they materialise, the port is the right moment to make them +streaming, since nothing downstream on main depends on the eager signature yet. + +### 2.3 `StarkHash` as the home — with one caveat + +`StarkHash` is the right home in shape: it is already the place where "how a +leaf becomes a digest" is named, and it already documents the +Batched/Pair split as "separate families because they hash different leaf +shapes, not because they are different hashes" (`config.rs:78-82`). An MMCS is a +third leaf shape. + +⚠ **But it does not fit `Batched` as declared, and the reason is worth +stating before someone tries.** `Batched`'s `Data = Vec>` is +a single-field, single-matrix leaf. A mixed-height MMCS leaf is not a `Vec` at +all — matrices are injected at *interior* levels, so the tree builder must know +the injection schedule; `IsStreamingLeafBackend` has no vocabulary for it. + +Two saving graces, both ✓ VERIFIED from the round structure: + +- **Per-round batching is field-homogeneous.** Main is all base, aux all ext, + parts all ext. So the *field* generic is not the problem — a `Mmcs` member + works. Had the design batched across rounds it would have needed a + heterogeneous leaf, and it cannot batch across rounds anyway (Fiat-Shamir, + §1.0). +- The digest type is already fixed: `Node` is "deliberately **not** an + associated type: it is `Commitment` for every implementation" + (`config.rs:88-91`), which is what keeps `StarkProof`'s rkyv derives + byte-identical. An MMCS root is one `Commitment`, so the wire format is + undisturbed by the *root*; what changes is that there is one root per round + instead of one per table (§3.2). + +**Recommended shape:** add a third member `type Mmcs` to `StarkHash` whose +implementation is built on the *same* `hash_bytes` as `Batched` — the same +discipline PA-PLAN §1.4 prescribes for the Pair/Batched invariant ("do not prove +that two independently-written encodings coincide; make them one function"), and +extend the existing invariant test (`tests/commitment_tests.rs:110-121`) with an +MMCS arm asserting that a single-matrix MMCS equals `Batched` on the same data. +That arm is the whole backward-compatibility argument in one test. + +### 2.4 The two recorded gaps — status after reading the branch + +- **The terminal-polynomial early-stop gap — ✓ CONFIRMED PRESENT.** (My own + first pass called this "not reproduced"; that was wrong, and the mistake is + instructive: I grepped `verifier.rs`, which handles the **non-batched** path + correctly, and never opened `batched.rs`.) The gap is in + `fri/batched.rs::batched_commit_phase` (`:90`): `num_committed_layers = + h_max.saturating_sub(1)` (`:129`), folding all the way to a **scalar** with + `transcript.append_field_element(&last_value)` (`:179`). The function takes no + `final_poly_log_degree` argument, and `terminal|FriFoldLayout|final_poly` + matches **one line** in the whole file. The non-batched + `commit_phase_from_evaluations` on the *same branch* does it right, returning + `final_poly_coeffs` from `terminal::coeffs_from_terminal_codeword` + (`fri/mod.rs:138-147`). Two stale doc comments date the lane: `batched.rs:87-89` + claims termination "mirrors `commit_phase_from_evaluations`" (it does not), and + `:186-188` says "#729 is not on this branch" — but #729 (`b3f85b79`) **is** on + it, merged via `1f3c0cec`, which is why `fri/terminal.rs` exists there at all. + + **Priced against this model: +2.5% to +3.7%** on the batched per-query cost + (2^21: 1,264 → 1,300, ratio 4.30× → 4.18×; 2^23: 1,457 → 1,493, 9.06× → + 8.84×). ~9 extra committed layers per proof, matching the campaign note's own + estimate of `k + blowup_log`. **So it is a real defect but a small one, and it + is a bug not to inherit rather than a tax on the design** — every projection in + §1 prices the *correct* construction with the early stop. The free acceptance + check: assert `num_committed(batched) == num_committed(largest leg)` using + `FriShape::effective_k = terminal_log − blowup_log` and `num_committed = + total_folds − 1` (`fri.rs:97-116`), plus the closed-form equality + `wrap_tests::the_census_fit_map_point` already asserts. +- **"Leaf-binding" — two different things were being conflated, and the + distinction changes the design.** `git grep -i 'leaf.?bind'` over `origin/main` + → zero hits; the campaign note's "leaf-binding fix required pre-production" + attaches to **PR #857 (LogUp-GKR port)**, not #768. + + On #768 the width/leaf-boundary binding is **implemented and tested**: + `verify_batch(root, iota, opening, heights, widths)` (`mmcs.rs:423-428`), with + widths derived AIR-side and never from the proof (`verifier.rs:2099-2128`), and + a negative test `batched_rejects_main_opening_width_mismatch` + (`batched_soundness_tests.rs:122`). That is #909's invariant, already honoured. + + **★ The residual gap is the TRANSCRIPT half, and it is a genuine design + requirement this plan must absorb** — flagged in-module at `mmcs.rs:78-81` + verbatim: *"`widths` and `heights` should ALSO be bound into the Fiat-Shamir + transcript by the consumer. Scope A's `absorb_height_histogram` currently binds + heights only; extending it to `(height, width)` pairs is a Task 4 / verifier + concern."* ✓ VERIFIED against `batched.rs:196`: the signature is + `absorb_height_histogram(transcript, heights: &[usize])` — heights only. + See §3.4's addendum, which folds this into the spec delta. + +**Also harvest:** `batched_soundness_tests.rs` (+237), and note PR #846 has since +landed "measure the verifier at real query counts, over real blocks" — that is +the instrument to settle §1.3's open denominator question with. + +--- + +## 3. DESIGN — MMCS in the LFM machine + +Scope note: §3.1–3.3 are the *machine's own* proof (application (b), the tower). +§3.5 is the emitter change, which is what application (a) needs and is the +larger of the two. They share the leaf construction in §3.4. + +### 3.1 Registry — `roots: [Commitment; 14]` → one root plus shape + +✓ VERIFIED today (`registry.rs:53-79`): + +```rust +pub struct LfmRegistryEntry { + kind, blowup_factor, + roots: [Commitment; NUM_LFM_CHIPS], // 14 + log_heights: [u8; NUM_LFM_CHIPS], + keccak_rnd_chunks: usize, + hasher: HasherKind, + program_id: Commitment, +} +``` + +and `build_artifacts_with_hasher` (`:151-200`) commits eleven instruction column +groups with `commit_group` plus `keccak_rc::preprocessed_commitment` and +`bitwise::preprocessed_commitment` into slots 12/13, leaving slot 11 +(`KECCAK_RND`, no preprocessed columns) as an all-zero sentinel, then derives +`program_id = lfm_program_id(&roots, &log_heights, keccak_rnd_chunks, hasher)` +(`:192`). + +**The change.** The 14 preprocessed roots become ONE batched preprocessed root +over the same 13 non-sentinel matrices (mixed heights: `log_heights` in the +registry runs 2…20). Keep `log_heights` — it is no longer only a height record, +it is the **injection schedule**, and the verifier needs it to build the walk. + +```rust +pub struct LfmRegistryEntry { + kind, blowup_factor, + prep_root: Commitment, // was [Commitment; 14] + log_heights: [u8; NUM_LFM_CHIPS], // now load-bearing: injection levels + prep_widths: [u16; NUM_LFM_CHIPS], // NEW: leaf shape per matrix (see §3.4) + keccak_rnd_chunks, hasher, program_id, +} +``` + +Three consequences, each of which has to be written down rather than inherited: + +1. **`program_id`'s preimage changes → re-bless.** `lfm_program_id` is fed the + root array today. Feeding it `prep_root` + `log_heights` + `prep_widths` + moves every digest. The regeneration path exists and is governed + (`cargo run --bin compute_lfm_registry --release`, drift tests on every PR, + "a drift failure is investigated, never re-blessed to silence the test", + `registry.rs:1-13`). **Sequence this re-bless INTO D0's** — the same argument + D8 makes for folding the RATE=4 re-bless in (PLAN.md §D1). Three separate + re-blesses of the same digest in one campaign is three chances to bless a + drift. +2. **The all-zero sentinel for `KECCAK_RND` stops being expressible** as a root + and becomes an *absence* in the injection schedule. The soundness argument at + `registry.rs:98-110` — the chip is program-independent in both directions, so + binding nothing is sound, and what the entry pins is `keccak_rnd_chunks` — is + unchanged in substance, but the mechanism moves from "a zero root" to "a + matrix not in the batched tree". Write it that way; a reader who greps for + the sentinel must land on the new statement. +3. **`prep_widths` is new registry data and is soundness-bearing.** Under the + per-table scheme, a group's width is implied by its own root plus the AIR. + Under an MMCS the widths determine how the leaf is parsed, so they must be + program shape, pinned, and folded into `program_id` — never read off the + proof. This is `verifier.rs:639`'s instruction ("do not re-derive it from the + proof") applied one level up, and it is the same rule COMMIT.md §1.3 states + for the header. + +Effort: **M**. + +### 3.2 `LfmArtifacts` / `verify_against` / `lfm_verify` + +✓ VERIFIED `verify_against` takes `roots: &[Commitment; NUM_LFM_CHIPS]` and +hands it to `LfmAirs::new_with_hasher(roots, options, keccak_rnd_chunks, hasher)` +(`proof.rs:251-291`); `lfm_prove` does the same via +`prove_traces_with_hasher` (`:170-175`). Both signatures change to +`(prep_root, log_heights, prep_widths)`. + +Two guards to preserve verbatim, because they are the shape of the soundness +argument and an MMCS refactor is exactly the kind of change that erodes them: + +- The `keccak_rnd_chunks == 0` rejection and the + `view.len() != num_lfm_airs(keccak_rnd_chunks)` length check + (`proof.rs:262-268`) — under batching the *AIR set* is still per-chip, so both + survive unchanged. Keep them; the batched root binds the preprocessed + matrices, not the chip count. +- The exhaustive `const _: () = match stark::config::COMMITMENT_HASH { ... }` + tripwire at `registry.rs:158-160`. PA-PLAN §4.2 already flags that this guard + becomes a half-truth once a second `StarkHash` exists. **Adding an `Mmcs` + member is a second reason to revisit it in the same pass** — the guard's job + is "the list of places that have to be revisited before this crate can commit + under two hashes" (`config.rs:55-62`), and "two leaf shapes" belongs on that + list too. + +Effort: **S** once §3.1 lands (mechanical signature threading). + +### 3.3 The prover's Round-1 commit + +✓ VERIFIED the seam. `multi_prove` commits each table independently under +`run_admitted` and then absorbs roots sequentially in index order — +`prover.rs:3240-3295`, with the comment stating the requirement exactly: "the +transcript only needs the roots absorbed in index order, done sequentially below +once every commit completed — the one ordering Fiat-Shamir requires before +sampling the shared challenges." + +Under batching that loop produces **one** tree and absorbs **one** root. + +**★★ The requirement that makes or breaks this: the batched tree build must be +STREAMING PER MATRIX.** This is the single most important implementation +constraint in the document, and getting it wrong silently undoes a large part of +the win. + +✓ VERIFIED the property at risk — the `Lde` struct's own doc +(`prover.rs:265-274`): main LDEs are all-N-live because Round 1 is a phase-wide +barrier, but **aux is produced and consumed inside the same fused task, so at +most `table_parallelism()` of them coexist**. Batching the aux round introduces +a *new* phase barrier at aux-commit. A naive implementation — materialise every +table's aux LDE, then build one tree over all of them — converts the aux term +from `O(k)` to `O(N)`, which per CENSUS Part 2 §1's table is 18.15 GiB per +`KECCAK_RND` chunk moving from ×21 to ×N. **That would give back a large +fraction of what batching buys, in the same commit.** + +The fix is available and is the same insight S3 rests on: retain the digest, +drop the buffer. Because COMMIT.md §1.2's absorption is a sequential **chain**, +a leaf can be accumulated matrix by matrix: + +``` +acc[leaf] = H0 // per-leaf accumulator, 16 B +for each matrix m, in commitment order: + compute m's LDE (one at a time, k-bounded exactly as today) + for each leaf: acc[leaf] = absorb(acc[leaf], m's header ‖ m's rows) + drop m's LDE +build the tree from acc[] // + injection levels +``` + +Retained state is `O(num_leaves × 16 B)` instead of `O(N × aux_cols × lde_size)` +— for a 2^19-row chunk that is ~4 MiB against 12.1 GiB. This is what "digests-only +MMCS by design" means in memory `pr768-memfix-mmcs-digest-only`, and it is why +that design note matters more than the code it describes. + +**Write it into the acceptance test, not the prose.** The falsifiable check is +the same shape as S3's: prove the same statement with per-table and batched +commitments and assert the batched run's peak anon is not higher. If the +streaming build was missed, that test fails loudly instead of the campaign +discovering it in a census three weeks later. + +The same treatment applies to the composition-parts round, and batched FRI needs +the analogous care: the combined codeword is a linear combination, so accumulate +`combined += α^i · quotient_i` one table at a time rather than materialising all +quotients. Both are `O(1)` in N if written that way and `O(N)` if not. + +**What does move: the per-table transcript fork.** The fork +(`prover.rs:3361-3370`, `t.append_bytes(&(idx as u64).to_le_bytes())`) exists so +that "aux build, aux commit and rounds 2-4 run FUSED per table … tables never +wait on a phase barrier" (`:3315-3326`). Batching reimposes barriers at +aux-commit, parts-commit and FRI, because a batched root cannot be absorbed +until every contributing matrix exists. So the fork survives only for the +per-table constraint/OOD work; the commitment and FRI stages rejoin the shared +transcript. **This is a restructure of `multi_prove`'s phase architecture, not +only of its R1 loop** — it is why M-4 is L and not M, and it is the reason the +sequencing in §4.3 puts it after S3 Phase A+B rather than beside them. + +**★ One verifier check that a batched preprocessed tree changes shape.** +✓ VERIFIED `verifier.rs:1288-1312`: for a preprocessed table the verifier +compares the proof's precomputed root against `air.precomputed_commitment()` — +"the critical soundness check - ensures prover used correct precomputed values" — +and absorbs BOTH the precomputed and the main root. Under a batched preprocessed +MMCS this becomes one comparison against the registry's `prep_root` instead of +one per table, and the per-table `precomputed_commitment()` accessor stops being +the thing that is checked. **That is a consolidation of a soundness check, which +is exactly the kind of change that quietly loses coverage**: the batched +comparison must still fail if *any* single table's preprocessed matrix is wrong, +which it does only if `prep_widths` and `log_heights` pin the parse (§3.1 +item 3). Put a tamper control on it per matrix, not just on the tree. + +⚠ **The protocol change this forces, stated plainly for the soundness review:** +today each sub-proof samples its own query indices from its own domain +(219 independent queries per table). Batched MMCS + batched FRI means **one +index per query, shared across all tables**, with shorter matrices opened at +`index >> (D − depth_t)`. That is the standard batched-FRI construction and it +is where the security parameters must be re-derived — not assumed to carry over. +It is the one part of this plan that is a protocol change rather than a +refactor, and per the house rule it goes to adversarial-debate review with +tamper controls in both directions plus an honest-path control. + +Effort: **L** (this is the center of mass on the prover side). + +### 3.4 ★ SPEC DELTA — the batched leaf, for Mauro's ratification + +This section is written to be lifted into `commit-spec/COMMIT.md` as a new +subsection under §1. It states what changes in the RATE=4 construction and what +does not. + +**What does not change.** The `LFML_row` function, the RATE, the tag, the +accumulator-in-the-message design, the chain-not-tree fold, `ROWS_PER_LEAF = 2`, +the node codec (§3.1), the strict decode (§3.2), the power-of-two leaf-count +assertion (§3.3). ✓ All of COMMIT.md §1.2's primitives are reused unchanged. + +**What changes: a leaf covers many matrices, so one header no longer describes +it.** COMMIT.md §1.2 sets `H = [LEAF_MARK, num_cols, kind, ROWS_PER_LEAF]` — +one header, one `num_cols`, one `kind`. An MMCS leaf interleaves matrices of +different widths. A single header binding a single `num_cols` would bind +*nothing* about how the felt stream splits between matrices, which reopens +§1.1's hazard in a new dress: not "moving columns between trees absorbed at +different times" but **moving columns between matrices inside one absorption**. + +**★ RECOMMENDED (option b): one header cell per MATRIX, at its injection +level.** The leaf at level ℓ absorbs, in matrix order: + +``` +for each matrix m injected at level l, in registry index order: + acc = LFML_row(acc, [LEAF_MARK, m.num_cols, m.kind, m.matrix_index]) + for each chunk c of RATE felts of serialize(m.rows): + acc = LFML_row(acc, c) +``` + +Why this one: + +- **COMMIT.md §1.3's argument survives verbatim, per matrix.** "The header binds + `num_cols` AND `kind`" and "the verifier must build the header from the AIR, + never from the opening" are unchanged statements; they now hold once per + matrix instead of once per leaf. The C2/C3/C4 executed collisions carry over + as-is, and the `m=1`-padded / `m=2`-unpadded collision (C4) is if anything more + necessary here, since adjacent matrices' padding meets inside one stream. +- **Matrix order is bound for free**, by the same property §1.3 already relies + on: "the chain binds chunk order for free." +- `ROWS_PER_LEAF` leaves the header (it is a global constant, already bound once + per proof) and `matrix_index` takes its slot — which is what closes the + reordering question the multi-matrix leaf introduces. Field count is + unchanged, so the header is still exactly one 4-felt cell and one compression. +- **It is cheap, and the cost is now measured rather than assumed:** + +| context | batched cost/query | + headers | overhead | +|---|---|---|---| +| tower, D1 fixture | 5,097 | +55 | **+1.1%** | +| tower, D1 real-2^21 | 6,670 | +58 | **+0.9%** | +| inner 2^21, blake3 | 2,405 | +100 | +4.2% | +| inner 2^23, blake3 | 2,838 | +196 | +6.9% | + + In the tower — where this spec applies — it is a rounding error. At the inner + layer (where the hash is byte-oriented, not LFML) the framing overhead is a + keccak/blake3 padding block per matrix and the same 4–14% band applies; still + far inside the 2.4–9.1× the batching buys. + +**Option (a), considered and not recommended:** one header per leaf binding a +*shape digest* over the ordered `(matrix_index, num_cols, kind)` vector. Cheaper +(the digest is program shape, computed once, ~0 per query), but it introduces a +second commitment object whose preimage rules need their own C-tests, and it +makes the leaf's binding indirect at exactly the point §1.3 argues it must be +direct. Take (a) only if the inner layer's 4–14% turns out to bind, and then +only there. + +**Injection structure and step typing.** The walk uses only the two existing +primitives: + +``` +digest = absorb(matrices at level 0) +for l in 1..=D: + digest = LFMC_compress(digest, sibling) + if any matrix injects at l: + digest = LFMC_compress(digest, absorb(matrices at l)) +``` + +No third step type and therefore **no new domain tag** — which matters, because +COMMIT.md §4.1.3 flags tag changes as a read-before-touching surface and the +crate anchor (C9) depends on the message staying a plain byte string. Injection +is expressed as one extra `LFMC_compress`, not as a compress-with-payload. + +**What the tree-shape rules become.** §3.3's "assert the leaf count is a power of +two — do not pad" still holds and gets *stronger*: the batched tree's leaf count +is the tallest matrix's `lde_size / 2`, and every injected matrix's own leaf +count must divide it exactly, which is automatic since all are powers of two. +Assert `D − depth_m == log2(leaves_D / leaves_m)` per matrix; it is free on the +honest path and it is what stops a matrix being walked in at the wrong level. + +**★ ADDENDUM — the transcript must bind the SHAPE, not only the leaf.** This is +the residual half of #768's leaf-binding item (§2.4) and it is the one place +where reading that branch changed this design rather than confirming it. + +The per-matrix header above binds `(num_cols, kind, matrix_index)` **inside the +leaf preimage**, so a mis-parsed opening fails authentication. That is +necessary and not sufficient: the injection schedule itself — *which* matrices +exist, at *which* heights, with *which* widths — is what the verifier builds its +walk from, and it must be pinned before any challenge that depends on it. +#768 pins heights and not widths: ✓ VERIFIED `absorb_height_histogram(transcript, +heights: &[usize])` (`batched.rs:196`), with its own module flagging the omission +(`mmcs.rs:78-81`). + +**Requirement: absorb `(height, width)` pairs, in commitment order, before the +batched root.** In the LFM machine this is nearly free because the shape is +already registry data — §3.1's `log_heights` and the new `prep_widths` are +exactly the two vectors, and they are already folded into `program_id`. So for +the tower the binding is *doubly* covered (program_id and transcript) and costs +one absorption per proof. For application (a) — the RV64 epoch proof, whose +table set is not a registry constant — it is the load-bearing one. + +Why it cannot be skipped on the grounds that "the verifier builds the walk from +the AIR anyway": that argument is exactly the one `verifier.rs:633-639` records +as having been a **live break** for aux opening widths — the check existed +upstream, but the root was absorbed after the challenge, so the prover got to +choose. An unbound shape vector reopens the same door one level up. Bind it, and +bind it before the root. + +**Open for ratification (the three decisions this section needs):** +- **M1.** Per-matrix headers (b) vs shape digest (a). Recommendation: (b). +- **M3.** Confirm `(height, width)` histogram absorption before the batched root + is the right placement, and that ordering it in commitment order (rather than + sorted) is what the walk reconstruction needs. +- **M2.** Does `matrix_index` replacing `ROWS_PER_LEAF` in the header satisfy + the reviewer that the multi-matrix reordering surface is closed, or does the + header need both (5 fields = 2 cells = 2× the header cost, still ~2% in the + tower)? This is the one place a second opinion is cheap now and expensive + later, exactly as §1.4.1 was. + +### 3.5 The EMITTER — `sub_proof.rs`'s leg structure + +This is where application (a)'s 4.3× materialises, and it is the largest single +code item in this plan. + +✓ VERIFIED today's structure. `SubProofShape::groups()` returns +`trace_groups ‖ parts_group` (`sub_proof.rs:128-132`); `emit_query_from_bits` +loops `for (commitment, opening) in commitments.iter().zip(openings)` calling +`emit_group_authentication` (`:446-448`), which does +`emit_leaf_hash` → `keccak_merkle_walk(leaf, bits, siblings)` → +`assert_word_eq_lanes` against the root (`:277-292`). The arena stride is +`values + 2·merkle_depth·groups` per query (`:146-150`), and +`emit_sub_proof_with_bits` declares `roots` as `2 · groups.len()` words +(`:548`). + +**The shape change.** `SubProofShape` describes ONE sub-proof; the batched +verifier's unit is a ROUND across sub-proofs. Introduce: + +```rust +pub struct MmcsRoundShape { + /// Matrices in commitment order, with their injection levels. + pub matrices: Vec<(GroupShape, /*depth*/ usize, /*matrix_index*/ u32)>, + pub depth: usize, // = max over matrices + pub log2_lde_length: u32, + pub coset_offset: FE, +} +``` + +and one emitter `emit_mmcs_query(b, round, root_lanes, openings, bits)` that +walks once, absorbing injected rows at their levels. Per query the wrap then +emits **4 walks instead of 100** (2^21) or **4 instead of 196** (2^23). + +Four properties of today's emitter that the change must preserve — all four are +load-bearing and all four are documented in the module header as things built by +construction rather than by convention: + +1. **The join.** `emit_group_authentication` "takes cells and cannot hint, so + the only values it can authenticate are the caller's, and `emit_query` hands + those same cells to the DEEP fold" (`sub_proof.rs:6-12`). The batched emitter + must keep the same discipline: the injected rows it absorbs are the same + cells DEEP folds. This gets *easier* under batching, not harder, because the + crossing described at `:14-36` ("the authentication groups by matrix and the + fold groups by point") is now one absorption in matrix order feeding one DEEP + fold in point order — the same two orders, one fewer tree. +2. **One index, shared.** `bits` are decomposed once and drive the walk *and* + the point derivation (`:38-52`, `:409`). Under batching this becomes + structurally true across *tables* as well, which removes a whole class of + hazard: there is no longer a per-table index that could disagree. + `QueryOutput::bits`/`point` (`:357-389`) keep serving the FRI join, and the + FRI join gets simpler for the same reason — one FRI, one bit vector. +3. **The two-consumer root hazard.** `GroupCommitment::from_lanes` exists so a + root reaches the leg "as the SAME cells the transcript absorbed rather than + as a second hint" (`:201-214`). With 4 roots instead of 100 this is 25× less + surface, but the constructor discipline must not be relaxed while the count + shrinks. +4. **The arena stride assertion** (`:621-625`, cursor must equal + `num_queries * query_words()`). Recompute `query_words` for the batched + layout — `values + 2·D` per round rather than `values + 2·depth·groups` — and + keep the assertion. It is the cheapest guard in the file. + +**Interplay with COMMIT.md §1.2/S1, stated for the record.** The emitter's +`emit_leaf_hash` (`:245-269`) currently renders base groups through +`edsl::keccak_leaf_hash` and ext groups by unpacking lanes 0..3 and byteswapping +— with an explicit note that lane 3 is not hashed and why that is sound +(`:236-244`). Under an MMCS leaf **that argument must be re-made per matrix**, +because it rests on "every extension value a query opens is also consumed as an +ext operand by the DEEP fold", which remains true but is now asserted across a +concatenated stream. Keep the note, scope it per matrix, and keep the +`debug_assert_eq!(len_bytes, shape.leaf_bytes())` (`:267`) as a per-matrix +assertion — under batching it becomes the thing that catches a mis-parsed +boundary between two matrices' felts. + +Effort: **L**. + +### 3.6 Item summary + +| # | item | effort | +|---|---|---| +| M-1 | `StarkHash::Mmcs` member + keccak instance + single-matrix-equals-`Batched` invariant test | **M** | +| M-2 | **PORT `fri/mmcs.rs` from #768** (1,015 lines + 7 tests), re-parameterize over `StarkHash::Mmcs`, **make the build streaming-per-matrix (§3.3)** with the peak-anon acceptance test | **M** (was L before the branch was read) | +| M-3 | Batched FRI — **port `fri/batched.rs`'s `combine_by_height`**; one instance over the largest domain, smaller matrices folded in at the matching layer; make the accumulation streaming | **M** (was L) | +| M-4 | `multi_prove` R1/aux/parts commit → one tree per round; one root absorbed | **L** | +| M-5 | Verifier mirror + shared query-index derivation | **M** | +| M-6 | Registry: `prep_root` + `prep_widths`, `program_id` re-bless (fold into D0's) | **M** | +| M-7 | `LfmArtifacts` / `verify_against` / `lfm_verify` signature threading | **S** | +| M-8 | Emitter: `MmcsRoundShape` + `emit_mmcs_query`, arena stride, FRI join | **L** | +| M-9 | COMMIT.md spec delta (§3.4) + C-tests for the per-matrix header | **M** | +| M-10 | Security-parameter re-derivation for shared query indices + adversarial review | **M** | + +| M-11 | **Settle §1.3's −57% denominator** before §1.1 is quoted as a schedule input. Step 1 is free: pin which measurement it is (`pr768-batched-fri-state.md`'s sim, not the PR's CI +3.61%-at-1-query). Only if that fails, re-measure with PR #846's harness | **S** | +| M-12 | Do not inherit #768's terminal-poly gap: batched FRI must stop at `fri_final_poly_log_degree`, with the free `num_committed` equality assertion (§2.4). Worth +2.5–3.7% | **S** | +| M-13 | Extend `absorb_height_histogram` to `(height, width)` pairs, absorbed before the batched root (§3.4 addendum / M3) | **S** | + +Whole item: **L**, comparable to P-a — but two of the three critical-path items +(M-2, M-3) drop from L to M once #768's primitives are ported rather than +rewritten, which is the main practical consequence of reading the branch. +**M-11 is S and should be done first**: it is the only item that can change the +size of the prize. M-12 and M-13 are both S and both come straight from #768's +defects — cheap to carry, expensive to rediscover. + +⚠ **One dependency that is easy to miss:** M-1 targets `StarkHash`, which lives +on `blake3-real-hash` and **not on main** (§2.1). Costing this item as +independent of D0 would be wrong. + +--- + +## 4. COLLISION MAP AND SEQUENCING + +### 4.1 Shared files + +| file | MMCS needs | S3 (in flight NOW) | P-a stages | +|---|---|---|---| +| `crypto/stark/src/prover.rs` R1 loop (`:3240-3295`) | **rewrite** — per-table commit → one tree | **rewrite** — `MainLdeSlot::{Retained,Dropped}`, Phase A | Stage 2 threads `H` (light here) | +| `crypto/stark/src/prover.rs` fused task | reads the batched opening | **rewrite** — recompute LDE, Phase B aux release | — | +| `crypto/stark/src/config.rs` | **new `Mmcs` member** | — | **new `Blake3StarkHash` instance** (Stage 1) | +| `crypto/stark/src/commitment.rs` | new mixed-height builder beside `commit_bit_reversed` | — | leaf backend swap (Stage 1) | +| `crypto/stark/src/fri/**` | **rewrite** — one batched instance | — | **thread `H`, ~13 sites** (Stage 2, §4.1) | +| `crypto/stark/src/verifier.rs` | batched path auth + shared index | — | `H::Batched` already threaded | +| `prover/src/lfm/registry.rs` | `prep_root`, `prep_widths`, re-bless | — | `COMMITMENT_HASH` tripwire (Stage 6) | +| `prover/src/lfm/proof.rs` | signature threading | **`ResidencyMode` threading (landed, `:103-212`)** | — | +| `prover/src/lfm/sub_proof.rs` + `epoch_verify.rs` + `fri.rs` | **rewrite** — batched leg | — | Stage 5 emitter switch (§4.6) | +| `crypto/math-cuda/**` | batched tree kernels | — | **nine blake3 kernels** (track G) | + +**Two hard collisions and one soft one:** + +- **`fri/**` is contested by MMCS (M-3, rewrite) and P-a Stage 2 (thread `H`, + ~13 sites, PA-PLAN §4.1).** These must not run concurrently. P-a Stage 2 is + the smaller and is already scheduled; MMCS's FRI rewrite should land *after* + it and inherit the threading. +- **`prover.rs`'s R1 loop is contested by MMCS (M-4) and S3 Phase A.** S3 is in + flight this week. MMCS must not touch that loop until Phase A lands. +- Soft: `config.rs` gets a new member from each of MMCS (`Mmcs`) and P-a + (`Blake3StarkHash`). Different axes of the same trait, mergeable, but they + should not be written in the same week by different agents — the + `const _: fn()` tie-in block (`config.rs:170-192`) is a magnet for conflicts. + +### 4.2 How MMCS and S3 interact — they are complements, not substitutes + +This is the most important scheduling fact in this document and it is easy to +get backwards. + +- **S3 attacks residency:** peak = `17.37·N + 30.2·k` → flat in N. It does not + reduce the work; it stops the work from being simultaneously resident. Cost: + one extra forward NTT per table (S3-RECOMPUTE-PLAN §4). +- **MMCS attacks the work:** N itself, 55 → 13 at 2^21, 133 → 15 at 2^23. + +Multiply them and the fit closes with margin from either side; take only one and +it is tight. Take only S3 and the flat floor is ~48–56 GiB *plus* whatever the +non-chunk base has grown to. Take only MMCS and N=13 at the measured +13.4 GiB/chunk marginal is ~205 GiB — better than 654, still over. + +**⚠ But there is one way they fight, and it is the aux term.** S3's Phase B +("free each table's aux columns when its fused task completes") is written +against the fused per-table task. Batching reimposes a barrier at aux-commit +(§3.3), so Phase B's "end of the fused task" moves and, if the batched builder +is not streaming, the aux LDEs become all-N-live — the exact property S3 is +trying to fix on the main side. **Whoever writes M-4 must read S3 Phase B +first**, and the streaming builder requirement in §3.3 is what keeps the two +compatible. If M-4 lands before S3 Phase B, Phase B's design has to be rewritten +against the new phase structure; if after, it is a small adaptation. That is a +second, independent reason for the ordering in §4.3. + +**MMCS also makes S3 Phase C less likely to be needed.** S3's own decision gate +says Phase C proceeds only "if the re-census disagrees" after P-a +(S3-RECOMPUTE-PLAN §3). Batching cuts N by a further 4–9×, which is a second +reason for that gate to come back negative. **Recommendation: re-run the S3 +Phase-C decision gate after MMCS, not only after P-a.** + +### 4.3 Proposed order + +``` +NOW ─────────────────────────────────────────────────────────────────────── + S3 Phase A (in flight) P-a Stages 1-3 [D0 blake3 switch] + │ │ + ├── S3 Phase B ├── P-a Stage 2 (fri/ threading) + │ │ │ + ▼ ▼ ▼ + ══ MMCS may start here ═══════════════════════════════════════════════ + M-11 settle the -57% denominator ← unblocked NOW, S, sizes the prize + M-10 security-parameter derivation ← unblocked NOW, could invalidate the plan + M-9 COMMIT.md spec delta+M1/M2/M3 ← unblocked NOW (spec work, no code) + M-1 StarkHash::Mmcs member ← needs S3 Phase A *and* D0's StarkHash + (which is on blake3-real-hash, NOT main) + │ + ▼ + M-2 PORT #768's fri/mmcs.rs ← needs M-1 + M-6 registry + re-bless ← FOLD INTO D0's re-bless pass + │ + ▼ + M-3 batched FRI ← WAITS for P-a Stage 2 (fri/ threading) + M-4 multi_prove batched rounds ← WAITS for S3 Phase A+B (prover.rs R1) + │ + ▼ + M-5 verifier mirror + M-8 emitter ← WAITS for P-a Stage 5 (same four sites) + M-7 signature threading +``` + +**What can begin the moment S3 Phase A lands:** M-1 (the `StarkHash` member) and +M-2 (porting the tree builder) — both are additive in `crypto/stark`, behind a +configuration, with keccak per-table remaining the default. Neither touches the +R1 loop. ⚠ **But both must be cut from `blake3-real-hash`, not main**, because +that is where `StarkHash` lives (§2.1 correction box) — so M-1 inherits D0's +merge risk, and the MMCS lane becomes a third passenger on the campaign branch +alongside P-a and S3. If D0 is expected to take a long time to reach main, the +alternative is to write M-2's port against the concrete backend first (as #768 +did) and parameterize it when D0 lands; that trades one refactor for +independence. + +**What can begin RIGHT NOW, before anything:** **M-11** (settle the −57% +denominator — S, and it sizes the prize), **M-10** (re-deriving the security +parameters for shared query indices — pure analysis, and the one item that could +invalidate the whole plan), and **M-9** (the COMMIT.md spec delta, §3.4 — spec +text and C-tests, and it should go to Mauro's ratification in the *same* pass as +D9/RATE=5 since batching changes D9's arithmetic, §1.4). All three are +analysis/spec work with no code dependency and no collision with S3 or P-a. + +**What waits for P-a:** M-3 (fri/) waits for Stage 2's threading; M-8 (emitter) +waits for Stage 5, which switches the same four emitter sites (PA-PLAN §4.6). +Doing them in the other order means writing the batched emitter twice. + +### 4.4 The July caveat, re-examined + +The recorded caveat is *"hash choice gates the batching decision"*, from the +July campaign. **It does not survive, and here is precisely why.** + +That claim was about RV64-**guest** economics, where the verifier's bill is +*cycles* and keccak's rate-17 sponge makes leaf absorption cheap relative to +tree walks — so which hash you pick changes which term dominates and therefore +whether batching is worth its complexity. + +For the LFM wrap the currency is **permutations/compressions**, i.e. chip cells, +i.e. memory. In that currency: + +| | batching buys | leaf share after batching | +|---|---|---| +| keccak inner | 3.46× – 9.06× | 74–77% | +| blake3 inner | 2.28× – 5.12× | 86–88% | + +Batching wins decisively under both hashes; the hash changes the multiplier by +about 1.6× and changes nothing structural. **What does survive of the caveat, +restated correctly:** the hash choice governs *what is left to optimise after* +batching. Under either hash the post-batching bill is 74–88% leaf absorption, so +after MMCS lands the only remaining levers anywhere in the stack are the leaf +RATE (tower) and the leaf payload itself (inner). That is a genuinely useful +reframing of the caveat — and it is an argument for doing MMCS *before* spending +more effort on Merkle/FRI micro-optimisation anywhere. + +--- + +## 5. Confidence ledger + +| claim | mark | +|---|---| +| The closed form `leaf + groups·depth + fri` and every constant in it | ✓ VERIFIED, `epoch_verify.rs:552-559`, `sub_proof.rs:88-90,:160-166`, `fri.rs:97-116,:133-144` | +| Model reproduces measured `query_permutations` exactly at 4 real points | ✓ MEASURED, census_logs | +| Leg shapes (widths, depths, FRI layers, sub-proof counts) | ✓ MEASURED, `ethrex_e2*_skip.log` | +| Spine permutations at real query counts | ✓ MEASURED, `ethrex_e2*_spine.log` | +| Per-round batching is forced (Fiat-Shamir), not chosen | ✓ VERIFIED, `prover.rs:3216`, `verifier.rs:1295-1317` | +| `StarkHash` shape, `Batched`/`Pair` members, `Node = Commitment` | ✓ VERIFIED, `config.rs:55-192` | +| Registry root array, `program_id` preimage, sentinel slot 11 | ✓ VERIFIED, `registry.rs:53-200` | +| R1 commit loop, root absorption order, per-table transcript fork | ✓ VERIFIED, `prover.rs:3240-3295, :3361-3370` | +| Emitter per-group walk, arena stride, join discipline | ✓ VERIFIED, `sub_proof.rs:128-150, :277-292, :424-492, :534-628` | +| All batched projections (§1.1–1.4 tables) | **DERIVED from the calibrated model** | +| 4.06× hash factor | ✓ MEASURED (campaign hash matrix), composed multiplicatively — ⚠ the fixed non-hash floor does not shrink, so the product is mildly optimistic | +| `KECCAK_RND` cell cost 72,672 / blake3 4,946 | MEASURED / ? INFERRED on the 630 aux width (`tower.py`'s own caveat) | +| #768 branch exists, OPEN, CONFLICTING, +5,097/−1,099 / 25 files, merge base `3ea4f916` (2026-07-17), 25 behind / 45 ahead | ✓ VERIFIED, `gh pr view 768` + `git merge-base` | +| #768's MMCS is wired (3 round instances, built, absorbed, opened) | ✓ VERIFIED `prover.rs:612-614, :2666-2667, :4413-4415` on the branch | +| #768's MMCS layout == this model's `batched_tree_cost` semantics | ✓ VERIFIED, `fri/mmcs.rs:1-56` module doc quoted in §1.3 | +| ~~`879bdc0f` (StarkHash) is a main-side commit~~ | **✗ FALSIFIED** — StarkHash is **not on main at all**; `git grep -c StarkHash origin/main` → 0, `git branch --contains 879bdc0f` → `origin/blake3-real-hash` only. It is a D0 campaign-branch artifact, so M-1 inherits D0's merge risk (§2.1 correction box) | +| ~~#768 = FRI-only, digests-only MMCS~~ | **✗ FALSIFIED** — my inference from memory notes, refuted by the branch's three wired round-MMCS instances. §1.3 records what it cost | +| ~~terminal-poly early-stop gap not reproduced~~ | **✗ FALSIFIED** — the gap is real, in `batched.rs:129,:179`; I had grepped only the non-batched `verifier.rs`. Priced at +2.5–3.7% (§2.4) | +| Width/leaf-boundary binding on #768 | ✓ VERIFIED **implemented and tested** (`mmcs.rs:423-428`, `verifier.rs:2099-2128`, negative test `batched_soundness_tests.rs:122`) — the campaign's "leaf-binding fix" note attaches to #857, not #768 | +| Transcript binds heights but NOT widths | ✓ VERIFIED `batched.rs:196` + the module's own flag `mmcs.rs:78-81` — folded into §3.4 as a design requirement (M3) | +| #768's batched lane contains no CUDA; #877 deleted `plan_table_chunks` which it calls | ✓ VERIFIED (§2.1) | +| A merge silently deletes #845's view machinery (≈ +136M guest cycles) | ✓ VERIFIED via `git merge-tree`, and documented on-branch as a TODO (§2.1) | +| The −57% ↔ model −76.7% discrepancy | **✗ OPEN** — and the target is itself a sim (`pr768-batched-fri-state.md`), not the PR's CI number (+3.61% cycles at 1 query). Gates quoting §1.1 end-to-end | +| CENSUS Part 2 §3's "+19%" D0 feedback | ✗ DOES NOT REPRODUCE — re-running `tower.py` gives +47%; flagged, not resolved | +| Batching factors composed with P-a | DERIVED × MEASURED | + +## 6. Reproduction + +Projections (the tooling sits beside `project.py` / `tower.py`, the calibrated +chip model it imports): + +``` +cd ~/workspace/lambda_vm_bench_cache/lfm_census_2026-08-12 +python3 mmcs_project.py # model validation + (a) and (b) headline +python3 mmcs2.py # four-corner decomposition, #768 comparison, composites +python3 mmcs3.py # header cost, prep sensitivity, RATE sweep +``` + +§2's branch facts: + +``` +gh pr view 768 --json headRefName,state,mergeable,additions,deletions,changedFiles +git fetch origin 'refs/heads/feat/batched-fri-per-epoch:refs/remotes/origin/feat/batched-fri-per-epoch' +git merge-base origin/main origin/feat/batched-fri-per-epoch # -> 3ea4f916 +git merge-base --is-ancestor 879bdc0f 3ea4f916 && echo pre || echo post # -> post +git diff --stat 3ea4f916..origin/feat/batched-fri-per-epoch +git show origin/feat/batched-fri-per-epoch:crypto/stark/src/fri/mmcs.rs | head -60 +git grep -n mmcs origin/feat/batched-fri-per-epoch -- crypto/stark/src prover/src +``` + +The tower discrepancy in §1.4: + +``` +python3 -c " +import sys; sys.path.insert(0,'.') +import tower; from tower import * +from project import BYTES_PER_CELL, GIB +tower.WIDTH['LFM_HASH']=(3056,630) +inv,_ = node_cost(REAL21_WRAP, REAL21_RND, 219, 'blake3') +print(inv*BLAKE3_CELLS_PER_COMPRESSION/0.935*BYTES_PER_CELL/GIB) # 559, not 452 +" +``` diff --git a/thoughts/shared/block-compression/PA-PLAN.md b/thoughts/shared/block-compression/PA-PLAN.md new file mode 100644 index 000000000..254f9d76c --- /dev/null +++ b/thoughts/shared/block-compression/PA-PLAN.md @@ -0,0 +1,1211 @@ +# PA-PLAN — the RV64 prover commits with BLAKE3 + +**Scoping record.** Read-only pass; no builds run, no edits made. +**Ground:** worktree `/Users/maurofab/workspace/lambda_vm-blake3-impl`, branch +`blake3-real-hash` @ `bad2d97d`. **Date:** 2026-08-12. + +**What P-a is:** move the RV64 STARK prover's commitment hash from keccak256 to +BLAKE3 across all four domains together — Merkle trees, FRI-layer trees, +Fiat–Shamir transcript, grinding PoW — because the LFM wrap re-derives the inner +proof's transcript and every domain must be hosted-recomputable. + +**Decisions already taken (Mauro, 2026-08-12), folded in throughout:** +1. **6-round BLAKE3 is the target** ("to see if this works"). 7r stays buildable + via the existing feature; the primary host implementation is the in-repo + reduced-round compression, and the `blake3` crate stays dev-only as the 7r + compression anchor. See §1.5, and §1.6 for the structural consequence. +2. **The CUDA kernels are a pre-authorized parallel workstream**, not a tail + stage. Kernel list, oracle and start condition are in §6.1; it appears as + track **G** in the stage table. + +**The one decision still owed** is §1.6: bare cv-chain vs standard chunk tree. +It blocks track G's chaining loop and Stage 5's emitter, so it wants answering +before either commits. + +Claims are ✓ VERIFIED (read the code, cited) / ? INFERRED / ✗ UNVERIFIED. +Everything marked ✓ below was read in this pass, not inherited. + +> **Provenance note.** Four delegated sweeps (host backends; transcript + +> grinding; guest/fixture/CI blast radius; GPU + emitters + D0 collision) were +> launched and none returned before this was written — the same failure +> D0-DESIGN.md records. Every load-bearing claim here was therefore read +> directly by the author, and inherited citations are marked as such and were +> re-checked (two were stale — see §2.1 and §5). §8 lists what is genuinely +> still open. + +--- + +## 0. Verdict — read this before scheduling anything + +**The prover-side switch is M. The thing that makes it worth doing is L, it is a +change to the MACHINE, not to the prover, and PLAN.md currently prices the whole +item off the prover half.** + +PLAN.md's rationale for putting P-a first says step 2's `StarkHash` +parameterization "makes it a second config instance rather than surgery" +(`PLAN.md:97-98`). That is true of `crypto/stark` and false of the payoff. Three +findings, in order of how much they move the schedule: + +1. **The 4.06× requires a chip the machine does not have.** The hash matrix + priced hosting at 4,946 base-equivalent cells per compression at **rate 8** + (64-byte message block). That is `blake3_chip`/`LFM_BLAKE3` — the *general* + BLAKE3 compression, taking `h`, all 16 message words, counter, `block_len` + and flags from columns (`blake3_chip.rs:504-525`). ✓ VERIFIED that chip is + **not a machine chip**: `LfmColumnGroups` has ten groups and none is blake3 + (`compiler.rs:112-123`); `LFM_BLAKE3` occurs only inside `blake3_probe.rs` + (`:141 .with_name("LFM_BLAKE3")`), a standalone measurement instrument. + The machine's live blake3 is the `LFM_HASH` **socket arm**, which pins + `h = IV`, pins counter/`block_len`/flags, and uses 8 of 16 message words + (`blake3_socket.rs:740-757`, `:725-732`). Hosting a wide-leaf absorption on + the socket runs at an amortized **rate 4**, not 8 + (`epoch_verify.rs:428-456`: `LFM_HASH_RATE_FELTS = 4`, "**this is 4.25× + worse than keccak's 17**", `:437`). Promoting `LFM_BLAKE3` to a registered, + program-callable group — layout, preprocessed prefix, eDSL emitters, + registry rows, admission-validator coverage — is the campaign's real P-a + cost and it is soundness-bearing. + +2. **The socket alternative is probably disqualified on digest width, not on + cost.** The socket truncates to four output words = one cell = **128 bits** + (`blake3_socket.rs:268-271`, `OUT_WINDOW = HASH_DIGEST_FELTS`; + `hash.rs:23`), against the RV64 proof's current `Commitment = [u8;32]` + (`config.rs:18-19`) and `KeccakDigest = [Cell;2]` in the guest + (`edsl`, used at `sub_proof.rs:228`, `fri.rs:314`). A 128-bit Merkle node is + a 64-bit collision bound. That is a decision for Mauro, but it is a + *security* decision, and it should not be taken as a side effect of picking + a cheaper hosting route. + +3. **Under `cuda`, a blake3 `StarkHash` instance cannot be written at all.** + ✓ VERIFIED `config.rs:116-122`: with `feature = "cuda"`, `StarkHash::Batched` + additionally requires `KeccakTreeBackend`. The step-0 H3 guard landed and + bound seven GPU tree entries to that marker (`gpu_lde.rs:701, 758, 802, 884, + 1120, 1171, 1573`). This is the guard working exactly as designed — and it + means the GPU regression window is not a runtime fallback but a compile-time + fork that P-a must decide explicitly. + +**Recommended shape.** Byte-oriented BLAKE3 **at 6 rounds** (decided — §1.5), +256-bit digests, one family across all four domains, hosted by a promoted +`LFM_BLAKE3`. It keeps the existing leaf byte encoding untouched, keeps +`Commitment = [u8;32]` and the rkyv wire format byte-identical, and is the only +option consistent with the 4.06× the campaign is planning against. + +**One question needs answering before Stage 1 commits an API** (§1.6): at 6 +rounds nothing in the world can recompute our hashes anyway, so standard +BLAKE3's chunk tree — 1024-byte chunks, per-chunk counter, flag schedule — buys +no interop and costs ~6% extra compressions plus a state machine in all nine +CUDA kernels *and* in the wrap emitter. Recommend a bare cv-chain over 64-byte +blocks instead, same construction at both round counts. This is the one decision +that blocks other people's work. + +**Effort, honestly:** whole item **L** (multi-week). Stages 1–3 (the prover) are +M and land behind a config with keccak still default. Stage 5 (chip promotion + +emitter switch) is L and is the center of mass. Stage 4 (guest leg) is L and is +gated on merging an unmerged branch. The CUDA kernels (track G) are M and run in +parallel from now. + +--- + +## 1. Host backends + +### 1.1 What exists + +✓ VERIFIED the `blake3` crate is **dev-only, in `prover` alone**: +`prover/Cargo.toml:48` — `blake3 = { version = "1.8.5", default-features = +false, features = ["std","pure"] }`, sitting after `[dev-dependencies]` +(`:38`), with `:44-45` stating the intent: "The external anchor for +`lfm::blake3` at 7 rounds and for the `LFM_HASH` BLAKE3 socket … Test-only on +purpose." `crypto/crypto` and `crypto/stark` have **no** blake3 dependency +(grep over all `Cargo.toml`: the only two hits are `prover/Cargo.toml:22` and +`:48`). + +✓ VERIFIED the round-generic host implementation is `prover/src/lfm/blake3.rs`: + +| item | location | shape | +|---|---|---| +| `blake3_compress_rounds(h, m, t, block_len, flags, rounds)` | `blake3.rs:125-148` | fully general compression, `rounds` a runtime argument, `u32` words | +| `BLAKE3_STANDARD_ROUNDS = 7` / `BLAKE3_SIX_ROUNDS = 6` | `blake3.rs:59, 63` | | +| `BLAKE3_ROUNDS` | `blake3.rs:83, 85` | `7` unless `feature = "blake3-6round"`, then `6` | +| `BLAKE3_IV`, `BLAKE3_MSG_PERMUTATION` | `blake3.rs:46, 52` | | +| `CANONICAL_VECTORS` + `CANONICAL_OUT_7ROUND` | `blake3.rs:198-462` | 10 KAT vectors across `block_len` 18–64, both round counts | + +✓ VERIFIED `blake3-6round` is declared **only** at `prover/Cargo.toml:22` +(`blake3-6round = []`), enabled by nothing, off by default. The house already +treats a split round count as a shipping hazard: `blake3_socket.rs:215` asserts +`SOCKET_ROUNDS == BLAKE3_ROUNDS` with the comment that a second `cfg` pair here +"is a silent pricing lie: the probe would measure one hash and the machine would +use another." + +There is **no byte-oriented `&[u8] -> [u8;32]` blake3 on the host** outside the +dev-only crate — `blake3.rs` stops at the compression function, and +`blake3_socket.rs` is cell-oriented and fixed at a 36-byte one-block message +(`BLOCK_LEN_LFMC = 36`, `:262`; `FLAGS_LFMC = 0x0B`, `:256`; `COUNTER_LFMC = 0`, +`:265`). + +### 1.2 The crate-layering problem, and the fix + +The Merkle backends live in `crypto/crypto/src/merkle_tree/backends/`; the +round-generic compression lives in `prover`, which depends on `crypto`. A +backend in `crypto` therefore **cannot** call `prover`'s blake3. + +**With 6 rounds decided, there is only one way out.** The `blake3` crate is +7-round only, so it cannot implement the pipeline's primary arm at all: + +- **(a) `blake3` as a real dependency of `crypto/crypto`.** ✗ **Ruled out by the + round-count decision** — it has no 6-round mode. It stays where it is + (`prover`, dev-only) as the **compression-level KAT anchor for the 7r arm**, + which is exactly the role `prover/Cargo.toml:44-45` already assigns it. +- **(b) Sink the compression core down. REQUIRED.** Move + `prover/src/lfm/blake3.rs`'s `blake3_compress_rounds` + `BLAKE3_IV` + + `BLAKE3_MSG_PERMUTATION` + `CANONICAL_VECTORS`/`CANONICAL_OUT_7ROUND` into + `crypto/crypto` (`hash/blake3/`), re-export upward so `lfm` keeps its current + API, and move the `blake3-6round` feature with it. ✓ Safe by construction: the + LFM chip, the socket and the new backend then share **one** compression + function — which is what `blake3_socket.rs:203-215` says the tree already + depends on, and the only way the CUDA kernels (§6.1) and the wrap emitter can + be checked against the same reference. + +✓ VERIFIED the 6-round implementation the backend will call already exists and +is the one the chip's trace filler uses. `blake3_compress_rounds(h, m, t, +block_len, flags, rounds)` (`blake3.rs:125-148`) takes `rounds` as a **runtime +argument**; `blake3_compress_6round` (`:108-115`) is the fixed-6 wrapper. The +chip fills its trace through the value interpretation of the same dataflow — +`ValueFlow`'s `input_h`/`input_v12`/`add3` at `blake3_chip.rs:650-665` read +`self.h[i]` / `self.v12[j]` / `self.m[m_idx]` — so host filler and backend hash +identically by sharing one function, not by agreeing. + +⚠ Note `make lint` does **not** build `blake3-6round` (recorded at +`thoughts/shared/lfm-real-hash/phase2-report.md:446, 520`). Moving the feature +into a lower crate widens that blind spot — the feature must be added to the +Makefile's combination matrix in the same change. + +### 1.3 What the two backends must implement + +✓ VERIFIED the contract, `crypto/crypto/src/merkle_tree/traits.rs`: + +```rust +pub trait IsMerkleTreeBackend { // :11-32 + type Node: PartialEq + Eq + Clone + Sync + Send; + type Data: Sync + Send; + fn hash_data(leaf: &Self::Data) -> Self::Node; // :16 + fn hash_leaves(..) -> Vec; // :20, defaulted + fn hash_new_parent(a: &Self::Node, b: &Self::Node) -> Self::Node; // :31 +} +pub trait IsStreamingLeafBackend: IsMerkleTreeBackend { // :47-59 + fn hash_bytes(data: &[u8]) -> Self::Node; // :54 + fn hash_data_from_slices(a: &[FE], b: &[FE]) -> Self::Node; // :58 +} +``` + +The contract is **byte-oriented** (`:52-58`: `hash_bytes` "Equals `hash_data` +applied to the elements `data` encodes"). That is a direct fit for standard +blake3 and a poor one for any felt-absorbing variant — a second, independent +argument for the byte-oriented choice. + +✓ VERIFIED the leaf **encoding does not move**: `leaves_bit_reversed_grouped` (`commitment.rs:55-110`) and `commit_bit_reversed_with` +(`commitment.rs:175-190`) are already backend-generic and serialize +`rows_per_leaf` bit-reversed rows column-by-column big-endian into a reused +buffer, then call `B::hash_bytes(buf)` (`:94`). Only the `keccak_*`-named +wrappers pin the alias (`commitment.rs:123, 137, 148, 165`). **Consequence: D0's +S1 wide-leaf spec item does not apply to P-a.** The RV64 leaf keeps the exact +byte layout it has today; what binds opening width remains the explicit I3 check +the verifier already performs (`verifier.rs:204-213`), not the hash. + +### 1.4 The Pair/Batched two-element invariant + +✓ VERIFIED the invariant is documented at `config.rs:93-106` and pinned by a +test that asserts `::hash_data(&vec![a,b]) == +::hash_data(&[a,b])` over three vectors (`tests/commitment_tests.rs:110-121`), +plus a second test that the streaming routes agree with `hash_data` +(`:124-152`). + +✓ VERIFIED it is load-bearing, not decorative, and the reason is asymmetric: +the prover builds FRI-layer trees with `Pair` (`fri/mod.rs:105`) and the +verifier authenticates those same openings with `Batched` +(`verifier.rs:736`, `verify_merkle_path::>`). The +verifier never uses `H::Pair` at all. + +**How the blake3 instance honours it: one family, both sides.** Define +`Blake3Batched` and `Blake3Pair` over the *same* serialize-then-`hash_bytes` +routine, with `Pair::hash_data(&[a,b])` implemented as +`hash_bytes(a.be ‖ b.be)` — literally the two-element case of the batched path. +Do not prove that two independently-written encodings coincide; make them one +function. Then keep the existing invariant test and add the blake3 arm to it. + +### 1.5 Round count — DECIDED: 6 rounds + +**Mauro, 2026-08-12: 6-round is the target ("to see if this works"); 7r stays +buildable.** All pipeline numbers below are 6r. + +The round count stays a **compile-time** knob, matching what exists: +`BLAKE3_ROUNDS` (`blake3.rs:83-85`). A backend generic over a `const ROUNDS: +usize` would let one build produce two hashes and is exactly the failure +`blake3_socket.rs:203-215` was written to prevent. So: one `Blake3StarkHash` +whose backends call the crate-global `BLAKE3_ROUNDS`, and the feature moves the +whole tree at once. + +⚠ **Do not invert the feature's polarity.** `blake3-6round` currently means +"6 instead of the default 7" (`blake3.rs:83-85`), and A6R-signoff / +`ORCHESTRATION.md:45` record the ratified framing as "7-round instantiated +baseline, 6 behind the feature". Flipping the flag's sense would silently change +what every existing measurement and report means by "default". Keep the name and +the polarity; make the campaign **build with `--features blake3-6round`** and add +it to the Makefile's lint/test matrix — `make lint` does not cover it today +(`thoughts/shared/lfm-real-hash/phase2-report.md:446, 520`), and PLAN.md:177 +already lists "blake3-6round OFF by default (+16% if forgotten)" as a live build +trap. It is now a trap on the P-a pipeline too. + +### 1.6 ★ What 6 rounds does to the leaf CONSTRUCTION (new decision surface) + +The round-count decision has a structural consequence that is easy to miss, and +it makes the work *smaller*. + +✓ VERIFIED the interop position, A6R-signoff `:104-106`: "7-round parent merges +are bit-compatible with published BLAKE3, so an external verifier can recompute +a tree. **6-round merges are computed by nothing else in the world.**" + +At 6 rounds there is therefore **no external verifier to be compatible with** — +and standard BLAKE3's chunk-tree machinery (1024-byte chunks, per-chunk counter +`t`, the `CHUNK_START`/`CHUNK_END`/`PARENT`/`ROOT` flag schedule) exists purely +for interop and parallelism, not for security. Keeping it at 6r buys nothing and +costs three times over: + +- ~6% extra compressions (one parent per 16 block compressions — §5, the + overhead the hash matrix does not model), +- a chunk-tree state machine in each of the nine CUDA kernels (§6.1), +- the same state machine again in the wrap's eDSL emitter (§4.6), where every + flag/counter case is emitted cells. + +**Recommendation: define the RV64 leaf/parent hash as a bare cv-chain over +64-byte blocks** — `cv₀ = IV`, `cv_{i+1} = compress(cv_i, block_i, t=0, +block_len, flags)` with one domain constant per role and the length bound into +the final block — and use **the same construction for both round counts**, with +the `blake3` crate anchoring the *compression function* at 7r rather than the +full hash. That is already how the socket is anchored +(`prover/Cargo.toml:44-45`: `blake3::hash(a ‖ b ‖ "LFMC")` is a one-block call), +and `CANONICAL_VECTORS` (`blake3.rs:198-462`) already KATs the compression at +both round counts across `block_len` 18–64. + +⚠ The cost of this recommendation: the 7r arm stops being a *tree-compatible* +BLAKE3. If the point of keeping 7r buildable is "an external party can recompute +our commitments", then the 7r arm must keep the standard chunk tree and the two +arms are **two constructions**, not one knob — which roughly doubles the kernel +and emitter work. **This is a question for Mauro and it should be answered before +Stage 1 commits an API**, because §6.1's kernel agent needs to know which +structure it is building. + +### 1.7 ★ DRAFT SPEC — the RV64 byte hash (`Blake3Chain`) + +**Status: DRAFT.** Implemented at Stage 1 as the working default, per Mauro's +standing decision to proceed on the cv-chain; **formally pending ratification**. +This subsection is what §1.6 said had to exist before Stage 1 committed an API, +and it is the reference track G's chaining loop and Stage 5's emitter build to. + +**Scope.** This is the *host, byte-oriented* hash the RV64 prover's Merkle +leaves, Merkle parents, FRI-layer leaves, transcript and grinding are built +from. It is **not** the LFM-native cell-oriented layer specified in +`commit-spec/COMMIT.md`, which chains `LFML_row` over cells inside the machine. +The two are different domains that happen to share a compression function. + +#### 1.7.1 The construction + +`Blake3Chain(M)` for a byte string `M`, at the crate-global round count +`BLAKE3_ROUNDS`: + +``` +n = max(1, ceil(|M| / 64)) # blocks; the empty message is ONE block +m_i = bytes [64i, 64i+64) of M, zero-padded to 64, read as 16 LE u32 words +L = |M| - 64·(n-1) # 0 when |M| = 0; 1..=64 otherwise +F_i = (CHUNK_START if i = 0 else 0) | (CHUNK_END | ROOT if i = n-1 else 0) + # CHUNK_START = 1, CHUNK_END = 2, ROOT = 8 + +cv_0 = BLAKE3_IV +cv_{i+1} = compress(cv_i, m_i, t = 0, block_len = 64, flags = F_i)[0..8] for i < n-1 +digest = compress(cv_{n-1}, m_{n-1}, t = 0, block_len = L, flags = F_{n-1})[0..8] +``` + +The digest is those low 8 output words written **little-endian** = 32 bytes. +`t = 0` on every block; the chaining value is never reset. + +In one sentence: **standard BLAKE3 restricted to a single chunk that never +ends.** + +#### 1.7.2 The five properties it was designed for + +- **P1 — the crate anchor is maximal.** For `|M| ≤ 1024` at 7 rounds this is + bit-for-bit `blake3::hash(M)`. Standard BLAKE3's first chunk *is* this chain + (t = 0, that flag schedule), and a message of at most one chunk has the + chunk's output as the root, so `ROOT` lands on the same compression. The + entire 0..=1024-byte range is therefore a known-answer test against the + official crate with **no oracle, no JSON and no transcription in between** — + the strongest external anchor available to any construction at this layer, and + strictly stronger than the compression-only anchor §1.6 assumed. +- **P2 — a 64-byte message degenerates to exactly the parent form.** One block, + first and last, so `flags = 0x0B`, `block_len = 64`, `h = IV`, `t = 0`. That is + precisely `blake3_hash_merkle_parent` (`kernels/blake3.cu:222`) and + `merkle_parent` (`tests/blake3_reference/mod.rs`). Note that is the *parent* + claim. The two-element **leaf** invariant of `config.rs:93-106` is separate and + easier — two Goldilocks elements are 16 bytes — and it holds **by + construction** for a different reason: `Pair` and `Batched` are the same + generic backend over the same digest, so `Pair::hash_data(&[a,b])` and + `Batched::hash_data(&vec![a,b])` are the same 16 bytes through the same + function. What P2 adds is that the parent layer needs no separate definition: + it is this same hash at a 64-byte message. +- **P3 — the divergence is stated, not discovered.** Above 1024 bytes this is + **not** standard BLAKE3: the standard would start chunk 1 (`t = 1`, `cv = IV`) + and build a chunk tree over chunk CVs. We keep one unbounded chunk. This buys + the ~6% of extra parent compressions §1.6 priced, and keeps the emitter and the + nine CUDA kernels free of a chunk-tree state machine. The 7r arm remains a + *compression-level* anchor, not a tree-compatible BLAKE3 — the cost §1.6 + already named and recommended accepting. +- **P4 — the framing is injective.** `(n, L)` determines `|M|`, and the blocks + are `M` zero-padded, so distinct messages give distinct compression-input + sequences. Two messages of different length never share a chain: they differ in + `L`, or in `n` (hence in which block carries `CHUNK_END|ROOT`), or in block + content. Padding introduces no cross-length collision. +- **P5 — parents are construction-independent.** A parent's message is one block, + so bare cv-chain and chunk tree agree on it bit-for-bit. Whatever §1.6 is + eventually ratified as, every parent hash in the tree is unchanged. + +#### 1.7.3 Design forks, and how each is resolved + +| # | fork | resolution | status | +|---|---|---|---| +| **F1** | `t` = block counter, or 0 throughout? | **0 throughout.** A block counter diverges from the crate at the second block and would cost P1 — the ≤1KiB anchor — for nothing: `t` carries no security here, it is the chunk index of a construction that has one chunk. | ✗ OPEN for ratification | +| **F2** | keep the `CHUNK_START`/`CHUNK_END`/`ROOT` schedule, or drop it for one constant? | **Keep.** Dropping it saves one selector in the emitter and breaks both P1 and P2 — and P2 is the invariant `config.rs` requires. The emitter cost is three constants selected by first/last, not a state machine. | ✗ OPEN for ratification | +| **F3** | domain-separate leaves from parents? | **No — inherit keccak's posture exactly.** ✓ VERIFIED the live keccak configuration does not separate them either: `hash_new_parent_bytes` (`field_element_vector.rs:74-92`) is the digest of the two concatenated 32-byte nodes, and an 8-element leaf is the digest of the same 64 bytes (`:217-227`). P-a therefore *inherits* this property rather than introducing it, and the argument that covers keccak's tree covers this one unchanged. Changing it is a change to both hashes, not to blake3. | ✗ OPEN — carried, not new | +| **F4** | 128-bit socket-style digest, or 256-bit? | **256-bit** (Mauro, decided). §0's finding 2 is why: 128 bits is a 64-bit collision bound, and it also keeps `Commitment = [u8; 32]` and the rkyv wire format byte-identical. | ✓ DECIDED | + +#### 1.7.4 The KAT schedule + +What the vectors have to discriminate, and the length at which each becomes +visible. Both round counts, every row. + +| # | input | what it pins | +|---|---|---| +| K1 | `|M| = 0` | the empty message is ONE block with `block_len = 0`, not zero blocks | +| K2 | `|M| ∈ 1..=63` | `block_len` is the true length; the tail is zero-padded | +| K3 | `|M| = 64` | **P2** — one block, `0x0B`, and equality with the parent form | +| K4 | `|M| = 65` | the first chain step: block 0 loses `CHUNK_END\|ROOT`, block 1 gains it | +| K5 | `|M| = 128` | an exact multiple of 64 does not emit a spurious empty final block | +| K6 | `|M| ∈ {192, 256, 1024}` | the interior blocks carry `flags = 0` | +| K7 | `|M| = 1088` | **P3** — the first length past one chunk, where we leave the standard | + +At **7 rounds**, K1–K6 are all specified by reference to an external artifact: +each equals `blake3::hash(M)` from the official crate (P1). They are checked that +way in the tests rather than transcribed, so there is nothing to mistype. K7 is +specified as `≠ blake3::hash(M)` — the negative control for P3, without which +"we implement the single-chunk chain" would be unfalsifiable. + +At **6 rounds** the vectors are generated from this construction and committed as +a table. §1.6 said no external artifact exists at 6 rounds; **that turns out to +be too pessimistic**, and the correction matters because it is the campaign's +weakest provenance link. + +✓ **Cross-checked, 2026-08-14.** #903's Python oracle +(`thoughts/blake3/blake3-oracle/blake3_ref.py`) is a full standard-BLAKE3 +implementation with the round count as a parameter — `blake3_hash(data, out_len, +rounds)`. Two facts make it usable as an independent reference here: + +1. At `rounds = 7` it reproduces the official `blake3` package (1.0.9) + bit-for-bit at every length checked, **including multi-chunk lengths** (1088, + 2048). So the oracle is standard BLAKE3, pinned from outside, not just at the + compression level but at the tree level. +2. Standard BLAKE3 over a message of at most one chunk *is* this construction + (P1) — at any round count, since P1's argument is structural and does not + mention the round function. + +So `blake3_hash(m, 32, 6)` is an independent computation of `Blake3Chain` at 6 +rounds for every `|m| ≤ 1024`. It was run over all twelve KAT messages: **the +eleven at `|m| ≤ 1024` all match**, and **1088 differs** — which is P3 confirmed +from the other side, and is strictly more than the 7-round negative control +gives. The 7r control says "we are not the standard at 7 rounds"; this says the +divergence is *the chunking*, because a reference that is standard at 6 rounds +too still parts from us at exactly the chunk boundary. + +That leaves the 6-round table cross-checked by implementations sharing no code, +over the whole range the prover actually hashes in. It is still not a *published* +vector — nothing published computes this — but "regression pin only" would now +understate it. + +✓ **Reproducible, not merely recorded.** Everything the cross-check needs is +tracked. `thoughts/blake3/blake3-oracle/` holds `blake3_ref.py` (vendored at +commit `65025095`), which exposes raw compression entry points — `compress`, +`compress_cv`, `compress_6round` — as well as `blake3_hash`, alongside +`canonical_6round_vectors.json`, `official_test_vectors.json` and +`test_oracle.py`. The cross-check is therefore runnable at compression level, not +only at full-hash level. + +★ **A second source, and it is the stronger one.** +`thoughts/blake3/reference-impl/` is upstream BLAKE3 1.8.5's own portable C with +its round loop parameterized; the entire edit is `PARAMETERISATION.diff`, which +replaces seven unrolled `round_fn` calls with a loop bounded by +`BLAKE3_ROUNDS_PARAM`. It reproduces `CANONICAL_VECTORS` at both round counts +(10/10, all 16 words) and the §1.7.5 chain digests at 6 rounds over every length +up to one chunk, and it diverges past one chunk — P3 confirmed from upstream's +side. Crucially it encodes the message schedule as an indexed `MSG_SCHEDULE[r]` +table where the Rust and CUDA compose a single permutation between rounds: +structurally different expressions of one convention, so agreement +cross-validates the schedule rather than restating it — a bug in the iterative +composition is exactly what a single source cannot catch. +`make test-blake3-second-source` runs it: a ~1 second C compile plus a 5000-case +randomised differential, no cargo and no GPU. + +⚠ **What these vectors do NOT pin, and it is not obvious.** Every message in this +subsection is hashed with `t = 0`, so nothing here constrains the counter split. A +compression with `v[12]` and `v[13]` transposed reproduces the official vectors at +all 65 single-block lengths *and* the multi-block chain vectors, and is caught +only by `CANONICAL_VECTORS`, whose ten vectors all carry `t ≥ 2^32`: 320 failing +words against 0 from either official table. Measured, not argued. The chain table +and the compression table cover different axes and neither is redundant with the +other — "the standard already covers it" is the reasoning that would retire the +only check on the counter split. Separately, the `r < rounds - 1` permutation +guard is **unobservable**: always permuting gives identical output at both round +counts, because the schedule permuted after the final round is never read. It is +an optimization, not a convention any known-answer test can validate. + +#### 1.7.5 The committed 6-round vectors + +Message of length `n` is bytes `37i + 11 (mod 256)`, `i` in `0..n` — the same +generator the existing compression-level anchor uses. Digests are +`Blake3Chain` at **6 rounds**, hex, in the byte order the digest has on the +wire. Live copy: `CHAIN_KAT_6ROUND` in +`crypto/crypto/src/hash/blake3/chain.rs`, asserted by +`six_round_chain_matches_the_committed_table`. The `oracle` column is the +independent cross-check described above. + +| len | digest | oracle @6r | +|---|---|---| +| 0 | `3C3BBB1F335A31EA86464B651C0206FC81D33262AE00EA1A65F3D1D04AFAEFC9` | agrees | +| 1 | `2A50E45B8921F9EFA008D9F39F7165600CF48A7F0E859C2122E3CCB6B9677EE5` | agrees | +| 31 | `C38BF62F506040B2600273778D281B8943621E2B8A9F59E2379F8FD7E5C85125` | agrees | +| 63 | `C373F51A5EB8B27EA05BB1F6F4E62E924FF4D8A279F0D05AFA5CD519391D6389` | agrees | +| 64 | `5900A1E398BB2BF6D3BA7F1A29197B79C86B71AD2C2631F4AC736C82DB043CB5` | agrees | +| 65 | `53953FCADC39B8623901AF7B534F2F6933E312F50299331334E6C0A7C9DBC2BE` | agrees | +| 127 | `9E0DD8168D199A04590C2CBA439B270776E42715D518F68655E56692483E505E` | agrees | +| 128 | `5CAFFC8784E817BBBA991B2108C26A3DFDF804245EF63AE1040A3C34F1B362FF` | agrees | +| 192 | `399D6B9ADEB2F88450775F773E9DEC08836C135713C2C5DD09F4CECEB0ED3888` | agrees | +| 256 | `FBCAB3699A4959FA37190E98CA5142DDBC88330F2E7D12335DB9C6C8881A0B87` | agrees | +| 1024 | `F395E7E2150363B6D200487515425B0204EEA424072183B701176ECCBE0FFE1B` | agrees | +| 1088 | `B4738EDE77A6EC166EE97667118D4793CBF2B08B45AAC7C6D52943B5D298C688` | **differs** — P3, as designed | + +#### 1.7.6 What Stage 1 built against this spec + +- `crypto::hash::blake3::chain::Blake3Chain` — the construction as a `digest` + hasher, so it drops into the Merkle backends and (at Stage 3) the transcript. +- `BatchBlake3Backend` / `PairBlake3Backend` — the *same* two generic backends + the keccak aliases are, with the digest swapped. P2 is therefore structural: + the two families are one function, not two encodings shown to agree. +- `stark::config::Blake3StarkHash` + `CommitmentHash::Blake3`, non-`cuda` only. +- Oracles: the 7-round anchor over all 1025 lengths; the P3 divergence control; + the parent-form check at both round counts; streaming-split agreement; the + committed table and its distinctness control; the two-element invariant with a + blake3 arm; a commit→open→verify round trip with a negative control. + +**Not** built, and why: a full prove→verify under `Blake3StarkHash`. `fri/` is +not parameterized over the configuration (§4.1), so the prover would build +keccak FRI trees and the verifier check them with blake3. That is Stage 2. + +--- + +## 2. Transcript + +### 2.1 What `DefaultTranscript` actually is + +✓ VERIFIED `crypto/crypto/src/fiat_shamir/default_transcript.rs`. It is a thin +`digest::Digest` wrapper, not a bespoke sponge: + +- `use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256;` (`:3`), + `use digest::Digest;` (`:5`) +- `pub struct DefaultTranscript { hasher: Keccak256, … }` (`:31-32`) +- doc `:19` — "Keccak-sponge Fiat-Shamir transcript with a Plonky3-style duplex + output buffer" +- squeeze (`:76-78`): `result_hash = hasher.finalize_reset(); hasher.update(result_hash)` +- `append_bytes` → `hasher.update(new_bytes)` (`:113-118`) +- `append_field_element` → `element.stream_bytes(&mut |b| self.hasher.update(b))` (`:121-125`) +- `state()` → `hasher.clone().finalize().into()` (`:128-129`) + +✓ VERIFIED `IsTranscript` has five methods +(`fiat_shamir/is_transcript.rs:7-26`): `append_field_element`, `append_bytes`, +`state() -> [u8;32]`, `sample_field_element`, `sample_u64`. `IsStarkTranscript` +adds `sample_z_ood*` (`:28+`), whose bodies are defaults over +`sample_field_element`. + +✓ VERIFIED the transcript is already injectable into the RV64 prove/verify path: +`multi_prove(… transcript: &mut (impl IsStarkTranscript + +Clone + Send) …)` at **`prover.rs:3055-3068`**, mirrored by +`multi_verify` at **`verifier.rs:1191-1200`** and `multi_verify_archived` at +**`:1211-1214`** (both `impl IsStarkTranscript<..> + Clone`). +⚠ Note these three line numbers correct D0-DESIGN.md §2, which cites +`prover.rs:3032-3044` / `verifier.rs:1219-1231` — stale after the intervening +commits, and it misses `multi_verify_archived` entirely. The in-place rkyv +verify path takes a transcript too, so a blake3 transcript has **two** verifier +entry points to satisfy, not one. + +**So `Blake3Transcript` is the smallest piece of P-a.** The honest change is to +make `DefaultTranscript` generic over `D: Digest + Clone` and add a +`Blake3Transcript` alias — every method body is already hash-agnostic. Estimated +S. + +### 2.2 The design decision the brief asks for: LFMT vs bytes + +This is a real fork, and it is **not** the same fork as D0's. State it plainly: + +| | **bytes-oriented blake3 transcript** | **B1 / LFMT construction** | +|---|---|---| +| what it is | `DefaultTranscript` with blake3 in place of keccak; 256-bit state; absorbs 64 B per compression | `blake3::hash(state ‖ operand ‖ "LFMT")` truncated to 128 bits (`blake3_socket.rs:245-254`); absorbs one cell (4 felts) per step | +| wrap hosts it with | promoted `LFM_BLAKE3` (does not exist as a chip yet) | the **existing** `LFM_HASH` socket arm | +| rate | 8 felts / compression | 4 felts / compression | +| state / challenge entropy | 256-bit state | 128-bit state, `squeeze_ext` takes 3 of 4 lanes ⇒ **96-bit challenges** (D0-DESIGN.md §3 item 3, unanalysed at production query counts) | +| `append_bytes` | native | ⚠ no byte-level absorb; needs a padding-and-length-bound byte→cell convention, specified not improvised | +| `state() -> [u8;32]` | native | ✗ no equivalent — and `state()` is what seeds grinding (§3) | + +**How much does it matter?** Less than the leaf decision, and the numbers say +so: the transcript/spine is 2,667 of keccak's 118,080 permutations = **2.3%** +of the hash bill (`others/lfm-hash-matrix-scope.md:208`, ✓ read). So the +transcript should **follow** the leaf/tree decision rather than drive it. + +**The reason to keep one family across all four domains is not elegance, it is +width.** If the leaves use the general chip and the transcript uses the socket, +the wrap's emitted program carries **both** AIRs — two hash tables, two sets of +preprocessed columns, and the tower re-absorbs both traces. PLAN.md's own D1 +census already found that `LFM_HASH` dominates the tower's leaf bill at 57% +(`PLAN.md:151-153`); adding a second hash table makes that worse, not better. + +**Recommendation: bytes-oriented, one family.** Take LFMT only if the socket +route wins the §1 digest-width decision, in which case all four domains go +socket-shaped together. + +--- + +### 2.3 ★ Two riders come due at Stage 3 + +✓ VERIFIED `others/lfm-migration-riders.md` (read in full). It lists changes that +are "cheap-to-free if they ride the transcript/hash rebuild … and not worth a +proof-breaking change on their own", with an explicit admission rule: an entry +belongs there only if "the migration has to touch that code anyway" (`:63-67`). +**P-a Stage 3 is that migration for both entries.** They were written for the +ecosystem hash migration; nothing in them is LFM-specific. + +**Rider 1 — constant-consumption challenge sampling (`:7-18`). Adopt it.** +`sample_field_element` loops on rejection, and "a straight-line machine cannot +follow a data-dependent consumption schedule, so the LFM transcript replay +encodes the no-rejection schedule and is **unprovable for a transcript that ever +rejects**" (`SOUNDNESS.md` §6.3, cited at `:12-15`). Cost of the fix is +completeness only, bounded `< 10^-6` per proof at production draw counts. + +⚠ **This does not go away by switching to blake3.** Rejection sampling lives in +the *field* layer, not the hash layer — D0-DESIGN.md §3 item 2 traces it to +`extensions_goldilocks.rs:575-581` calling the base sampler three times, each an +unbounded `loop` at `goldilocks.rs:548-555`. A byte-oriented `Blake3Transcript` +inherits it unchanged. So P-a either adopts the rider or ships a blake3 RV64 +transcript that carries the same standing unprovability restriction into every +future wrap — having just paid the proof-breaking cost that would have removed +it. **Adopted, for the BLAKE3 configuration only** (`TranscriptHash:: +CANDIDATES_PER_COORDINATE`); keccak keeps the unbounded schedule so existing +proofs do not move. + +⚠ **"The cheapest item in the whole plan" understates the cost.** A coordinate +draws `n = 2` candidates where the unbounded schedule draws ~1, so **challenge +sampling consumes twice the squeeze bytes** — a cubic-extension element goes +from ~3 candidates (0.75 squeezes) to exactly 6 (1.5). The transcript is 2.3% of +the hash bill, so it is small in the prover, but it lands in the recursion guest, +which replays every challenge and is exactly what the recursion campaign has been +optimizing. `n = 1` would be free — it is today's modal cost — but leaves a +≈ 2⁻³² per-coordinate tail, one proof in a few hundred thousand, which is not +negligible enough to call the schedule fixed. `n = 2` puts the tail at ≈ 2⁻⁶⁴. + +⚠ **The fallback may NOT be a modular reduction.** Reducing an out-of-range +candidate mod `p` is free and total, but biases challenges by ≈ 2⁻³² per draw; +over ~10⁴ draws that is ~2⁻¹⁹ of statistical distance, which would *dominate* +the ~92 proven bits SECURITY-LEVELS establishes. The implementation draws on +instead, keeping the distribution exactly uniform. Failing instead would make +challenge sampling fallible on the verifier's replay path — an `Option` return +through every caller, and a panic risk where the no-prod-panic policy forbids one. + +**Rider 2 — statement pad (`:20-61`). ✓ RE-DERIVED, and the premise below was +wrong: the arithmetic does NOT move, so P-a is not this rider's forcing +function.** The continuation-epoch statement encodes to `207 + L` bytes +(`L = |public_output|`, one byte per COMMIT); the inherited cursor shift is +`(3 + L) mod 4`, so Phase-A root absorbs land misaligned and need splicing — +"2 roots × 8 halves × T tables … at T = 24 that is 384 `BitDec` + ~13k `BALU` +rows per proof", ~0.2% of instructions but "low single-digit percent of the +machine's fixed trace floor" (`:51-57`). Zero for the ~1-in-4 workloads whose +`L` lands on a boundary. + +✓ VERIFIED **the `mod 4` is the machine's half width, not the sponge's rate.** +`epoch_statement_cursor_is_three_plus_output_len` (`machine_tests.rs:2200`) +asserts the shift modulo `keccak_host::BYTES_PER_HALF`, and that constant is +**4** (`keccak_host.rs:15`) — the eDSL packs absorbed bytes into 4-byte halves. +The rate appears only in `padded_len` / `num_blocks`, i.e. in how many +compressions an absorb costs, never in where a root lands. Since 4 divides both +136 and 64 and the message bytes are identical either way, **the shift and the +splice cost are invariant under keccak → blake3**. The earlier claim that "the +shift arithmetic is absorb-granularity-specific and P-a moves the granularity" +conflated the sponge rate with the half width. + +What *does* move is the compression count for the same absorb: keccak takes +`floor(n/136) + 1` permutations (2 for `n = 207`), `Blake3Chain` takes +`ceil(n/64)` compressions (4). More compressions, each ≈ 1/13.7 the cell cost, so +≈ 6.9× cheaper for the statement absorb — but that is the §5 census, not this +rider. + +⚠ **"One-byte pad" is a misnomer.** `L` is workload-determined, so a fixed byte +cannot align anything; the pad has to be the 0–3 bytes that take `207 + L` to +the next multiple of 4. The file's own second correction implies this, but its +title does not. + +**✗ OPEN — for Mauro, not forced by P-a.** Under the riders file's own admission +rule an entry belongs there only if "the migration has to touch that code +anyway" (`:63-67`), and Stage 3 does not: the statement encoder is hash-agnostic +and its arithmetic is unchanged. The natural host is **Stage 5**, which does +rewrite these emitters, or Stage 6, which is the proof-breaking moment the tag +bump wants. The numbers to decide on are above; the cost of waiting is that +~3-in-4 workloads keep paying ~0.2% of epoch-verify instructions. + +Note the file carries three self-corrections on rider 2 (`:33-49`), including +that the `16R` term is dead because `runtime_page_ranges` is always empty for +continuation epochs. It is not re-imported above. + +## 3. Grinding + +✓ VERIFIED `crypto/stark/src/grinding.rs` in full. Two-layer keccak PoW over +`digest::Digest`: + +``` +inner = Keccak256( PREFIX(8) ‖ seed(32) ‖ grinding_factor(1) ) // :80-90, 41 bytes +valid = u64be( Keccak256( inner(32) ‖ nonce_be(8) )[..8] ) < 2^(64-gf) // :66-76, 40 bytes +PREFIX = 0x0123456789abcded // :6 +``` + +`is_valid_nonce(seed: &[u8;32], nonce: u64, grinding_factor: u8)` (`:21`) is +seeded by `transcript.state()`. + +**What P-a needs here is small and mostly mechanical.** + +- Both hashed inputs are 41 and 40 bytes — one keccak block each, and equally + **one blake3 compression each**. The wrap-hosted re-check is therefore + **2 compressions per proof**, i.e. cost-irrelevant either way. Grinding is not + a reason to choose anything. +- Because the file is written against `digest::Digest` (`:2`) and takes/returns + `[u8;32]`, swapping the hash is a type substitution with **no signature + change** — `is_valid_nonce`'s seed stays `[u8;32]` because the blake3 + transcript's `state()` is also 32 bytes. +- ⚠ **Do not scope grinding out for RV64.** D0-DESIGN.md §3 item 1 recommends + `grinding_factor: 0` — that is right for *LFM* proofs and wrong here. + `MIN_PROOF_OPTIONS` sets `grinding_factor: 1` on the RV64 recursion presets + (`prover/src/recursion.rs:39-45`, cited in D0-DESIGN.md §3), and grinding is + part of the RV64 proof's claimed security budget. Port the PoW; do not delete + it. +- The wrap must emit the two compressions. ✓ VERIFIED **it already does — there + is no gap.** The search above looked in `epoch_verify.rs` and the check is not + there; it lives in the challenge spine, `prover/src/lfm/epoch.rs`, which is + where the transcript absorbs are and therefore the right place. + `emit_grinding_check` (`:350-406`, called at `:514` exactly when a nonce is + present) builds the inner keccak over `PREFIX ‖ state ‖ factor` and the outer + over `inner ‖ nonce_be`, then bit-decomposes the digest's first lanes and + asserts the top `factor` bits are zero — `is_valid_nonce`'s predicate, done as + a bit decomposition plus zero assertions rather than a comparison, because the + bound is a power of two. Its own doc states the stake: "the nonce is absorbed, + so the query indices depend on it, and an unchecked nonce is a free re-roll of + every query index at zero cost." + + ⚠ **Consequence for Stage 5, and it is a site §4.6 does not list.** The check + reaches keccak through `edsl::keccak256` (`edsl.rs:439`) → + `keccak256_absorb_all` (`:483`), which is a **sponge-framing** emitter: it + loops over `num_blocks`/`BLOCK_HALVES` and splices `pad_half`, all of which + encode keccak's 136-byte rate and `pad10*1`. A BLAKE3 port needs the framing + rewritten to 64-byte blocks and the chain's length-in-final-block convention, + not just a compression swap. Two compressions per proof, so the *cost* is + irrelevant — the work is in the framing. + +--- + +## 4. Blast radius + +### 4.1 `crypto/stark` — the parameterization + +The whole host commitment path takes the configuration. The main trace commits +through ` as IsStreamingLeafBackend>::hash_bytes` +(`prover.rs:893`) and the verifier checks it through `H::Batched` (`verifier.rs:594, +598, 677, 684`). FRI commits its layer trees with `H::Pair` +(`fri/mod.rs:commit_phase_from_evaluations`, `fri/batched.rs:batched_commit_phase`) +and opens them through `query_phase`, also on `H`. + +The join between those two families is what makes a proof verifiable: the +prover builds FRI layer trees with `H::Pair` and the verifier re-hashes each +opened pair with `H::Batched` (`verifier.rs:736`), so the `StarkHash` +two-element invariant — `Batched::hash_data(&vec![a, b]) == +Pair::hash_data(&[a, b])` — is load-bearing rather than decorative. Naming one +`H` is what makes the two sides agree; a configuration that broke the invariant +would reject every honest proof at its first FRI query, loudly. + +⚠ **The `cuda` fork covers FRI as well.** `StarkHash::Pair` carries the same +`KeccakTreeBackend` bound `Batched` does under `cuda`, because `gpu_lde`'s FRI +commit drives the whole commit phase on device and hashes every layer with the +keccak kernels, labelling the result with the backend type it was handed. So a +cuda build has no BLAKE3 configuration for FRI layers either, which is +consistent with `Blake3StarkHash` not existing under `cuda` at all (§4.5, R4). + +The `FriLayerMerkleTree` / `FriLayerMerkleTreeBackend` aliases survive as the +*default* configuration's names — the `const _` assertions in `config.rs` pin +them to `KeccakStarkHash`'s members, and `math-cuda`'s parity tests build +reference trees with them. + +### 4.2 The `CommitmentHash` tripwire + +✓ VERIFIED `CommitmentHash` has one variant (`config.rs:63-67`) and the +crate-global `COMMITMENT_HASH` is pinned to it (`:71`), tied to +`KeccakStarkHash` by a `const _` assert (`:193-196`). + +✓ VERIFIED the only external consumer is the H1 guard: +`build_artifacts_with_hasher` opens with an exhaustive `const _: () = match +stark::config::COMMITMENT_HASH { CommitmentHash::Keccak256 => () };` +(`registry.rs:158-160`), documented at `:136-151` as the thing that "cannot +compile until someone decides here what the artifacts should say." + +⚠ **Note the guard's direction.** It fires when a `Blake3` *variant* is added, +and again when the *aliases* flip (because `COMMITMENT_HASH` describes the +aliases, not the active `H`). It does **not** catch "prover ran under a blake3 +`H` while the global const still reads Keccak256" — the const is global, the +configuration is per-type. If P-a keeps keccak as the default alias while a +blake3 `H` exists (which is the plan), `COMMITMENT_HASH` becomes a +half-truth for the duration. Either make the guard read `H::COMMITMENT_HASH` at +the call site, or write down that the global const describes the *default* +configuration only. + +### 4.3 Recursion guests — the hard external dependency + +✓ VERIFIED the guest acceleration mechanism, and it is clean: +`crypto/crypto/src/hash/platform_keccak.rs:1-5` — "Keccak-256 implementation +selected per target: the `keccak_permute` precompile on the riscv64 guest, +plain software `sha3::Keccak256` on host. Wraps +`lambda_vm_syscalls::keccak::Keccak256` with the `digest` crate traits so it's a +drop-in replacement anywhere a `D: Digest` is expected (Merkle tree backends, +Fiat-Shamir transcript)." The riscv64 arm is at `:7-45`. + +✓ VERIFIED the guest-side sponge is `syscalls/src/keccak.rs` — "High-level +Keccak-256 hasher backed by the lambda-vm `keccak_permute` precompile" (`:1-5`), +rate 136 (`:37`), domain byte 0x01 (`:43-44`). + +✓ VERIFIED **there is no blake3 syscall**: `syscalls/src/` contains +`allocator.rs, ef_io.rs, entrypoint.rs, keccak.rs, lib.rs, random.rs, +syscalls.rs` (on `origin/main`), and grep for `Blake3|BLAKE3` over `syscalls/` +returns nothing. + +⚠ **A silent-desync hazard sits directly on P-a's path.** +`platform_keccak.rs:14-21` carries a load-bearing invariant: "this adapter must +remain a PURE PASSTHROUGH … The TypeId specializations in +`crypto/crypto/src/merkle_tree/backends/field_element_vector.rs` bypass it and +drive the syscall sponge directly, on the assumption that both paths hash +identically. Adding ANY behavior here … silently desyncs the specialized +branches from the generic path — and the failure surfaces as **in-guest proof +rejection, not as a host test failure**." A blake3 backend must either avoid +that specialization or get its own, and the check is a guest run, not a host +test. + +✓ VERIFIED **#903 is unmerged and is exactly the missing piece.** Commit +`35038501 feat(prover,executor): BLAKE3 6-round compression accelerator` lives +on `feat/blake3-accelerator` (local + `origin/`), is **not** on `origin/main` +and **not** on `blake3-real-hash` (`prover/src/tables/` here contains only +`keccak.rs, keccak_rc.rs, keccak_rnd.rs`). Its message states the deliverable: +syscall `u64::MAX-2`; ABI `x10 → 8-aligned 176-byte region, h[32] | m[64] | +t[8] | len,flags[8] | out[64]`; executor implementation; chip +`prover/src/tables/blake3.rs` at 3,219 main columns / 1,397 sends / ~5,316 +cell-equivalents per compression ≈ 1/13.7 of a post-#889 keccak-f. ⚠ It is the +**6-round internal variant**, resting on the A6R assumption, "to be ratified in +the spec before production use." + +**Consequence for staging:** the RV64 recursion track (#844/#845/#846/#847) runs +a guest verifier whose hashing is keccak-precompile-accelerated. Flip the +prover's hash without merging #903 and that guest computes blake3 in RV64 +software — a large cycle regression in the exact place that campaign has been +optimizing. Stage 4 is therefore gated on merging #903, and #903 itself carries +an unratified round-count assumption. + +### 4.4 program_id / ELF digests / fixtures / CI + +- ✓ `registry.rs:128-171` derives artifacts through `commit_group` and the two + `preprocessed_commitment` helpers, all of which commit through `stark`'s + Merkle layer. **Every LFM registry root moves** when the aliases flip, and + `program_id` is derived from `hasher` (`:130-134`). Regeneration is + `cargo run --bin compute_lfm_registry --release`, under the standing policy + that "a drift failure is investigated, never re-blessed to silence the test." + ⚠ For P-a this is an *intended* move, so the re-bless is legitimate — say so + in the commit, and keep the Test-hasher rows as the honest control. +- ✓ Checked-in `.bin` files are ethrex **block inputs** + (`executor/tests/ethrex_{10_transfers,bench_4,empty_block,simple_tx}.bin`), + not proof bytes — commitment-hash-independent. ? INFERRED no checked-in proof + blobs exist; I found none via `git ls-files`. +- ✓ **CI census.** `.github/workflows/pr_main.yaml` has ten jobs — `lint:19`, + `test-executor:51`, `test-cli:158`, `test:186` (the gate, `if: always()`), + `test-disk-spill:223`, `test-stark-cuda-lib:282`, `build-prover-tests:309`, + `test-prover:344`, `test-prover-comprehensive:436`, `seed-elf-cache:522`. + **Nothing pins proof bytes**, and `cross_verify_vm.sh` is not wired into CI — + it is operator-run, so Stage 6's positive control is a manual gate. Two jobs + matter to P-a: + - `test-cli:178` — "Run syscalls host tests (keccak differential vs sha3)". A + blake3 syscall needs the twin of this differential, against the `blake3` + crate as the reference. + - `test-stark-cuda-lib:282` — compiles the `cuda` feature **on every PR**. + This is what will enforce R4's discipline: the blake3 `StarkHash` instance + must be `#[cfg(not(feature = "cuda"))]` or this job goes red. Useful, not + an obstacle — it turns the GPU fork into a compile error a reviewer sees. + GPU execution lives in the separate `gpu-tests.yml` workflow (consistent with + the standing note that it runs on `merge_group` against the rented box). +- ✓ VERIFIED **continuation chaining binds NO commitment-hash-derived value + across epochs.** The epoch N→N+1 carry is `reg_fini: Vec`, a plain + register file (`continuation.rs:438-456`, consumed at `:1703` as the next + epoch's `register_init`); the rest of the carry is the GlobalMemory LogUp bus, + which is field elements. No Fiat-Shamir state crosses either — every epoch + builds a fresh `DefaultTranscript` seeded from the statement alone, and the + `elf_digest` in that statement is an *independent* `PlatformKeccak256` over raw + ELF bytes that does not move when the commitment hash does. `EpochProof`'s one + `Commitment` field, `l2g_root`, binds an epoch to the **global** proof inside + the same bundle — both sides move together under a flip, so it is inert. + + ⚠ **The format surface is a pinned constant, not a chained root.** + `static_zero_page_commitment` (`prover/src/tables/page.rs:411-430`) is a + hardcoded per-blowup commitment sitting directly on `verify_global`'s + continuation path, and it is deliberately never supplied via private input + ("zero-init pages use a compile-time constant and are never listed", + `recursion.rs:113-115`) — so it is compiled into the host verifier *and* baked + into the recursion guest ELFs. It moves on the flip. Regenerate with + `cargo run --bin compute_static_commitments --release`, under the standing + policy that a drift failure is investigated, never re-blessed to silence a + test — which that function's own doc states. + + ✓ VERIFIED, upgrading the `? INFERRED` above: no checked-in proof blobs exist. + The LFM `FixtureArchive` is a regenerable `/tmp` cache, untracked. + + ✓ The rkyv wire format does **not** move. `Commitment` is `[u8; 32]` and + `StarkHash::Node` is deliberately not an associated type precisely so a + configuration change leaves `StarkProof`'s derives byte-identical + (`stark/src/config.rs:20-21, 109-112`), so #845's in-place verify path is + untouched by a flip. + +### 4.5 GPU + +✓ VERIFIED the compile-time fork described in §0.3. Additionally, ✓ the +tree-less entry the CPU-trees phase needs exists: `try_expand_columns_batched` (`gpu_lde.rs:430`) takes **no** backend parameter and builds **no** tree — +GPU does the LDE, host does leaves and tree. + +⚠ ✓ VERIFIED `device_only_gate` (`gpu_lde.rs:195-215+`) is **entirely +hash-agnostic** — field tower, env disables, power-of-two `lde_size`, LDE and +barycentric thresholds, `!is_preprocessed`, contiguous offsets, uniform +zerofier. That is the hazard, not the relief: under blake3 it still evaluates +**true**, and device-only residency drops the host trace that CPU leaf hashing +needs. Its own doc (`:178-186`) says a violated precondition hits a +`host_trace_empty` "hard-abort … the prove aborts loudly". So during the +accept-CPU-trees phase it **must be forced false** under the blake3 +configuration — otherwise blake3 GPU proving aborts rather than falling back. +⚠ `:188-194` adds a LOCKSTEP obligation: the gate must imply the runtime +dispatch checks, and "a fallback condition added to a dispatch without a mirror +here turns every gate-true table into a hard-abort". Forcing it false is safe in +that direction; adding a blake3 condition to a dispatch without mirroring it here +is not. + +✓ VERIFIED in this pass (upgraded from inherited): `grep -ril blake3 +crypto/math-cuda/` returns **nothing**. The eleven kernel sources are +`arith.cu, barycentric.cu, constraint_interp.cu, deep.cu, ext3.cuh, fri.cu, +goldilocks.cuh, inverse.cu, keccak.cu, logup.cu, ntt.cu` — `keccak.cu` is the +only hash. Everything hash-agnostic survives untouched (LDE/NTT, constraint +composition, barycentric, DEEP, FRI fold arithmetic, LogUp, inverse/arith). +The full kernel and wrapper inventory is in §6.1, where the parallel track needs +it. + +### 4.6 The LFM wrap's hosted-verify emitters — where the 4× materializes + +✓ VERIFIED the five emission sites, all keccak today: + +| domain | emitter | site | +|---|---|---| +| leaf absorption | `emit_leaf_hash(b, shape, values) -> KeccakDigest` | `sub_proof.rs:245` | +| trace Merkle paths | `edsl::keccak_merkle_walk(b, leaf, bits, &opening.siblings)` | `sub_proof.rs:289` | +| FRI-layer paths | `edsl::keccak_merkle_walk(..)` | `fri.rs:564` | +| transcript | `keccak_absorb` / `keccak_absorb_rev` | `builder.rs:414, 426` | +| ★ register commitment | `edsl::keccak_leaf_hash` + `edsl::keccak_merkle_tree_root` | `programs.rs:1266, 1270` | + +★ The fifth is different in kind from the other four: `emit_register_commitment` +**builds a whole Merkle tree in eDSL**, over the cross-epoch register carry, +rather than walking or absorbing one. `keccak_merkle_tree_root` has exactly one +caller, so a grep for `merkle_walk` misses it. Note the interaction with §4.4: +the carry itself is hash-free on the host path, but the LFM wrap feeds it +through the commitment hash *in-machine* — deliberately, per `RootCells::from_digest` +(`lfm/epoch.rs:118-125`): "computing it from those cells is what binds them." +A BLAKE3 twin of this emitter is therefore part of Stage 5, not optional. +⚠ Scope check: the emitter ships, but the assembled epoch verifier is +`#[cfg(test)]` and no `LfmProgramKind` reaches it yet (`programs.rs:1149-1154`). + +and the LFM-native counterparts that exist today — `edsl::leaf_hash_pair` and +`edsl::merkle_walk` over **one-cell** digests, used by the fixture programs +(`programs.rs:638-648`), with the host mirror `fixture.rs:145 +host_leaf_hash_pair`. `KeccakDigest` is `[Cell;2]` (`sub_proof.rs:228`, +`fri.rs:314`) against the native one-cell digest — the "half as many cells per +Merkle level" payoff, and the 128-bit digest question from §0.2, are the same +fact seen twice. + +**This is the switch that produces the win, and it is the stage that needs the +new chip.** `edsl::merkle_walk` compresses through the *socket* +(`blake3_socket.rs:86, 586`), so reusing it verbatim buys the socket's rate 4 +and its 128-bit digest. Emitting against a promoted `LFM_BLAKE3` needs new eDSL +emitters, which do not exist. + +--- + +## 5. The numbers, and what I checked about them + +The campaign is planning against "blake3-6r is 4.06×" (`CENSUS.md:143`). I +traced that number to source rather than inheriting it, because P-a is being +sequenced on it. + +✓ VERIFIED the composition (`others/lfm-hash-matrix-scope.md:60-95`, A6R-signoff +`:108-133`, F7 `:10-30`): + +``` +keccak epoch-verify total 11,165,806,868 base-equiv cells (census of the EMITTED program) + hash term 9,381,609,472 (84.02%) = P 118,080 × 77,992 × 1.01871 padding + residue + BITWISE 1,784,197,396 (15.98%) +blake3 hash term 967,402,978 = 195,593 compressions × 4,946 cells + total 2,751,600,246 → 4.06× at 6 rounds + → 3.85× at 7 rounds (5,714 cells/compression) +``` + +**Three things worth knowing before quoting 4.06×:** + +1. ✓ **The rate penalty IS already in the model — I checked, because it is the + obvious way for a figure like this to be wrong.** Keccak absorbs 17 felts per + permutation, a rate-8 candidate 8, so absorption-bound work costs more + invocations. `others/lfm-hash-matrix-scope.md:191-210` measures the split + rather than assuming it: keccak 115,413 legs (67,671 leaves + 47,742 + paths/FRI) vs rate-8 187,902 (140,160 + 47,742) = **1.63×**, not the 2.125× + ceiling, because 41.4% of the bill is path/FRI work that is 1:1 at any rate. + `P_candidate ∈ [190,569, 193,569]`. The keccak side reproduces the ledger + exactly (115,413 = entry 10's legs figure), which is what makes the candidate + side trustworthy. **The 4.06× survives this check.** + +2. ⚠ **But it is a rate-8 figure, and rate 8 is the general chip.** + `blake3_probe.rs:683` computes it as + `query_permutations_at_rate(&l.verify, 8)`, and the comment at `:686-694` + says exactly what that means: "Rate 8 is BLAKE3's own: its socket absorbs two + cells of message per compression … It is NOT the field-native chain's rate — + that is `epoch_verify::LFM_HASH_RATE_FELTS`, which is 4 because the chain + absorbs one cell per step. The two were the same number while the sponge was + a three-cell duplex, and this line used to say 'blake and field-native' on + that basis; **they have since diverged**." Wide-leaf absorption through the + socket — chained or tree-shaped — amortizes to 4 felts per compression. + **DERIVED** (my arithmetic, not the repo's): back-solving the leaf term from + the two measured points gives ≈3,248 (query, group) terms over ≈1.095M leaf + felts, so rate 4 lands at ≈277k leaf permutations and `P ≈ 330,000` ≈ 1.72× + the rate-8 count; folding in the socket arm's slightly narrower 2,964 main + columns (`PLAN.md:167-169`) puts socket-hosted blake3 at roughly **3.3× at 6 + rounds**, not 4.06×. + **This is free to settle exactly and should be Stage 0**: the closed form is + already rate-parameterized (`epoch_verify.rs:484-541` — + `blocks_at_rate`, `leaf_permutations_at_rate`, + `fri_leaf_permutations_at_rate`, `query_permutations_at_rate`), so changing + the `8` at `blake3_probe.rs:683` to `4` and re-running the ignored instrument + prices the socket route with no proving. + +3. ⚠ **Provenance caveats already on record, which P-a inherits.** Only the + keccak row is a census of a real artifact; the blake3 row is + `measured residue + measured BITWISE + hardcoded P × measured AIR width`, + with `let p = 192_000u64;` at `blake3_probe.rs:711` never asserted against + the instrument's own computed interval (F7.1). And every figure is for a + 16-cycle fibonacci fixture epoch at blowup 8 / 73 queries — **3.9× under a + production-sized epoch** (F7.2), while `CENSUS.md` applies the ratio at + blowup2/219q and blowup4/110q. The ratio is probably more portable than the + absolute, but neither has been checked at the presets the campaign will use. + +**The 6-round decision is the cheaper arm of the matrix**: 4,946 cells per +compression and **4.06×**, against 5,714 and 3.85× at 7 rounds — the +15.5% +per-compression / +5.5% epoch-column delta derived at A6R-signoff `:118-133`. +So the decision moves the plan's headline number the right way, and every figure +in this document is the 6r arm unless it says otherwise. + +**A standard chunk tree would add ~6% the model does not show** (? INFERRED, my +arithmetic): above 1024 bytes it costs one parent compression per 16 block +compressions. The model's 1.01871 factor is `KECCAK_RND` chunking waste, a +different thing. **§1.6 argues this 6% should simply not be incurred** — at 6 +rounds the chunk tree has no interop purpose, and #903's ABI exposes raw compress +(`h[32] | m[64] | t[8] | len,flags[8] | out[64]`), so a bare cv-chain is +directly buildable on the guest, the host, the device and the chip alike. + +--- + +## 6. Staging + +Keccak stays the default through Stage 5. Every stage has an oracle that can +fail. + +**On the king gate — the brief pointed at the wrong one for P-a.** ✓ VERIFIED +`prover/tests/d0_king_gate.rs` proves and verifies **LFM** proofs (`lfm_prove` / +`lfm_verify` over `trivial_program`, `:36-39`), and its own header says it "is +the LFM-side counterpart of `scripts/cross_verify_vm.sh`, which does the same +for RV64 ELF proofs in both directions" (`:7-9`). **P-a's king gate is +`scripts/cross_verify_vm.sh REF_OLD REF_NEW`** (`:1-36`): builds `bin/cli` at +both refs in an isolated worktree and exchanges real VM proofs per ELF, both +directions. + +Its polarity inverts at the flip, and that is the point: +- Stages 1–4 (keccak still default): cross-verify must **PASS** both directions + — that is the proof the refactor is inert. +- Stage 6 (flip): cross-verify must **FAIL** both directions, and a same-ref + blake3 round trip must pass. A passing cross-verify after the flip would mean + the hash did not actually move. + +### 6.1 ★ PARALLEL TRACK — the blake3 CUDA kernels + +**Pre-authorized by Mauro as a parallel workstream, not a tail stage.** This is +the right call: it is the only part of P-a with no dependency on the machine-chip +work (Stage 5) or the guest work (Stage 4), and leaving it to the end is what +would create the GPU regression window described in §0.3/R4. + +**Start condition — two options, and the earlier one is real.** + +- **Earliest (can start immediately):** the kernels depend on the *compression + function* and the *leaf byte layout*, both of which are already frozen and + readable today — `blake3_compress_rounds` (`blake3.rs:125-148`) and + `leaves_bit_reversed_grouped` (`commitment.rs:55-110`, which serializes + `rows_per_leaf` bit-reversed rows column-by-column big-endian and hashes the + buffer once). Neither moves in Stage 1. **An agent can begin on the device + compression function plus the two simplest leaf kernels right now.** +- **Blocking on one answer:** the *chaining construction* — §1.6's open question + (bare cv-chain vs standard chunk tree). The device compression function and + the byte serialization are identical either way, so roughly 60% of the work is + unblocked; the leaf-kernel chaining loop and the tail handling are not. + **Dispatch now, scoped to the compression function + serialization + the + level/tail compressors; hold the multi-block leaf chaining until §1.6 is + answered.** + +**The kernel list.** ✓ VERIFIED firsthand against `keccak.cu` — nine hash +kernels to mirror, one device helper to replace, one kernel that needs nothing: + +| keccak kernel | line | blake3 mirror needed | +|---|---|---| +| `keccak_f1600` (device helper) | `:50` | → `blake3_compress` device fn, 6r, from `blake3.rs:125-148` | +| `keccak256_leaves_base_batched` | `:152` | yes | +| `keccak256_leaves_base_row_pair_batched` | `:196` | yes | +| `keccak256_leaves_ext3_batched` | `:237` | yes | +| `keccak_comp_poly_leaves_ext3` | `:277` | yes | +| `keccak_fri_leaves_ext3` | `:326` | yes | +| `keccak_merkle_level` | `:394` | yes (parent compressor) | +| `keccak_merkle_tail` | `:408` | yes (parent compressor) | +| `keccak256_leaves_base_row_major_row_pair` | `:473` | yes | +| `keccak256_leaves_base_row_major_row_pair_range` | `:511` | yes (column-subset variant) | +| `merkle_gather_paths` | `:433` | **none — hash-agnostic**, reusable as is | + +✓ VERIFIED the Rust wrappers that need blake3 twins, `crypto/math-cuda/src/merkle.rs`: +`keccak_leaves_base:33`, `keccak_leaves_ext3:83`, +`build_merkle_tree_on_device:316`, `build_comp_poly_tree_from_slabs_dev:494`, +`build_comp_poly_tree_from_evals_ext3_keep:544`, +`build_fri_layer_tree_from_evals_ext3:564`. `gather_merkle_paths_dev:358` is +hash-agnostic and needs no twin. Note tree *building* is on-device too, not only +leaf hashing. + +**The parity oracle.** ✓ VERIFIED the template already exists — mirror these +rather than inventing a harness: `crypto/math-cuda/tests/keccak_leaves.rs`, +`merkle_root_parity.rs`, `fri_layer_tree.rs`, `comp_poly_tree.rs`, +`merkle_tree.rs`, `merkle_gather.rs`. The blake3 versions assert device output +against **the host 6-round implementation** (`blake3_compress_rounds` at +`BLAKE3_SIX_ROUNDS`), which is the same reference the chip's trace filler uses +(§1.2) — so device, host backend and in-circuit chip are all checked against one +function. Seed the compression-level check with `CANONICAL_VECTORS` + +`CANONICAL_OUT_7ROUND` (`blake3.rs:198-462`), which cover `block_len` 18–64 at +both round counts; ⚠ there is no 6-round expected-output constant table beside +`CANONICAL_OUT_7ROUND`, so the 6r arm's KATs are pinned by the host +implementation only — **generating and committing a `CANONICAL_OUT_6ROUND` table +is a prerequisite for the kernel agent to have an independent oracle at all.** + +**Ordering guard.** The existing keccak parity tests must stay green throughout +— keccak remains the default until Stage 6, and the cuda-feature job runs on +every PR (`pr_main.yaml:282`). + +**Effort: M.** Nine kernels, but they are structurally uniform, the byte +serialization is shared with the CPU path, and the parity harness is a template +rather than new design. The risk is not difficulty, it is the §1.6 answer +arriving late and forcing the chaining loop to be rewritten. + +### 6.2 Stage table + +| # | stage | oracle | effort | +|---|---|---|---| +| **0** | **Price the fork before building.** Re-run the census at rate 4 vs rate 8 (`blake3_probe.rs:683`), and settle §0.2's digest-width question with Mauro. Zero proving. | the instrument's own printout; `p_lo ≤ p ≤ p_hi` asserted (fixes F7.1 in passing) | **S** | +| **1** | Sink the compression core into `crypto/crypto` (§1.2b); **generate and commit `CANONICAL_OUT_6ROUND`** (R13 — the 6r arm has no independent KAT table today, and it is the arm we are shipping); add `Blake3Batched`/`Blake3Pair` + `Blake3StarkHash`; keccak still the alias | `CANONICAL_VECTORS` × both round counts (7r against `CANONICAL_OUT_7ROUND`, `blake3.rs:407`, itself anchored to the `blake3` crate; 6r against the new table); the Pair/Batched invariant test (`commitment_tests.rs:110-121`) extended with the blake3 arm; `make lint` incl. `blake3-6round` | **M** | +| **2** | Thread `H` through `fri/` (§4.1, ~13 sites); prove+verify round trip under `Blake3StarkHash` behind config. Close the §4.4 continuation-chaining question here | same-ref blake3 round trip passes; `cross_verify_vm.sh` still passes keccak↔keccak both directions | **M** | +| **3** | `Blake3Transcript` (make `DefaultTranscript` generic over `D: Digest + Clone`); port grinding to blake3 (§3); **adopt rider 1 — constant-consumption sampling — and re-derive rider 2's cursor arithmetic under blake3's 64-byte block (§2.3)** | transcript KATs; a grinding KAT; honest-path control — blake3 proofs with `grinding_factor: 1` verify; rider 1 pinned by a test that the draw consumes a fixed candidate count | **M** (was S–M; the riders add scope but remove a standing restriction) | +| **4** | **Guest leg.** Merge #903 (`feat/blake3-accelerator`); add `platform_blake3.rs` mirroring `platform_keccak.rs`; audit the TypeId specialization (§4.3) | an in-guest verify of a blake3-committed proof, measured in cycles against the keccak baseline. ⚠ host tests cannot see this failure | **L** | +| **5** | **Promote `LFM_BLAKE3` to a machine chip group**; new eDSL emitters; switch the four emitter sites (§4.6); re-census | adversarial-debate review (new chip group = soundness surface, house rule); re-census against Stage 0's projection; tamper controls both directions + honest-path control | **L** | +| **G** | **PARALLEL: blake3 CUDA kernels** (§6.1). Nine kernels + six wrappers; keccak stays default throughout | mirrored parity tests vs the host 6r implementation; `CANONICAL_OUT_6ROUND` committed first; existing keccak parity tests stay green | **M** | +| **6** | **Flip:** default aliases, registry re-bless, and the GPU fork resolved — if track G has landed, `StarkHash`'s `cuda` `KeccakTreeBackend` bound (`config.rs:116-122`) comes off; if not, blake3 stays `cfg(not(cuda))` and GPU proving stays keccak-only | `cross_verify_vm.sh` fails both directions (positive control); same-ref blake3 round trip passes; full suite green; `compute_lfm_registry` re-blessed deliberately | **S** code / **M** judgement | + +Round count is no longer a Stage-6 decision — 6r is decided (§1.5), which is why +Stage 6's judgement load drops from L to M. What remains open for Mauro is +**§1.6's construction question (bare cv-chain vs standard chunk tree)**, and that +one is needed *early*, before Stage 1 commits an API and before track G writes a +chaining loop. + +Stages 1–3 are independent of 4 and 5 and can run in parallel with them. Stage 5 +does not depend on Stage 4. **Track G runs alongside everything and gates +nothing except Stage 6's GPU fork** — its start condition is in §6.1, and the +unblocked ~60% can begin immediately. Stage 6 depends on 1–5. + +--- + +## 7. Risk register + +| # | risk | status | +|---|---|---| +| **R1 ★** | **The 4× needs a chip the machine does not have.** §0.1. Scoping P-a as a `crypto/stark` config instance under-prices it by the whole of Stage 5 | Pinned by nothing. **This is the finding that should change the schedule.** | +| **R2 ★** | **Socket route drops Merkle nodes to 128-bit** = 64-bit collision bound, on the production RV64 proof (§0.2) | Pinned by nothing. Security decision, needs Mauro | +| **R3** | **FRI is still keccak-concrete** (§4.1) — a blake3 `H` silently mixes hashes at the type level and rejects every honest proof at the first FRI query | Fails loudly at test time; no guard | +| **R4** | **cuda + blake3 does not compile** (§0.3) — deliberate, via the step-0 H3 guard. Track G (§6.1) is what retires it; until then blake3 must be `cfg(not(cuda))` and the PR-time cuda job (`pr_main.yaml:282`) enforces that | Guarded at compile time (`config.rs:116-122`, `gpu_lde.rs:701…`) | +| **R5** | **Guest desync is invisible to host tests** — the TypeId specialization bypass (`platform_keccak.rs:14-21`) surfaces as in-guest proof rejection only | Documented, not tested. Stage 4 needs a guest-run oracle | +| **R6** | **#903 unmerged and unratified** — the guest syscall P-a needs is on a side branch and is the 6-round A6R variant "to be ratified in the spec before production use" | Branch `feat/blake3-accelerator` | +| **R7** | **Domain separation across the four domains.** Keccak gets none today (leaves, parents, FRI leaves and transcript are all plain keccak over distinct byte shapes). Blake3 offers tags cheaply (the socket already does this: `TAG_LFMC/LFML/LFMT`, `blake3_socket.rs:227-254`) | ⚠ **Decide explicitly.** Adding tags is a strict improvement but changes the hash; inheriting "no separation" is defensible but should be a written choice, not an oversight | +| **R8** | **`COMMITMENT_HASH` becomes a half-truth** while a blake3 `H` coexists with keccak aliases (§4.2) | Guard exists but reads the global const, not `H` | +| **R9** | **Fixture/registry regeneration** — every LFM root moves; the re-bless is legitimate here but collides with the standing "never re-bless to silence" policy unless stated | `registry.rs:5-8` | +| **R10** | **Round-count split.** With 6r the target but `blake3-6round` OFF by default (`blake3.rs:83-85`) and `make lint` not building it, the pipeline's primary arm is the one CI never compiles. Sinking the core into `crypto/crypto` widens the blind spot | `blake3_socket.rs:215` asserts single-knob; Makefile matrix **must** be extended in Stage 1 | +| **R12 ★** | **6 rounds rests on an unratified assumption.** A6R-signoff `:104-106` — 6r "is computed by nothing else in the world"; #903's own commit message says the variant "rests on the named A6R assumption … to be ratified in the spec before production use". Mauro's framing is exploratory ("to see if this works"), which is a fine reason to build it and not a reason to skip the ratification | Recorded in `thoughts/blake3/blake3-chip/IMPLEMENTATION.md` per #903; spec ratification still owed | +| **R13** | **The 6r arm has no independent KAT table.** `CANONICAL_OUT_7ROUND` exists (`blake3.rs:407`); there is no `CANONICAL_OUT_6ROUND`, so 6-round expected outputs are pinned by the host implementation alone. Track G would then be checking a device port against the same code path it was derived from | **Blocking prerequisite for §6.1** — generate and commit the 6r table first | +| **R14** | **§1.6 unanswered blocks two workstreams.** The chaining construction determines the leaf kernel loop (track G) and the emitter's flag/counter cases (Stage 5). Answering it late forces rework in both | Needs Mauro; ~60% of track G is unblocked meanwhile | +| **R11** | **Collision with in-flight D0 steps 3–4.** Both P-a and D0 add `StarkHash` instances and both touch `config.rs`, `registry.rs` and the backends directory | See below | + +### On R11 — how P-a and D0 avoid colliding + +They want **different instances of the same trait**, which is the good case: +D0's is cell-oriented over `LfmWord` (LFML leaves / LFMC parents, 128-bit, +socket-hosted); P-a's is byte-oriented (256-bit, general-chip-hosted). Both +keep `Node = Commitment = [u8;32]`, so neither moves the wire format. + +Three shared files need sequencing rather than merging: `config.rs` (both add a +`CommitmentHash` variant and an instance), `registry.rs:158` (the H1 guard's +exhaustive match breaks for whichever lands first), and +`merkle_tree/backends/`. **Recommendation: land P-a's Stage 1 §1.2b core sink +first** — D0's Blake3 backends can then be built on the same compression +function instead of a second one, which is the same argument +`blake3_socket.rs:203-215` makes about the probe and the socket. + +⚠ D0's `d0_king_gate.rs` "must compile *unchanged* across the refs being +compared — that is itself the API-stability half of the test" (`:29-33`). P-a +Stage 1's crate move must not touch the API surface that file names +(`lfm_prove`, `lfm_verify`, `build_artifacts`, `LfmWord`, `MultiProof`). + +--- + +## 8. What I did not close + +Stated so the next pass does not assume coverage: + +- **Continuation chaining** (§4.4) — whether any commitment-hash-derived value + is bound across epochs. Fold into Stage 2. +- **Whether the wrap re-checks the inner grinding nonce today** (§3). If it does + not, that is a pre-existing hosted-verify gap to file separately. +- **The rate-4 figure in §5.2 is my arithmetic**, marked DERIVED. Stage 0 + replaces it with the repo's own closed form at no cost. +- **§1.6's construction question is open, not unverified** — it needs a decision + from Mauro, and it blocks track G's chaining loop and Stage 5's emitter. + +Closed since the first draft: the CUDA kernel inventory (§4.5/§6.1) and +`device_only_gate` (§4.5) are now ✓ VERIFIED firsthand rather than inherited; +the CI census (§4.4) is complete. diff --git a/thoughts/shared/block-compression/PLAN.md b/thoughts/shared/block-compression/PLAN.md new file mode 100644 index 000000000..09204b701 --- /dev/null +++ b/thoughts/shared/block-compression/PLAN.md @@ -0,0 +1,233 @@ +# PLAN — compress the Ethereum bench block (25368371) with the LFM machine + +**Objective:** one proof attesting block 25368371 (74.8M cycles). A blowup-2/219q LFM +STARK as the single output is the campaign target; a further "small final proof" layer is +Stage E, explicitly optional and decided later. + +**Grounding (all measured unless marked projected):** +- Block proves as N continuation epochs: 9 × 2^23 / 13-18 × 2^22 / ~36 × 2^21. Epoch size is + a free knob of `prove_continuation`. +- The LFM wrap proves + verifies ONE epoch-verify today — but only for the 16-cycle fixture + at the 1-query diagnostics preset (7.2 s on a 5090 with `LAMBDA_VM_GPU_LDE_THRESHOLD=262144`, + BOX-RESULTS.md). Secure inner presets are blowup2/219q and blowup4/110q. +- Per-epoch verify cost has a floor independent of epoch size (queries × ~25-31 table proofs × + Merkle depth). Smaller epochs shrink each wrap but grow the total. +- The wrap's legs recompute the INNER prover's commitment hash. RV64 epoch proofs commit with + keccak → base-layer wraps pay the hosted keccak family (84% of cells at production shape). + Hash matrix (measured): epoch-verify 11.17B cells under keccak vs 2.75B under blake3-6r. +- The LFM proof's OWN commitments/transcript are `DefaultTranscript` (keccak) today; the + machine's native real-blake3 domains (LFMC/LFML/LFMT) are what its programs compute, and + FriToyV0 already proves+verifies blake3-shaped proofs. ✓ VERIFIED in `lfm/proof.rs`. +- Only production-shape census on record: blowup-8/73q single-epoch wrap → 350.6 GiB projected + peak (unprovable). blowup2/219q and blowup4/110q have NEVER been censused. +- Census is free: `query_permutations` closed form + `projected_peak_bytes` — no proving needed. + +--- + +## Shape: two tracks that meet + +``` +TRACK 1 (BASE, real epochs in) TRACK 2 (TOWER, N→1) +A census fit map D0 LFM-proof hash decision +B one real ethrex epoch wrapped D1 LFM-proof-verifier emitter +C all N epochs wrapped + chaining D2 aggregate 2→1 (on fixture wraps!) + \ D3 binary tree + \ / + block → N base wraps → tower → ONE proof [E: small final proof] +``` + +Track 2 starts immediately in parallel: D1/D2 prototype against TODAY'S fixture wrap — +they never wait on real epochs. + +--- + +## Track 1 — real epochs into the wrap + +### A. Census fit map (effort S, ~1-2 days, zero proving) +Sweep epoch_log2 ∈ {20, 21, 22, 23} × {blowup2/219, blowup4/110} × hash {keccak, +blake3-6r-modelled}. Per point: emitted-program cells, KECCAK_RND chunk count, projected +peak RSS. Trace-length profiles per epoch size come from EXECUTING the block (cheap), +not proving it. Harness: `real_epoch_with` + `report_census` generalized over the profile. +- **Gate A:** some (epoch size, preset) fits ~90-110 GiB (the box / rigs). If keccak-inner + fits nowhere → the inner-hash switch (RV64 commits blake3-6r) is promoted from + optimization to prerequisite and goes to Mauro as a decision. +- **★ GATE A VERDICT (2026-08-12, measured — CENSUS.md): FAIL AT EVERY POINT.** Cheapest + real point (2^20/blowup4) projects 1,199 GiB — 13× over budget; the 219q program cannot + even be EMITTED (OOM at 89 GiB during emission on the 16-cycle fixture). Scaling: linear + in queries and sub-proof count, only logarithmic relief from epoch size (2^23→2^20 buys + 3.2×). KECCAK_RND is 92.5% of cells. **The inner-hash switch is NECESSARY BUT NOT + SUFFICIENT** — blake3-6r's 4.06× leaves 295 GiB at the cheapest point (3.2× over). The + coefficient-free floor after blake3 fits at exactly one point (2^20/blowup4 → 70 GiB), so + the residual is PROVER RESIDENCY: peak is the SUM over 23-133 chunks in one multi_prove. + Track 1 therefore adds two structural prerequisites: **(P-a) inner RV64 → blake3-6r**, + **(P-b) bounded-residency proving**, and likely **(P-c) streamed emission** (the emitter + itself OOMs first). ⚠ P-b CORRECTED: "one chunk ≈ 50 GiB flat" holds ONLY if nothing but + the root survives per chunk; one full chunk's working set is 35.5 GiB, and if each + chunk's main LDE + tree must survive Fiat-Shamir to answer openings, the floor is + retained×N (267 GiB at N=23 … 1,542 GiB at N=133). Real P-b is likely RE-DERIVATION + (retain roots, recompute chunk LDE+tree at query time, ~2× prover hash time for O(1) + memory). Note Gate-A ran with `disk-spill` compiled OUT (not a default feature). +- **★ P-b RESOLVED BY AUDIT (residency-seam-audit.md; CENSUS.md Part 2 §1 is the + independent second read): a real refactor with named seams, NOT a + flag.** Nothing bounds residency today (`TABLE_PARALLELISM` bounds only aux/R2-4 + transients; disk-spill never touches the LDE and is unreachable from the LFM path). + Fiat-Shamir forces only the ROOTS before the shared LogUp challenge — LDE retention is + a perf choice, so the refactor is protocol- and wire-compatible. Peak model, KECCAK_RND + family: **17.37·N + 30.2·k GiB** (N=23 → ~430 GiB today). Bounding only the LDE lands + at 309-819 GiB; the flat floor needs the TRACE streamed too — chunks are pure functions + of their `round_ops` slice (zero cross-chunk logic), so regeneration is trivially + available → **~48-56 GiB flat regardless of N, at ? +40-60% wall time**. Seams S1-S7 + named in residency-seam-audit.md (multi_prove takes a per-index producer; LfmTraces + goes lazy; drop-and-recompute LDE/trees). Also corrects Gate A's coefficient: 33.7 B/cell is ~2.1× + high for the KECCAK_RND shape — the Gate-A band reads ~560-3,200 GiB; verdicts unchanged. +- **P-b LADDER (census Part 2, reconciled with the seam audit):** existing levers reach + ~654 GiB at 2^21/blowup2 (`TABLE_PARALLELISM=1` → 972; + disk-spill on traces → 654 — + note disk-spill is currently UNREACHABLE from the LFM path: feature off + `lfm/proof.rs` + hardcodes Ram, so wiring is part of P-b). The missing piece either way is **main-LDE + re-derivation at query time** (drop each LDE once its root is absorbed; Round-1 barrier + requires only the ROOTS by soundness) → ~35 GiB with spill, ~48-56 GiB with the pure + regeneration variant. Aux side is k-bounded per the `Lde` doc (`prover.rs:265-274`), + which halves the big-epoch Gate-A figures: corrected band **1,300-2,692 GiB** — still + 14-29× over, verdicts unchanged. ~~**P-b is the highest-value item in the campaign.**~~ + **⛔ REORDERED BY MAURO (2026-08-12, verbatim: "Instead of doing weird streaming stuff, + change the hash of the prover to blake3 first"): P-a GOES FIRST.** P-b demoted to + fallback — after P-a lands, re-census and take the cheapest sufficient memory measure + (existing flags + spill wiring first; streaming only if the numbers still demand it). + Rationale that holds: P-a is needed at every layer forever, shrinks the workload at the + source, and step 2's StarkHash parameterization makes it a second config instance + rather than surgery. Kept visible: ÷4.06 alone projects ~320 GiB at the cheapest point + (still over 93-124 GiB boxes), so SOME memory measure likely remains; the P-b seam + analysis stays valid for that day. P-a staged plan: PA-PLAN.md (scoping in flight). + Flip-time decisions RESOLVED by Mauro (2026-08-12): **6-round blake3** ("I'd prefer the + 6 round to see if this works") — 6r is the target, 7r stays buildable via the existing + feature structure; and **blake3 CUDA kernels are pre-authorized as an agent dispatch + whenever needed** ("send an agent to do the blake3 cuda kernels whenever it's needed") + — closes the GPU regression window; kernel list from the GPU audit (row-pair leaves, + column-range leaves, ext3 comp-poly leaves, FRI leaves, level/tail compressors), parity + oracle = the in-repo host 6r implementation. +- **★ P-c RESOLVED BY AUDIT: the 89 GiB emission OOM is NOT the instruction stream** + (271M × 80 B = 21.7 GB, ~24%). Dominant: the per-instruction `Vec>` row + intermediate (~47 GB, with 80% capacity waste on 10-wide rows landing at cap 18) plus a + drained-but-unshrunk `read_counts` HashMap (~18.3 GB) held by scope through the peak. + **Two nearly-free wins: `drop(read_counts)` before `emit_column_groups` (−18.8 GB, one + line) and a flat-append `ColumnGroupBuilder` (−27 GB, ~50 lines) → peak ~99-102 GB → + ~53-56 GB, zero semantic change** (program_id commits over matrices, bit-identical). + Full per-leg streaming: seams named (builder `instrs` field; compile merges into the + builder; executor needs a 10-way merge by destination — the one new algorithm). ⚠ But + emission is not the last wall: even streamed, execute wants ~21 GB memory + ~10 GB + records, and LFM_BALU pads to 2^28 rows at 219q — the P-b prover streaming remains + load-bearing. Trap for Stage B/C: real 2^23 + inner proves die on the 5090 via #927-class cliff panics; workaround = disable device + paths (57.5 s CPU). + +### B. First real rung (effort M, ~2-4 days) +1. Generalize the `RealEpoch` builder: parameterize ELF + private input + epoch_log2 + + options (today it hardcodes the fibonacci fixture + empty input). +2. Prove ONE real-block epoch at the Gate-A geometry (RV64 continuation prove, GPU box). +3. Census the real epoch-verify program; falsify A's projection against it. +4. **Gate B: wrap it — prove + verify on the box.** First real compression artifact: + one real-block epoch proof → one LFM proof. Everything downstream is scale-out. + +### C. The whole block as N wraps (effort S code / compute-bound) +- Wrap all N epochs sequentially on GPU. +- Chaining: verify the emitter publishes the epoch boundary state (the spine already binds + continuation roots per #844-adjacent design — VERIFY, don't assume). If the publics need + additions, that is emitter/soundness surface → adversarial-debate review before merge + (house rule from the merge-fix lesson). +- **Gate C:** block attested by N LFM proofs + host adjacency check, all verifying. + +## Track 2 — the tower (N→1) + +### D0. LFM-proof hash decision — **DECIDED 2026-08-12: Blake3 (Mauro: "Switch the blake3, yes")** +Tower legs recompute the LFM proof's OWN trees. Today that's keccak (`DefaultTranscript`) +→ the tower would pay the expensive chips forever. Switching the LFM proof's +commitments/transcript/FRI to the machine's native blake3 domains makes every tower layer +~4× cheaper in cells — and the chips already exist, z3-gated, proven in FriToyV0. +Proof-breaking for LFM proofs only (no RV64 impact). **Recommend: switch before D1 so the +emitter targets one format.** + +**★ GATE D1 VERDICT (2026-08-12, projected on a 4-leg-validated model): FAILS as spec'd, +FIXABLE in the spec.** D1 node (verify one fixture wrap, blake3 legs, 110q) = 124 GiB +(1.3× over); real-wrap inner 227 GiB; D2 2-proof node 248-454 GiB. blake3 buys 5.2× vs +keccak here — but non-uniformly: Merkle parents 14.7×, **leaf absorption only 1.73×** +(LFML takes 2 felts/compression vs keccak's 17/permutation), and leaf absorption is 69.8% +of the node bill. **The dominant lever is the LFML leaf RATE — ×2 makes the D1 fixture +node FIT (81 GiB), ×4 → 59 GiB.** Spec census (COMMIT.md §1.5) refines it two ways: +(1) **LFM_HASH itself dominates the tower's leaf bill at 57%** (3,457 cols under Blake3-7r, +2.3× KECCAK_RND) — the tower spends most of its budget re-absorbing the hash chip's own +trace; (2) the missing ×2 is located precisely: the LFMC fold costs one compression per +4 felts because the socket pins the chaining value to IV. **D7 SUPERSEDED → RATE=4 ADOPTED (COMMIT.md board 85/85; the intermediate RATE=5 draft +was found UNBUILDABLE by the chip read — hash rows read whole 4-felt CELLS +(`instr.rs:99-110` num_input_cells gates the LFM_HASH bus; `word.rs:15`), so felts/row +must be a multiple of 4).** RATE=4 = accumulator cell + one felt cell in ONE compression +(13 of 16 message words), landing on the EXISTING 2-cells-in/1-out bus arity — the frozen +bus shape does not move. ⚠ REFUTE PASS RESULTS (2 refuted, 1 refuted-as-stated, 3 confirmed): the "multiple of 4" +argument is a NON-SEQUITUR (cell receives bind all four felts to memory; unused felts are +sound) — the real constraint is the compile-time lane map, and re-packing via +Pack/Unpack makes **RATE=5 buildable after all (~19% cheaper on an UNPRICED sketch)**; +"the frozen bus shape doesn't move" is also wrong — arity stays but the receive +MULTIPLICITY moves (Sum3 → 4-way selector) and `num_input_cells(Leaf)=2` panics +`emit_unread_input_pins` as written. CONFIRMED: the +16-col arithmetic (exact), the +per-lane-range gate hazard at `blake3_socket.rs:1304` (acc lanes constrained, felt-half +lanes not), and the two-chips width reconciliation (socket arm 2,964 main @6r is what +the tower pays). **★ Ship-breaking hazard found: a SILENT release-mode constraint-index +collision** (lane identities 6..17 overlap unused-output pins at 14+; `EmitTracker` +asserts only under debug_assertions; constraint COUNT unchanged so every count-based +guard is blind — lanes 8-11 would lose their identities). **Gate D1 ≈81 GiB / ~13% +margin STANDS at RATE=4** (working default). **NEW DECISION D9 (Mauro): RATE=4 +(fully priced) vs RATE=5 (block ceiling, ~19% sketch, unpriced LfmMem/padding/re-pack +costs) — an optimization decision, not a fit decision.** D8 (sequencing, +Mauro): fold the RATE=4 re-bless INTO D0's re-bless pass (zero marginal cost) vs a +second re-bless later. Spec REQUIRES blake3-6round ON for tower builds. Build traps: `blake3-6round` OFF by default (+16% if forgotten); the BLAKE3 +chip is the machine's widest table under D0. + +### D1. LFM-proof-verifier emitter (effort L — the campaign's center of mass) +Same emitter machinery as the epoch verifier, pointed at an LFM proof: 14 fixed tables, +known log-heights, wrap options fixed → the program shape is static per (K, options). +Census first (closed form), then emit, then prove. Prototype input: the FIXTURE wrap's +proof — exists today, no Track-1 dependency. +- **Gate D1:** census says the 1-proof verifier fits comfortably (expected: yes — 14 tables + vs ~25-31, blake3 legs vs keccak). + +### D2. Aggregate 2→1 (effort M) +One program verifying TWO LFM proofs + consistency of their published words. +- **Gate D2:** wrap-of-two-fixture-wraps proves + verifies, tamper controls reject + (both falsification directions, honest-path control per house rule). + +### D3. Binary tree (effort S code / compute) +N base wraps → ⌈log2 N⌉ layers → one proof. With N ≤ 36 that is ≤ 6 layers; per-layer +cost is the D1 census number × 2. +- **Gate D3 = THE OBJECTIVE:** one LFM proof attesting block 25368371, with the boundary + publics chaining genesis→final state. + +## E. Small final proof (deferred, decide after D3) +A high-blowup/low-query wrap of the last aggregate (or an outer SNARK later). The wrap's +own options at blowup 8 multiply ITS trace memory ×4 — needs its own census. Not on the +critical path: D3's single STARK already IS "the block, compressed". + +--- + +## Cross-cutting + +- **GPU:** the −57% threshold env var is operational on every wrap; the permanent + admission-token gate (BOX-RESULTS.md Stage-3 design) lands as its own reviewed PR. +- **Fit levers if a census gate fails:** smaller epochs (Track 1 only), inner-hash switch + (4× on base-layer cells), `max_rows`/chunk-cap tuning (parked memory: max-rows-should-be- + tunable), lever-2 D2H skip (host-RAM relief). Escalate hash decisions to Mauro; they gate + batching/design choices per the July campaign. +- **House rules in force:** census before prove; ABBA for any perf claim; adversarial + debate on emitter/soundness diffs; honest-path controls beside every falsification; + checkpoint measurements off rented boxes as produced; no artifacts in the PR diff. +- **Known trap:** the fibonacci fixture ELF drift (BOX-RESULTS.md) — pin or fix before it + bites another box; Track-1 work stops depending on the fixture at Gate B anyway. + +## Order of operations (first two weeks) + +1. A census sweep (box is warm now) — days 1-2. +2. D0 decision + D1 census — days 1-3, parallel. +3. B real-epoch feeding + Gate B first real wrap — days 3-7. +4. D1 emitter on fixture wraps — week 2+. +5. C scale-out whenever B lands; D2/D3 when D1 lands. + +Single biggest unknown: Gate A / Gate D1 census numbers. Both are free to compute and +both are scheduled first — the plan self-corrects on real numbers before any large build. diff --git a/thoughts/shared/block-compression/S3-RECOMPUTE-PLAN.md b/thoughts/shared/block-compression/S3-RECOMPUTE-PLAN.md new file mode 100644 index 000000000..7e5fb9762 --- /dev/null +++ b/thoughts/shared/block-compression/S3-RECOMPUTE-PLAN.md @@ -0,0 +1,223 @@ +# S3 — Recompute-instead-of-retain: bounded-memory proving for the wrap + +**Status: DRAFT — awaiting Mauro's read. The hunted design is FOUND and this plan is its +revival; provenance below governs naming and salvage.** + +## 0-pre. Provenance — this design already existed, was built, and was benched + +The alternative continuation design = **"Approach 1: Prove-and-retire"** from the +streaming spec (spec PR #642, branch `spec/streaming`; today a 4-line footnote in +`streaming.typ` — "additional engineering complexity and re-executions"). The "initial +discussion" that chose Approach 2 (today's continuations, PR #685) was never written +down on GitHub. But the design has three recorded homes: + +1. **PR #647 "Feat/streaming prover" (diegokingston, Jun 2026, closed unmerged; branch + `origin/feat/streaming-prover` still live, 10 commits):** a COMPLETE implementation — + `LAMBDA_STREAM_LDE=1` retire-LDE recomputing on demand via `reconstruct_round1` (M1), + leaf-drop Merkle trees keeping internal nodes (T1 — the same keep-the-tree choice §2 + makes), `Executor::snapshot/from_snapshot` VM checkpoints (B), deterministic trace + builds via sorted dedup (C.2a — a HARD prerequisite: HashMap-order row + nondeterminism breaks commit-vs-rebuild root equality), on-demand per-table trace + rebuild (C.2b ≈ Phase C here), batched per-lde_size FRI. **Byte-identical proofs + flag-on vs flag-off, verified** (grinding disabled). +2. **The kill bench, and why it does NOT carry to the wrap:** #647 benched on + monolithic `fib_iterative_8M`: **−0.9% peak heap / +14.1% prove time** → closed with + "We are now using #685". On that workload the retired LDE cache was ~1% of peak + (peak lived in the trace builder/executor). On the WRAP, the retained LDEs are the + MEASURED dominant term (11.56 of the 13.4 GiB/chunk marginal; 532–1,538 GiB summed + over chunks). Same trade, opposite workload shape: the +14% time now buys the entire + fit. The historical verdict was correct FOR ITS WORKLOAD and is not a verdict on this one. +3. **`memory/streaming-proving-vs-zisk.md`** (2026-06-03): the axis analysis. Approach + 1's 2× re-execution was forced by a single global proof's FS ordering. NOTE: S3 does + not inherit that — the wrap's tables already have per-table transcript forks; S3's + recompute sits entirely below the transcript. + +**Salvage map (verified against today's branch):** `reconstruct_round1` SURVIVES in-tree +(`prover.rs:1358`, debug-checks path) — the recompute engine exists and Phase A largely +promotes it out of cfg(debug-checks) under the new mode. From `feat/streaming-prover` +(June-era, big drift vs main — cherry-pick ideas/tests, not rebase): the byte-identical +oracle test, the two-pass round split, T1's tree handling, and for Phase C the +`VmSnapshot` machinery (never landed on main) + the C.2a determinism fix — **check +whether main's LT/MUL/DVRM/BRANCH builders still have HashMap-order row nondeterminism; +if yes it is a live Phase-C precondition** (it changes proof output = a re-bless). +Naming follows #647's (M1/T1/C.x) where it overlaps. + +Grounding: `residency-seam-audit.md` (the verified retention map; every file:line there), +`CENSUS.md` Part 2 §1 (independent second read) and Part 3 (the measured spill ladder that +this plan's marginal predictions extend). Measured anchors: **13.4 GiB marginal per full +KECCAK_RND chunk with spill on** (q=12→16 differencing); the audit's model 17.37·N + 30.2·k +GiB validated within the expected anon delta. + +--- + +## 0. The one-sentence design + +Commit each table's Round-1 root exactly as today, then **drop the main LDE** (keep the +32 B/row Merkle tree and the trace); when that table's fused task (aux → R2 → R3 → R4) +runs after the shared challenge, **recompute the LDE from the trace** into a task-local +buffer that dies with the task — turning the N-way LDE retention into a k-way transient, +with zero change to roots, transcript order, or proof bytes. + +## 1. Why this is sound (the protocol argument, verified in code) + +- Fiat–Shamir requires all main roots to be absorbed before the shared LogUp challenge + (`prover.rs:3196-3225`; verifier mirror `verifier.rs:1295-1317`). It requires the + ROOTS — nothing about the LDE buffers. Retention is a performance choice, stated as + such by the `Lde` struct's own doc (`prover.rs:263-274`). +- After the per-table transcript fork (`prover.rs:3263-3271`) tables are independent; + each `StarkProof` is self-contained (`prover.rs:3856-3887`). +- Recomputation is deterministic: same trace + same twiddles (process-cached, + `prover.rs:517-574`) → bit-identical LDE → identical opening values against the KEPT + tree. The tree is never recomputed, so there is no "recomputed root must match" hazard + at all — the root that entered the transcript is the root openings are checked against. + +## 2. What is dropped, kept, recomputed — and why the tree is KEPT + +| buffer (per KECCAK_RND chunk, blowup 2) | size | S3 decision | rationale | +|---|---|---|---| +| main LDE | 11.56 GiB | **DROP after R1 commit; RECOMPUTE in fused task** | the binding buffer; one extra NTT to recompute | +| main Merkle tree | 0.03 GiB | **KEEP** | keeping it makes recompute = one NTT, NOT NTT + full leaf re-hash; R4 auth paths read the tree, only opening VALUES read the LDE | +| main trace | 5.78 GiB | keep (Phase A); lazy-regenerate (Phase C) | it is the recompute input | +| aux trace | 6.05 GiB | keep (Phase A); **free at fused-task end (Phase B)** | written into the caller's TraceTable (`lookup.rs:1209-1211`) and today never freed; nothing reads it after the table's proof is done | +| aux LDE, composition, DEEP, FRI | ~12.4 GiB | unchanged | already k-bounded inside the fused task | + +**Marginal-per-chunk prediction (falsifiable on the box):** today with spill ≈ **13.4** +(measured). Phase A → **≈ 11.9** (trace 5.78 + aux trace 6.05 + tree 0.03). Phase A+B → +**≈ 5.8**. Phase A+B+spill (traces to mmap) → **≈ 0.03 resident** — the flat floor. +If the measured Phase-A marginal is not ≈ LDE-sized lower than 13.4, the implementation +missed a retention point; that is the acceptance test, not wall-clock. + +## 3. The phases + +### Phase A — core S3 (effort M; crypto/stark only, no public-signature changes) + +1. `ResidencyMode { Retain, RecomputeLde }` — a NEW enum next to `StorageMode`, no cargo + feature (pure code path, no disk dependency), default `Retain` so every existing + caller is byte-identical. Threaded like storage_mode into `multi_prove`; LFM call + site opts in via env (`LAMBDA_VM_RESIDENCY=recompute`) through `auto_storage::decide_lfm` + (the c5ffadf3 seam). +2. R1: after the commit produces `(root, tree, lde)`, under `RecomputeLde` push root+tree + as today but drop the LDE instead of accumulating it into `main_ldes` + (`prover.rs:3144-3145, :3201`). The `main_lde_cells` accounting (`:3306-3316`) + follows the mode. +3. Fused task entry: under `RecomputeLde`, recompute the table's main LDE from its trace + (same `coset_lde_full_expand_row_major` the commit used, minus tree building) into a + task-local; every downstream consumer inside the task (aux build's `columns_main`, + R2 evaluator, R3 barycentric, R4 DEEP + opening values `gather_main_row_range`) + reads it exactly as it reads the retained buffer today — same type, different lifetime. +4. Preprocessed tables: the precomputed-columns tree stays process-cached (untouched); + the multiplicity LDE gets the same drop/recompute treatment. Verify the cache path + (`prover.rs:1151-1159`) is mode-independent. +5. cfg surfaces that ASSUME retention get gated: `debug-checks` reconstruction + (`prover.rs:1338+`) forces `Retain` (mirroring how `device_only_gate` already returns + false under debug-checks); the cuda `device_only`/handle paths are DISJOINT from this + mode in Phase A — `RecomputeLde` is documented CPU-prove-oriented, and under cuda it + forces the host path per-table (same posture as spill; the fit story is CPU proving, + per Mauro's own framing). + +**Oracles A:** full suites at exact baselines (prover 859/34, stark 241/0, lfm 307/19); +a NEW roots-equality test — same trace, same statement: `Retain` and `RecomputeLde` +produce IDENTICAL commitment roots (roots are diffable even though whole proofs are not, +per the house never-diff-proof-bytes rule); cross-mode verify (proof made under +`RecomputeLde` verifies with the standard verifier — same bytes format, this is nearly +tautological and that is the point); `make lint`/`fmt`; then the BOX LADDER re-run at +q=12/16/20 — q=16 must complete with marginal ≈ 11.9 GiB/chunk, q=20 (which paged out +at 52 GiB anon) should now complete. + +### Phase B — aux-trace release (effort S-M) + +Free each table's aux columns from the caller-owned `TraceTable` when its fused task +completes (they are dead weight after the table's proof exists). This mutates +caller-visible state, so it is part of the documented `RecomputeLde` contract, not a +silent change to `Retain`. Oracle: suites + marginal drops to ≈ 5.8 GiB/chunk on the box. + +### Phase C — lazy chunk traces (effort L; ONLY if post-P-a numbers demand it) + +The audit's S1/S2/S6: `multi_prove` takes a per-index trace producer; `LfmTraces` stops +materializing all KECCAK_RND chunks (each is a pure function of its `round_ops` slice, +`chunking.rs:12-21` — regeneration trivially available); each chunk's trace is generated +twice (R1 commit, fused task) and never coexists with its siblings. Floor → tree-roots +only, ~0.03 GiB/chunk marginal + one working set. This changes the `AirTracePair` +signature — a real API refactor, separately reviewed, and the point where the hunted +alternative continuation design (if found) must be reconciled first. + +**Decision gate for C:** ~~after P-a lands, re-census. Model says Phase A+B post-blake3 at +the cheapest geometry ≈ fits the 124 GiB rigs with margin (traces are the only O(N) term +left and they divide by the hash shrink too); if the re-census disagrees, C proceeds.~~ + +> ### ★ GATE C RESOLVED BY MEASUREMENT (2026-08-13) — S6 is box-class-dependent +> +> The gate no longer waits on a re-census. ✓ MEASURED on a 60 GiB / 32-core box: +> the real-block wrap (block 25368371, epoch 0 at 2^16, inner blowup4 / **110 +> queries**) is **OOM-killed at 56.91 GiB anon, BEFORE proving starts** — spill +> volume 0.00 GiB, disk untouched, `RssFile` peak 0.01 GiB. Emission succeeds +> and prints its full census first, so neither the emitter nor the prover is the +> wall. The wall is `build_traces_with_hasher` +> (`prover/src/lfm/trace.rs:162-167` = **S6**), which materialises all 15 +> `KECCAK_RND` chunk traces into one `Vec` — 87 GiB — before `multi_prove` is +> called. **Phases A and B bound residency inside `multi_prove` and are never +> reached.** +> +> So the gate splits by box class rather than by census: +> +> | box RAM | verdict on S6 at 110q | +> |---|---| +> | 64-128 GiB | **REQUIRED.** 87 GiB of eager trace alone; build-side spill does not rescue it either (87 GiB against a 61 GiB disk). | +> | ~258 GiB | **NOT required.** The eager build fits; A+B then bound the prove. | +> +> S6 is therefore the enabler for the 64-128 GiB class, not an optimisation, and +> the campaign can reach the secure inner preset today by using a big-memory box +> instead of building it. Phase C's cost/benefit is now a hardware-procurement +> question rather than a proving-architecture one. +> +> Corollary worth carrying: the arithmetic that made build-side spill look +> pointless — "a main LDE is `blowup` × its trace, so Σ traces is at most half of +> Σ main LDEs" (CENSUS Part 3 §5) — compares two quantities that are only both +> alive if the prove is reached. Σ traces is what must be resident *to call* +> `multi_prove`, so at large N it binds first regardless of the LDE side. + +## 4. Cost model (stated honestly) + +Recompute cost = ONE extra forward NTT per table per prove (the tree is kept, so no +re-hashing — this roughly halves the audit's +40-60% wall estimate, which priced +LDE+tree recompute; ? MODELED, the box ladder measures it). k concurrent recomputed +LDEs bounded by TABLE_PARALLELISM exactly as today's transients are. + +## 5. Composition with everything else in flight + +- **Spill (c5ffadf3):** composes — spill moves the traces Phase A retains onto mmap; + spill+A+B is the best CPU configuration short of Phase C. +- **P-a:** orthogonal (hash choice never appears in this plan); the ÷4 multiplies. +- **GPU:** untouched in Phase A (mode forces host path per-table under cuda). + **★ PHASE A2 — DEVICE-RECOMPUTE (promoted from deferred to the designated follow-up, + Mauro 08-13):** the same seam, re-expanding into VRAM instead of host RAM — drop the + device LDE handle after the root is absorbed, re-expand on device at fused-task entry + (one NTT on the card), composed with the existing VRAM admission gate scheduling + tables through the 32 GiB budget. This is the GPU-native bounded-memory prover and + the endgame configuration under the 64-GiB-preferred production budget: host holds + traces (lazy via S6 or spilled), VRAM holds one table's working set. The +7.8% + CPU-side recompute cost shrinks toward noise on device. Sequenced after P-a's GPU + stages (needs the blake3 kernels for blake3-committed tables; works under keccak + immediately). +- **D0/tower:** unaffected; tower nodes already fit without S3. + +## 6. Risks + +1. A downstream consumer reading the LDE OUTSIDE the fused task that the audit missed — + the loud guard: under `RecomputeLde`, poison the dropped buffer path (the existing + `host_trace_empty`-style assert pattern) so a missed consumer aborts instead of + silently reading empty data. +2. debug-checks / test-utils paths that reconstruct or cross-check from retained LDEs — + gated to `Retain` (step A5); the suites run both modes to keep coverage honest. +3. The disk-spill + recompute interaction on the SAME table (spilled trace → recompute + reads through mmap = page-cache pressure instead of anon): measured on the box, not + assumed. +4. `StorageMode::Disk` disabling the precomputed-tree cache (spill ladder finding) + compounds if both modes are on — measure the preprocessed-heavy fixture point. + +## 7. What this is NOT + +Not a protocol change, not a proof-format change, not the full streaming redesign, not +epoch-level checkpoint/re-execution (that is the hunted alternative design's territory — +if it surfaces, it likely replaces Phase C, not Phases A/B, since A/B live entirely +below the epoch abstraction). diff --git a/thoughts/shared/block-compression/SOLUTION-ARRAY.md b/thoughts/shared/block-compression/SOLUTION-ARRAY.md new file mode 100644 index 000000000..fb990dca9 --- /dev/null +++ b/thoughts/shared/block-compression/SOLUTION-ARRAY.md @@ -0,0 +1,94 @@ +# The solution array — memory × throughput exploration plan + +**Mandate (Mauro, 2026-08-13):** "I don't mind each solution tbh — batched FRI first is +fine, improving disk spill is fine, cleverly sending tables to the 5090 and keeping +others in memory is fine too. Make a plan to explore the solution array and get some +conclusions." + +**The question this campaign answers:** what is the production wrap-prover configuration +at the endgame budget (5090 mandatory; **64 GiB RAM preferred, 128 acceptable**), +minimizing GPU-hours per block — and in what order should the remaining levers be built? + +## 1. The array + +| # | Lever | What it does | Status | Build effort | +|---|---|---|---|---| +| A | S3 host-recompute (Phase A+B) | drop LDE after root; recompute on CPU; free dead aux | **LANDED** (4 commits, oracles green) | — | +| B | S3 **device**-recompute (Phase A2) | same seam; re-expand into VRAM; one NTT on card | designed | **M (small)** — delta over A | +| C | Disk spill (traces+trees) | mmap page-out; measured ladder exists | **LANDED** (c5ffadf3) | — | +| C+ | **LDE spill** (new) | mmap-backed `LDETraceTable` — page LDEs out instead of recomputing them | not built | M | +| D | VRAM residency scheduling | admission gate + `device_only` + threshold lever ("send some tables to the 5090, keep others in RAM") | exists as knobs (gate, threshold, per-table heuristics) | S per-heuristic | +| E | Batched FRI | one FRI for all tables: 2.0-2.8× fewer leg perms | scoped (port #768 primitives) | M | +| F | Batched MMCS | shared commitment trees: +1.3× | scoped (same port) | M (with E) | +| G | P-a blake3-6r inner | ÷~4 on everything | in flight (separate track) | — | +| H | TABLE_PARALLELISM / k | measured: k≥4 saturates time; k=1 minimizes memory | exists | — | + +Existing evidence folded in (NOT re-measured): the spill ladder (CENSUS Part 3), the S3 +CPU cost (+7.8% single-pair; the mission's box ladder refines it), the GPU threshold +lever (−57% on the fixture wrap), the MMCS/FRI projections (unit-exact model), the +Gate A/D1 censuses. + +## 2. The benchmark protocol (common to every cell) + +- **Two fixed points**: MID = the largest epoch the current 60 GiB box completes + (mission Phase 3 determines it); LARGE = the largest epoch the Japan box (258 GiB / + 5090 / 2.8 TB NVMe) completes. Same block (25368371), same inner params + (blowup4/110q), same commit. +- **Metrics per cell**: peak host RSS (`time -v`) + peak anon, peak VRAM (1 Hz sampler), + wall, verify green + falsifications, spill/disk volume, and the derived + **$/wrap at vast prices**. +- **Repeat policy**: single run to place a cell; ABBA pairs only where two cells land + within 15% of each other AND the difference would change a conclusion. +- **Fit verdicts judged against 64 and 128 GiB**, not the box's actual RAM. + +## 3. The rounds + +### Round 1 — measure what exists (no new code) +The matrix on both points: {Retain+GPU+threshold-lever, Retain+GPU+gate-default, +A (cpu recompute), A+C (recompute+spill), C alone+TP1, D variants (threshold sweep × +device_only on/off)} × {k=1, k=4}. ~12-16 cells, most are minutes each. The mission's +Phase-3 config table seeds this; Round 1 completes it on the Japan box. +**Interim conclusion 1:** the best NO-NEW-CODE config at 64 and at 128 GiB, and the gap +to close (if any). + +### Round 2 — the head-to-head the array actually turns on: B vs C+ +Both attack the same buffer (the LDE) by opposite means: **recompute it on the GPU** vs +**page it to NVMe**. Build both (each M), measure at both points, same matrix slots. +Decision rule, stated now: **if B holds VRAM under budget via the admission gate and +lands within 15% wall of the best Round-1 config, B is the production mode and C+ is +discarded for the hot path** (kept only if B fails on VRAM pressure or the #927 cliff +class resurfaces). If both fail at 64 GiB, the trace side (S6 lazy traces) joins Round 2. + +### Round 3 — the throughput lever: batched FRI (E), then MMCS (F) if E confirms +Port #768's primitives per MMCS-PLAN (M-12 terminal-poly fix + M-13 width absorption +included; streaming-per-matrix acceptance test mandatory). Measure the SAME matrix +winner ± batching. Decision rule: **batching ships if it improves $/wrap ≥2× at the +LARGE point** (the projection says 2-2.8× for E alone; a measured <1.5× means the model +missed something — stop and reconcile before F). + +### Round 4 — conclusions document +- The Pareto table (memory × wall × $/wrap) across all measured cells. +- **The production recommendation**: one named config for 64 GiB and one for 128 GiB, + each with its measured numbers and its failure modes. +- The discard list — levers measured and retired, with the number that retired them. +- The build order for whatever remains (e.g., "E after G lands; F with E; C+ retired"). + +## 4. Sequencing against in-flight work + +- Round 1 starts when the Japan box lands (mission Phase 3 seeds it from the current box + meanwhile). No code, no worktree contention. +- Round 2's builds queue on the branch AFTER the mission's commits (same worktree); + B before C+ (B is the smaller delta and the posture favorite). +- Round 3 serializes with P-a Stage 2 (both rewrite fri/ — MMCS-PLAN M-3's rule). +- P-a (G) proceeds independently; every Round re-runs its winner under G when G lands + (the multipliers compose, the ORDER of winners shouldn't change — if it does, that is + itself a finding). + +## 5. What would change the plan + +- The mission's Phase-3 numbers landing far from the census model (>2×) → re-anchor + before Round 1. +- The M-11 reconciliation (−57% vs −76.7%) resolving AGAINST the model → shrink Round-3 + expectations before building. +- A 64-GiB fit from Round 1 alone → Rounds 2-3 become pure economics, run at lower + priority behind the tower. diff --git a/thoughts/shared/block-compression/commit-spec/COMMIT.md b/thoughts/shared/block-compression/commit-spec/COMMIT.md new file mode 100644 index 000000000..a1a351748 --- /dev/null +++ b/thoughts/shared/block-compression/commit-spec/COMMIT.md @@ -0,0 +1,1287 @@ +# The LFM-native commitment layer — specification + +> # ⚠ DRAFT — PENDING MAURO RATIFICATION +> +> **The decision points are closed; the construction is not.** D1–D6 were ruled +> on by Mauro on 2026-08-12 and are recorded with provenance in §7. What still +> needs his read is **the S1 wide-leaf construction itself** (§1) — the part +> nobody has ratified because nobody had specified it before this document. +> +> **★★ Read §1.4.1 first.** The leaf **RATE** (`LFML_FELTS_PER_ROW = 4`) is the +> single most consequential number here: leaf absorption is 69.8% of a tower +> node's bill, and this parameter decides whether the recursion tower fits on +> real hardware. Gate D1 was projected to **FAIL at 124 GiB** against a ~93 GiB +> budget at the old rate; at `RATE = 4` it lands at **≈81 GiB**. It is **✗ OPEN +> (D8)** and it is a chip change, so it wants a deliberate yes/no. +> +> > **★★ NEW — D9: rate 4 or rate 5? An OPTIMIZATION call, not a fit call.** +> > A refute pass on 2026-08-12 (§1.4.2a) found that the argument retiring +> > `RATE = 5` — *"a hash row reads whole cells, so felts per row must be a +> > multiple of 4"* — is a **non-sequitur**: a receive binds all four felts of a +> > cell to memory, so a row may read three cells and use nine felts soundly. The +> > true ceiling is **5**, not 4, and three message words sit dead at rate 4. +> > `RATE = 4` **stays the adopted working default** and Gate D1 already fits at +> > it with ~13% margin, so nothing is blocked. But rate 5 is ? ~19% cheaper on +> > an **unpriced** sketch and someone should decide whether to price it. +> > **✗ OPEN (D9)**, §7. +> +> **⚠ Before anyone writes the chip change, read §1.4.4** — nine verified +> implementation hazards. **H1 is a silent one:** at `NUM_LANES = 12` the lane +> identities collide with the output pins, the constraint *count* does not move, +> and the only assert that would catch it is disabled in release builds. +> +> This is step 1 of the D0 change list (`../D0-DESIGN.md` §6), written **before +> any Rust exists**, in the same discipline as `lfm-real-hash/leaf-spec/LEAF.md` +> and `lfm-real-hash/transcript-spec/TRANSCRIPT.md`. Three sub-questions remain +> **✗ OPEN** (**D8**, **D9** and **D6a**, §7); nothing has been silently +> defaulted. + +**Date:** 2026-08-12. **Depends on:** ratified `LFMC` (Merkle parent), `LFML` +(leaf/felt mode, LEAF.md) and `LFMT` (B1 transcript, TRANSCRIPT.md). +**Allocates no new socket tag.** + +**What it covers** — the three things no ratified doc covers when the LFM +machine's own proof moves to the machine's native hashing scheme: + +1. the **wide leaf**: an arbitrary-width row pair → a chained `LFML` sequence, + with the shape bound inside the construction (D0 §7 **S1**, the gating item, + and **S3**) — and its **RATE** (§1.4.1), the parameter that decides whether + the recursion tower fits on real hardware; +2. the **byte→cell absorb** encoding for the B1 transcript (D0 §3 item 4); +3. the **node codec** — `pack_digest` into `[u8;32]` plus a strict decode + (**S2**) — and the tree's arity/padding rule (**S6**); +4. **grinding under B1** (§4.1) — added after the D3 ruling, since B1 has no + `state() -> [u8;32]` and cannot express the keccak PoW it replaces. + +Claims are ✓ EXECUTED (ran it, output in `run-kats.log`) / ✓ VERIFIED (read the +code, cited) / ? INFERRED / ✗ OPEN. + +--- + +## 0. Board + +✓ EXECUTED, `python3 commit_kats.py`, full log in `run-kats.log`. + +| id | check | result | +|---|---|---| +| **C1** | wide leaf over a BASE matrix, both round counts, cost formula | **PASS 4/4** | +| **C2** | wide leaf over an EXT3 matrix; same felt count, different kind ⇒ different leaf | **PASS 4/4** | +| **C3** | ★ width binding: the recorded live break, plus the honest leg | **PASS 4/4** | +| **C4** | ★ padding is unambiguous *because* the header binds the count | **PASS 3/3** | +| **C5** | byte→cell encoding: O1 automatic, injective under zero-pad | **PASS 6/6** | +| **C6** | ★ node codec: round-trip + four rejection flavours + honest leg | **PASS 8/8** | +| **C7** | tree arity/padding: power-of-two asserted, not padded | **PASS 3/3** | +| **C8** | the 96-bit question, both options costed in compressions | **PASS 4/4** | +| **C9** | ★ the crate anchor survives — `LFML` rows are still plain `blake3` @7r | **PASS 2/2** | +| **C10** | ★ the header is load-bearing (construction-level domain separation) | **PASS 3/3** | +| **C11** | ★ B1 grinding: honest mine, factor/seed/marker binding, both cross-domain directions, range discipline, **and the absorb identity §4.1.3 depends on** | **PASS 14/14** | +| **C12** | ★★ the leaf RATE: 4 felts/compression (a whole machine cell), anchor intact, header properties survive, per-query cost 6,048 → 3,024 | **PASS 11/11** | +| **PIN** | all 19 vectors match `commit_kats.json` | **PASS 19/19** | +| | **TOTAL** | **85/85 PASS** | + +> ⚠ **Two pinned vectors were re-blessed on 2026-08-12** — `C12.per_query_old` +> 6,062 → **6,048** and `C12.per_query_new` 3,031 → **3,024** — when §1.5's +> `LFM_HASH` census row was corrected from `NUM_COLUMNS` (3,457) to main columns +> (**3,444**), the preprocessed prefix being committed in the precomputed tree +> rather than the main tree. `commit_kats.py:457`'s width list carries the +> correction and its reason. **No cryptographic vector moved**: the re-pin diff is +> exactly those two integers, and the other 17 digests are byte-identical. +> Recorded here rather than absorbed silently, because re-pinning a KAT to match +> a new belief is how a regression gets laundered — this one is a scope fix with +> a stated reason and a checkable diff. + +Run order: `python3 commit_kats.py --write` once, then `python3 commit_kats.py` +to check. Plain `python3`, no cargo, no third-party packages. + +--- + +## 1. The wide leaf (S1 — the gating item) + +### 1.1 The problem, stated from the code + +Production hashes a leaf as `evaluations ‖ evaluations_sym`, streamed **with no +length prefix and no separator**. ✓ VERIFIED — `verifier.rs:204-206` says it in +those words, and `verify_opening_pair` (`verifier.rs:569-594`) is the single +generic implementation, instantiated at `Field` for the main and precomputed +trees and at `FieldExtension` for the aux and composition trees. + +The consequence was a **live break**, and the code records it rather than +alluding to it. ✓ VERIFIED `verifier.rs:633-639`: + +> *"This authenticates the opening against the aux root; it does NOT constrain +> how many columns that opening has. Nothing here did, and that was a live +> break: the aux root is absorbed only after the shared LogUp challenges, so a +> prover that moved main columns into the aux tree got to choose them after +> seeing `z`/`alpha` (`tests::aux_opening_width_tests`). The width is pinned +> upstream by `trace_opening_widths_well_formed`; do not re-derive it from the +> proof."* + +So the hazard is not "a wrong width" in the abstract — it is **moving columns +between trees that are absorbed at different times**, buying the prover a choice +after a challenge that should precede it. Today that is closed by an *external* +check (I3, `trace_opening_widths_well_formed`), not by the hash. + +Rebuilding the leaf under `LFML`/`LFMC` is the moment to decide whether the hash +carries its own shape. **It should.** + +### 1.2 The construction + +``` +RATE = LFML_FELTS_PER_ROW = 4 ★ the spec parameter, §1.4.1 +H = [ LEAF_MARK, num_cols, kind, ROWS_PER_LEAF ] one header cell +F = serialize(evaluations) ‖ serialize(evaluations_sym) +F' = F ‖ 0^r r = (−|F|) mod RATE zero-pad to RATE +acc = H + for each chunk c of RATE felts: acc = LFML_row(acc, c) +leaf = acc + +LFML_row(acc, c) = BLAKE3( LE32(acc[0..4]) + ‖ LE32(lo_i)‖LE32(hi_i) for each felt in c + ‖ "LFML" )[0..16] 52 bytes, ONE block +``` + +**The accumulator rides in the message, so there is no separate fold** — each +row absorbs `RATE` felts *and* chains, in one compression. That is the whole of +§1.4.1, and it is the parameter that decides whether the tower fits. + +- `kind` ∈ {1 = base, 3 = ext3} — the felts-per-element count doubles as the + kind tag: injective over the kinds that exist, and the number the serializer + needs anyway. +- `serialize` writes a row column by column; an ext3 element contributes its + three components in order `(c0, c1, c2)`, mirroring `write_bytes_be` + (✓ VERIFIED `sub_proof.rs:234-236`). +- `ROWS_PER_LEAF = 2` (✓ VERIFIED `commitment.rs:42`). It is **not** a parameter + of the function — the two-slice signature *is* the row pair — but it is bound + in the header so a future layout could not collide with this one. + +### 1.3 Why a header cell, and why these fields + +**The header binds `num_cols` AND `kind`.** Binding the width alone would not +close §1.1: 6 base columns and 2 ext3 columns serialize to the **same twelve +felts**, so under a width-only header those two openings still share a preimage +— which is the main↔aux confusion in miniature. ✓ EXECUTED (**C3**): the two +produce different leaves under this construction, and (**C2**) the same holds +for the 18-felt pair. + +**The verifier must build the header from the AIR, never from the opening.** +This is the whole load-bearing condition and it is the exact analogue of the +instruction already in the code at `verifier.rs:639` — *"do not re-derive it +from the proof."* A verifier that set `num_cols = len(evaluations)` would +reproduce the prover's own choice and bind nothing at all. The reference +enforces this shape by taking `num_cols` as an argument and *checking* the data +against it (✓ EXECUTED, **C3**: a disagreeing width is refused). + +**Zero-padding is safe here, and only here.** Two felt streams that agree after +padding must have differed in `(num_cols, kind)`, which the header separates. +✓ EXECUTED (**C4**) on a constructed collision: `m=1` padded and `m=2` unpadded +share the felt stream `[7, 9, 0, 0]` and produce different leaves. Without the +header, that collision is real. + +**The fold is a sequential chain, not a balanced tree.** A balanced tree over +`k` chunk digests costs `k−1` compressions against the chain's `k` — but needs +`k` padded to a power of two, reintroducing exactly the shape ambiguity the +header exists to remove. One compression is not worth a second padding rule, and +the chain binds chunk order for free. + +### 1.4 Cost + +``` +compressions = ceil( 2 · num_cols · kind / RATE ) RATE = 4 +``` + +So **`0.5 · num_cols` compressions per base leaf** and `1.5 · num_cols` per ext3 +leaf. ✓ EXECUTED (**C1**, **C2**, **C12**). + +### 1.4.1 ★★ The leaf RATE — the parameter Mauro must ratify + +> **This single number decides whether the recursion tower fits on real +> hardware.** Leaf absorption is **69.8%** of a tower node's bill (Gate D1 +> census), so the rate scales ~70% of the cost linearly. The Gate D1 node was +> projected at **124 GiB against a ~93 GiB budget — a 1.3× FAIL** at the old +> rate. + +**What the chip actually supports** — ✓ VERIFIED, not taken on faith. +`message_word_ref` (`blake3_socket.rs:725-731`) maps `m[0..8]` to the eight +input lanes' byte columns, `m[8]` to the mode-selected tag, and **`m[9..16]` to +`WordRef::Const(0)`**. Seven of BLAKE3's sixteen message words are dead; the +socket uses nine (`BLOCK_LEN_LFMC = 36`, `blake3_socket.rs:261`). + +There is headroom to spend — but not as much as the block alone suggests: + +> ### ⚠⚠ The binding constraint is the machine's CELL structure, not the block +> +> ✓ VERIFIED `instr.rs:99-110`: `HashMode::num_input_cells` is **2** for +> Compress/Transcript, **1** for Leaf, 3 for Permute — and the doc is explicit +> that *"the `LFM_HASH` bus receives are gated by exactly this"*. A hash row +> reads whole **cells** from memory, and a cell is **four felts** +> (`LfmWord`, `word.rs:15`). +> +> **So the felts per row must be a multiple of 4.** An earlier revision of this +> section set `RATE = 5` from block headroom alone — accumulator cell plus five +> felts. That is 1.25 cells of felt input and is **unbuildable**: the machine +> cannot read it. The error was reasoning from BLAKE3's block size while +> ignoring the machine's word size, and it is the reason this section is now +> written from `instr.rs` rather than from byte counts. +> +> --- +> +> > ### ⛔ SUPERSEDED by §1.4.2a (1) — the rule above is a NON-SEQUITUR +> > +> > **Kept in place because it is what the RATE-4 adoption was reasoned from, and +> > a reader who meets `RATE = 4` elsewhere needs to find the retraction here.** +> > +> > The quoted text is accurate: `instr.rs:100-103` does say the receives are +> > gated by `num_input_cells`. But read what it constrains — a mode must not +> > **receive** a cell it does not **read**. Nothing says a row must **use** every +> > felt of a cell it does receive, and nothing could: all four felts of a +> > received cell are bound to memory by the `LfmMem` receive +> > (✓ VERIFIED `chips.rs:628-642`), so ignoring three of them is sound, not +> > underconstrained. A 5-felt row is buildable as a **3-cell read** — accumulator +> > cell plus two felt cells, 12 felts received and 9 used. The third receive +> > already exists (`chips.rs:638-642`, multiplicity `Column(MODE_P)`); it would +> > need `MODE_L` added, which is the **same edit** the adopted RATE-4 construction +> > already needs on the second receive (see the next banner). +> > +> > **The real constraint is the compile-time lane map, and it is a stronger +> > argument.** `leaf_lo_lane(i) = 2i` / `leaf_hi_lane(i) = 2i+1` +> > (✓ VERIFIED `blake3_socket.rs:680-687`) are `const fn`s, identical on every +> > row. A rate that does not divide the 4-felt cell puts each row's felts at a +> > *different offset* inside the cells it reads, and the AIR has exactly one +> > mapping. That forces either a rotating per-row lane map or a **re-packed felt +> > stream** — and re-packing exists: `Instr::Unpack { input, outs: [Addr; 4] }` +> > and `Instr::Pack { lanes: [Addr; 4], out }` (✓ VERIFIED `instr.rs:229-241`) +> > are a felt-granular scatter/gather, running on `LFM_LANES` at `PREP_WIDTH + 4` +> > columns (`chips.rs:1267-1271`) against `LFM_HASH`'s 3,460. +> > +> > **So RATE = 5 is a COST question, not an impossibility.** It is now **D9** +> > (§7). `RATE = 4` remains the adopted working default — this banner does not +> > change it. + +Enumerating what actually fits, given both constraints: + +| | construction | words | rate | vs old | verdict | +|---|---|---:|---:|---:|---| +| A | 7 felts + keep the `LFMC` fold | 15 | 3.5 | 1.75× | ✗ 7 is not a multiple of 4 | +| ~~B~~ | accumulator + 5 felts, no fold | 15 | 5.0 | 2.5× | ⛔ verdict RETRACTED — see below | +| **★ C** | **accumulator cell + ONE felt cell (4 felts), no fold** | **13** | **4.0** | **2.0×** | ✓ **ADOPTED (working default)** | +| — | accumulator + two felt cells | 21 | 8.0 | 4× | ✗ > 16 words | +| — | two felt cells, keep the fold | 17 | 4.0 | 2× | ✗ > 16 words | +| — | 6 felts + keep the fold | 13 | 3.0 | 1.5× | ✗ dominated by C | + +> ### ⛔ SUPERSEDED by §1.4.2a (2) — "4 is the maximum" and the headroom claim +> +> **The block ceiling is 5, not 4.** 16 message words − 1 tag − 4 accumulator +> lanes = 11 words ⇒ **5 half-pairs, with one word spare.** Row ~~B~~ above *is* +> the ceiling; it was struck only by the multiple-of-4 rule the previous banner +> retracts. At the adopted `RATE = 4` the socket uses 13 of 16 words and **three +> are dead**, so the statement below that 4 "is the whole of the available +> headroom" is false as written. +> +> **Two ways out that do NOT exist**, checked and closed so nobody re-opens them: +> the tag cannot stop consuming a word — moving it to `flags`/`t`/`h` breaks the +> crate anchor (✓ VERIFIED `blake3_socket.rs:35-41`) and it is what mechanically +> discharges O5 (`:127-142`); and the accumulator cannot overlap it — a 3-lane +> (96-bit) accumulator frees a word but drops the chain to 48-bit collision +> resistance against the socket's recorded 128-bit/64-bit posture (`:150-154`), +> while folding `acc[3]` into the tag word is *expressible* (`ModeSelected` is a +> linear form; `word_expr` would take `ModeSelected + Cols` at degree 1) but stops +> the message being a plain byte string, killing the C9 anchor — the same +> objection §1.4.1 uses against `h`-chaining. Note a *free* tag reaches only 6 +> felts, so under the retracted rule it would have bought nothing either. +> +> **✗ OPEN as D9** (§7): nobody has priced RATE 5 end to end. See §1.4.2a (2) for +> the ? INFERRED ~19% sketch and, more importantly, for what it does **not** +> cover. + +**`RATE = 4` is the working default, and it has one property the rate-5 route +does not:** its felt input is a whole machine cell, so the leaf program reads the +opening stream in its natural 4-per-cell layout with no re-packing pass at all. +Canonicity witnesses stay at 4 felts — **no change** — because the accumulator +lanes are a previous digest, hence `u32` by construction: they need byte +decomposition but no canonicity gate. (⚠ That last clause holds **only** if the +lanes-0–3 identity is gated on the full `mu`; see §1.4.4 hazard **H6**.) + +> ### ⛔ SUPERSEDED by §1.4.2a (3) — "the frozen bus arity does not move at all" +> +> The retracted sentence read: *"it lands on the **existing** two-cells-in / +> one-cell-out bus contract (`num_input_cells == 2`, the same arity Compress and +> Transcript already use), so the frozen `LFM_HASH` bus arity does not move at +> all."* +> +> **The arity does not move. The MULTIPLICITY does.** ✓ VERIFIED `chips.rs:626`: +> the second input cell's receive is +> `reads_two() = Multiplicity::Sum3(cols::MODE_C, cols::MODE_T, cols::MODE_P)` — +> **`MODE_L` is deliberately absent**, and `chips.rs:620-622` says so in those +> words, because today a leaf row reads one cell. Under construction C a leaf row +> **must** receive cell 1, so that multiplicity gains `MODE_L`; and +> `Multiplicity::Sum3` is exactly three columns +> (✓ VERIFIED `crypto/stark/src/lookup.rs:1458`), so it must become the four-way +> `selector_sum(MODE_C, NUM_SELECTORS)` (`chips.rs:52-61`) the *first* receive +> already uses. +> +> **And raising `num_input_cells(Leaf)` to 2 panics AIR construction as the code +> stands.** ✓ VERIFIED `chips.rs:722-733`: `emit_unread_input_pins`' `slot = 1` +> pass filters modes with `num_input_cells() <= 1`; that set becomes **empty**, +> the fold returns `None`, and +> `.expect("some mode reads fewer than three input cells")` fires. +> +> This is a real edit to the frozen contract, not a no-op. It is tracked as +> hazards **H2** and **H3** in §1.4.4. + +**✓ The crate-KAT anchor survives.** A full row is `16 + 32 + 4 = 52` bytes — +still **one** BLAKE3 block, so `block_len` moves 36 → 52 and nothing else about +the framing does. For any input +under 64 bytes `blake3::hash` is exactly one compression with `h = IV`, `t = 0`, +`block_len = len`, `flags = CHUNK_START|CHUNK_END|ROOT`, so a 52-byte row is a +plain library call just as the 36-byte row was. ✓ EXECUTED (**C12**), asserted +against `blake3_oracle` directly. Carrying the accumulator in the chaining value +`h` instead (the earlier D7 sketch, now **superseded**) would have made the row a +chunk *continuation* and split that anchor for the **same** rate of 4.0 — strictly +worse, since it also costs `h`-as-witness and an O3 revision. + +**Chip cost of the widening** — ✓ VERIFIED against the chip, see §1.4.2 for the +audit. `NUM_LANES` appears in exactly **four** non-test places, all generic: + +- byte columns `4 × NUM_LANES` (`blake3_socket.rs:618-622`): 8 → **12** lanes, **+16** +- `AreBytes` sends, 2 per lane (`blake3_socket.rs:947-957`): **+8** +- canonicity witnesses: still 4 felts, **+0** +- **the mixing core does not move**: `NUM_G = rounds × 8` G-blocks of `G_SIZE = 60` + (3,360 cells at 7r) is driven by the round count, not by how many message + words are live — a `Const(0)` word still feeds an `add3`. + +So ≈ **+16 columns on a 3,457-column chip (+0.5%) for a 2.0× cut in ~70% of the +tower's cost.** That ratio is why this is worth a chip change at all. + +**Measured effect** — ✓ EXECUTED (**C12**), at the real widths of §1.5: + +| | per-query main-tree leaf compressions | ×219 q | ×110 q | +|---|---:|---:|---:| +| old rate (2 felts/compression) | 6,048 | 1,324,512 | 665,280 | +| **RATE = 4** | **3,024** | **662,256** | **332,640** | + +> ⚠ These figures moved by −14 / −7 on 2026-08-12 when §1.5's `LFM_HASH` row was +> corrected from 3,457 to **3,444** (the preprocessed prefix is committed in the +> *precomputed* tree, not the main tree — §1.5 note **(4)**). The superseded +> figures were 6,062 / 3,031. `commit_kats.py`'s width list was corrected with +> them and the two `C12.per_query_*` vectors re-pinned; **no cryptographic vector +> moved** and the board stayed 85/85. + +Against the Gate D1 sensitivity (×2 → 81 GiB fits; ×4 → 59 GiB), **2.0× lands +the node at ≈81 GiB — inside the ~93 GiB budget with ~13% margin.** The census +correction is −0.23% and does not move that number visibly. + +> ⛔ The sentence that stood here — *"That is the whole of the available +> headroom: the enumeration above shows 4 is the ceiling, so if 13% proves too +> tight the next lever is not the leaf rate"* — is **SUPERSEDED by §1.4.2a (2)**. +> The block ceiling is 5, three message words are dead at `RATE = 4`, and a +> ? INFERRED sketch puts RATE 5 ~19% cheaper again. **If 13% proves too tight, +> the leaf rate IS still a lever — it is D9.** What is true, and worth keeping, +> is that Gate D1 already **fits** at `RATE = 4`, so D9 is an optimization +> decision and not a fit decision. + +> **✗ OPEN (D8) — FOR MAURO'S RATIFICATION READ.** `RATE = 4` is a **chip** +> change (`NUM_LANES` 8 → 12, `block_len` 36 → 52, `MODE_L` semantics widened). +> It moves every `LFML` digest, so all vectors and all six registry entries +> re-bless — but that re-bless is already happening under D0, so the marginal +> protocol cost is zero **provided it is sequenced into the same pass**. That +> sequencing is the decision: taking it later costs a second re-bless. + +### 1.4.2 ✓ The chip audit — does `NUM_LANES` 8 → 12 constrain cleanly? + +Read of `blake3_socket.rs` (the `LFM_HASH` arm) and `blake3_chip.rs`. +**Verdict: mechanical except for one real constraint change, named below.** + +> ⛔ **That verdict is too optimistic — SUPERSEDED by §1.4.2a and §1.4.4.** This +> section is a *site survey*: it enumerates where `NUM_LANES` appears, and on +> that it is correct and complete. But three of the breaks are not greppable — +> they are arithmetic on constraint **indices**, a bus **multiplicity**, and a +> mode's **cell count** — so a survey cannot see them. The count is **nine** +> hazards, not one. §1.4.4 is the register; read it instead of this line. + +**`NUM_LANES` is used in exactly four non-test places, all generic:** + +| site | use | generic? | +|---|---|---| +| `blake3_socket.rs:618` | `pub const NUM_LANES: usize = 8` | the definition | +| `:622` | `G = LANES + 4 * NUM_LANES` — byte-column base | ✓ arithmetic | +| `:913` | `Vec::with_capacity(… + 2 * NUM_LANES)` | ✓ capacity hint only | +| `:947` | `for lane in 0..NUM_LANES` — the `AreBytes` sends | ✓ loop | +| `:1304` | `for lane in 0..NUM_LANES` — the lane/message identity | ✓ loop, but see below | + +`lane_byte(lane, b) = LANES + 4·lane + b` (`:649-651`) is generic in `lane`. + +**Hardcoded 8s that must move** — mechanical, but they are real edits: +`message_word_ref`'s `0..=7` arm (`:726`), and the `[u32; 8]` lane arrays in +`socket_values` (`:864`), `bitwise_ops_for` (`:970`), `lanes_from_row` (`:1084`) +and `fill_canonicity_witness` (`:1100`). + +> **★ The one substantive change, and it is NOT mechanical.** The lane/message +> identity at `:1304` is gated on `digest_mu` — the *digest* modes — and the +> comment at `:1299-1303` says why: *"On a LEAF row the eight message lanes are +> four felts' halves, so `IN_lane` and `m[lane]` are deliberately NOT the same +> field element … Gating this on mu instead would make every leaf row +> unprovable."* +> +> Under `RATE = 4` a leaf row's twelve lanes are **mixed**: lanes 0–3 are the +> accumulator (a digest — the identity *should* hold) and lanes 4–11 are the +> four felts' halves (it must *not*). So the gate stops being per-mode and +> becomes **per-lane-range**. That is a genuine constraint change with a +> soundness face: get the split wrong in the permissive direction and the +> accumulator lanes go unconstrained. It needs its own control in the chip's +> gate suite, in the style of WA1/WA2. + +**The `with_capacity(1_259)` figure is not the socket's.** ✓ VERIFIED it lives in +`blake3_chip.rs:913` and is pinned by `blake3_probe.rs:351` +(`predicted_interactions(6) == 1_259`) — the standalone chip, not the `LFM_HASH` +arm. The socket sizes its own vector at `blake3_socket.rs:913` and that +expression is already generic in `NUM_LANES`. **No unasserted constant blocks +the widening.** + +✓ Both halves of that are confirmed by the end-to-end read. +`blake3_chip::bus_interactions` emits exactly `107 + 24·NUM_G` +(7 receivers + 4 senders + `4·(4·NUM_G + 16)` `ByteAlu` + `4·(2·NUM_G)` +`AreBytes` + 32 message `AreBytes`), which is 1,259 at `NUM_G = 48` and 1,451 at +`NUM_G = 56`; `blake3_probe.rs:365-366` asserts the *built* length against the +formula at the compiled round count, so it is pinned, not merely predicted. It +is `NUM_LANES`-independent because the standalone chip has no lanes — all +sixteen of its message words are `Cols` and always draw 32 `AreBytes` +(`blake3_chip.rs:968-979`). ⚠ The literal is nonetheless the **6-round** one in a +file that compiles at 7 by default, so at the default it under-allocates by 192; +capacity hint only, one realloc, no correctness effect. Same for +`bitwise_ops_for`'s `1_248` (`blake3_chip.rs:991`) `= 24·NUM_G + 96`. + +#### 1.4.2a ★ The end-to-end read — three breaks a site survey cannot see + +The survey above enumerates where `NUM_LANES` *appears*. Reading +`blake3_socket::eval` and `blake3_chip::run_flow` end to end finds three further +breaks, none of which appears in any `NUM_LANES` grep: they are arithmetic on +constraint **indices**, a bus **multiplicity**, and a mode's **cell count**. + +**✓ First, the shared dataflow itself generalises cleanly — question (ii) +answered.** A `Const(0)` message word and a `Cols` one flow through the identical +path, differing only in the operand source, and this is structural rather than +incidental: **message words reach `add3` and nothing else.** ✓ VERIFIED — the +schedule indices `mx`/`my` are consumed at exactly two call sites, +`blake3_chip.rs:327` and `:333`, both `f.add3(…)`. `word_expr` +(`blake3_chip.rs:1032-1048`) handles `Cols`, `Const` and `ModeSelected` +uniformly at degree ≤ 1, so the `add3` sum identity stays degree 2 under the +mu gate and the chip's max degree of 3 does not move. `Add3Wire.m` is already +typed `WordRef` for precisely this reason (`blake3_chip.rs:439-446`). The +`unreachable!`s in `WordRef::byte` and `WordRef::rotr_bytes` +(`blake3_chip.rs:395-424`) are never reachable from a message word. **The G-block +wiring is fully index-agnostic; the mixing core's sends, constraints and degree +are functions of the round count alone.** + +> **★ Break 1 — the lane identities collide with the output pins, and in a +> release build the collision is SILENT.** ✓ VERIFIED. +> +> `eval` numbers its framing constraints by hand: the lane identities are +> `b.emit_base(6 + lane, …)` over `0..NUM_LANES` (`blake3_socket.rs:1304-1309`), +> then the unused-output pins are `b.emit_base(14 + j, …)` for `j ∈ 0..8` +> (`:1315-1318`), the digest recompositions `22 + i` (`:1325-1330`), and +> `UNREAD_IDX = 26` (`:1223`). At `NUM_LANES = 12` the lane block runs 6..17 and +> **overlaps the output pins at 14..17**. +> +> `EmitTracker::mark` asserts `"constraint {idx} emitted twice"` — but only under +> `#[cfg(debug_assertions)]` (`crypto/stark/src/constraints/builder.rs:492-504`), +> and this workspace declares no `[profile.release]` override, so under the house +> convention `cargo test --release` the tracker is a no-op and the second write +> simply overwrites the first (`builder.rs:614-617`). The lane loop runs first, +> so **lanes 8–11 lose their identity entirely and nothing fails.** +> `assert_complete` does not catch it either: every index in `0..NUM_CONSTRAINTS` +> is still written, and `NUM_CONSTRAINTS` (`:1214`) does not reference +> `NUM_LANES`, so the declared count never moves. +> +> The failure mode is exactly the soundness hole below, which makes this the most +> dangerous item in the change: the four constraints that go missing are the four +> that matter. **Fix: derive the framing indices from `NUM_LANES` instead of +> writing 14/22/26 as literals, and add a test that the emitted index set is +> `0..NUM_CONSTRAINTS` without repeats** — the debug tracker is not enough, +> because the suite runs in release. + +> **★ Break 2 — raising `HashMode::Leaf` to two input cells panics the pin +> emitter.** ✓ VERIFIED. Construction C reads an accumulator cell *and* a felt +> cell, so `num_input_cells` for `Leaf` goes 1 → 2 (`instr.rs:104-110`). Then in +> `emit_unread_input_pins` the `slot = 1` iteration filters modes with +> `num_input_cells() <= 1` — **which becomes empty**, the fold returns `None`, and +> `.expect("some mode reads fewer than three input cells")` fires +> (`chips.rs:722-733`). AIR construction panics. +> +> Consequences, all mechanical once seen: the loop must skip slots no mode +> under-reads; `NUM_UNREAD_INPUT_PINS` goes 8 → 4 (`chips.rs:678`), which moves +> `UNREAD_IDX`, `LEAF_IDX`, `CORE_IDX` and `NUM_CONSTRAINTS`; and the leaf felts +> move from cell 0 to cell 1, so `leaf_lo_lane(i) = 2i` / `leaf_hi_lane(i) = 2i+1` +> (`:680-687`) become `4 + 2i` / `4 + 2i + 1` and the felt source `IN0 + i` at +> `:1369` becomes `IN0 + 4 + i`. + +> **★ Break 3 — the second `LfmMem` receive excludes `MODE_L`, so the felt cell +> would never be read.** ✓ VERIFIED `chips.rs:626`: the second input cell's +> multiplicity is `reads_two() = Multiplicity::Sum3(MODE_C, MODE_T, MODE_P)` — +> `MODE_L` is deliberately absent, because today a leaf row reads one cell +> (`chips.rs:620-622` says so in those words). Under construction C a leaf row +> **must** receive cell 1, so that multiplicity has to include `MODE_L`. +> `Multiplicity::Sum3` is exactly three columns (`crypto/stark/src/lookup.rs:1458`), +> so this becomes the four-way `selector_sum(MODE_C, NUM_SELECTORS)` +> (`chips.rs:52-61`) that the first receive already uses. +> +> ⚠ **This is a correction to §1.4.1's claim that "the frozen `LFM_HASH` bus arity +> does not move at all."** The *arity* does not — still three receives, three +> sends. The *multiplicity* of the second receive does. That is a smaller change +> than a new tuple, but it is a change to the frozen contract and it must be +> stated as one, because a reader who takes "does not move at all" literally will +> not look at `lfm_mem_interactions`. + +**★ A free win the survey also misses: the four new lanes are pinned for you.** +Lane bytes reach only two kinds of constraint — the identity at `6 + lane`, gated +on `digest_mu = MODE_C + MODE_T` (`:1304-1309`), and the leaf halves binding, +gated `mode_l` (`:1360-1388`) — plus the `AreBytes` sends (`:947-957`), which +bound each byte below `2^8` but say nothing about its value. So on a Compress or +Transcript row, four *unconstrained* lanes would hand the prover `m[9..13]` +outright and the parent digest would stop being a function of `(a, b)`: Merkle +parents forge. **At `NUM_LANES = 12` the existing code already closes this**, and +by luck rather than design: `b.main(0, cols::IN0 + lane)` for `lane ∈ 8..12` +lands on the **third input cell**, `IN8..IN12` (`IN0 = PREP_WIDTH = 13`, +`S8 = PREP_WIDTH + 12`, `chips.rs:482-488`) — which `emit_unread_input_pins` +pins to zero on every digest row. The identity then reads `0 = Σ bytes·2^{8k}`, +and with the `AreBytes` bound in hand that forces all sixteen bytes to zero. +✓ So the required pin is free **provided Break 1 is fixed**; if it is not, those +are exactly the four identities that get silently overwritten. Note also that 12 +is the last lane count for which this holds: at 13 lanes `IN0 + 12` is `S8`, and +the identity would start reading the capacity-state columns as input felts. + +**Question (iii) — `block_len` 36 → 52 flows through as a plain framing constant, +with one hard caveat.** ✓ VERIFIED: one definition (`blake3_socket.rs:261`) feeds +three consumers — the host reference (`:304`), the wire interpretation +`input_v12` (`:752`) and the value interpretation (`:873`) — so changing the +constant moves all three together and they cannot desynchronise. `36` is assumed +nowhere else load-bearing: the only other occurrences are one test assertion +(`leaf_tests.rs:179`, `assert_eq!(msg.len(), 36)`) and doc headers in the KAT +tables. No canonicity gate and no mode selection reads it. +⚠ **But it cannot be made mode-dependent.** `block_len` is `v[14]`, which +`G_INDICES[2] = (2,6,10,14)` makes the `vd` operand of round-0 G #2, and `vd` +goes straight into `f.xor(g, 0, vd, a1)` (`blake3_chip.rs:328`) — an XOR, whose +byte extraction `WordRef::byte` panics on `ModeSelected` (`:395-404`). So all +three domains move to 52 together: **compress and transcript digests re-bless +too**, and their messages gain sixteen zero bytes — which is what makes the pin +above load-bearing rather than cosmetic. + +**Question (iv) — the verified arithmetic.** From the layout constants +(`PREP_WIDTH = 13`, `layout.rs::hash`; `SHARED_VALUE_COLUMNS = 28`, +`chips.rs:494`; `G_SIZE = 60`; `OUT_WINDOW = HASH_DIGEST_FELTS = 4`; +`NUM_G = 8·rounds`), the socket's width is + +``` +NUM_COLUMNS = PREP_WIDTH + SHARED_VALUE_COLUMNS + 4·NUM_LANES + + 60·NUM_G + 4·OUT_WINDOW + 2·FELTS_PER_LEAF +``` + +| | 7r, 8 lanes | 7r, 12 lanes | Δ | +|---|---:|---:|---:| +| lane bytes | 32 | 48 | **+16** | +| canonicity witnesses | 8 | 8 | 0 | +| mixing core | 3,360 | 3,360 | 0 | +| `NUM_COLUMNS` | 3,457 | 3,473 | **+16** | +| main (census) columns | 3,444 | 3,460 | **+16** | +| `AreBytes` sends (`2·NUM_LANES`) | 16 | 24 | **+8** | +| bus interactions | 1,382 | 1,390 | +8 | +| census cells (`main + 3·⌈n/2⌉`) | 5,517 | 5,545 | +28 | + +✓ The **+16 columns / +8 sends** in §1.4.1 are exact. What the estimate omits is +the constraint delta — the framing block is renumbered and grows (Breaks 1–3), +at **zero column cost**, since every fix is a constraint or a multiplicity. + +**⚠ One correction to §1.5's census table — ✓ APPLIED 2026-08-12.** It listed +`LFM_HASH` at **3,457**, which is `cols::NUM_COLUMNS` *including* the 13 +preprocessed columns. Those are committed in the precomputed tree, not the main +tree, so the main-tree row is **3,444** — the figure §1.5's own prose already +named. At `RATE = 4` that is `⌈2·3444/4⌉ = 1,722` compressions rather than 1,729, +i.e. **−7 per query**; the per-query totals move 6,062 → **6,048** and +3,031 → **3,024**. Propagated to §1.4.1's measured-effect table, §1.5's table and +totals, §0's board line, and `commit_kats.py:457`'s width list (two +`C12.per_query_*` vectors re-pinned; no cryptographic vector moved; board still +85/85). ⚠ **This is a re-attribution, not a saving** — the 13 columns are still +absorbed, in the precomputed tree, which this census does not count at all +(§1.5 note **(4)**). + +### 1.4.3 ✓ Reconciled: 2,964 vs 3,056 are two DIFFERENT chips + +The census track's 3,056 and this document's 2,964 are both correct and measure +different tables — ✓ VERIFIED: + +| | file | `NUM_COLUMNS` | `PREP_WIDTH` | main | in the LFM AIR set? | +|---|---|---:|---:|---:|---| +| `LFM_HASH` (Blake3 arm) | `blake3_socket.rs:638` | 2,977 @6r / 3,457 @7r | 13 | **2,964 / 3,444** | **yes** (`chips.rs:592`) | +| standalone BLAKE3 chip | `blake3_chip.rs:162` | 3,072 | 16 (`:151`) | **3,056** | **no** — `airs.rs` never references it | + +They are not variants of one chip: the standalone one takes `h`, `t`, +`block_len` and `flags` as *inputs* (`blake3_chip.rs:157`, seven input machine +words), which is the general compression function; the socket pins all four and +reads two cells. **The tower pays the socket's width, so 2,964 @6r is the figure +for every tower number in this document.** ✗ RESOLVED — nothing left open here. + +*(Aside, now moot: the standalone chip already carries `h` as a variable input, +so the D7 sketch was buildable — just against a 3,056-column chip reading seven +words, for the same rate 4 the socket reaches with 12 lanes.)* + +### 1.4.4 ⚠ Implementation hazard register — read before writing the chip change + +Every item is ✓ VERIFIED against `lambda_vm-blake3-impl@blake3-real-hash`. They +apply to the **adopted `RATE = 4` / `NUM_LANES = 12`** construction; D9 moving to +5 would add to this list, not shorten it. **H1 is the one that ships broken.** + +| id | hazard | where | fails how | +|---|---|---|---| +| **H1** | ★★ constraint-index collision | `blake3_socket.rs:1304-1318` | **SILENT in release** | +| **H2** | `num_input_cells(Leaf)` = 2 panics the pin emitter | `chips.rs:722-733` | loud panic | +| **H3** | 2nd `LfmMem` receive excludes `MODE_L` | `chips.rs:626` | leaf never reads its felts | +| **H4** | leaf lane/felt offsets shift by one cell | `blake3_socket.rs:680-687`, `:1369` | binds the wrong felts | +| **H5** | `lanes_from_row` is all-or-nothing | `blake3_socket.rs:1084-1093` | witness ≠ AIR | +| **H6** | lanes 0–3 gate must be `mu`, not `digest_mu` | `blake3_socket.rs:1304-1309` | **accumulator unconstrained** | +| **H7** | `admits`' Leaf arm inspects the wrong cell | `blake3_socket.rs:529-537` | prover panic, not rejection | +| **H8** | `LfmHasher::leaf` signature ripples to Test/Poseidon | `hash.rs:109-114` | silent semantic change | +| **H9** | `block_len` 52 is a **tri-domain** re-bless | `blake3_socket.rs:261` | scope under-counted | + +> ### ★★ H1 — the lane identities collide with the output pins, and every guard is blind +> +> `eval` numbers its framing constraints by hand: the lane identities are +> `b.emit_base(6 + lane, …)` over `0..NUM_LANES` +> (✓ VERIFIED `blake3_socket.rs:1304-1309`), then the unused-output pins are +> `b.emit_base(14 + j, …)` for `j ∈ 0..8` (`:1315-1318`), the digest +> recompositions `22 + i` (`:1325-1330`), and `UNREAD_IDX = 26` (`:1223`). +> **At `NUM_LANES = 12` the lane block runs 6..17 and overlaps the output pins at +> 14..17.** +> +> `EmitTracker::mark` asserts `"constraint {idx} emitted twice"` — but only under +> `#[cfg(debug_assertions)]` +> (✓ VERIFIED `crypto/stark/src/constraints/builder.rs:492-504`), and the +> workspace declares **no `[profile.release]` override**, so under the house +> convention `cargo test --release` the tracker is a no-op and the second write +> silently overwrites the first (`builder.rs:614-617`). The lane loop runs first, +> so **lanes 8–11 lose their identity entirely and nothing fails.** +> +> **Why every existing guard misses it.** The constraint *count* does not move: +> lane identities go 8 → 12 (+4) while the unread pins go 8 → 4 (−4, per **H2**), +> so `NUM_CONSTRAINTS` (`:1214`), `CORE_IDX` and +> `predicted_constraints(rounds) = 50 + 16·(8·rounds)` +> (`blake3_socket_tests.rs:104-106`) all still hold. `assert_complete` sees no +> gap either, because every index in `0..NUM_CONSTRAINTS` is still written by +> *something*. **The only thing that would have caught it is a debug-only assert +> the release suite disables.** +> +> **And the four constraints lost are exactly the four that matter** — see +> **H6**: lanes 8–11's identity is what pins `m[9..13]` to zero on digest rows. +> Losing it hands the prover four free message words in a Merkle parent. +> +> ⛔ INDEX CORRECTION (2026-08-13, at implementation): the NORMATIVE layout is +> §1.2 / `commit_ref.py` — lanes at `m[0..12]`, **tag LAST at `m[12]`** — so the +> words this aside calls `m[9..13]` are `m[8..12]` in the implemented layout +> (this aside and two other mentions predate the resolution; the substantive +> argument is unchanged — the free pin comes from the lane→COLUMN map, not the +> message index). Resolved toward §1.2 by the RATE-4 implementation. +> +> **Required:** derive the framing indices from `NUM_LANES` instead of the 14/22/26 +> literals, **and** add a release-visible test that the emitted index multiset is +> exactly `0..NUM_CONSTRAINTS` with no repeats. The debug tracker is not +> sufficient, because the suite that would run it does not. + +**H2 / H3 — the frozen bus contract does move.** Both are stated in full in the +third supersession banner in §1.4.1. In short: `reads_two()` must gain `MODE_L` +and outgrow `Multiplicity::Sum3`, and `emit_unread_input_pins`' `slot = 1` pass +must stop assuming some mode reads fewer than two cells. + +**H4 — the felts move from cell 0 to cell 1.** `leaf_lo_lane(i) = 2i` / +`leaf_hi_lane(i) = 2i+1` (`:680-687`) become `4 + 2i` / `4 + 2i + 1`, and the +felt source `b.main(0, cols::IN0 + i)` (`:1369`) becomes `IN0 + 4 + i`. +`fill_canonicity_witness` (`:1100-1115`) reads through the same helpers, so it +follows automatically — which is the trap: fix the helpers and the filler moves +with them, fix `:1369` alone and it does not. + +**H5 — the row is a HYBRID and no current code can express one.** +`lanes_from_row` (`:1084-1093`) branches on `MODE_L` and applies **one** reading +to all eight lanes. Construction C needs cell 0 through `lanes_of` (u32 lanes) +and cell 1 through `leaf_lanes` (felt halves) **on the same row**. The same +all-or-nothing shape is in the constraint gating, which is **H6**. + +> **★ H6 — the gate re-cut, and the one direction that is a soundness break.** +> §1.4.2's blockquote has this right; here is the exact split and why the +> "+0 canonicity witnesses" claim depends on it. +> +> - **lanes 0–3 → gate on full `mu`.** On a leaf row they are the accumulator; on +> a digest row they are `a` = `IN0..IN4`. **The same identity is correct for +> both readings**, which is why one gate serves. This is also the *only* thing +> that range-checks the accumulator: identity + `AreBytes` forces +> `IN_lane < 2^32`, which is what "the accumulator lanes are a previous digest, +> hence `u32` by construction" cashes out to. **Gate these on `digest_mu` and a +> leaf row's accumulator lanes carry no identity at all — the prover picks the +> chain's message words freely and the whole leaf chain unbinds.** +> - **lanes 4–11 → gate on `digest_mu`.** On a leaf row they are felt halves and +> the identity must NOT hold; the halves binding covers them instead. +> +> **A free win worth not throwing away:** at exactly 12 lanes, +> `b.main(0, cols::IN0 + lane)` for `lane ∈ 8..12` lands on the **third input +> cell**, which `emit_unread_input_pins` pins to zero on every digest row. The +> identity then reads `0 = Σ bytes·2^{8k}`, and with the `AreBytes` bound in hand +> that forces all sixteen bytes to zero — so the pin that keeps `m[9..13]` out of +> the prover's hands costs nothing. ⚠ **12 is the last lane count for which this +> works:** at 13, `IN0 + 12` is `S8` +> (✓ VERIFIED `chips.rs:482-488`) and the identity would start reading the +> capacity-state columns as input felts. A D9 move to 14 lanes must supply these +> pins explicitly. + +**H7 — `admits` would inspect the accumulator and call it the felts.** The Leaf +arm checks `leaf_lanes` over `state[0..4]` (`:529-537`), which under construction +C is the **accumulator cell**, not the felts. Left as is, a non-canonical felt +passes execution and fails later in the filler or the AIR — a prover panic where +the house rule wants a clean rejection ("reject, never reduce"). It needs to +check `lanes_of(acc)` **and** `leaf_lanes(felts)`. + +**H8 — the trait change is not local to BLAKE3.** `LfmHasher::leaf(&self, felts: +&LfmWord)` (`hash.rs:114`) takes one cell; construction C needs `(acc, felts)`. +The default `leaf_out` delegates to `compress_out(felts, &[zero; 4])` +(`hash.rs:109-110`), so **the `Test` and `Poseidon` arms' leaf semantics change +too** — silently, since they compile either way. Both already carry the recorded +weakening that they do not domain-separate leaves from parents; this widens it. + +**H9 — `block_len` 52 re-blesses THREE domains, not one.** ✓ VERIFIED it cannot +be made mode-dependent: `block_len` is `v[14]`, which +`G_INDICES[2] = (2,6,10,14)` makes the `vd` operand of round-0 G #2, and `vd` +goes straight into `f.xor(g, 0, vd, a1)` (`blake3_chip.rs:328`) — an XOR, whose +byte extraction `WordRef::byte` panics on `ModeSelected` (`:395-404`). So +`LFMC` and `LFMT` move to 52 with `LFML`: **every Merkle parent and every +transcript step re-blesses, and their messages gain 16 zero bytes** — which is +what makes H6's pin load-bearing rather than cosmetic. Scope this into D8's +re-bless pass, not just the leaf vectors. On the credit side it flows from one +constant (`:261`) into the host reference (`:304`), the wire interpretation +(`:752`) and the value interpretation (`:873`), so the three cannot +desynchronise; and 52 < 64 keeps every row a single block, so the C9 crate anchor +survives for all three domains. + +**✓ What is NOT a hazard: the shared dataflow.** Message words reach `add3` and +nothing else — the schedule indices are consumed at exactly two call sites, +`blake3_chip.rs:327` and `:333`, both `f.add3(…)`. `word_expr` (`:1032-1048`) +handles `Cols`, `Const` and `ModeSelected` uniformly at degree ≤ 1, so the sum +identity stays degree 2 under the mu gate and the chip's max degree of 3 does not +move. `Add3Wire.m` is already typed `WordRef` for exactly this reason +(`:439-446`). **The G-block wiring is fully index-agnostic; the mixing core's +sends, constraints and degree are functions of the round count alone.** + +### 1.5 ★ Chain-depth census — and the rate problem it exposes + +The build-time question §6 flagged, answered. Main-tree widths ✓ VERIFIED from +source (`chips.rs` + `layout.rs`; `keccak_rnd.rs:95`, `keccak_rc.rs:36`, +`bitwise.rs:94`). `LFM_HASH` under Blake3 has `cols::NUM_COLUMNS = 3457` +(`blake3_socket.rs:616-638` with `SHARED_VALUE_COLUMNS = 28`, `NUM_G = 56` at 7 +rounds), of which **3,444 are main-tree value columns** — the figure that +reproduces the leaf-impl-report exactly, and the one this table uses. + +Per leaf = one row pair, so `felts = 2·num_cols·kind`, `chunks = ceil(felts/4)`, +**chain depth = chunks**, `compressions = 2·chunks` (§1.4). + +| chip (main tree) | cols | felts | chain depth | compressions | +|---|---:|---:|---:|---:| +| **`LFM_HASH` (Blake3, 7r)** | **3444** | 6888 | **1722** | **3444** | +| `KECCAK_RND` (per chunk) | 1480 | 2960 | 740 | 1480 | +| `LFM_KECCAK` | 792 | 1584 | 396 | 792 | +| `LFM_BITDEC` | 196 | 392 | 98 | 196 | +| `LFM_SELECT` / `LFM_XALU` / `BITWISE` | 25 / 23 / 21 | | 13 / 12 / 11 | 26 / 24 / 22 | +| `LFM_LANES` / `LFM_BALU` / `KECCAK_RC` | 16 / 14 / 10 | | 8 / 7 / 5 | 16 / 14 / 10 | +| `LFM_CONST` / `LFM_PUBLIC` / `LFM_HINT` / `LFM_RANGE` | 7 / 7 / 6 / 2 | | 4 / 4 / 3 / 1 | 8 / 8 / 6 / 2 | +| **total, main trees, `C = 1`** | | | | **6,048** | + +**Four findings, in order of how much they matter.** + +**(1) ★ The widest table is the BLAKE3 chip itself, not `KECCAK_RND`.** At 3,444 +main columns it is 2.3× `KECCAK_RND` and **56.9%** of the per-query main-tree leaf +cost. The tower spends most of its leaf budget re-absorbing the trace of the +hash chip that made the proof cheap. This inverts the natural assumption and is +a property of the *tower*, not of the base layer — where `KECCAK_RND` still +dominates at 92.5% of cells. + +**(2) ★★ The cost formula was not a depth problem, it was a RATE problem — +and §1.4.1 fixes it.** At the old construction, for a base tree +**compressions per leaf ≈ `num_cols`**, because 2 rows × `m` felts cost +`2·ceil(2m/4) ≈ m`. That was **2 felts per compression** — four felts per `LFML` +row, halved by the `LFMC` fold. Keccak absorbs **17 felts per permutation** +(rate 136 B ÷ 8 B), which is why the native scheme was only ~1.7× better on leaf +absorption despite ~14.7× on Merkle parents. + +**`RATE = 4` (§1.4.1) takes it to 4 felts per compression, a 2.0× cut.** The +table above is the OLD-rate census, kept because it is what located the problem; +the columns are unchanged, so the new per-chip cost is `0.5 × cols` (e.g. +`LFM_HASH` 3,444 → **1,722**), and the per-query total is **6,048 → 3,024**. + +Chain *depth* is fine — 1,722 sequential steps is nothing for a fully-unrolled +straight-line program, and depth carries no soundness cost since the header +binds the shape (§1.3). The compression *count* is the whole story: + +| | per query, main trees, `C = 1` | ×219 queries | ×110 queries | +|---|---:|---:|---:| +| leaf-absorption compressions | 6,048 | **1,324,512** | **665,280** | + +**(3) It is the Gate D1 lever, and §1.4.1 pulls it.** The Gate D1 verdict +(PLAN.md) independently measures leaf absorption at **69.8% of the tower node +bill** and finds the node fails at 124 GiB against ~93 GiB, with **×2 → fits**. +`RATE = 4` delivers **2.0×** → ≈81 GiB. ⚠ And it is **not** the last turn of this +lever — see D9 (§7) and the second supersession banner in §1.4.1. + +**(4) ⚠ This census counts MAIN trees only, and that is a real scope limit.** +The row above is 3,444 rather than `NUM_COLUMNS`'s 3,457 because the 13 +preprocessed columns are committed in the **precomputed** tree. They are still +absorbed — this table just does not count them, nor the aux or composition trees +(`verifier.rs:605-650` confirms three separate trees). ✓ The correction was +applied 2026-08-12 (6,062 → 6,048 old, 3,031 → 3,024 new; −0.23%, invisible at +Gate D1's ≈81 GiB). **Do not read the total as the whole tower leaf bill** — it +is the main-tree component of it, which is what §1.4's formula is scoped to. + +> ⚠ **The earlier D7 sketch in this section — carry the accumulator in the +> chaining value `h` — is SUPERSEDED and should not be built.** It reached only +> 4 felts/compression, required `h` to become a witness, forced a revisit of +> obligation **O3**, and split the crate-KAT anchor. §1.4.1's in-message +> accumulator is strictly better on all four counts. The premise that made D7 +> look necessary — "the socket's eight lanes are full, so an accumulator cannot +> ride in the message" — was **wrong**: it counted the *lanes* the socket +> currently reads, not the *message words* BLAKE3 has, and seven of those are +> `WordRef::Const(0)`. + +### 1.5.1 Two build traps this spec must state + +**(a) `blake3-6round` is OFF by default.** ✓ VERIFIED `SOCKET_ROUNDS = +BLAKE3_ROUNDS` (`blake3_socket.rs:202`) and `BLAKE3_ROUNDS = +BLAKE3_STANDARD_ROUNDS` unless the `blake3-6round` feature is set +(`blake3.rs:83-85`). Every number in this document is quoted at **7 rounds**, +which is the compiling default and **+16%** on every tower figure. The campaign +intends 6 rounds. **The spec therefore states a build requirement: tower +proving builds must enable `blake3-6round`, and any census that does not must +say so.** At 6 rounds `NUM_G` is 48 rather than 56, so the `LFM_HASH` arm is 2,977 columns +total / **2,964 main** (2,993 total with the §1.4.1 widening) against 3,457 / +3,444 at 7r. ✓ The 3,056 figure circulating on the census track is the +**standalone** BLAKE3 chip, a different table — §1.4.3 reconciles them. + +**(b) The `LFM_HASH` chip is the widest table under D0.** Finding (1) above: +3,444 main columns at 7r (2,964 at 6r), 2.3× `KECCAK_RND`, **56.9%** of the +old-rate per-query leaf bill. Each tower layer pays to re-absorb it. `RATE = 4` +cuts the absolute cost 2.0× but does **not** change the share — the chip is still +the widest table, and any future widening of the socket lands on the tower with +that ~57% multiplier. Worth remembering before adding socket columns for anything +else — including the **+16** this very construction adds (§1.4.1), which is ++0.5% on the widest table and therefore ~+0.3% on the whole per-query leaf bill. + +### 1.6 What the I3 width check still guards afterward + +? INFERRED, and stated conservatively on purpose. + +With the header built from the AIR, an opening of the wrong width produces a +different leaf digest and fails authentication — so for the leaf path the check +becomes **defence in depth rather than the primary mechanism**. + +It is still needed, for two reasons. First, it runs *before* any opening is +indexed (✓ VERIFIED `verifier.rs:213-215`: "Runs once per table, before any +opening is read"), so it is what stops a malformed opening from being indexed at +all. Second, it pins widths for things no leaf covers — the OOD tables. + +> **⚠ Do not delete it as redundant.** This is the M8 lesson from +> TRANSCRIPT.md §3.3, where a paragraph that *looked* like it made the +> registrar's one-hot check redundant was wrong, and a reader who trusted it +> could have removed the load-bearing check while every constraint still passed. +> The same trap is available here. **DECIDED (D2): the check stays** — §7. + +--- + +## 2. The byte→cell absorb encoding + +### 2.1 The problem + +`DefaultTranscript` absorbs **bytes** (`append_bytes`); B1 absorbs **cells** of +four u32 lanes. `absorb_lfm_statement` feeds raw byte strings — a tag, a +`program_id`, little-endian integers (✓ VERIFIED `statement.rs:79-89`). An +encoding is required, and it must be injective. + +### 2.2 The construction + +``` +header = [ BYTES_MARK, len & 0xFFFFFFFF, len >> 32, 0 ] +body = data zero-padded to a multiple of 16, each 16 bytes read as + four LITTLE-ENDIAN u32 lanes +absorb = absorb(header) then absorb(each body cell) +cost = 1 + ceil(len / 16) compressions +``` + +**O1 compliance is automatic, and that is the point.** Every lane is exactly +four bytes, hence `< 2^32` by construction — no canonicity gate, no rejection, +no `MODE_L` row. A byte block is already digest-shaped. This is precisely why +bytes take *this* path while field elements take `absorb_felts` (`LFML`), where +the canonicity gate lives. ✓ EXECUTED (**C5**). + +**The length prefix is what makes it injective**: without it `b"\x01"` and +`b"\x01\x00"` absorb identically. ✓ EXECUTED (**C5**). + +Little-endian to match `word_of`'s stated convention — ✓ VERIFIED +`blake3_socket.rs:441-443`: *"one felt = one u32 = four little-endian bytes"*. +Note this is **not** `pack_digest`'s eight-bytes-per-lane serialization; the two +conventions coexist in the codebase and the same doc comment already warns about +it. + +--- + +## 3. Node codec and tree shape + +### 3.1 The embedding + +`pack_digest` (✓ VERIFIED `word.rs:44-50`) writes four canonical u64 lanes +little-endian: 32 bytes. Under BLAKE3 every lane is `< 2^32` (✓ VERIFIED +`word_of`, `blake3_socket.rs:443`), so **the high four bytes of each 8-byte +chunk are zero** — sixteen bytes of padding. + +That padding is what lets a 128-bit digest ride inside the existing 32-byte +`Commitment` without moving the proof format: `StarkProof`'s commitment fields +and the rkyv derives stay byte-identical (D0 §2). Parents hash **cells**, never +the padded bytes, so nothing enters a preimage that the guest must re-pad. + +### 3.2 ⚠ The strict decode (S2) + +`unpack_digest` (✓ VERIFIED `word.rs:52-61`) reads each chunk as a u64 and +**reduces mod p**. Many distinct 32-byte strings therefore decode to one node — +any lane may be offset by a multiple of `p`, and far more cheaply, any of the +sixteen padding bytes may be set. Node-level malleability inside a Merkle path +is a proof-format forgery surface. + +**The rule: every lane must be `< 2^32`; reject otherwise.** `< 2^32` implies +`< p`, so one test covers both. This mirrors `lanes_of` (✓ VERIFIED +`blake3_socket.rs:431-438`), which already rejects rather than reduces on the +host. + +✓ EXECUTED (**C6**), four rejection flavours plus two honest legs: a set high +byte in lane 0, a set top byte in lane 3, a lane congruent to 1 mod p, and a +short commitment all reject; the round trip and an all-zero digest still decode. +The honest legs are not optional — a decoder that rejected everything would pass +a rejection-only suite. + +### 3.3 Tree arity and padding (S6) + +**Binary, and assert the leaf count is a power of two — do not pad.** + +The leaf count is always `lde_size / 2` and `lde_size` is always a power of two +(✓ VERIFIED the prover debug-asserts exactly this, `commitment.rs:67-70`), so +the assertion is always satisfiable on the honest path and costs nothing. +Padding would add a duplicate-leaf second-preimage surface for a case that does +not arise — an unreachable branch that weakens the tree. `HostTree::build` +already asserts the same (✓ VERIFIED `fixture.rs:163-175`). ✓ EXECUTED (**C7**). + +--- + +## 4. Scope-outs, stated in the spec rather than assumed + +### 4.1 Grinding under B1 (S7) — **DECIDED: grinding STAYS** + +> **Ruling (Mauro, 2026-08-12), verbatim:** *"Grinding should help you, we need +> 128 security for sure."* +> +> This **reverses** an earlier recommendation in this document to set +> `grinding_factor: 0` and scope grinding out. §7.1/§7.2 show that +> recommendation was wrong on the numbers: dropping grinding costs +41 queries +> at blowup 2 — **+222,794 tower permutations per wrap verify, forever**, in +> exactly the cost centre D0 exists to shrink. Grinding is not overhead here; it +> is the cheapest 20 bits in the protocol. + +So B1 needs a PoW it can express. This section specifies one. + +#### 4.1.1 What it replaces + +✓ VERIFIED `grinding.rs:67-89` — **two** keccak256 hashes over byte buffers: + +``` +inner = Keccak256( PREFIX(8) ‖ seed(32) ‖ factor(1) ) 41 bytes +valid = u64_be( Keccak256( inner(32) ‖ nonce_be(8) )[..8] ) < 2^(64−factor) +``` + +Neither layer is a 2-to-1 compress, and both run through the hosted keccak +family. The seed is `transcript.state()` (`prover.rs:2093`, +`verifier.rs:1587`) — a `[u8;32]` B1 does not have. + +#### 4.1.2 The construction + +``` +GRIND_MARK = "GRD0" as a little-endian u32 +N(nonce, factor) = [ nonce_lo, nonce_hi, GRIND_MARK, factor ] one cell +W = compress_T( state, N(nonce, factor) ) ONE compress +valid ⟺ ( W[0] + 2^32·W[1] ) mod 2^factor == 0 +``` + +- **One cell, one compression.** The whole PoW is a single `compress_T`, which + is the design target: verification cost is O(1) in the difficulty. +- **The difficulty is in the preimage.** Without `factor` in the operand a + prover mines once at factor 1 and presents the result at factor 20. + ✓ EXECUTED (**C11**): the same nonce at factors 12 and 13 gives different + digests. +- **The seed is the transcript state cell**, not a 32-byte digest — the B1-shaped + analogue of today's `transcript.state()` seed. ✓ EXECUTED (**C11**). +- **The difficulty predicate reads `W[0] ‖ W[1]` as one 64-bit value**, covering + the whole documented `1..=64` range (`grinding.rs:22`) under one rule. For a + realistic `factor ≤ 32` it touches lane 0 only. The alternative — a lane-0 + rule with a second rule bolted on above 32 — is two cases where one will do. + +#### 4.1.3 ⚠ Domain separation — read before changing the tag + +The construction **reuses `LFMT`** and allocates no fourth domain. The argument +is the one B1 already relies on for absorb-vs-squeeze, quoting TRANSCRIPT.md +§1.1: the operation sequence is a compile-time constant of the program, so *"a +prover cannot perform a squeeze where the program says absorb"* — and equally +cannot present a PoW evaluation where the program says absorb. `GRIND_MARK` sits +on exactly the same footing as `SQUEEZE_MARK`, which that section is explicit is +**defence in depth, not the load-bearing argument**. + +Sharing the tag costs nothing cryptographically: to satisfy the difficulty a +prover must still search operands at a state it does not control, and no +transcript step computed elsewhere helps. It saves a tag, a fifth preprocessed +selector (`MODE_G`), `PREP_WIDTH` 13 → 14, and a registry re-bless. + +✓ EXECUTED (**C11**), both cross-domain directions plus the marker: a PoW step +equals neither an `LFMC` parent nor an `LFML` leaf of the same cells, and +dropping `GRIND_MARK` changes the digest. + +> ### ⚠ The one separation the hash does NOT provide +> +> **A PoW step and an ABSORB of its operand cell are the same function** — both +> are `compress_T(state, cell)`. Against a Merkle parent and a leaf the tag +> separates them; against a transcript absorb **nothing in the hash does**, and +> `GRIND_MARK` only means an *honest* absorb is unlikely to collide, not that a +> chosen one cannot. +> +> That separation is carried entirely by the program's compile-time operation +> sequence — the same mechanism B1 already accepts for absorb-vs-squeeze. It is +> asserted as an **identity** in **C11** rather than left in prose, so the +> reliance is visible on the board: if D6a is ever taken, that leg flips and says +> so. A reader who wants the separation to hold without the fixed-sequence +> premise wants D6a. + +> **✗ OPEN (D6a)** — whether to spend the tag + selector + re-bless anyway, for a +> separation that does not lean on the fixed-sequence argument. Recommendation: +> no, on consistency grounds — if the fixed sequence is good enough for +> absorb/squeeze it is good enough here, and a fifth selector is not free. + +#### 4.1.4 The payoff, stated honestly + +The guest verifies PoW with **one blake3 compression plus one `LFM_BITDEC` row** +(to expose the low bits), against **two keccak sponge invocations** through the +hosted keccak family. + +⚠ **That saving is O(1) per proof and therefore small in absolute terms.** It is +not the reason grinding stays. The reason is §7.1: grinding buys back 41 queries +— ~222,794 tower permutations per wrap verify — for a one-off mining cost the +*prover* pays once. Quoting the compression saving as the justification would +overstate a real but minor effect and understate the actual argument. + +### 4.2 Challenge entropy (S8) + +> **DECIDED: squeeze-twice (~192-bit).** ⚠ **Decided *by implication* of the +> 128-bit total-security requirement, not by an explicit ruling on this +> question** — 96-bit challenges are below target, so the upgrade follows. +> **Flagged for explicit confirmation** rather than recorded as settled, because +> a decision nobody consciously made is the kind that gets silently reversed. +> Priced in §7.1 at **254 permutations** per tower wrap verify — about two orders +> of magnitude below D3. The analysis below stands as written; only the "not +> recommended either way" framing is resolved. + +The ratified B1 `squeeze_ext` takes lanes 0–2 of one squeezed cell. Each lane is +a u32, so an extension challenge carries **96 bits**, against the ~192 that +`DefaultTranscript` delivers (three near-full Goldilocks coordinates, +✓ VERIFIED `extensions_goldilocks.rs:575-581`). TRANSCRIPT.md §4.1 bounds the +*state* (128 bits, ~64-bit collision) but does not analyse *per-challenge* +entropy at production query counts. + +**The alternative, costed** — `squeeze_ext_2` in the reference: + +``` +c0, c1 = squeeze(), squeeze() +coef_i = ( lanes[2i] + 2^32 · lanes[2i+1] ) mod p i ∈ 0..3 +``` + +- **Cost: a flat +1 compression per extension challenge** (2 instead of 1). + ✓ EXECUTED (**C8**). +- **Query-index sampling is unaffected**: `squeeze_bits` reads lane 0 only, so + the query loop — the dominant squeeze run — pays nothing. +- **No rejection loop, deliberately.** A uniform 64-bit value reduced mod p is + biased by ~2^-32, negligible for a Fiat–Shamir challenge. An exact rejection + loop is *unimplementable* in the fully-unrolled eDSL ("nothing loop-shaped + reaches the machine", TRANSCRIPT.md §1.1 citing `edsl.rs:1-4`). The bias is + the right trade and the reason is structural. + +> **DECIDED (D4): squeeze-twice.** 96 bits is below the 128-bit total-security +> requirement, so the upgrade follows by implication — see the banner above for +> why that is flagged for confirmation rather than filed as settled. + +--- + +## 5. What this construction does *not* change + +- **No new socket tag.** The wide leaf is `LFML` rows folded by `LFMC` parents — + the two domains LEAF.md already ratified. `LEAF_MARK`/`BYTES_MARK` are **lane + constants inside a header cell**, not `m[8]` tags, so no new hash domain is + created and no new domain analysis is owed. (**DECIDED D1**, §7.) The B1 PoW + (§4.1) follows the same rule, reusing `LFMT` with a `GRIND_MARK` lane constant. +- ⛔ ~~**No change to the chip.** Nothing here needs a constraint that does not + already exist; the wide leaf is a *program shape* built from existing rows.~~ + **RETRACTED — this was true only of the pre-§1.4.1 construction.** Adopting a + `RATE > 2` puts the accumulator in the message, and that **is** a chip change: + `NUM_LANES` 8 → 12, `block_len` 36 → 52, the lane/message identity re-cut per + lane range, `reads_two()` gaining `MODE_L`, and `emit_unread_input_pins` + restructured. Nine verified hazards in §1.4.4. The bullet is kept because it is + what §5 promised before D8 existed, and a reader who takes §5 as the change + budget would under-scope the work by an order of magnitude. +- **The crate anchor survives.** ✓ EXECUTED (**C9**): the `LFML` rows are still + byte-identical to a plain `blake3::hash` call at 7 rounds — now of a 52-byte + message rather than 36. ⚠ There is no longer a *fold*: the accumulator rides in + the message (§1.4.1), so an `LFML` chain is a sequence of rows and not rows + plus `LFMC` parents. The `LFMC` socket is unchanged as a *function*, but its + `block_len` moves to 52 with everything else (§1.4.4 **H9**), so Merkle-parent + and transcript digests re-bless too. + +--- + +## 6. Open items that are *build-time*, not decisions + +| item | status | +|---|---| +| the same vectors against the Rust `blake3` crate | ✗ DEFERRED — needs cargo | +| the exact production grouping of precomputed/main/aux trees per AIR | ✓ VERIFIED as three separate trees (`verifier.rs:605-650`); the per-AIR widths still need reading off `trace_layout` at build time | +| whether any LFM AIR has `num_cols` large enough to make the chain depth a cost concern | ✓ **DONE — §1.5.** Depth is a non-issue; the **rate** was, and §1.4.1 fixes it (**✗ OPEN D8**, and how far to push it is **✗ OPEN D9**) | +| the end-to-end cost of a `RATE = 5` leaf: extra `LfmMem` traffic, the ~60%-larger padded opening buffer, and the re-packing `Pack`/`Unpack` rows | ✗ **OPEN — this is what D9 needs.** §1.4.2a's ~19% is a ? INFERRED sketch over trace columns only; the leaf program does not exist yet, so nothing here is measured | +| a release-visible test that the emitted constraint-index set is exactly `0..NUM_CONSTRAINTS` with no repeats | ✗ **REQUIRED by §1.4.4 H1.** The existing `EmitTracker` duplicate check is `#[cfg(debug_assertions)]` and the house convention runs `cargo test --release` | + +--- + +## 7. Decision points + +> **Status 2026-08-12 — D1–D6 are RULED ON by Mauro** (D4 by implication, see +> its row). **D8, D9 and D6a remain open.** The construction itself (§1, the S1 +> wide leaf) stays DRAFT pending his read. The quantification in §7.1 was +> produced *after* the D3/D4 rulings and is recorded because it **supports** +> them — and because it falsifies this document's own earlier recommendation +> on D3. +> +> **D9 is new (2026-08-12) and it exists because this document was wrong twice +> about the same number.** The first RATE draft said 5 from block headroom while +> ignoring the machine's cell structure; the correction said 4 and justified it +> with a cell argument that does not follow. Both retractions are marked in place +> in §1.4.1 rather than edited away, because the pattern — *an open item carried +> into a recommendation stops looking open* — is the one §7.2 already names, and +> this is its second instance in the same section. + +| id | decision | **ruling** | provenance / note | +|---|---|---|---| +| **D1** | `LEAF_MARK`/`BYTES_MARK` as lane constants, or genuine `m[8]` socket tags? | **lane constants** | Mauro: *"do whatever feels simpler."* Matches the recommendation (§5). No new hash domain, no new domain analysis owed. | +| **D2** | Does the I3 `trace_opening_widths_well_formed` check stay after the header lands? | **stays** | Uncontested. As recommended (§1.6). The M8 lesson from TRANSCRIPT.md §3.3 holds: a mechanism that *looks* like it subsumes a check is how a load-bearing check gets deleted. | +| **D3** | Grinding: `grinding_factor: 0`, or compensate with more queries? | **grinding STAYS** | Mauro, verbatim: *"Grinding should help you, we need 128 security for sure."* ⚠ **Reverses this document's own earlier recommendation**; §7.2 records why it was wrong. Consequence: the B1 PoW is now **specified** in §4.1, not scoped out. | +| **D4** | Keep 96-bit `squeeze_ext`, or pay +1 compression for ~192-bit? | **squeeze-twice (~192-bit)** | ⚠ Decided *by implication* of the 128-bit requirement, **not** by an explicit ruling — flagged for confirmation (§4.2). §7.1 prices it ~2 orders of magnitude below D3. | +| **D5** | Zero-pad the felt stream to 4, or forbid non-multiple-of-2 widths? | **zero-pad** | Mauro: no opinion → the recommendation stands (§1.3). | +| **D6** | Grinding needs a B1-expressible PoW | **specified** (§4.1), KATs **C11** | Raised by this document after the D3 ruling; discharged in the same pass. | +| **D8** | ★★ Adopt `RATE = LFML_FELTS_PER_ROW = 4` (§1.4.1)? | ✗ **OPEN — the consequential one** | Decides whether the tower fits: 2.0× on 69.8% of the node bill, Gate D1 124 GiB → ≈81 GiB. A **chip** change (`NUM_LANES` 8 → 12, `block_len` 36 → 52) costing ≈+16 columns on 3,444 main (+0.5%). ⛔ "4 is the CEILING" is **RETRACTED** — the ceiling is 5, see **D9**. Moves every `LFML` digest → re-bless — and ⚠ **`LFMC` and `LFMT` too**, since `block_len` cannot be made mode-dependent (§1.4.4 **H9**). D0 is re-blessing anyway **if sequenced into the same pass**. Supersedes the D7 `h`-chaining sketch (same rate, breaks the C9 anchor). **Before implementing, read §1.4.4** — 9 hazards, H1 silent in release. | +| **D9** | ★★ Stay at `RATE = 4`, or price and take `RATE = 5`? | ✗ **OPEN — optimization, NOT fit** | **Nothing is blocked either way: Gate D1 already fits at rate 4 (≈81 GiB vs ~93 GiB, ~13% margin).** This is about whether to chase ~19% more. **Rate 4 (adopted default):** buildable today, fully priced (+16 cols, +8 sends, all costs in §1.4.1/§1.4.4), felt input is one whole machine cell so the leaf program reads the opening in its natural layout with **no re-packing pass**. Needs the multiplicity change (H3), the pin-emitter fix (H2), the per-lane-range gate (H6) and the rest of §1.4.4. **Rate 5:** the true block ceiling (16 words − tag − 4 accumulator lanes = 5 half-pairs); ? INFERRED **~19% cheaper** per §1.4.2a (2). ⚠ **UNPRICED, and the gaps are not small:** the extra `LfmMem` traffic, the ~60%-larger padded opening buffer, and the re-packing program's own `Pack`/`Unpack` rows. Needs the **third** receive to admit `MODE_L` plus either a rotating per-row lane map or a repacked felt stream, and §1.4.4 **H6**'s free digest-row pin stops working at >12 lanes. **Recommendation: ratify 4 now, and treat 5 as a follow-up only if Gate D1's 13% margin proves too thin** — taking it later costs a second re-bless, which is the same sequencing argument D8 makes. | +| **D6a** | Give the PoW its own `LFMG` tag + `MODE_G` selector? | ✗ **OPEN** | Recommendation: no (§4.1.3). Costs `PREP_WIDTH` 13 → 14 and a re-bless for a separation the fixed-sequence argument already carries — the same argument B1 accepts for absorb/squeeze. **But note what it buys:** a PoW step *is* an absorb of its operand cell (C11's identity leg), so PoW-vs-absorb separation is the one direction the hash does not give you. Take D6a if that premise should not be load-bearing. | + +### 7.1 The D3 / D4 arithmetic + +✓ VERIFIED formula — `options.rs:121-125`: + +``` +rate = 1 / blowup +proximity = 1 − sqrt(rate) − 1/300 +bits_per_query = −log2(1 − proximity) +queries = ceil( (security_bits − grinding_factor) / bits_per_query ) +``` + +✓ EXECUTED, and the formula reproduces the recorded presets exactly (219 at +blowup 2, 110 at blowup 4 — `prover/src/recursion.rs`'s `Blowup2`/`Blowup4`), +which is what makes it safe to extrapolate: + +| blowup | bits/query | q @ grinding 20 | q @ grinding 0 | Δ | +|---|---|---|---|---| +| **2** | 0.493215 | **219** | **260** | **+41 (+18.7%)** | +| 4 | 0.990414 | 110 | 130 | +20 (+18.2%) | + +**Is `grinding_factor: 0` admissible?** ✓ VERIFIED **yes**, on both counts: +`security_bits <= grinding_factor` is `128 <= 0` = false, so `with_params` +returns `Ok` (`options.rs:114-119`); and every grinding call site is gated on +`security_bits > 0` (`prover.rs:2092`, `verifier.rs:1584`, `verifier.rs:1666`), +so `grinding.rs:22`'s `debug_assert!((1..=64).contains(..))` is never reached. +Admissible — just expensive. + +**The cost of dropping grinding, three ways** (blowup 2, 128-bit): + +| axis | effect | +|---|---| +| **(a) proof size** | +18.7%. Every query contributes, per tree, `evaluations ‖ evaluations_sym` plus a Merkle path; all of it scales linearly in query count. | +| **(b) prover work** | +18.7% on the query phase (path gathers, FRI openings). Total prover time grows by *less*, since LDE and commit are query-independent. | +| **(c) ★ tower verifier permutations** | **+222,794 per wrap verify** — 41 extra queries × **5,434 leg permutations per query** (✓ measured, `CENSUS.md` §2b, closed-form checked against `epoch_verify::query_permutations`). | + +For scale: `CENSUS.md` §3 records that the 93 GiB box budget holds ~**40.8k +permutations in total**. The grinding-0 delta *alone* is ~**5.5× the entire box +budget** — and the tower pays it on every layer, for every proof, forever. + +> ⚠ **Caveat on the absolute figure.** 5,434 perms/query is the *epoch* +> verifier (keccak inner) at 2^21/blowup2. The tower's LFM-proof verifier has +> not been censused (Gate D1). The **+18.7% is exact and hash-independent**; the +> 222,794 is an order-of-magnitude anchor from the nearest measured shape. + +**The cost of D4's squeeze-twice.** ✓ VERIFIED the extension challenges a verify +actually samples: + +- per table — `beta` (`verifier.rs:1470`), `z` (`:1501`), `gamma` (`:1535`), and + `zetas` = one per committed FRI root (`:1557`) plus one final-fold challenge + (`:1572`), i.e. `total_folds` of them; +- once per multi-proof — `LOGUP_NUM_CHALLENGES = 2` (`lookup.rs:105`, + `verifier.rs:1312`). + +Everything else is free: the boundary, transition, trace-term and DEEP +coefficients are **powers** of `beta`/`gamma` (`verifier.rs:1538-1541`), not +squeezes, and query indices go through `sample_u64` → `squeeze_bits`, which +reads lane 0 only and needs no extra entropy. + +`total_folds = lde_log − min(blowup_log + k, lde_log)` with `k = 7` +(`fri/terminal.rs:45-55`), so a 2^22-row table at blowup 2 gives +`num_committed = 14` — ✓ consistent with `CENSUS.md` §3's observed "14 FRI +layers" at that shape. + +| shape | ext challenges | **Δ permutations (+1 each)** | +|---|---|---| +| tower: 14 LFM tables, 2^22 rows | 14 × 18 + 2 | **254** | +| epoch verifier: 64 sub-proofs, 2^22 rows | 64 × 18 + 2 | **1,154** | + +### 7.2 The conclusion, and a correction + +**D3 costs ~200–900× what D4 costs** (222,794 against 254–1,154 permutations per +wrap verify). Both rulings take the cheap-per-bit option: keep the security +grinding buys for a flat one-time PoW, and buy the entropy `squeeze_ext` lacks +for ~254 permutations. + +> **§4.1's recommendation ("`grinding_factor: 0`; grinding out of scope") was +> wrong, and this section is why.** It reasoned qualitatively — *a guest +> recomputing keccak PoW defeats the purpose* — which is true but answers the +> wrong question. The right comparison is one PoW recomputation per wrap verify +> against 222,794 extra permutations per wrap verify, forever, in exactly the +> cost centre D0 exists to shrink. The correct move is the one Mauro took: keep +> grinding and **specify a blake3 PoW**, so the guest recomputes a cheap PoW +> instead of paying 41 extra queries. +> +> The lesson is the one this campaign keeps relearning: an open item carried +> into a recommendation stops looking open. §4.1 recorded the compensating-query +> cost as "not quantified here" and then recommended anyway. + +**D6 — discharged in this pass.** The B1 PoW the D3 ruling requires is specified +in **§4.1**, with a domain-separation argument (§4.1.3) and thirteen KAT legs +(**C11**), including both cross-domain directions and the factor/seed/marker +bindings. One sub-question stays open, **D6a**: whether to give the PoW its own +tag and selector rather than lean on the fixed-sequence argument. + +--- + +## 8. Files + +| file | what | +|---|---| +| `commit_ref.py` | the reference — wide leaf, byte absorb, node codec, tree, both squeeze options | +| `commit_kats.py`, `commit_kats.json` | C1–C10 + 13 pinned vectors, both round counts | +| `run-kats.log` | the executed board, 54/54 | + +Imports resolve relatively to `../../lfm-real-hash/{gate-oracle,leaf-spec,transcript-spec}`; +no absolute paths, no worktree assumptions. diff --git a/thoughts/shared/block-compression/commit-spec/commit_kats.json b/thoughts/shared/block-compression/commit-spec/commit_kats.json new file mode 100644 index 000000000..699b656bd --- /dev/null +++ b/thoughts/shared/block-compression/commit-spec/commit_kats.json @@ -0,0 +1,29 @@ +{ + "C1.base.m5.r6": "e4de1ba3f273bba4c1d0da85c28957ed", + "C1.base.m5.r7": "806c1f1975c2a7b04f3f0dc89d1c024e", + "C11.pow.r6": "b43d97e1a77a70fad01192b572b8b90d", + "C11.pow.r7": "a7b9d7f34c4ec0df950b5141ba2e0583", + "C12.per_query_new": 3024, + "C12.per_query_old": 6048, + "C12.row.r6": "61d2905bca9474b6ecd679c9e63eb0c7", + "C12.row.r7": "98fac621148797b93c1b3fc843686fcb", + "C2.ext3.m3.r6": "6fcdd59d1cbf8ab32eee6aaeae8b3816", + "C2.ext3.m3.r7": "1190621f28c484c3ce5619f6653e9811", + "C4.pad.m1": "ada1fa7dd1e9f8b72e9185bf9180f146", + "C4.pad.m2": "0122606e7f11746d5986b9ad9aa06d95", + "C5.absorb.r6": "361e045d27573487578eb67262269371", + "C5.absorb.r7": "e64918f40c0ec458ab437ac9bd10fae7", + "C6.pack": "6745230100000000efcdab89000000000000000000000000ffffffff00000000", + "C7.root.n8.r6": "4b000a65c2b6f26548a89340f3cbf2c2", + "C7.root.n8.r7": "3b9b427c80bd2d449d67e879a6c28d84", + "C8.ext1": [ + "ff4f7652", + "b00b7366", + "b27cf36d" + ], + "C8.ext2": [ + "b00b7366ff4f7652", + "4b1f4918b27cf36d", + "6b37f0dcfb3d5744" + ] +} \ No newline at end of file diff --git a/thoughts/shared/block-compression/commit-spec/commit_kats.py b/thoughts/shared/block-compression/commit-spec/commit_kats.py new file mode 100644 index 000000000..fffac180f --- /dev/null +++ b/thoughts/shared/block-compression/commit-spec/commit_kats.py @@ -0,0 +1,502 @@ +""" +KATs for the LFM-native commitment layer — C1..C10. + +DRAFT — PENDING MAURO RATIFICATION. + +Run: python3 commit_kats.py (check against commit_kats.json) + python3 commit_kats.py --write (regenerate the vectors) + +Discipline, inherited from `leaf_kats.py` / `transcript_kats.py`: every negative +control is paired with an HONEST leg, because a construction that rejected +everything would pass a negative-only suite. +""" + +from __future__ import annotations + +import json +import os +import sys + +import commit_ref as cr + +_HERE = os.path.dirname(os.path.abspath(__file__)) +VECTORS = os.path.join(_HERE, "commit_kats.json") + +ROUNDS = (6, 7) +P = cr.P + +results: list[tuple[str, str, str]] = [] # (id, what, verdict) +vectors: dict = {} + + +def check(cid: str, what: str, ok: bool) -> None: + results.append((cid, what, "PASS" if ok else "FAIL")) + if not ok: + print(f" !! {cid} FAILED: {what}") + + +def hexw(w: list[int]) -> str: + return "".join(f"{x:08x}" for x in w) + + +# --- fixtures --------------------------------------------------------------- + +def base_row(m: int, seed: int) -> list[int]: + return [(seed * 1000 + i * 7 + 1) % P for i in range(m)] + + +def ext_row(m: int, seed: int) -> list[list[int]]: + return [[(seed * 1000 + i * 7 + c + 1) % P for c in range(3)] + for i in range(m)] + + +# =========================================================================== +# C1 — the wide leaf over a BASE matrix, both round counts +# =========================================================================== +def c1() -> None: + m = 5 # 2*5*1 = 10 felts -> 3 chunks (pad 2) + ev, sym = base_row(m, 1), base_row(m, 2) + for r in ROUNDS: + d = cr.wide_leaf(ev, sym, cr.KIND_BASE, m, rounds=r) + vectors[f"C1.base.m{m}.r{r}"] = hexw(d) + check("C1", f"base m={m} r={r} digest is 4 u32 lanes", + len(d) == 4 and all(0 <= x <= cr.MASK32 for x in d)) + # determinism + a = cr.wide_leaf(ev, sym, cr.KIND_BASE, m) + b = cr.wide_leaf(ev, sym, cr.KIND_BASE, m) + check("C1", "deterministic", a == b) + check("C1", "cost formula matches the chain length (10 felts / rate 4)", + cr.wide_leaf_compressions(m, cr.KIND_BASE) == 3) + + +# =========================================================================== +# C2 — the wide leaf over an EXT3 matrix +# =========================================================================== +def c2() -> None: + m = 3 # 2*3*3 = 18 felts -> 5 chunks (pad 2) + ev, sym = ext_row(m, 3), ext_row(m, 4) + for r in ROUNDS: + d = cr.wide_leaf(ev, sym, cr.KIND_EXT3, m, rounds=r) + vectors[f"C2.ext3.m{m}.r{r}"] = hexw(d) + check("C2", f"ext3 m={m} r={r} digest well formed", + len(d) == 4 and all(0 <= x <= cr.MASK32 for x in d)) + check("C2", "cost formula matches the chain length (18 felts / rate 4)", + cr.wide_leaf_compressions(m, cr.KIND_EXT3) == 5) + # base and ext3 over the SAME felt count must differ (the kind is bound) + m_b = 9 # 2*9*1 = 18 felts, same as above + d_base = cr.wide_leaf(base_row(m_b, 3), base_row(m_b, 4), cr.KIND_BASE, m_b) + d_ext = cr.wide_leaf(ev, sym, cr.KIND_EXT3, m) + check("C2", "same felt count, different kind -> different leaf", + d_base != d_ext) + + +# =========================================================================== +# C3 — ★ WIDTH BINDING: the recorded live break, and the honest leg +# =========================================================================== +def c3() -> None: + # The break (verifier.rs:633-639): a prover moves one column from the main + # tree into the aux tree, choosing it after the LogUp challenges. Under the + # keccak leaf both leaves still hash the bytes they were given and nothing + # in the leaf noticed. Here the width is IN the preimage. + m = 6 + ev, sym = base_row(m, 5), base_row(m, 6) + + honest = cr.wide_leaf(ev, sym, cr.KIND_BASE, m) + check("C3", "HONEST leg: the true width still verifies", + honest == cr.wide_leaf(ev, sym, cr.KIND_BASE, m)) + + # A verifier that built the header from the AIR (m) while the prover shipped + # a different width gets a different leaf -> authentication fails. + shrunk = cr.wide_leaf(ev[:m - 1], sym[:m - 1], cr.KIND_BASE, m - 1) + check("C3", "a narrower opening yields a different leaf", shrunk != honest) + + # ★ The decisive one, and it is the main<->aux confusion in miniature: + # 6 BASE columns and 2 EXT3 columns both serialize to the SAME 12 felts. + # Under the keccak leaf those two openings are byte-identical preimages, so + # one leaf hash authenticates both — exactly the shape that let a prover + # move columns between the main (base) and aux (ext) trees. The header + # separates them because it binds the KIND as well as the width. + a, b, c, d, e, f = ev + g, h, i, j, k, l = sym + same_felts_base = cr.wide_leaf(ev, sym, cr.KIND_BASE, 6) + same_felts_ext = cr.wide_leaf([[a, b, c], [d, e, f]], + [[g, h, i], [j, k, l]], cr.KIND_EXT3, 2) + check("C3", "★ identical felt stream, base vs ext3 -> different leaf " + "(the main<->aux confusion, closed by the header)", + same_felts_base != same_felts_ext) + + # The reference REFUSES to derive the width from the data. + try: + cr.wide_leaf(ev, sym, cr.KIND_BASE, m + 1) + check("C3", "a width disagreeing with the data is refused", False) + except AssertionError: + check("C3", "a width disagreeing with the data is refused", True) + + +# =========================================================================== +# C4 — padding is unambiguous BECAUSE the header binds the count +# =========================================================================== +def c4() -> None: + # m=1 base: 2 felts, padded with 2 zeros. m=2 base: 4 felts, no padding. + # If the padded stream of m=1 equalled the stream of m=2 with two zero + # columns, only the header would separate them. Construct exactly that. + ev1, sym1 = [7], [9] # -> [7, 9, 0, 0] + ev2, sym2 = [7, 9], [0, 0] # -> [7, 9, 0, 0] + d1 = cr.wide_leaf(ev1, sym1, cr.KIND_BASE, 1) + d2 = cr.wide_leaf(ev2, sym2, cr.KIND_BASE, 2) + check("C4", "★ colliding padded felt streams separated by the header", + d1 != d2) + vectors["C4.pad.m1"] = hexw(d1) + vectors["C4.pad.m2"] = hexw(d2) + + # honest leg: padding is stable, not random + check("C4", "HONEST leg: padded leaf is deterministic", + d1 == cr.wide_leaf(ev1, sym1, cr.KIND_BASE, 1)) + + # a zero-width matrix is still a well-defined (header-only) leaf + d0 = cr.wide_leaf([], [], cr.KIND_BASE, 0) + check("C4", "zero-width leaf is the bare header", + d0 == cr.leaf_header(0, cr.KIND_BASE)) + + +# =========================================================================== +# C5 — the byte -> cell absorb encoding +# =========================================================================== +def c5() -> None: + # O1: every lane is exactly four bytes, hence < 2^32, with no gate. + blob = bytes(range(37)) + cells = cr.bytes_to_cells(blob) + check("C5", "every lane is a u32 (O1 automatic)", + all(0 <= x <= cr.MASK32 for c in cells for x in c)) + check("C5", "cell count is header + ceil(len/16)", + len(cells) == 1 + 3 and cr.absorb_bytes_compressions(37) == 4) + + # ★ injectivity under zero-padding — the reason for the length prefix. + check("C5", "★ b'\\x01' and b'\\x01\\x00' encode differently", + cr.bytes_to_cells(b"\x01") != cr.bytes_to_cells(b"\x01\x00")) + check("C5", "empty string is header-only", len(cr.bytes_to_cells(b"")) == 1) + + # a pinned end-to-end vector through the B1 chain + import transcript_ref as tr + for r in ROUNDS: + t = tr.Transcript(rounds=r) + cr.absorb_bytes(t, b"LAMBDAVM_LFM_STATEMENT_V1") + vectors[f"C5.absorb.r{r}"] = hexw(t.state) + check("C5", "absorb advances the chain", True) + + # HONEST leg: the encoding round-trips the bytes it claims to carry + recovered = b"" + for c in cells[1:]: + for lane in c: + recovered += int(lane).to_bytes(4, "little") + check("C5", "HONEST leg: body bytes recover the input under its length", + recovered[:len(blob)] == blob) + + +# =========================================================================== +# C6 — node embedding and STRICT decode (S2 malleability) +# =========================================================================== +def c6() -> None: + word = [0x01234567, 0x89abcdef, 0x00000000, 0xffffffff] + packed = cr.pack_digest(word) + check("C6", "pack is 32 bytes", len(packed) == 32) + check("C6", "HONEST leg: pack -> strict_unpack round-trips", + cr.strict_unpack_digest(packed) == word) + check("C6", "the sixteen padding bytes are zero", + all(packed[8 * i + 4:8 * i + 8] == b"\x00" * 4 for i in range(4))) + vectors["C6.pack"] = packed.hex() + + # ★ every non-canonical flavour must REJECT, not reduce. + def rejects(b: bytes, label: str) -> None: + try: + cr.strict_unpack_digest(b) + check("C6", f"★ rejects {label}", False) + except ValueError: + check("C6", f"★ rejects {label}", True) + + ba = bytearray(packed); ba[4] = 0x01 + rejects(bytes(ba), "a set high byte in lane 0 (the cheap forgery)") + ba = bytearray(packed); ba[8 * 3 + 7] = 0x80 + rejects(bytes(ba), "a set top byte in lane 3") + # the lane p + 1, which unpack_digest would reduce to 1 + ba = bytearray(packed); ba[0:8] = (P + 1).to_bytes(8, "little") + rejects(bytes(ba), "a lane congruent to 1 mod p") + rejects(packed[:31], "a short commitment") + + # and the honest control that the fix is not "reject everything" + check("C6", "HONEST leg: an all-zero digest still decodes", + cr.strict_unpack_digest(b"\x00" * 32) == [0, 0, 0, 0]) + + +# =========================================================================== +# C7 — tree arity and padding (S6) +# =========================================================================== +def c7() -> None: + leaves = [[i, i + 1, i + 2, i + 3] for i in range(8)] + for r in ROUNDS: + root = cr.merkle_root(leaves, rounds=r) + vectors[f"C7.root.n8.r{r}"] = hexw(root) + check("C7", "HONEST leg: a power-of-two tree builds", + len(cr.merkle_root(leaves)) == 4) + check("C7", "a single leaf is its own root", + cr.merkle_root([leaves[0]]) == leaves[0]) + try: + cr.merkle_root(leaves[:7]) + check("C7", "★ a non-power-of-two leaf count is refused", False) + except AssertionError: + check("C7", "★ a non-power-of-two leaf count is refused", True) + + +# =========================================================================== +# C8 — the 96-bit question, costed +# =========================================================================== +def c8() -> None: + import transcript_ref as tr + t1 = tr.Transcript() + t1.absorb([1, 2, 3, 4]) + before = t1.compressions + e1 = cr.squeeze_ext_1(t1) + cost1 = t1.compressions - before + check("C8", "squeeze_ext_1 costs one compression", cost1 == 1) + check("C8", "★ its coordinates are u32-bounded (96 bits total)", + all(0 <= x <= cr.MASK32 for x in e1)) + + t2 = tr.Transcript() + t2.absorb([1, 2, 3, 4]) + before = t2.compressions + e2 = cr.squeeze_ext_2(t2) + cost2 = t2.compressions - before + check("C8", "squeeze_ext_2 costs two compressions (+1 flat)", cost2 == 2) + check("C8", "its coordinates span the full field", + all(0 <= x < P for x in e2) and any(x > cr.MASK32 for x in e2)) + vectors["C8.ext1"] = [f"{x:08x}" for x in e1] + vectors["C8.ext2"] = [f"{x:016x}" for x in e2] + + +# =========================================================================== +# C9 — the crate anchor survives: LFML rows are still plain blake3 at 7 rounds +# =========================================================================== +def c9() -> None: + import leaf_ref as lr + felts = [1, 2**32, P - 1, 0] + word = lr.leaf_compress(felts, 7) + byte = lr.leaf_compress_bytelevel(felts, 7) + check("C9", "★ the wide leaf's LFML rows keep the byte-level anchor @7r", + word == byte) + # the fold is the ratified LFMC socket, unchanged + import socket_ref as sk + a, b = [1, 2, 3, 4], [5, 6, 7, 8] + check("C9", "the fold is the honest LFMC socket", + sk.socket_digest_wordlevel(a, b, sk.Framing(rounds=7)) + == sk.socket_digest(a, b, sk.Framing(rounds=7))) + + +# =========================================================================== +# C10 — the header is load-bearing (domain separation of the construction) +# =========================================================================== +def c10() -> None: + import socket_ref as sk + import leaf_ref as lr + # A one-chunk wide leaf must NOT equal the bare LFML digest of those felts, + # nor an LFMC of them: the header fold is what separates them. + felts = [11, 22, 33, 44] + wide = cr.wide_leaf([11, 22], [33, 44], cr.KIND_BASE, 2) + bare = lr.leaf_compress(felts, 7) + check("C10", "★ a wide leaf is not the bare LFML digest", wide != bare) + check("C10", "★ a wide leaf is not an unheaded LFMC fold", + wide != sk.socket_digest_wordlevel([0, 0, 0, 0], bare, + sk.Framing(rounds=7))) + # honest leg: at RATE = 4 these four felts are exactly one row, no padding. + check("C10", "HONEST leg: it is exactly lfml_chain_row(header, felts)", + wide == cr.lfml_chain_row(cr.leaf_header(2, cr.KIND_BASE), felts)) + + +# =========================================================================== +# C11 — grinding under B1 (D3 ratified: grinding STAYS) +# =========================================================================== +def c11() -> None: + import socket_ref as sk + import leaf_ref as lr + state = [0x11111111, 0x22222222, 0x33333333, 0x44444444] + FACTOR = 12 # ~4096 trials, fast in python + + nonce = cr.find_nonce(state, FACTOR) + check("C11", "HONEST leg: a mined nonce satisfies the difficulty", + nonce is not None and cr.pow_is_valid(state, nonce, FACTOR)) + check("C11", "the difficulty actually bites (mining was not trivial)", + nonce is not None and nonce > 0) + for r in ROUNDS: + vectors[f"C11.pow.r{r}"] = hexw(cr.pow_digest(state, 338, FACTOR, r)) + + # ★ the difficulty is IN the preimage: a nonce mined at one factor is + # worthless at another. Without `factor` in the operand a prover mines once + # at factor 1 and presents the result at factor 20. + check("C11", "★ the factor is bound into the digest", + cr.pow_digest(state, 338, 12) != cr.pow_digest(state, 338, 13)) + + # ★ the seed is bound: a nonce is not portable across transcript states. + other = [0x11111111, 0x22222222, 0x33333333, 0x44444445] + check("C11", "★ the transcript state is bound into the digest", + cr.pow_digest(state, 338, 12) != cr.pow_digest(other, 338, 12)) + + # ★ GRIND_MARK is load-bearing (defence in depth, per the docstring). + import transcript_ref as tr + unmarked = tr.compress_t(state, [338, 0, 0, 12], 7) + check("C11", "★ GRIND_MARK changes the digest", + cr.pow_digest(state, 338, 12) != unmarked) + + # ★★ THE IDENTITY THE FIXED-SEQUENCE ARGUMENT MUST CARRY. + # A PoW step and an ABSORB of the operand cell are the SAME FUNCTION — both + # are compress_T(state, cell). No KAT can separate them and none pretends to: + # the separation is the program's compile-time operation sequence, exactly as + # TRANSCRIPT.md §1.1 says for absorb-vs-squeeze. This leg asserts the identity + # so the reliance is VISIBLE in the board rather than buried in prose — if a + # future change gives the PoW its own tag (D6a), this leg flips and says so. + op = cr.grind_operand(338, 12) + t_absorb = tr.Transcript() + t_absorb.state = list(state) + t_absorb.absorb(op) + check("C11", "★★ a PoW step IS an absorb of its operand cell — separation " + "rests on the fixed program sequence, NOT on the hash (D6a)", + cr.pow_digest(state, 338, 12) == t_absorb.state) + + check("C11", "★ a PoW step is not an LFMC Merkle parent of the same cells", + cr.pow_digest(state, 338, 12) + != sk.socket_digest_wordlevel(state, op, sk.Framing(rounds=7))) + check("C11", "★ a PoW step is not an LFML leaf of the same felts", + cr.pow_digest(state, 338, 12) != lr.leaf_compress(op, 7)) + + # the difficulty rule: lane 0 for factor <= 32, lane 1 above. + w = cr.pow_digest(state, 338, 20) + check("C11", "factor <= 32 reads lane 0 only", + cr.pow_is_valid(state, 338, 20) == (w[0] % (1 << 20) == 0)) + check("C11", "factor > 32 requires lane 0 fully zero (so this sample fails)", + w[0] != 0 and not cr.pow_is_valid(state, 338, 33)) + + # range discipline, mirroring grinding.rs:22's 1..=64 + for bad, label in ((0, "factor 0"), (65, "factor 65")): + try: + cr.grind_operand(1, bad) + check("C11", f"★ rejects {label}", False) + except ValueError: + check("C11", f"★ rejects {label}", True) + try: + cr.grind_operand(2**64, 20) + check("C11", "★ rejects an out-of-range nonce", False) + except ValueError: + check("C11", "★ rejects an out-of-range nonce", True) + + check("C11", "verification is ONE compression, independent of difficulty", + cr.pow_verify_compressions() == 1) + + +# =========================================================================== +# C12 — ★ THE LEAF RATE (the parameter that decides Gate D1) +# =========================================================================== +def c12() -> None: + check("C12", "the spec parameter is 4 felts/row with a 4-lane accumulator", + cr.LFML_FELTS_PER_ROW == 4 and cr.LFML_ACC_LANES == 4) + # ★★ the binding constraint: a hash row reads whole CELLS of 4 felts + # (instr.rs:99-110 + word.rs:15), so the rate MUST be a multiple of 4. + check("C12", "★★ the rate is a multiple of 4 (whole machine cells)", + cr.LFML_FELTS_PER_ROW % 4 == 0) + check("C12", "★★ it fits the EXISTING 2-cells-in bus contract " + "(acc cell + one felt cell = 12 lanes + tag = 13 of 16 words)", + cr.LFML_ACC_LANES + 2 * cr.LFML_FELTS_PER_ROW + 1 <= 16) + + # ★ one compression per row, not two: the fold is gone. + m = 10 # 20 felts -> 5 rows exactly + ev, sym = base_row(m, 11), base_row(m, 12) + check("C12", "★ rate is 4 felts/compression (was 2)", + cr.wide_leaf_compressions(m, cr.KIND_BASE) == 5 + and cr.wide_leaf_v0_compressions(m, cr.KIND_BASE) == 10) + check("C12", "★ that is a 2.0x improvement on the dominant cost", + cr.wide_leaf_v0_compressions(m, cr.KIND_BASE) + / cr.wide_leaf_compressions(m, cr.KIND_BASE) == 2.0) + + # the widened row is ONE blake3 block: 16 acc + 40 felt halves + 4 tag = 60 + d = cr.lfml_chain_row([1, 2, 3, 4], [7, 8, 9, 10]) + check("C12", "★ a full row is 52 bytes — still one BLAKE3 block (<= 64)", + len(d) == 4) + for r in ROUNDS: + vectors[f"C12.row.r{r}"] = hexw(cr.lfml_chain_row([1, 2, 3, 4], + [7, 8, 9, 10], r)) + + # ★ the crate anchor survives the widening — this is the reason to prefer + # the in-message accumulator over carrying it in the chaining value h. + import blake3_oracle as ora + msg = (b"".join(int(x).to_bytes(4, "little") for x in [1, 2, 3, 4]) + + b"".join(int(x).to_bytes(4, "little") for x in + [7, 0, 8, 0, 9, 0, 10, 0]) + + b"LFML") + full = ora.hash_bytes(msg, 32, rounds=7) + check("C12", "★ row == plain blake3::hash(52 bytes) @7r (anchor intact)", + len(msg) == 52 + and cr.lfml_chain_row([1, 2, 3, 4], [7, 8, 9, 10], 7) + == [int.from_bytes(full[4 * i:4 * i + 4], "little") for i in range(4)]) + + # the header still binds shape at the new rate (C3/C4 properties survive) + a, b, c, dd, e, f = base_row(6, 5) + g, h, i, j, k, l = base_row(6, 6) + check("C12", "★ base-vs-ext3 separation survives the rate change", + cr.wide_leaf([a, b, c, dd, e, f], [g, h, i, j, k, l], cr.KIND_BASE, 6) + != cr.wide_leaf([[a, b, c], [dd, e, f]], [[g, h, i], [j, k, l]], + cr.KIND_EXT3, 2)) + check("C12", "★ padding still separated by the header at rate 4", + cr.wide_leaf([7], [9], cr.KIND_BASE, 1) + != cr.wide_leaf([7, 9], [0, 0], cr.KIND_BASE, 2)) + + # non-canonical felts are still REJECTED, not reduced + try: + cr.lfml_chain_row([1, 2, 3, 4], [cr.P, 1, 2, 3]) + check("C12", "★ a non-canonical felt still rejects", False) + except ValueError: + check("C12", "★ a non-canonical felt still rejects", True) + + # per-query tower cost at the real widths, old rate vs new. + # LFM_HASH is 3444 MAIN columns, not cols::NUM_COLUMNS' 3457: the 13 + # preprocessed columns are committed in the precomputed tree, not the main + # tree this census is scoped to (COMMIT.md §1.5 note (4)). + widths = [3444, 1480, 792, 196, 25, 23, 21, 16, 14, 10, 7, 7, 6, 2] + old = sum(cr.wide_leaf_v0_compressions(w, cr.KIND_BASE) for w in widths) + new = sum(cr.wide_leaf_compressions(w, cr.KIND_BASE) for w in widths) + check("C12", f"★ per-query main-tree leaf cost {old} -> {new} " + f"({old / new:.2f}x)", old > new) + vectors["C12.per_query_old"] = old + vectors["C12.per_query_new"] = new + + +def main() -> int: + write = "--write" in sys.argv + for fn in (c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12): + fn() + + if write: + with open(VECTORS, "w") as f: + json.dump(vectors, f, indent=2, sort_keys=True) + print(f"wrote {VECTORS} ({len(vectors)} vectors)") + elif os.path.exists(VECTORS): + with open(VECTORS) as f: + pinned = json.load(f) + for k, v in vectors.items(): + check("PIN", f"{k} matches the pinned vector", pinned.get(k) == v) + else: + print("no vector file yet — run with --write") + + width = max(len(w) for _, w, _ in results) + by_id: dict[str, list[int]] = {} + for cid, _, verdict in results: + by_id.setdefault(cid, [0, 0]) + by_id[cid][0 if verdict == "PASS" else 1] += 1 + print() + for cid, what, verdict in results: + print(f" {cid:<4} {what:<{width}} {verdict}") + print() + npass = sum(1 for _, _, v in results if v == "PASS") + print(f"BOARD: {' | '.join(f'{k} {v[0]}/{v[0]+v[1]}' for k, v in by_id.items())}") + print(f"TOTAL: {npass}/{len(results)} PASS") + return 0 if npass == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/thoughts/shared/block-compression/commit-spec/commit_ref.py b/thoughts/shared/block-compression/commit-spec/commit_ref.py new file mode 100644 index 000000000..bf0132674 --- /dev/null +++ b/thoughts/shared/block-compression/commit-spec/commit_ref.py @@ -0,0 +1,560 @@ +""" +THE LFM-NATIVE COMMITMENT LAYER — reference implementation. + +DRAFT — PENDING MAURO RATIFICATION. Nothing here is ratified; every open +decision is marked `OPEN:` in the docstrings and listed in COMMIT.md §7. + +This is the reference for the three things no ratified doc covers when the LFM +machine's OWN proof moves to the machine's native hashing scheme: + + 1. WIDE LEAF — an arbitrary-width row pair -> an LFML chain folded by LFMC, + with the width bound INSIDE the construction. + 2. BYTE ABSORB — a canonical byte-string -> cell encoding for the B1 + transcript, since `DefaultTranscript` absorbs bytes and B1 + absorbs 4xu32 cells. + 3. NODE CODEC — `pack_digest` into [u8;32] plus a STRICT decode that rejects + the non-canonical encodings `unpack_digest` would silently + reduce. + +It builds on the ratified LFMC / LFML / LFMT sockets and adds no new tag: the +wide leaf is LFML rows folded by LFMC parents, exactly the two domains +`leaf-spec/LEAF.md` already ratified. + +Runnable with plain python3; no cargo, no third-party packages. +""" + +from __future__ import annotations + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_LFM = os.path.join(_HERE, "..", "..", "lfm-real-hash") +for _p in ("gate-oracle", "leaf-spec", "transcript-spec"): + sys.path.insert(0, os.path.join(_LFM, _p)) + +import blake3_oracle as ora # noqa: E402 +import socket_ref as sk # noqa: E402 +import leaf_ref as lr # noqa: E402 +import transcript_ref as tr # noqa: E402 + +P = 2**64 - 2**32 + 1 # Goldilocks +MASK32 = 0xFFFFFFFF +LANES = sk.DIGEST_LANES # 4 +FELTS_PER_LFML_ROW = lr.FELTS_PER_LEAF_ROW # 4 + +# The production row-pair leaf: leaf i covers bit-reversed rows 2i, 2i+1. +# `stark::commitment::ROWS_PER_LEAF = 2`. +ROWS_PER_LEAF = 2 + +# --- Element kinds ---------------------------------------------------------- +# A committed matrix is uniform in element type: `verify_opening_pair` is +# instantiated at `Field` for the main and precomputed trees and at +# `FieldExtension` for the aux and composition trees (verifier.rs:605-650). +# The felts-per-element count doubles as the kind tag: it is injective over the +# kinds that exist and is the number the serializer actually needs. +KIND_BASE = 1 # one Goldilocks felt per element +KIND_EXT3 = 3 # three components per element + +# --- Domain markers --------------------------------------------------------- +# These are LANE CONSTANTS inside a header cell, not new socket tags. No new +# tag is allocated: `LFMW`/`LFMB` would be new hash domains needing their own +# analysis, and the construction does not need one — the header cell is an +# ordinary LFMC operand at a program-fixed chain position. +# +# OPEN (D1): whether these should instead be genuine `m[8]` tags. Argued +# against in COMMIT.md §2.4; the counter-argument is that a lane constant is +# only as good as the chain position that carries it. +LEAF_MARK = int.from_bytes(b"LFW0", "little") # wide-leaf header +BYTES_MARK = int.from_bytes(b"LFB0", "little") # byte-absorb header + + +# =========================================================================== +# 1. THE WIDE LEAF +# =========================================================================== + +def serialize_elements(elements: list, kind: int) -> list[int]: + """One row's elements -> a flat felt sequence. + + Base elements contribute one felt; ext3 elements contribute their three + components in order (c0, c1, c2). This mirrors the existing byte layout, + where `write_bytes_be` writes an extension element's components 0, 1, 2 in + that order (`sub_proof.rs:234-236`). + """ + out: list[int] = [] + for e in elements: + if kind == KIND_BASE: + assert isinstance(e, int), "a base element is one felt" + out.append(e) + elif kind == KIND_EXT3: + assert len(e) == 3, "an ext3 element is three components" + out.extend(int(c) for c in e) + else: + raise ValueError(f"unknown element kind {kind}") + return out + + +def leaf_header(num_cols: int, kind: int, + rows_per_leaf: int = ROWS_PER_LEAF) -> list[int]: + """The header cell that binds the leaf's SHAPE. + + H = [ LEAF_MARK, num_cols, kind, rows_per_leaf ] + + THIS IS THE WHOLE POINT OF THE CONSTRUCTION. The keccak leaf it replaces + streams `evaluations ‖ evaluations_sym` with no length prefix and no + separator (`verifier.rs:204-206`), which is why a prover could move columns + between the main and aux trees and choose them AFTER the LogUp challenges + the aux root is absorbed behind (`verifier.rs:633-639`, the recorded live + break, `tests::aux_opening_width_tests`). + + ⚠ The header only closes that break if the VERIFIER BUILDS IT FROM THE AIR, + never from the opening it received. That is the exact analogue of the + existing instruction at `verifier.rs:639` — "The width is pinned upstream by + `trace_opening_widths_well_formed`; do not re-derive it from the proof." + A verifier that read `num_cols` off `len(evaluations)` would reproduce the + prover's own choice and bind nothing. + """ + assert 0 <= num_cols <= MASK32 + assert kind in (KIND_BASE, KIND_EXT3) + assert 0 <= rows_per_leaf <= MASK32 + return [LEAF_MARK, num_cols, kind, rows_per_leaf] + + +# =========================================================================== +# ★ THE LEAF RATE — the single most consequential parameter in this spec +# =========================================================================== +# +# It decides whether the recursion tower fits on real hardware: leaf absorption +# is 69.8% of a tower node's bill (Gate D1 census), so the rate scales ~70% of +# the cost linearly. +# +# ✓ VERIFIED from the chip, not assumed. `message_word_ref` +# (`blake3_socket.rs:725-731`) maps m[0..8] -> the eight input lanes' byte +# columns, m[8] -> the mode-selected tag, and **m[9..16] -> `WordRef::Const(0)`**. +# Seven message words are dead. BLAKE3's block is 16 words / 64 bytes; the +# socket uses 9 (block_len 36). +# +# Two ways to spend the headroom: +# +# ⚠⚠ THE BINDING CONSTRAINT IS THE MACHINE'S CELL STRUCTURE, NOT THE BLOCK. +# ✓ VERIFIED `instr.rs:99-110`: `HashMode::num_input_cells` is 2 for +# Compress/Transcript, 1 for Leaf, 3 for Permute, and the doc is explicit that +# "the LFM_HASH bus receives are gated by exactly this". A hash row reads whole +# CELLS from memory, and a cell is FOUR felts (`LfmWord`, `word.rs:15`). +# +# So the felt count per row must be a MULTIPLE OF 4. An earlier draft of this +# spec set the rate to 5 (accumulator cell + 5 felts, 14 lanes) purely from +# block headroom. That is 1.25 cells of felt input and is UNBUILDABLE — the +# machine cannot read it. Enumerating what actually fits: +# +# accumulator cell (4 lanes) + ONE felt cell (4 felts = 8 lanes) + tag +# = 13 of 16 words -> 4 felts/compression ✓ 2.0x ADOPTED +# accumulator cell + TWO felt cells = 4 + 16 + 1 = 21 words ✗ > 16 +# no accumulator, two felt cells = 16 + 1 = 17 words ✗ > 16 +# +# 4 IS THE MAXIMUM. And it lands on the EXISTING two-cells-in/one-cell-out bus +# contract (`num_input_cells == 2`, same as Compress/Transcript), so the frozen +# LFM_HASH bus arity does not move at all — a better outcome than the rate-5 +# draft, which would have needed a bus contract the machine does not have. +LFML_FELTS_PER_ROW = 4 # ★ the spec parameter — a MULTIPLE OF 4 by construction +LFML_ACC_LANES = 4 # the chained accumulator, one digest cell + + +def _u32le(x: int) -> bytes: + return int(x).to_bytes(4, "little") + + +def lfml_chain_row(acc: list[int], felts: list[int], + rounds: int = 7) -> list[int]: + """One widened LFML row: absorb `felts` AND chain `acc`, in one compression. + + msg = LE32(acc[0..4]) ‖ LE32(lo_i)‖LE32(hi_i) for each felt ‖ "LFML" + digest = BLAKE3(msg)[0..16] as four LE u32 lanes + + At `LFML_FELTS_PER_ROW = 4` that is 16 + 32 + 4 = **52 bytes** — still ONE + BLAKE3 block (64), so `block_len` moves 36 -> 52 and nothing else about the + framing does. + + ✓ THE CRATE-KAT ANCHOR SURVIVES, which is the reason to prefer this over + carrying the accumulator in the chaining value `h` (the D7 sketch). For any + input under 64 bytes `blake3::hash` is exactly one compression with `h = IV`, + `t = 0`, `block_len = len`, `flags = CHUNK_START|CHUNK_END|ROOT` — so a + 60-byte row is a plain library call just as the 36-byte row was. Moving the + accumulator into `h` would have made the row a chunk *continuation* and split + the anchor (C9). + + The accumulator lanes need byte decomposition (they are message words) but + NOT canonicity: they are a previous digest, hence u32 by construction. That + is why B costs fewer witness columns than A despite the higher rate. + """ + assert len(acc) == LFML_ACC_LANES + assert 1 <= len(felts) <= LFML_FELTS_PER_ROW + msg = b"".join(_u32le(x) for x in acc) + for v in felts: + lo, hi = lr.felt_halves(v) # REJECTS non-canonical, never reduces + msg += _u32le(lo) + _u32le(hi) + msg += lr.TAG_LFML_ASCII + assert len(msg) == 4 * LFML_ACC_LANES + 8 * len(felts) + 4 + full = ora.hash_bytes(msg, 32, rounds=rounds) + return [int.from_bytes(full[4 * i:4 * i + 4], "little") for i in range(4)] + + +def wide_leaf(evaluations: list, evaluations_sym: list, kind: int, + num_cols: int, rounds: int = 7) -> list[int]: + """The wide-leaf digest: an LFML chain folded into an LFMC chain. + + H = [LEAF_MARK, num_cols, kind, rows_per_leaf] + F = serialize(evaluations) ‖ serialize(evaluations_sym) + F' = F ‖ 0^r r = (-len F) mod 4 (zero-pad to 4 felts) + d_j = LFML(F'[4j : 4j+4]) + acc = H ; for each j: acc = LFMC(acc, d_j) + leaf = acc + + `num_cols` is passed in rather than read off the inputs, and the lengths are + CHECKED against it — a reference that derived the width from the data would + encode the very bug this construction exists to remove. + + ZERO-PADDING IS SAFE HERE, and only because the header binds the exact + element count: two different felt sequences that agree after padding must + have had different (num_cols, kind, rows_per_leaf), which the header + separates. Without the header, zero-padding is ambiguous. + + THE FOLD IS A SEQUENTIAL CHAIN, not a balanced tree. A balanced tree over k + chunk digests costs k-1 compressions against the chain's k, but needs the + chunk count padded to a power of two — reintroducing exactly the + shape-ambiguity the header was added to remove. One compression is not worth + a second padding rule. The chain also binds chunk ORDER for free. + + `rows_per_leaf` is NOT a parameter: the two-slice signature mirrors + production's `hash_data_from_slices(evaluations, evaluations_sym)` + (`verifier.rs:583`), which structurally IS the row pair. It is still bound in + the header, as the constant it is, so a future layout that changed it could + not collide with this one. + """ + assert len(evaluations) == num_cols, ( + f"evaluations has {len(evaluations)} columns, AIR pins {num_cols}") + assert len(evaluations_sym) == num_cols, ( + f"evaluations_sym has {len(evaluations_sym)} columns, AIR pins {num_cols}") + + felts = (serialize_elements(evaluations, kind) + + serialize_elements(evaluations_sym, kind)) + expected = ROWS_PER_LEAF * num_cols * kind + assert len(felts) == expected, f"{len(felts)} felts, shape implies {expected}" + + if not felts: + return leaf_header(num_cols, kind) + + pad = (-len(felts)) % LFML_FELTS_PER_ROW + felts = felts + [0] * pad + + acc = leaf_header(num_cols, kind) + for j in range(0, len(felts), LFML_FELTS_PER_ROW): + acc = lfml_chain_row(acc, felts[j:j + LFML_FELTS_PER_ROW], rounds) + return acc + + +def wide_leaf_compressions(num_cols: int, kind: int, + rows_per_leaf: int = ROWS_PER_LEAF) -> int: + """Compressions one wide leaf costs: ONE per `LFML_FELTS_PER_ROW` felts. + + The fold is gone — each row absorbs and chains in the same compression — so + this is `ceil(felts / rate)`, not `2 * ceil(felts / 4)`. + """ + felts = rows_per_leaf * num_cols * kind + return -(-felts // LFML_FELTS_PER_ROW) # ceil + + +def wide_leaf_v0_folded(evaluations: list, evaluations_sym: list, kind: int, + num_cols: int, rounds: int = 7) -> list[int]: + """The SUPERSEDED 4-felt + LFMC-fold construction, kept for the rate KAT. + + 2 felts/compression. Retained only so C12 can measure the improvement + against something executable rather than against a remembered number. + """ + felts = (serialize_elements(evaluations, kind) + + serialize_elements(evaluations_sym, kind)) + pad = (-len(felts)) % FELTS_PER_LFML_ROW + felts = felts + [0] * pad + acc = leaf_header(num_cols, kind) + fr = sk.Framing(rounds=rounds) + for j in range(0, len(felts), FELTS_PER_LFML_ROW): + d = lr.leaf_compress(felts[j:j + FELTS_PER_LFML_ROW], rounds) + acc = sk.socket_digest_wordlevel(acc, d, fr) + return acc + + +def wide_leaf_v0_compressions(num_cols: int, kind: int) -> int: + felts = ROWS_PER_LEAF * num_cols * kind + return 2 * (-(-felts // FELTS_PER_LFML_ROW)) + + +# =========================================================================== +# 2. THE BYTE -> CELL ABSORB ENCODING +# =========================================================================== + +def bytes_to_cells(data: bytes) -> list[list[int]]: + """A byte string -> a length-prefixed cell sequence, for B1 absorb. + + header = [BYTES_MARK, len & 0xFFFFFFFF, len >> 32, 0] + body = data zero-padded to a multiple of 16, each 16 bytes read as + four LITTLE-ENDIAN u32 lanes + + O1 COMPLIANCE IS AUTOMATIC AND THAT IS THE POINT: every lane is exactly four + bytes, so every lane is `< 2^32` by construction. No canonicity gate, no + rejection, no `MODE_L` row — a byte block is already digest-shaped. This is + why bytes go through THIS path and field elements go through `absorb_felts` + (`LFML`), which is where the canonicity gate lives. + + The length prefix is what makes the encoding injective under zero-padding: + without it `b"\\x01"` and `b"\\x01\\x00"` would absorb identically. + + Little-endian to match `word_of`'s convention (`blake3_socket.rs:441-443`: + "one felt = one u32 = four little-endian bytes"). + """ + n = len(data) + assert n < 2**64, "byte strings are length-prefixed with 64 bits" + header = [BYTES_MARK, n & MASK32, (n >> 32) & MASK32, 0] + + pad = (-n) % 16 + padded = data + b"\x00" * pad + body = [] + for i in range(0, len(padded), 16): + block = padded[i:i + 16] + body.append([int.from_bytes(block[4 * k:4 * k + 4], "little") + for k in range(4)]) + return [header] + body + + +def absorb_bytes(t: "tr.Transcript", data: bytes) -> "tr.Transcript": + """Absorb a byte string into a B1 transcript under the encoding above. + + Costs `1 + ceil(len/16)` compressions — the header cell plus one per block. + """ + for cell in bytes_to_cells(data): + t.absorb(cell) + return t + + +def absorb_bytes_compressions(nbytes: int) -> int: + return 1 + (-(-nbytes // 16)) + + +# =========================================================================== +# 3. NODE EMBEDDING AND STRICT DECODE +# =========================================================================== + +def pack_digest(word: list[int]) -> bytes: + """LfmWord -> the 32-byte `Commitment`, mirroring `word.rs:44-50`. + + Four canonical u64 lanes, little-endian, in lane order. Under BLAKE3 every + lane is `< 2^32` (`word_of`, `blake3_socket.rs:443`), so bytes 4..8 of each + 8-byte chunk are ZERO — the padding that lets a 128-bit digest ride inside + the existing 32-byte proof format without moving the rkyv wire layout. + """ + assert len(word) == LANES + out = b"" + for lane in word: + assert 0 <= lane < P, "a digest lane must be a canonical felt" + out += int(lane).to_bytes(8, "little") + return out + + +def strict_unpack_digest(b: bytes) -> list[int]: + """[u8;32] -> LfmWord, REJECTING everything `unpack_digest` would reduce. + + ⚠ THIS IS THE MALLEABILITY FIX (COMMIT.md S2). `word.rs:52-61`'s + `unpack_digest` reads each 8-byte chunk as a u64 and reduces mod p, so MANY + distinct 32-byte strings decode to ONE node: any lane may be offset by a + multiple of p, and — more cheaply — any of the sixteen zero padding bytes may + be set to anything below the reduction boundary. Node-level malleability in a + Merkle path is a proof-format forgery surface, not a cosmetic issue. + + The strict rule mirrors `lanes_of` (`blake3_socket.rs:431-438`), which + already rejects rather than reduces on the host: EVERY lane must be `< 2^32`, + i.e. the high four bytes of every chunk must be zero. `< 2^32` implies + `< p`, so one test covers both. + """ + if len(b) != 32: + raise ValueError(f"a commitment is 32 bytes, got {len(b)}") + word = [] + for i in range(LANES): + chunk = b[8 * i:8 * i + 8] + if chunk[4:] != b"\x00\x00\x00\x00": + raise ValueError( + f"lane {i} has non-zero high bytes {chunk[4:].hex()}: a BLAKE3 " + f"digest lane is a u32 (reject, never reduce)") + word.append(int.from_bytes(chunk[:4], "little")) + return word + + +# =========================================================================== +# 4. THE MERKLE TREE — arity and padding +# =========================================================================== + +def merkle_root(leaves: list[list[int]], rounds: int = 7) -> list[int]: + """Binary LFMC tree over wide-leaf digests. ASSERTS a power-of-two count. + + ARITY 2, NO PADDING, BY DECISION. The leaf count is always `lde_size / 2` + and `lde_size` is always a power of two (the prover debug-asserts exactly + this at `commitment.rs:67-70`), so the assertion costs nothing and is + ALWAYS satisfiable on the honest path. Padding to a power of two would add a + duplicate-leaf second-preimage surface for a case that does not arise, which + is the wrong trade: an unreachable branch that weakens the tree. + + Mirrors `fixture.rs:163-175`'s `HostTree::build`, which asserts the same. + """ + assert leaves, "a tree needs at least one leaf" + n = len(leaves) + assert n & (n - 1) == 0, ( + f"{n} leaves is not a power of two; the wide-leaf tree asserts rather " + f"than pads (COMMIT.md S6)") + fr = sk.Framing(rounds=rounds) + level = [list(x) for x in leaves] + while len(level) > 1: + level = [sk.socket_digest_wordlevel(level[i], level[i + 1], fr) + for i in range(0, len(level), 2)] + return level[0] + + +# =========================================================================== +# 5. CHALLENGE SAMPLING — the 96-bit question, both options +# =========================================================================== + +def squeeze_ext_1(t: "tr.Transcript") -> list[int]: + """The ratified B1 shape: lanes 0-2 of ONE squeezed cell. 1 compression. + + Each lane is a u32, so each coordinate is `< 2^32`: the extension challenge + carries 96 bits, not the ~192 `DefaultTranscript` delivers. TRANSCRIPT.md + §4.1 bounds the STATE (128 bits, ~64-bit collision) but does not analyse + per-challenge entropy at production query counts. + """ + return t.squeeze()[0:3] + + +def squeeze_ext_2_DECIDED(t: "tr.Transcript") -> list[int]: + """Alias marking the ratified choice. See `squeeze_ext_2`.""" + return squeeze_ext_2(t) + + +def squeeze_ext_2(t: "tr.Transcript") -> list[int]: + """The alternative: TWO squeezed cells -> three ~64-bit coordinates. + + c0, c1 = squeeze(), squeeze() + lanes = c0 ‖ c1 (8 lanes) + coef_i = (lanes[2i] + 2^32 * lanes[2i+1]) mod p for i in 0..3 + + COST: 2 compressions per extension challenge instead of 1 — a flat +1. + Query-index sampling is UNAFFECTED: `squeeze_bits` reads lane 0 only and + needs no extra entropy, so the query loop (the dominant squeeze run) does + not pay. + + NO REJECTION LOOP, deliberately. A uniform 64-bit value reduced mod p is + biased by about 2^-32 towards the low `2^32 - 1` residues, which is + negligible for a Fiat-Shamir challenge; a rejection loop would be exact but + is UNIMPLEMENTABLE in the fully-unrolled eDSL ("nothing loop-shaped reaches + the machine", TRANSCRIPT.md §1.1 citing `edsl.rs:1-4`). Bias is the right + trade here and the reason is structural, not lazy. + + OPEN (D4): whether 96 bits is in fact insufficient. This function exists so + the cost of the answer is known before the question is decided. + """ + lanes = t.squeeze() + t.squeeze() + return [(lanes[2 * i] + (lanes[2 * i + 1] << 32)) % P for i in range(3)] + + +# =========================================================================== +# 6. GRINDING UNDER B1 — the proof-of-work construction (D3, D6) +# =========================================================================== +# +# DECIDED (Mauro, 2026-08-12): "Grinding should help you, we need 128 security +# for sure." Grinding STAYS, so B1 needs a PoW it can express. The keccak PoW it +# replaces is TWO keccak256 hashes over byte buffers (`grinding.rs:67-89`): +# +# inner = Keccak256( PREFIX(8) ‖ seed(32) ‖ factor(1) ) 41 bytes +# valid = u64_be( Keccak256( inner(32) ‖ nonce_be(8) )[..8] ) < 2^(64-factor) +# +# Neither layer is expressible as a 2-to-1 compress, and both run through the +# hosted keccak family — the chips D0 exists to stop paying. + +GRIND_MARK = int.from_bytes(b"GRD0", "little") + + +def grind_operand(nonce: int, factor: int) -> list[int]: + """The PoW operand cell: `[nonce_lo, nonce_hi, GRIND_MARK, factor]`. + + ONE cell, so the whole PoW is ONE `compress_T` — that is the design target. + Both the nonce AND the difficulty live in the operand, so a nonce found at + one difficulty is worthless at another (KAT C11.d): without `factor` in the + preimage a prover could mine once at factor 1 and present the result at + factor 20. + """ + if not 0 <= nonce < 2**64: + raise ValueError("the nonce is a u64") + if not 1 <= factor <= 64: + raise ValueError("grinding_factor is in 1..=64 (`grinding.rs:22`)") + return [nonce & MASK32, (nonce >> 32) & MASK32, GRIND_MARK, factor] + + +def pow_digest(state: list[int], nonce: int, factor: int, + rounds: int = 7) -> list[int]: + """`W = compress_T(state, [nonce_lo, nonce_hi, GRIND_MARK, factor])`. + + ⚠ DOMAIN SEPARATION — read this before changing the tag. + + This reuses the TRANSCRIPT tag `LFMT`; it does NOT allocate a fourth domain. + The separation argument is exactly the one B1 already relies on for + absorb-vs-squeeze, quoting TRANSCRIPT.md §1.1: the operation sequence is a + compile-time constant of the program, so "a prover cannot perform a squeeze + where the program says absorb", and equally cannot present a PoW evaluation + where the program says absorb. `GRIND_MARK` is defence in depth on the same + footing as `SQUEEZE_MARK` — which that section is explicit is NOT the + load-bearing argument. + + Sharing the tag costs nothing cryptographically here: to satisfy the + difficulty a prover must still search operands at a state it does not + control, and no transcript step it computes elsewhere helps. It saves a tag, + a fourth preprocessed selector (`MODE_G`), `PREP_WIDTH` 13 -> 14 and a + registry re-bless. + + OPEN (D6a): whether to spend those anyway for an unconditional separation. + Costed in COMMIT.md §4.1. + """ + return tr.compress_t(state, grind_operand(nonce, factor), rounds) + + +def pow_is_valid(state: list[int], nonce: int, factor: int, + rounds: int = 7) -> bool: + """The difficulty predicate: the low `factor` bits of `W[0] ‖ W[1]` are zero. + + Reading the two lanes as one 64-bit value `W[0] + 2^32·W[1]` covers the whole + documented range `1..=64` under ONE rule, and for the realistic `factor <= 32` + it touches lane 0 only. The alternative — a rule on lane 0 with a second rule + bolted on above 32 — is two cases where one will do. + + GUEST COST: one `compress_T` plus one `LFM_BITDEC` row to expose the low + bits. Against the keccak PoW's two sponge invocations through the hosted + keccak family. + """ + w = pow_digest(state, nonce, factor, rounds) + combined = w[0] + (w[1] << 32) + return combined % (1 << factor) == 0 + + +def find_nonce(state: list[int], factor: int, rounds: int = 7, + limit: int = 1 << 24) -> int | None: + """Mine a nonce. Expected 2^factor trials — the honest prover's cost.""" + for nonce in range(limit): + if pow_is_valid(state, nonce, factor, rounds): + return nonce + return None + + +def pow_verify_compressions() -> int: + """PoW verification is ONE compression, independent of the difficulty. + + ⚠ Read the honest framing in COMMIT.md §4.1: this saving is O(1) per proof + and therefore small in absolute terms. The reason grinding stays is NOT this + compression — it is the 41 queries (222,794 tower permutations) that + grinding buys back, §7.1. + """ + return 1 diff --git a/thoughts/shared/block-compression/commit-spec/run-kats.log b/thoughts/shared/block-compression/commit-spec/run-kats.log new file mode 100644 index 000000000..6836e3229 --- /dev/null +++ b/thoughts/shared/block-compression/commit-spec/run-kats.log @@ -0,0 +1,89 @@ + + C1 base m=5 r=6 digest is 4 u32 lanes PASS + C1 base m=5 r=7 digest is 4 u32 lanes PASS + C1 deterministic PASS + C1 cost formula matches the chain length (10 felts / rate 4) PASS + C2 ext3 m=3 r=6 digest well formed PASS + C2 ext3 m=3 r=7 digest well formed PASS + C2 cost formula matches the chain length (18 felts / rate 4) PASS + C2 same felt count, different kind -> different leaf PASS + C3 HONEST leg: the true width still verifies PASS + C3 a narrower opening yields a different leaf PASS + C3 ★ identical felt stream, base vs ext3 -> different leaf (the main<->aux confusion, closed by the header) PASS + C3 a width disagreeing with the data is refused PASS + C4 ★ colliding padded felt streams separated by the header PASS + C4 HONEST leg: padded leaf is deterministic PASS + C4 zero-width leaf is the bare header PASS + C5 every lane is a u32 (O1 automatic) PASS + C5 cell count is header + ceil(len/16) PASS + C5 ★ b'\x01' and b'\x01\x00' encode differently PASS + C5 empty string is header-only PASS + C5 absorb advances the chain PASS + C5 HONEST leg: body bytes recover the input under its length PASS + C6 pack is 32 bytes PASS + C6 HONEST leg: pack -> strict_unpack round-trips PASS + C6 the sixteen padding bytes are zero PASS + C6 ★ rejects a set high byte in lane 0 (the cheap forgery) PASS + C6 ★ rejects a set top byte in lane 3 PASS + C6 ★ rejects a lane congruent to 1 mod p PASS + C6 ★ rejects a short commitment PASS + C6 HONEST leg: an all-zero digest still decodes PASS + C7 HONEST leg: a power-of-two tree builds PASS + C7 a single leaf is its own root PASS + C7 ★ a non-power-of-two leaf count is refused PASS + C8 squeeze_ext_1 costs one compression PASS + C8 ★ its coordinates are u32-bounded (96 bits total) PASS + C8 squeeze_ext_2 costs two compressions (+1 flat) PASS + C8 its coordinates span the full field PASS + C9 ★ the wide leaf's LFML rows keep the byte-level anchor @7r PASS + C9 the fold is the honest LFMC socket PASS + C10 ★ a wide leaf is not the bare LFML digest PASS + C10 ★ a wide leaf is not an unheaded LFMC fold PASS + C10 HONEST leg: it is exactly lfml_chain_row(header, felts) PASS + C11 HONEST leg: a mined nonce satisfies the difficulty PASS + C11 the difficulty actually bites (mining was not trivial) PASS + C11 ★ the factor is bound into the digest PASS + C11 ★ the transcript state is bound into the digest PASS + C11 ★ GRIND_MARK changes the digest PASS + C11 ★★ a PoW step IS an absorb of its operand cell — separation rests on the fixed program sequence, NOT on the hash (D6a) PASS + C11 ★ a PoW step is not an LFMC Merkle parent of the same cells PASS + C11 ★ a PoW step is not an LFML leaf of the same felts PASS + C11 factor <= 32 reads lane 0 only PASS + C11 factor > 32 requires lane 0 fully zero (so this sample fails) PASS + C11 ★ rejects factor 0 PASS + C11 ★ rejects factor 65 PASS + C11 ★ rejects an out-of-range nonce PASS + C11 verification is ONE compression, independent of difficulty PASS + C12 the spec parameter is 4 felts/row with a 4-lane accumulator PASS + C12 ★★ the rate is a multiple of 4 (whole machine cells) PASS + C12 ★★ it fits the EXISTING 2-cells-in bus contract (acc cell + one felt cell = 12 lanes + tag = 13 of 16 words) PASS + C12 ★ rate is 4 felts/compression (was 2) PASS + C12 ★ that is a 2.0x improvement on the dominant cost PASS + C12 ★ a full row is 52 bytes — still one BLAKE3 block (<= 64) PASS + C12 ★ row == plain blake3::hash(52 bytes) @7r (anchor intact) PASS + C12 ★ base-vs-ext3 separation survives the rate change PASS + C12 ★ padding still separated by the header at rate 4 PASS + C12 ★ a non-canonical felt still rejects PASS + C12 ★ per-query main-tree leaf cost 6048 -> 3024 (2.00x) PASS + PIN C1.base.m5.r6 matches the pinned vector PASS + PIN C1.base.m5.r7 matches the pinned vector PASS + PIN C2.ext3.m3.r6 matches the pinned vector PASS + PIN C2.ext3.m3.r7 matches the pinned vector PASS + PIN C4.pad.m1 matches the pinned vector PASS + PIN C4.pad.m2 matches the pinned vector PASS + PIN C5.absorb.r6 matches the pinned vector PASS + PIN C5.absorb.r7 matches the pinned vector PASS + PIN C6.pack matches the pinned vector PASS + PIN C7.root.n8.r6 matches the pinned vector PASS + PIN C7.root.n8.r7 matches the pinned vector PASS + PIN C8.ext1 matches the pinned vector PASS + PIN C8.ext2 matches the pinned vector PASS + PIN C11.pow.r6 matches the pinned vector PASS + PIN C11.pow.r7 matches the pinned vector PASS + PIN C12.row.r6 matches the pinned vector PASS + PIN C12.row.r7 matches the pinned vector PASS + PIN C12.per_query_old matches the pinned vector PASS + PIN C12.per_query_new matches the pinned vector PASS + +BOARD: C1 4/4 | C2 4/4 | C3 4/4 | C4 3/3 | C5 6/6 | C6 8/8 | C7 3/3 | C8 4/4 | C9 2/2 | C10 3/3 | C11 14/14 | C12 11/11 | PIN 19/19 +TOTAL: 85/85 PASS diff --git a/thoughts/shared/block-compression/emitter-memory-audit.md b/thoughts/shared/block-compression/emitter-memory-audit.md new file mode 100644 index 000000000..a79d6f73c --- /dev/null +++ b/thoughts/shared/block-compression/emitter-memory-audit.md @@ -0,0 +1,117 @@ +# LFM epoch-verifier emission: where 89 GiB goes (the "emitter audit") + +> **⚠ MEASURED OUTCOME (2026-08-12, box, `97124d18`; full table +> `~/workspace/lambda_vm_bench_cache/lfm_census_2026-08-12/pc_emitter_memory_results.md`):** +> the 219q OOM point now emits — **89.06 GiB OOM → 58.36 GiB exit 0**. Attribution at +> q=96 (uncensored): **Win 2 (flat-append builder) is the ENTIRE memory win** (−15.74 GiB, +> −35.8%); **Win 1 (drop read_counts) is ~0 bytes of peak** — §1g summed the map and the +> row intermediate as co-resident, but the phases are SEQUENTIAL and the emitter's peak +> dominates — Win 1 is a WALL-TIME lever instead (−30%, it stops hashing ~446M +> addresses). The with_capacity item was deliberately not implemented (the §1g virtual- +> allocation note is correct; measured ~0). §3's caveat stands: emission no longer walls, +> P-b remains load-bearing for provability at 219q. + +Delegated audit, 2026-08-12. Worktree `/Users/maurofab/workspace/lambda_vm-blake3-impl` +@ `2a8552f2`. Read-only in the tree; type sizes measured on a faithful standalone +replication of the `Instr` enum (scratchpad `rustc`, not a project build). This is the +document task #29 and PLAN.md's P-c entries cite; CENSUS.md Part 2 §2 carries the census +agent's independent (and partially superseded — see its ⚠ boxes) read. + +## 0. Measured type sizes + +`FE` = 8 B (`math/src/field/element.rs:50-52`, `goldilocks.rs:73`). `Instr` = **80 B** +align 8 (replicated from `prover/src/lfm/instr.rs:178-252`); largest variant `Hash` +(72 B of u64 arrays + HashMode); `KeccakOperands` boxed 432 B. 271M × 80 B = **21.7 GB — +only ~24% of the observed 89.1 GiB. The instruction vector is NOT the dominant term.** + +## 1. What dominates + +1a. `compile()` does NOT copy the instruction stream — REFUTED suspect +(`compiler.rs:136-223` destructures and moves; no clone). + +1b. **`emit_column_groups` (`compiler.rs:231-425`) builds a second, FATTER +materialization**: ten `Vec>` — one heap allocation per instruction — all ten +alive until the struct literal at :413-424 consumes them. Measured actual capacities: +`vec![…]` + `.extend(sels)` + `.push(mult)` lands a 10-wide BALU row at **cap 18** +(144 B heap + 24 B header = 168 B) — **80% waste**; XALU cap 20; BITDEC 130-wide = 1,064 B. +An ALU instruction costs 80 B as an `Instr` and ~168-184 B as a retained row Vec. + +1c. The mix engine: `felt_be_halves` (`transcript_replay.rs:743-761`) = 1 `BitDec` + +64 `BaseAlu` per leaf felt (const pool makes weights free); both leaf paths reach it +(`edsl.rs:235-246` once per value; `sub_proof.rs:245-269` 3× per ext value). ? INFERRED +mix ≈ 95% BaseAlu / 1.5% BitDec; conclusion insensitive (168→184 B at the extreme). + +1d. A `BitDec` is a ~2.2 KB instruction: 80 B in the Vec + 1,024 B `bits` Vec (retained +for program life, `builder.rs:273-280`) + 1,040 B for its 130-wide row + 24 B header. + +1e. ★ **`read_counts` (~18.3 GB) held alive across `emit_column_groups` by scope** +(`compiler.rs:137-143`; drained via `remove` at :155, asserted empty at :207 — but +HashMap does not shrink on removal; `emit_column_groups` is called at :213 inside the +scope). ~2 addrs/instruction → ~534M addresses → 16 B/entry + control at 7/8 load → +2^30 buckets ≈ 18.3 GB (+~27 GB transient at the last rehash). `written: vec![false]` ++0.5 GB. + +1f. Arena schema: ~1.5% of instructions, <1 GB — not a factor. + +1g. Budget at 219q (? INFERRED arithmetic over verified unit costs): Vec 21.7 + +BitDec bits 4.4 + read_counts 18.3 + written 0.5 + **row intermediate ~47** + flat BALU +~20.6 + flat BITDEC ~4.4 → **peak ~99-102 GB inside `from_rows(balu_rows)`**. The +89.1 GiB OOM lands in that window (exact death point ambiguous from RSS alone). +Allocator caveat: the Vec power-of-two capacity (2^29 slots = 42.9 GB) is virtual +until touched; glibc realloc uses mremap (no copy spike). The test binary does NOT use +jemalloc (`#[global_allocator]` only in `bin/cli/src/main.rs:11`). + +## 2. Materialization + +Everything is built into one `LfmBuilder.instrs` Vec (`builder.rs:97-104`, pushed at 13 +sites, handed out whole by `finish()` :490-498); the 219×25 loop (`epoch_tests.rs: +1284-1341` → `epoch_verify.rs:198`, per-query loop :314-366) appends to the same +builder; nothing is ever freed. **But the query body is already streaming-shaped** — +only `fri_terminal.push` (8 B/query) escapes the iteration. + +## 3. Could emission stream? + +- Machine is straight-line ✓ (`instr.rs:1-8`; eleven data-op variants, no + branch/jump/halt). +- Every consumer is a single forward scan ✓: compile pass 1 (:157), + emit_column_groups (:243), execute (`executor.rs:227`), build_traces + (`trace.rs:132-139`), validate (`validator.rs`, 5 passes). No consumer indexes instrs. +- The program digest commits the MATRICES, not the stream ✓ + (`registry.rs:151-203` reads only `program.groups`; `commit.rs:56`; + `statement.rs:50-68`) — so committed bytes are identical under any emitter shape. +- Blockers: LFM_HINT group is lossy (`compiler.rs:384-386` drops arena/index → need a + 33 MB side-stream); multiplicity backfill needs an addr→(chip,row) side table + (~4.3 GB vs the 21.7 GB stream it replaces; in-row slot recoverable since multi-write + outputs are consecutive); groups are per-CHIP not per-leg (append rows per leg, free + that leg's Instrs — cannot commit-and-free per leg). +- ⚠ Emission is not the only wall: even streamed, `execute` needs + `memory: Vec>` ≈ 21 GB + records ~10.4 GB, and LFM_BALU pads to 2^28 + rows at 219q. Fixing emission does not make 219 queries provable — P-b remains + load-bearing. + +## 4. Verdict: (b), with two nearly-free wins first + +★ Win 1 — one line, ~18.8 GB: `drop(read_counts); drop(written);` before +`compiler.rs:213`. + +★ Win 2 — local, ~27 GB: replace `ColumnGroup::from_rows(width, rows: Vec>)` +(`compiler.rs:38-52`) with a flat-append `ColumnGroupBuilder { width, real_rows, +data: Vec }` — removes headers, malloc chunk overhead, the 80% capacity waste, and +~271M malloc/free pairs (~50 lines, compiler.rs-local). + +Together: projected peak ~99-102 → **~53-56 GB**. + +Streaming seams, named: (1) `builder.rs:98` `instrs` field + `finish()` — replace with +ten flat group matrices + read_counts + addr→(chip,row) table + hint side-stream; every +`push` becomes `emit_row`. (2) `compiler.rs:136` — passes merge into the builder. +(3) `compiler.rs:84` `LfmProgram.instrs` consumers: executor = the hard one (10-way merge +by destination address, monotone within a chip, + hint side-stream); trace.rs trivial +(recover hash modes from the one-hot MODE_* columns); validator checks re-expressed +(note `Instr::writes()/reads()` allocate a fresh Vec per call ~3×/instruction — ~800M +transient allocations — want SmallVec regardless; `check_multiplicities` builds a second +full ~18 GB HashMap that should be a dense Vec). (4) `epoch_verify.rs:198/:314` — already +streaming-correct. + +Nothing about soundness, the AIR set, bus topology, or program_id moves — committed +matrices are bit-identical. Work concentrates in builder.rs + compiler.rs (mechanical) +and executor.rs (the one genuinely new algorithm). diff --git a/thoughts/shared/block-compression/residency-seam-audit.md b/thoughts/shared/block-compression/residency-seam-audit.md new file mode 100644 index 000000000..8ebc08154 --- /dev/null +++ b/thoughts/shared/block-compression/residency-seam-audit.md @@ -0,0 +1,141 @@ +# Bounded-Residency Proving — Code Audit (the "seam audit") + +Delegated audit, 2026-08-12. Worktree `/Users/maurofab/workspace/lambda_vm-blake3-impl` +(branch `blake3-real-hash`). Read-only; nothing built, nothing edited. This is the +document PLAN.md's P-b entries cite; CENSUS.md Part 2 §1 carries the census agent's +independent read of the same question and the reconciliation boxes. + +**Headline: the linearity assumption is TRUE, but the coefficient is wrong (too high by +~2.1× for a `KECCAK_RND` chunk). Peak really is a SUM over all sub-proofs for the main +trace + main LDE + main tree + aux trace; the aux LDE and all round-2-to-4 buffers are +`k`-bounded, not summed. Nothing in the prover bounds residency to a few tables at a +time.** + +## 1. Does the prover hold ALL table traces resident simultaneously? + +✓ VERIFIED — yes, unconditionally, structurally forced by the API. + +- `crypto/stark/src/prover.rs:50-54`: `AirTracePair<'a,…> = (&'a dyn AIR, &'a mut + TraceTable, &'a PI)`; `multi_prove` takes `Vec` (`prover.rs:3032-3036`) — + every trace must exist and be borrowed for the entire call. No iterator/factory/ + TraceSource exists; the only producers are `VmAirs::air_trace_pairs` + (`prover/src/lib.rs:542`) and `LfmAirs::air_trace_pairs` + (`prover/src/lfm/airs.rs:551-583`), both building a complete Vec. +- LFM wrap: `prover/src/lfm/trace.rs:43` `keccak_rnd: Vec`; all N chunk + traces built eagerly at `trace.rs:162-167`; `lfm/proof.rs:140-145` passes them whole. +- The one streaming precedent — `trace_builder.rs:2922-2949` `chunk_and_generate` under + `StorageMode::Disk` — spills each chunk's trace to mmap after build (VM only, trace + only, never the LDE) and still returns a Vec of all chunks. + +## 2. Where peak accretes (verified round structure, `prover.rs:3032-3689`) + +| Stage | Residency class | Bytes | +|---|---|---| +| Trace build (caller) | global — alive past return | rows·cols·8 | +| R1 main LDE (`commit_main_trace` → drained to `main_ldes` at :3145, :3201) | **retained for all N** | rows·blowup·cols·8 | +| R1 main Merkle (`TableCommit.tree` :114; cells :3306-3309) | retained for all N | ≈32 B/LDE row | +| FS boundary (:3190-3204 roots absorbed → :3219-3225 shared LogUp challenges) | — | — | +| Aux trace (`lookup.rs:1209-1211` writes into caller-owned TraceTable) | **global — never freed inside multi_prove** | rows·aux_cols·24 | +| Aux-build transients (`lookup.rs:1272` full `columns_main()` copy; :1287-1337 committed_columns) | per-table, ≤ k | rows·(main·8+aux·24) | +| Aux LDE + aux tree (:3377-3516, inside `aux_stage`) | ≤ k | rows·blowup·aux_cols·24 | +| Composition/DEEP/FRI (R2-4) | ≤ k | ~160 B/LDE row | + +The retention is documented in the code's own words — `prover.rs:263-274` (`Lde` struct +doc): main LDEs "all N tables' … live at once (O(N × main_cols × lde_size))"; aux "at +most `table_parallelism()` of them coexist". The barrier is real: `run_admitted` +(:689-722) joins a thread scope before roots are absorbed and challenges sampled, before +the second `run_admitted` for the fused phase at :3628. The independent in-repo model +agrees: `prover/src/auto_storage.rs:243-266` / :53-95. + +**⚠ Fiat-Shamir forces the ROOTS, not the LDEs** (`prover.rs:3196-3225`, verifier mirror +`verifier.rs:1295-1317`). After the shared challenge, each table gets a private +transcript fork (:3263-3271 / :1349-1356) and every later round is per-table +independent; each `StarkProof` is self-contained (:3856-3887). Retaining the LDE is a +performance choice, not a protocol constraint — this is the seam that makes bounded +residency possible. + +cuda note: `device_only` relocates the main LDE to VRAM without reducing N-way retention +(? INFERRED that this makes cuda strictly worse for this workload). + +## 3. What disk-spill / StorageMode bound TODAY + +Spilled to mmap under `StorageMode::Disk`: main trace Table (:3113-3122, trace_builder +:2938-2949, :3658-3680); aux trace Table (:3366-3371); main/precomputed/mult Merkle trees +(:1172, :1186, :1239, :1273, :1293 via `spill_tree` :1316-1331); aux Merkle CPU path only +(:3501; the GPU aux arms return early at :3419-3423/:3457-3461 without spilling). + +NEVER spilled (✓ VERIFIED — spill_tree has exactly 6 call sites; `LDETraceTable` has no +mmap field, `trace.rs:316-343`): main LDE, aux LDE, composition evals, composition tree, +FRI layers, the `columns_main()` copy. + +Wiring: the only live selector is `auto_storage::decide` on the monolithic VM path +(`lib.rs:1221-1226`); `continuation.rs` (:791-796, :991-992, :1282-1283) hardcodes Ram; +**the wrap (`lfm/proof.rs:140-145`) passes `Default::default()` = Ram**; feature +`disk-spill` is off by default (`prover/Cargo.toml:8`, opt-in :17) — under a normal build +the parameter does not exist (cfg at `prover.rs:3035`). + +**Spilling moves allocation (trace + trees only); the main LDE — the largest N-way +retained buffer — stays on the heap in every configuration, and Disk is unreachable from +the LFM path anyway.** + +## 4. Verdict + +(a) available behind flags? **NO.** `TABLE_PARALLELISM=1` bounds only aux/R2-4 +transients; `FORCE_DISK_SPILL` unreachable from LFM and wouldn't touch the LDE. + +(b)/(c): **a real refactor with named seams** — deeper than "moderate", shallower than +"fundamental". Not forced by FS ordering, not by the proof struct, not by chunk pairing +(`chunking.rs:12-21`: zero cross-chunk logic — a chunk trace is a pure function of its +`round_ops` slice, so **regeneration is trivially available**). + +What forces retention: (1) the `Vec<(&dyn AIR, &mut TraceTable, &PI)>` signature; (2) the +eagerly-built `LfmTraces.keccak_rnd`; (3) the deliberate R1 LDE cache (:3145, :3201, +:3311-3316); (4) `allocate_aux_table` writing into the caller-owned trace. + +### The seams + +| # | File:line | Change | +|---|---|---| +| S1 | `prover.rs:3032-3036` | `multi_prove` takes a per-index producer (`Fn(usize) -> (AIR, TraceTable, PI)` + `num_airs`); `run_admitted` already dispatches by index | +| S2 | `prover.rs:50-54` | `AirTracePair` stops carrying `&mut TraceTable` — the task owns its trace | +| S3 | `prover.rs:3144-3145, 3190-3204, 3311-3316` | delete `main_ldes`/`main_lde_cells`; drop LDE at end of R1 task, recompute at top of aux_stage (twiddles process-cached :517-574) | +| S4 | `prover.rs:3306-3309` | trees: keep (32 MiB/chunk), spill via existing `spill_tree`, or recompute → root-only peak | +| S5 | `lookup.rs:1209-1211` | aux trace freed with the owned TraceTable (automatic once S2 lands) | +| S6 | `lfm/airs.rs:551-583` + `lfm/trace.rs:162-167` | `air_trace_pairs` → lazy per-index generator over `chunking.split(&round_ops)`; keep `round_ops` (~92 MB) instead of N traces | +| S7 | `lfm/proof.rs:140-145` | call-site update | + +Wire compatibility preserved: verifier ordering depends only on root-absorption order and +the index-domain-separated fork, both unchanged. + +### The floor + +Per KECCAK_RND chunk (2^19 rows × 1480 main + 516 aux ext, blowup 2): main trace 5.78 / +main LDE 11.56 / aux trace 6.05 / aux LDE 12.09 / columns_main copy 5.78 / +committed_columns 6.05 / trees+R2-4 ≈0.2 GiB. Peak ≈ **17.37·N + 30.2·k GiB**: + +| N | today (k=1) | S3 only (recompute LDE, keep traces) | S3+S6 (regenerate traces; roots-only) | +|---|---|---|---| +| 23 | ~430 GiB | ~309 GiB | **~49 GiB** | +| 133 | ~2,341 GiB | ~819 GiB | **~56 GiB** | + +**Bounding only the LDE does NOT reach ~50 GiB** — the main+aux traces are still summed; +the flat floor requires the trace streamed too (regenerate each chunk twice: R1 commit + +aux build). Cost of the floor: 2× chunk trace generation + 2× main coset LDE per chunk, +serialization to k=1 — ? INFERRED roughly +40-60% wall on the KECCAK_RND family. + +### Coefficient correction to the census projection + +`MEASURED_BYTES_PER_CELL = 33.7` (`wrap_tests.rs:629-644`) prices a KECCAK_RND chunk at +49.8 GiB; the true persistent cost is 23.4 GiB — **~2.1× high** for this shape (aux LDE +priced persistent when k-bounded; slice-0 calibration mix). Linearity in N correct, slope +not. Gate-A band reads roughly ~560-3,200 GiB. **No verdict moves.** + +## Confidence ledger + +✓ VERIFIED by reading: round structure and R1 barrier; AirTracePair signature + both +producers; the Lde doc; auto_storage's split; all spill call sites; no mmap on +LDETraceTable; every StorageMode argument at every multi_prove call site; +allocate_aux_table; Table::columns() full copy; verifier transcript order; KECCAK_RND +1480 cols (`tables/keccak_rnd.rs:95`), 1031 interactions → 516 aux (`lookup.rs:117-125`). +? INFERRED: all GiB arithmetic; +40-60% wall; cuda VRAM note. ✗ NOT CHECKED: other 13 +chips' rows (totals are the chunk family's contribution); VramGate interaction. diff --git a/thoughts/shared/gpu-recursion/BOX-RESULTS.md b/thoughts/shared/gpu-recursion/BOX-RESULTS.md new file mode 100644 index 000000000..1709be9fb --- /dev/null +++ b/thoughts/shared/gpu-recursion/BOX-RESULTS.md @@ -0,0 +1,113 @@ +# GPU box measurements — EXPLORATION.md Stages 0/1 (2026-08-12) + +Box: a rented vast.ai instance (endpoint in the session notes, not committed here). +RTX 5090 32 GiB (GPU-8798ec09, driver 595.71.05, compute cap 12.0), CUDA toolkit **13.1**, +cgroup quota **30.7 cores** (nproc 32 — whole machine, not a fractional slice), 93 GiB RAM, +64 GiB disk. Code: `blake3-real-hash` @ `77cc7df1` (PR #930 tip + handoff docs commit), +shipped as a git bundle (no credentials on the box), toolchain 1.94.0 + +nightly-2026-02-01 for guest ELFs. + +## Health gates (Stage 0 step 0) — GREEN + +- `make test-math-cuda`: 87 tests pass on device. EXIT=0. +- `make test-cuda-integration`: 7/7 pass in 13.3 s — the R1-R4 dispatch counters all fire + on a real RV64 prove and the proof verifies. The stack works on CUDA 13.1 + sm_120. + +## Stage 0 — baseline wrap counters (default threshold 2^19) + +New harness: `lfm::wrap_tests::the_wrap_reports_gpu_counters` (`#[cfg(feature = "cuda")]`, +`#[ignore]`d) — resets the process-global counters AFTER the inner epoch is built (the inner +RV64 continuation prove has its own GPU traffic) and prints all 15 counters after `lfm_prove`. + +### ★ Found the root cause of the "19 pre-existing fibonacci.elf failures" + +A fresh `make compile-recursion-elfs` builds a `recursion/fibonacci.elf` of **1,344 bytes** +that **finishes in ≤16 cycles** → `real_epoch_with` panics `"wanted an INTERMEDIATE epoch"` +(`epoch_tests.rs:659`). The fixture premise (`proof_fixture.rs:41-49`: guest runs 17–64 +cycles, splits only at a 16-cycle epoch) was measured against an older build — the ELF in +`lambda_vm_2`/`lambda_vm_3` worktrees (**1,368 bytes**, Jul 21, sha `4346975f…`) still works. +Codegen drift shaved ~6 instructions and broke the split. This is why the whole +fixture-dependent lfm suite reports 19 failures on any machine that rebuilds the ELF. +Workaround on the box: shipped the Jul-21 ELF. Real fix is Mauro's call (re-measure +`FIXTURE_EPOCH_LOG2`, or pin the fixture guest against drift). + +RESULT (chip log-heights `[11, 21, 17, 11, 15, 2, 12, 16, 15, 8, 16, 0, 5, 20]` — BALU 2^21, +BITWISE 2^20, KECCAK_RND 2^17, matches the census): + +``` +lfm_prove 16.8 s, peak VRAM 5,789 MiB +lde 470 / leaf_hash 13 / merkle_tree 13 / extend_halves 2 / logup 10 +composition 2 / comp_poly_tree 2 / parts_lde 0 / bary 20 / deep 2 +batch_invert 8 / fri 2 / opening_gather 12 / device_only 0 +``` + +**All four Stage 0 falsifiable predictions CONFIRMED**: composition ≥ 2 (=2, BITWISE+BALU), +merkle_tree ≥ 4 cold (=13), device_only == 0 (threshold-caused), lde far below what +KECCAK_RND's 1,480 columns would add. §1's map survives falsification. + +## Stage 1(a) — LAMBDA_VM_GPU_LDE_THRESHOLD sweep + +| threshold | device_only | composition | lde | prove s (runs) | peak VRAM | verify | +|---|---|---|---|---|---|---| +| default 2^19 | 0 | 2 | 470 | **16.8, 16.7, 16.7** | 5.8 GiB | ✓ | +| 262144 (2^18) | **1** | 4 | 3533 | **7.2, 7.1, 7.2** | 12.7 GiB | ✓ | +| 131072 (2^17) | 1 | 6 | 3557 | 7.2 | 12.9 GiB | ✓ | +| 4096 (2^12) | 1 | 11 | 4601 | 7.1 | 11.3 GiB | ✓ | + +★ **LEVER 1 CONFIRMED, ABBA-tight (A-B-B-A run order): −57% prove time (16.73 → 7.17 s +mean, sd ≈ 0.05 s), zero code changed.** `LAMBDA_VM_GPU_LDE_THRESHOLD=262144` flips +`device_only` 0→1 — KECCAK_RND (88.1% of main cells, the one non-preprocessed chip) sails +through `device_only_gate` once past the size gate; its whole R1→FRI pipeline moves +on-device (`fri` 2→4, `merkle_tree` 13→17, `deep` 2→4) and the proof verifies. + +**The knee is exactly 2^18**: thresholds 2^17 and 2^12 stay at ~7.2 s — KECCAK_RND is the +entire win, and admitting every remaining chip (composition 2→11 across the sweep) neither +helps nor hurts at this scale. `device_only` is pinned at 1 at EVERY threshold: the other +13 chips are preprocessed and excluded by `&& !is_preprocessed` — that ceiling is lever 2. + +Attribution note (open): `composition` went +2 at 2^18 though only one chip (KECCAK_RND) newly +clears the ROW gate — the R2 fused path's own gate admits by a different rule than R1's +(sweep: 2→4→6→11). Doesn't affect the conclusion; worth settling when writing the permanent gate. + +## Stage 1(b) — GPU composition A/B at threshold 2^18 + +`LAMBDA_VM_DISABLE_GPU_COMPOSITION=1`: **13.8 s vs 7.2 s — disabling it nearly doubles prove +time.** EXPLORATION's prediction ("small, per the VM's −2.7%") is **falsified**: on the LFM +machine the fused composition path is a co-headline win, which is what you'd expect from +16k-IR-node × 1,480-column constraint programs. Note `composition 0 / device_only 0` in that +run — `device_only_gate` requires `!gpu_composition_disabled()`, so the kill switch also +demotes KECCAK_RND to host copies, yet 13.8 s still beats the 16.8 s baseline (the GPU +LDE + trees keep helping). + +## Stage 1(c) — TABLE_PARALLELISM at threshold 2^18 + +tp=1: 8.4 s / tp=4: 7.1 s / tp=14: 7.2 s / default (cores·2/3): 7.2 s. +Saturates at ≥4; even fully serial costs only +1.2 s. No lever here; default is fine. + +## Stage 3 correction — the cell-aware gate is BIGGER than EXPLORATION §4 scoped + +EXPLORATION said "3 sites + the `device_only_gate` mirror". The audit says otherwise: +`gpu_lde_threshold()` has **18 consumer sites** across R1/R2/R3/R4/FRI, and several +re-derive admission downstream even when they already hold the device handle +(e.g. `gpu_lde.rs:1170` checks `handle.lde_size < gpu_lde_threshold()`). A cell-admitted +table (KECCAK_RND: LDE 2^18 < row default 2^19) would pass R1 and then be REFUSED by +downstream row-checks — for a device-only table that is the documented LOCKSTEP hazard +(`gpu_lde.rs:185-188`): hard abort, or a silent fallback that forfeits the win. FRI folds +are width-1, so a naive `lde_size × m` rule degenerates to the row rule exactly where the +device-only pipeline must keep firing. + +The right permanent shape is **admission decided once at R1, device-handle presence as the +admission token downstream** — handle-bearing sites stop re-checking the size threshold. +That is a proper reviewed PR (~6-10 sites + the LOCKSTEP audit), with this box's counter +test as the oracle (device_only=1, ~7.2 s, verify green at DEFAULT env). NOT rushed here. + +**Operational recommendation until then:** run wrap proves with +`LAMBDA_VM_GPU_LDE_THRESHOLD=262144`. That exact configuration is what was proved end to +end here, ABBA-tight, verify green, 12.7 GiB peak VRAM on a 32 GiB card. (The env var is +process-global: in the wrap test process it also lowers the gate for the inner RV64 epoch +prove, which the t4096 run shows is harmless at this scale.) + +## Evidence + +Raw logs + 1 Hz VRAM traces: `~/workspace/lambda_vm_bench_cache/gpu_lfm_wrap_2026-08-12/` +(26 files). Counter harness committed on `blake3-real-hash` as `c495e9fc` (signed, unpushed). diff --git a/thoughts/shared/gpu-recursion/EXPLORATION.md b/thoughts/shared/gpu-recursion/EXPLORATION.md new file mode 100644 index 000000000..fd1a4e69a --- /dev/null +++ b/thoughts/shared/gpu-recursion/EXPLORATION.md @@ -0,0 +1,503 @@ +# GPU for the LFM recursion machine — post-merge map, remaining levers, box plan + +**Date:** 2026-08-12 +**Worktree read:** `/Users/maurofab/workspace/lambda_vm-blake3-merge`, branch +`blake3-real-hash-mainmerge`, **mid-merge** (`MERGE_HEAD` = `58160b6f` = `origin/main` +tip "Feat/hint ecall (#876)"). The merged content is staged but not committed, so +every line number below is against the *working tree* of that worktree. ✓ VERIFIED +by `git rev-parse MERGE_HEAD` + `git status`. +**Method:** read-only. No cargo, no edits. Every claim is marked ✓ VERIFIED (read the +code) or ? INFERRED (derived, not executed). + +--- + +## 0. Headline + +The premise of the earlier finding — *"the LFM machine gets ~zero GPU because the +GPU main-LDE path excludes preprocessed tables"* — **is now largely obsolete.** +Main's #863 (`d83b4d9e`, "halve GPU continuation proving time") added a dedicated +**GPU split-tree path for preprocessed tables**, and #875 (`5749a956`, "device-resident +rounds 2-4") made R2 fully device-resident. Neither is preprocessed-gated. + +Post-merge, for a preprocessed table that clears the 2^19 LDE threshold — which for +LFM means **BITWISE, the table that is 97–99% of all committed cells** — the LDE, both +Merkle trees, the LogUp aux build, the composition polynomial, the OOD barycentric, +DEEP and FRI **all run on device**. + +What is still excluded is narrower and precisely locatable: **the host D2H is never +skipped for a preprocessed table**, because `device_only_gate` still carries +`&& !is_preprocessed`. + +**But that is not the biggest lever for recursion.** The measured census of the real +epoch-verifier wrap (§3.1) shows the LFM machine's shape is **short and very wide** — +`KECCAK_RND` is 1,480 columns and carries **88.1% of all main cells** — whereas the +GPU LDE gate is a **row-count** threshold that is completely blind to width +(`lde_size = n × blowup`, no column term, `gpu_lde.rs:698-700, 799-801`). In the only +LFM wrap that has actually been proved end to end, that threshold drops the +88%-of-cells chip to the CPU while admitting BITWISE, which is 4.8%. **Fixing the +threshold to be cell-aware is the top lever, it needs no new kernels, and the +experiment costs one env var** (§4). + +**Three claims in the earlier LFM-GPU note should be retired:** (a) "no chip reaches +the GPU main-LDE/composition path" — false post-merge; (b) "R1 GPU main-LDE is inside +`if precomputed.is_none()`" — that `if` is now a two-way route, not an exclusion; +(c) "value scratch is 1.5 MiB per IR node → KECCAK_RND wants 23.9 GiB" — superseded by +liveness slot reuse (§3). The note's *suggested fix* ("lift `!is_preprocessed`") +survives, but it now buys a D2H skip rather than the whole GPU pipeline. + +--- + +## 1. Current map: what runs on GPU for an LFM per-table proof + +The LFM machine proves through the *same* `Prover::multi_prove` as the RV64 VM +(`prover/src/lfm/proof.rs:140`), so this is the generic prover path, read for the +preprocessed case. + +| Stage | Preprocessed table today | Gate (file:line) | Status | +|---|---|---|---| +| R1 main LDE (row-major NTT) | **GPU** | `prover.rs:1127-1157` → `gpu_lde.rs:779-858` → `math-cuda/src/lde.rs:718-830` | ✓ VERIFIED | +| R1 precomputed-columns Merkle tree | **GPU** (built on device, nodes D2H'd once, then process-cached) | `math-cuda/src/lde.rs:782-789`; cache `prover.rs:175-214` | ✓ VERIFIED | +| R1 multiplicity Merkle tree | **GPU, stays resident** (host tree is root-only) | `math-cuda/src/lde.rs:793-802`; `gpu_lde.rs:839-845` | ✓ VERIFIED | +| R1 main-LDE host copy (D2H) | **CPU / always paid** | `math-cuda/src/lde.rs:806-807` (unconditional), comment at `:804-805` | ✓ VERIFIED — **the remaining exclusion** | +| R1 LogUp aux build | **GPU**, reading the resident trace | `logup_gpu.rs:418-453`, threshold `1<<10` at `logup_gpu.rs:26`; preprocessed handles threaded at `prover.rs:3253-3260` | ✓ VERIFIED | +| R1 aux LDE + aux Merkle | **GPU** | `prover.rs:3394-3424` (resident) / `:3428-3463` (fused) | ✓ VERIFIED | +| R1 aux host copy (D2H) | **CPU / always paid** (same `device_only` flag) | `prover.rs:3407`, `:3449` — `!device_only` | ✓ VERIFIED | +| R2 composition `H(row)` | **GPU**, device-resident (`GpuCompH`) | `evaluator.rs:388-421` → `:302-332`; **no `is_preprocessed` check** | ✓ VERIFIED | +| R2 decompose + half-extend (d=2) | **GPU** | `prover.rs:1589-1621`, `gpu_lde.rs:580` | ✓ VERIFIED | +| R2 composition-parts host copy | **CPU / always paid** | `prover.rs:1606` — retain = `!host_trace_empty()` | ✓ VERIFIED | +| R2 composition Merkle tree | **GPU from the device parts handle** | `prover.rs:1737-1756` | ✓ VERIFIED | +| R3 OOD barycentric (main) | **GPU** off the handle | `trace.rs:766-776`; gate `gpu_lde.rs:1198-1233` (no prep check), threshold `1<<14` at `gpu_lde.rs:1020` | ✓ VERIFIED | +| R3 OOD barycentric (aux) | **GPU** | `trace.rs:824-834` | ✓ VERIFIED | +| R4 DEEP composition | **GPU** (device inv-denoms + device parts) | `prover.rs:2250-2278`, `:2190` | ✓ VERIFIED | +| R4 Merkle authentication paths | **GPU** — including the preprocessed table's multiplicity tree | `prover.rs:2705-2731` (no prep check) | ✓ VERIFIED | +| R4 opening **values** | **CPU host gather** for preprocessed | `prover.rs:2758` `(!is_preprocessed)`; consumer `prover.rs:2836-2876` `gather_main_row_range` | ✓ VERIFIED — **second exclusion** | +| FRI commit + query phase | **GPU** | `fri/mod.rs:62`, `:164`; `prover.rs:2008` | ✓ VERIFIED, not prep-gated | + +### The two `if precomputed.is_none()` / `is_preprocessed` sites, precisely + +1. **`prover.rs:1075` — `if precomputed.is_none()`.** This is *no longer an + exclusion*. It is a two-way route: `:1075` takes the plain fused path for normal + tables, `:1128` takes the **split-tree** path for preprocessed ones. Both end in + `return Ok((commit, main_data, Some(handle)))` — the preprocessed table gets a + device handle, which is what unlocks everything downstream. ✓ VERIFIED +2. **`gpu_lde.rs:210` — `&& !is_preprocessed` inside `device_only_gate`.** This one + *is* still a real exclusion, and it is the only place the preprocessed property + changes GPU behaviour in R1. Its consequence is a *host copy*, not a CPU + computation. ✓ VERIFIED +3. **`prover.rs:2758` — `(!is_preprocessed)` on `main_dev_values`.** Downstream of + (2): preprocessed R4 openings read values from the host LDE. ✓ VERIFIED + +`crypto/stark/src/verifier.rs:225` and `:1274` also mention `is_preprocessed` — those +are verify-side and irrelevant here. ✓ VERIFIED + +--- + +## 2. What main's split-tree path actually does on device + +`math_cuda::lde::coset_lde_row_major_split_trees` (`crypto/math-cuda/src/lde.rs:718`), +read line by line: ✓ VERIFIED + +- `:747` one H2D of the row-major trace → `expand_row_major_on_stream` (fused + row-major NTT), retaining the trace-domain column-major snapshot. +- `:759-778` a closure that builds **one subset Merkle tree per column range** by + launching `keccak_base_row_major_row_pair_range(col_start, col_end)` over the + resident LDE, then `build_inner_tree_levels`. Leaves are bit-identical to the CPU + `commit_rows_bit_reversed_subset` pair (asserted by the wrapper's doc, + `gpu_lde.rs:766-777`). +- `:782-789` precomputed tree: built on device; nodes D2H'd **only when + `build_precomputed` is true**, which is `cached_pre.is_none()` at `prover.rs:1156` + — i.e. only on a process-cache miss. +- `:793-802` multiplicity tree: built on device and **kept resident**; only the + 32-byte root comes back. +- `:806-807` **the row-major LDE is D2H'd unconditionally**, with the comment + `"preprocessed tables always keep the host copy — they are excluded from the + device-only gate"`. This is the single line the whole remaining lever hangs on. +- `:810` row→column-major transpose on device, producing the `GpuLdeBase.buf` every + downstream round reads. + +The precomputed-tree cache (`prover.rs:175-214`) is process-wide, type-erased and +keyed by the commitment root, so a long-lived prover process builds BITWISE's +precomputed tree **once**, not once per proof. ✓ VERIFIED. (A fresh process per proof +— e.g. a CLI invocation — rebuilds it, but on GPU.) + +--- + +## 3. Does the LFM machine actually reach these paths? + +Working through the gates against the LFM chip set: + +- **All 14 chips are the same generic AIR type** (`AirWithBuses`, + `prover/src/lfm/airs.rs:31`), proved by one `multi_prove` call + (`prover/src/lfm/proof.rs:140-146`). ✓ VERIFIED +- **13/14 preprocessed** — `build_air` chains `.with_preprocessed` + (`airs.rs:331-349`); only `KECCAK_RND` uses `build_air_no_prep` (`airs.rs:313`, + slot constant at `:74`). ✓ VERIFIED (unchanged from the earlier finding) +- **BITWISE: 2^20 rows, 21 columns, 11 preprocessed** (`prover/src/tables/bitwise.rs:98, + 94, 101`) — fixed in *every* LFM proof by the fixed-machine principle + (`airs.rs:34-49`). ✓ VERIFIED +- **KECCAK_RC: 32 rows, 10 columns, 9 preprocessed** (`prover/src/tables/keccak_rc.rs:46, + 36, 40`). ✓ VERIFIED +- **LFM_RANGE: 2^16 rows**, fixed and program-independent + (`prover/src/lfm/layout.rs:261-266`) — the second-largest always-present table. + At blowup 2 its LDE is 2^17, **4× below the 2^19 threshold**, so it is entirely + CPU today. ✓ VERIFIED +- All other chips are `padded_rows(real_rows) = real_rows.next_power_of_two().max(4)` + (`prover/src/lfm/layout.rs:271-276`) — program-dependent. ✓ VERIFIED +- **Blowup = 2** in every registry entry (`prover/src/lfm/registry.rs:196, 280, 364, + 448, 532, 616`). ✓ VERIFIED. So **LDE size = 2 × trace rows**, and a chip needs + **≥ 2^18 trace rows** to clear the 2^19 LDE threshold (`gpu_lde.rs:45`). +- **`split_col` precondition** `0 < split_col < m`: BITWISE gives 11 < 21 ✓; + KECCAK_RC gives 9 < 10 ✓. ✓ VERIFIED +- **Composition parts = `max_degree − 1`** (`lookup.rs:1110-1126`). LogUp batched + terms are degree 3, so every LFM chip is ≥ degree 3 ⇒ **2 parts**, which is exactly + the `number_of_parts == 2` condition the device-resident R2 path requires + (`prover.rs:1592`). ? INFERRED (degree-3 LogUp is stated at `lookup.rs:1113`; I did + not enumerate each chip's `max_degree()`). +- **`end_exemptions == 0` everywhere** — stated and measured across all tables at + `prover/src/lfm/constraints.rs:783-785`, and `RowDomain::ALL` is the only row + domain used in `prover/src/lfm/`. So `zerofier_uniform` holds. ✓ VERIFIED +- **`transition_offsets = [0, 1]`** for every `AirWithBuses` (`lookup.rs:959`) ⇒ + `offsets_are_contiguous` ✓. ✓ VERIFIED +- **`has_aux_trace` + non-empty `constraints_meta`**: even chips built on + `EmptyConstraints` (BITWISE, KECCAK_RC, LFM_CONST, LFM_LANES, LFM_HINT, LFM_PUBLIC, + LFM_RANGE) get LogUp metas appended by the framework (`lookup.rs:931-937`), and + BITWISE has 10 bus interactions ⇒ 6 aux columns. Both preconditions at + `prover.rs:1031` hold. ✓ VERIFIED + +## 3.1 What the *recursion* workload actually looks like (measured, in-repo) + +This is the part that changes the conclusion, and it is measured rather than derived. +**Do not reason about LFM-on-GPU from the registered programs** — they are toy-sized. +The recursion target is the assembled **epoch-verifier wrap**. + +**The wrap that has actually been proved** — inner epoch min preset, wrap options +blowup 2 / 219 queries / grinding 20, 14 LFM sub-proofs, prove 19.5 s, verify 0.09 s, +peak RSS 15.1 GiB on an 11-core laptop +(`others/lfm-agent-status.log:183, 186`) ✓ VERIFIED (checked-in measurement): + +| chip | rows | cols | main cells | share | LDE @ blowup 2 | ≥ 2^19? | +|---|---|---|---|---|---|---| +| **KECCAK_RND** | 131,072 = 2^17 | 1,480 | 193,986,560 | **88.1%** | 2^18 | **✗ misses by 2×** | +| LFM_BALU | 2^21 | 4 (+10 prep) | 8,388,608 | 3.8% | 2^22 | ✓ | +| BITWISE | 2^20 | 10 (+11 prep) | 10,485,760 | 4.8% | 2^21 | ✓ | +| LFM_XALU | 2^17 | — | — | — | 2^18 | ✗ | +| LFM_LANES, LFM_RANGE | 2^16 | — | — | — | 2^17 | ✗ | + +Totals for that wrap: 220,107,920 main + 87,073,068 aux ext = 481,327,124 base-field +equivalents. The "fixed-machine floor" (empty program) is 10,560,752 main = **4.8%**. + +**Two consequences, and they point in opposite directions from the old note:** + +1. **BITWISE is ~5% of the recursion workload, not 97–99%.** That 97–99% figure came + from the *registered toy programs* (trivial / keccak_sponge / statement_replay), + where the fixed machine is the whole proof. Carrying it into a recursion argument + is a category error. ✓ VERIFIED by the census above. +2. **The dominant chip is `KECCAK_RND` — the one non-preprocessed chip** — and it is + short and enormously wide (2^17 × 1,480). The row-based threshold sees only 2^17 + rows and refuses it, even though it is ~9× BITWISE's cell count. + +**Chunking is what decides whether this bites.** `KECCAK_RND_MAX_CHUNK_ROWS = 1 << 19` +(`prover/src/lfm/chunking.rs:41`, 24 rows/permutation at `:34`) ✓ VERIFIED. So a +**full** chunk is 2^19 rows → LDE 2^20 at blowup 2 → **clears the threshold**. It is +**partial / small-epoch chunks that fall off the GPU**, which is exactly the case in +the only wrap proved end to end. A production-sized epoch verify (the blowup-8 census +at `others/lfm-hash-matrix-scope.md:1146-1151` — 6 chunks, 2,883,584 rows, plus +LFM_BALU 2^27, BITDEC 2^21, LANES 2^21) would have its chunks at the cap and would +clear — but that configuration **has never been proved** (350.6 GiB projected peak; +`wrap_tests.rs` `the_wrap_census_at_blowup_8` is `#[ignore]`d). ✓ VERIFIED + +**Net:** at blowup 2, `LFM_BALU` and `BITWISE` clear every gate for the GPU split path +today and (being preprocessed) fail only `device_only_gate`. `KECCAK_RND` fails the +size gate at small chunk heights and clears it at full ones — and because it is *not* +preprocessed, once it clears it also qualifies for full device-only residency with +**no code change at all**. + +### A correction to the previous finding's cost model + +The earlier note recorded *"value scratch is 1.5 MiB per IR node → `KECCAK_RND` at +full chunk height wants 23.9 GiB"*. **That is stale.** Main's device lowering now does +**liveness slot reuse**: slots are freed at an operand's last use and the kernel +allocates `num_base_slots`/`num_ext_slots` per thread, not one slot per node +(`crypto/stark/src/constraint_ir/device.rs:24-30`, `:168-171`; +`crypto/math-cuda/src/constraint_interp.rs:99-101`). The working set is now the +*live* set, not the node count. ✓ VERIFIED. Whether that brings `KECCAK_RND` under +VRAM is a box measurement, not a code read — but the 23.9 GiB figure should not be +carried forward. + +--- + +## 4. The remaining levers, in priority order + +### Lever 1 (top) — make the GPU LDE gate cell-aware, not row-only + +The threshold is `lde_size < gpu_lde_threshold()` where `lde_size = n × blowup` +(`gpu_lde.rs:698-700`, `:799-801`, `:881-883`) — **there is no column term anywhere in +it**. Its own doc justifies this as "the check is on lde size, not trace length, +because that's what determines the FFT workload" (`gpu_lde.rs:36-44`) — true for the +RV64 VM's tall-and-narrow tables, **false for the LFM machine**, whose chips are short +and extremely wide. The work is `num_cols` FFTs of length `lde_size`, so the honest +cost proxy is `lde_size × num_cols`. + +What it costs today, on the wrap that has actually been proved: + +| chip | LDE × cols | bytes | on GPU? | +|---|---|---|---| +| **KECCAK_RND** | 2^18 × 1,480 | **~3.0 GiB** | **✗ CPU — 88.1% of main cells** | +| LFM_BALU | 2^22 × 14 | ~469 MiB | ✓ | +| BITWISE | 2^21 × 21 | ~352 MiB | ✓ | + +? INFERRED arithmetic over ✓ VERIFIED row/column counts. + +**Why this lever is the best one:** `KECCAK_RND` is the **one non-preprocessed chip** +(`airs.rs:313`, `:74`). Once it clears the size gate it satisfies `device_only_gate` +outright — no `!is_preprocessed` problem, no new kernel, **no code change at all**. +Everything from §1 (device LDE, device Merkle, GPU composition, R3, R4, FRI) plus full +device-only residency applies to it immediately. + +**Test it with one env var** — `LAMBDA_VM_GPU_LDE_THRESHOLD=262144` (2^18) admits the +2^17-row chunk at blowup 2. The permanent fix is a cell-aware gate; the env var proves +the thesis first. + +**Caveat, stated honestly:** full `KECCAK_RND` chunks are capped at 2^19 rows +(`chunking.rs:41`), so at blowup 2 a *full* chunk already clears at 2^20. This lever +therefore matters most for **partial chunks and small epochs** — which is precisely +the only configuration anyone has proved end to end. On a production-sized epoch the +chunks sit at the cap and this lever shrinks; lever 2 grows correspondingly. + +### Lever 2 — lift `!is_preprocessed` from `device_only_gate` (`gpu_lde.rs:210`) + +Per proof of the actually-proved wrap, at blowup 2, from the two preprocessed chips +that are already on GPU: + +| Buffer | Size | Gate that retains it | +|---|---|---| +| BITWISE main LDE host copy | 2^21 × 21 × 8 B = **352 MiB** | `math-cuda/src/lde.rs:806` | +| BITWISE aux LDE host copy (6 ext3 cols) | 2^21 × 6 × 24 B = **288 MiB** | `prover.rs:3407` / `:3449` | +| BITWISE composition parts (2 ext3 parts) | 2^21 × 2 × 24 B = **96 MiB** | `prover.rs:1606` | +| LFM_BALU main LDE host copy | 2^22 × 14 × 8 B = **469 MiB** | same | +| **total (main+aux+parts, both chips)** | **≳ 1.2 GiB** | | + +? INFERRED arithmetic. For calibration: the same D2H skip on the RV64 VM measured +**−12.5%** prove time (memory `gpu-constraint-eval`, 5 ABBA pairs, RTX 5090). This +lever **grows** on production-sized epochs, where more and larger preprocessed chips +clear the size gate. + +**Cost:** three changes, all small, all in already-guarded code. + +1. `gpu_lde.rs:210` — drop `&& !is_preprocessed`. +2. Thread a `retain_host_lde` flag into `try_expand_split_trees_row_major_keep` + (`gpu_lde.rs:779`) and `coset_lde_row_major_split_trees` + (`math-cuda/src/lde.rs:718`), so the D2H at `lde.rs:806` becomes conditional — + exactly mirroring what `coset_lde_row_major_with_merkle_tree_keep` already does + for the plain path. +3. `prover.rs:2758` — let preprocessed tables use `main_dev_values`. The device + gather returns **full** rows (`math-cuda/src/barycentric.rs:357-390`, no column + range), and preprocessed openings need only columns + `[num_precomputed_cols, total_cols)`, so this needs a range-slicing variant of + `device_row_pair` (`prover.rs:2633`) — the values are already in hand, it is a + slice, not a new kernel. + +The safety net already exists: every host-read fallback carries a +`host_trace_empty()` hard-abort (`evaluator.rs:240`, `:1637`, `trace.rs:786`, `:844`, +`prover.rs:2320`, `:2617`, `:2627`), so a missed precondition aborts loudly instead of +producing a wrong proof. ✓ VERIFIED + +### Lever 3 — the hash, which is a machine-design question, not a GPU one + +**84.0% of the production-shaped epoch verify's cells are the hash** +(`LFM_KECCAK + KECCAK_RND`), at 36,256 main + 13,912 aux cells per permutation +(`others/lfm-agent-status.log:196`) ✓ VERIFIED. No amount of GPU work changes that +ratio — it is what the BLAKE3 column exists to attack. Note also that the +production-shaped wrap is **not provable at 124 GiB** (350.6 GiB projected, +`lfm-agent-status.log:198`), so "make recursion fit" may outrank "make recursion fast". + +**Note on BITWISE:** the earlier framing — "BITWISE is 97–99% of cells, shrinking it +dominates everything" — holds only for the *registered toy programs*. In the recursion +wrap BITWISE is 4.8%. Do not spend design effort there on recursion's account. + +--- + +## 5. Staged plan for the GPU box + +### Stage 0 — establish the baseline and falsify the map (do this first) + +The single most valuable measurement, and it is a *falsification test of §1*: + +**Measure the WRAP, not the registered programs.** The registered programs are +toy-sized (§3.1) and will tell you almost nothing. The target is the assembled +epoch-verifier wrap in `prover/src/lfm/wrap_tests.rs` — the harnesses there are +`#[ignore]`d, so they need `--ignored`. Use the **min inner preset** (the one that has +actually been proved: 19.5 s, 15.1 GiB peak on an 11-core laptop); the blowup-8 +production shape is not provable at 124 GiB. + +**Falsifiable predictions.** On an LFM *wrap* prove under `--features cuda`, post-merge: +- `gpu_composition_calls()` **≥ 2** (BITWISE and LFM_BALU both clear the size gate). + The earlier finding predicted **0** for everything; if it is still 0, §1 is wrong. +- `gpu_merkle_tree_calls()` **≥ 4** cold (two subset trees each for BITWISE and + LFM_BALU), fewer warm as the precomputed-tree cache fills. +- `gpu_device_only_calls()` **exactly 0** — BITWISE and LFM_BALU are preprocessed, and + `KECCAK_RND` (the only non-preprocessed chip) is at 2^17 rows, under the gate. This + is the sharpest prediction in the document: **it is 0 only because of a threshold, + and one env var should flip it to ≥ 1.** +- `gpu_lde_calls()` should **exclude** `KECCAK_RND`'s 1,480 columns — i.e. the counter + should be far below the machine's total column count. That is lever 1's evidence. + +Commands (do **not** run locally — these are for the box): + +```bash +# 0. Confirm the box's own GPU stack is healthy BEFORE touching LFM. +# Both targets are ✓ VERIFIED in the Makefile (:565, :572). +make test-math-cuda +make test-cuda-integration + +# 1. Build. Do not add RUSTFLAGS; sccache cache keys must stay stable. +cargo build --release -p lambda-vm-prover --features cuda + +# 2. THE measurement: the assembled epoch-verifier wrap, min inner preset. +# Counter-gated work shares global atomics, so keep --test-threads=1 throughout. +cargo test --release -p lambda-vm-prover --features cuda \ + lfm::wrap_tests -- --ignored --nocapture --test-threads=1 + +# 3. A cheap smoke check first, if the wrap is slow to iterate on. +cargo test --release -p lambda-vm-prover --features cuda \ + lfm::machine_tests::machine_proves_the_sample_replay -- --nocapture --test-threads=1 +``` + +There is **no counter-printing harness for LFM in-tree** — the PR #915 artifacts +(`prover/src/lfm/device_parity_tests.rs`, `prover/tests/gpu_lfm_constraint_interp.rs`, +`thoughts/shared/lfm-gpu/`) live on branch `lfm-gpu-experiments` / +`/Users/maurofab/workspace/lambda_vm-lfm-gpu` and are **not present in this worktree** +(✓ VERIFIED by `ls`). **Stage 0's real deliverable is a ~30-line test** that copies the +shape of `prover/tests/cuda_path_integration.rs:22-36` — `reset_all_gpu_call_counters()` +(`gpu_lde.rs:70`), prove one LFM program, print/assert every counter, verify. Note +those tests are `#[ignore]`d and need `--ignored`; follow the same convention so +no-GPU CI keeps skipping. + +Timing baseline, for the A/B in Stage 1: + +```bash +# Per-phase timings (feature `instruments`) — shows where LFM prove time actually goes. +cargo test --release -p lambda-vm-prover --features cuda,instruments \ + lfm::machine_tests::machine_proves_the_sample_replay -- --nocapture --test-threads=1 + +# Per-chip geometry, so §6 open question 1 stops being open. `lfm_chip_census` +# (airs.rs:138) already computes rows/main_cols/aux_cols per chip. +cargo test --release -p lambda-vm-prover lfm:: -- --nocapture --test-threads=1 2>&1 | grep -i census +``` + +### Stage 1 — A/B the levers that need no code change + +```bash +# (a) ★ LEVER 1, the headline experiment. 2^18 admits the 2^17-row KECCAK_RND chunk +# (88.1% of main cells) at blowup 2. Expect gpu_device_only_calls() to go 0 -> >=1 +# with ZERO code changed, because KECCAK_RND is the non-preprocessed chip. +LAMBDA_VM_GPU_LDE_THRESHOLD=262144 cargo test --release -p lambda-vm-prover \ + --features cuda lfm::wrap_tests -- --ignored --nocapture --test-threads=1 +# then 2^17 (adds XALU/LANES/RANGE) and 2^12 (everything) to find the real knee: +LAMBDA_VM_GPU_LDE_THRESHOLD=131072 ... +LAMBDA_VM_GPU_LDE_THRESHOLD=4096 ... + +# (b) Is the GPU composition path worth anything on LFM? In-binary A/B. +LAMBDA_VM_DISABLE_GPU_COMPOSITION=1 cargo test --release -p lambda-vm-prover \ + --features cuda lfm::wrap_tests -- --ignored --test-threads=1 --nocapture + +# (c) Table concurrency: 14 AIRs of wildly unequal size. Default is cores*2/3 under +# cuda (prover.rs:588-616). Sweep it. +TABLE_PARALLELISM=1 ... ; TABLE_PARALLELISM=4 ... ; TABLE_PARALLELISM=14 ... + +# (d) Other knobs available (grep-verified, gpu_lde.rs / logup_gpu.rs / prover.rs): +# LAMBDA_VM_DISABLE_DEVICE_ONLY, LAMBDA_VM_GPU_BARY_THRESHOLD, +# LAMBDA_VM_NO_GPU_LOGUP, LAMBDA_VM_VRAM_BUDGET_MB, LAMBDA_VM_LOGUP_TIMING +``` + +**Watch VRAM on (a).** `KECCAK_RND` at 2^18 LDE × 1,480 columns is ~3.0 GiB of LDE +before scratch; `estimate_table_vram_bytes` (`prover.rs:622`) will price it around +6–7 GiB. That fits a 5090, but it is the first LFM table big enough to make the +admission gate matter — if it thrashes, `LAMBDA_VM_VRAM_BUDGET_MB` is the knob. + +**Prediction for (a):** **large** — this is the one to bet on. It moves 88.1% of the +machine's main cells from CPU to a fully device-resident path in one env var. If it +does *nothing*, the most likely explanations are (i) `KECCAK_RND`'s constraint program +does not fit the interpreter's per-thread slot scratch and silently falls back to CPU +(a fallback, not an error — check `gpu_composition_calls()` did not rise), or (ii) the +1,480-column shape breaks a kernel launch assumption. Both are worth knowing. + +**Prediction for (b):** small, per the VM's −2.7%-vs-−12.5% split. + +### Stage 2 — build the lever (the three-part change in §4, lever 2) + +Order matters, because each step is independently verifiable: + +1. Make the split-path D2H conditional (`lde.rs:806`, `gpu_lde.rs:779`) but keep + `device_only_gate` unchanged. **Behaviour-neutral** — nothing sets the flag yet. + Gate: existing `cuda_path_integration` suite still green. +2. Add the range-slicing device opening for preprocessed tables (`prover.rs:2758`, + `:2633`) with the release cross-check at `prover.rs:2640` left on. **Still + behaviour-neutral** for correctness; the cross-check is the oracle. +3. Flip `gpu_lde.rs:210`. Now `gpu_device_only_calls()` should go from 0 to ≥1 on an + LFM prove, and no `host_trace_empty` assert may fire. + +**Falsifiable prediction for Stage 2:** after step 3, an LFM prove reports +`gpu_device_only_calls() ≥ 1`, zero guard panics, verify green, and prove time drops. +If a `host_trace_empty` assert fires, a precondition in `device_only_gate` is not +implied by some dispatch — that is the documented LOCKSTEP hazard at +`gpu_lde.rs:185-188`, and the message names the round. + +**Gate before believing any speedup:** proofs are non-deterministic, so never diff +bytes. Use prove→verify plus cross-version verify, per the house rule. + +### Stage 3 — make the threshold permanently cell-aware + +If Stage 1(a) confirms lever 1, replace the row-only check with a cell-aware one +(`gpu_lde.rs:698-700`, `:799-801`, `:881-883`, and the `device_only_gate` mirror at +`:207-209`). The proxy is `lde_size × num_cols` — every call site already has +`num_cols` in scope. Recalibrate the constant against the box rather than reusing +2^19, whose doc says it was calibrated on a 46-core machine for the VM's shape +(`gpu_lde.rs:36-44`). Keep the row check too if a minimum FFT length matters for +launch efficiency; the point is that width must enter the decision. + +--- + +## 6. Open questions I could not settle from code alone + +1. ~~**Heights of the program-dependent chips.**~~ **SETTLED** by the checked-in + census (§3.1). Fixed tables: BITWISE 2^20, LFM_RANGE 2^16, KECCAK_RC 32. Proved + wrap: KECCAK_RND 2^17×1,480, LFM_BALU 2^21, LFM_XALU 2^17. Blowup-8 production + shape (never proved): BALU 2^27, BITDEC 2^21, LANES 2^21, KECCAK_RND 6 chunks. + All ✓ VERIFIED. +2. **Does `KECCAK_RND`'s constraint program actually run on the GPU interpreter?** + This is now the load-bearing unknown, because lever 1 depends on it. It is the + biggest program in the machine (16,317 IR nodes on the old count) and the kernel + allocates per-thread slot scratch. Liveness reuse (§3.1 correction) should make it + fit, but "should" is a code read. **A silent OOM here is a CPU fallback, not an + error** — so the counter must be checked, not just the wall clock. +3. **Whether the R2 device path fires for each chip.** The gate is + `number_of_parts == 2` and `max_degree` is per-chip; I derived degree-3 from LogUp + but did not enumerate each chip. Stage 0's counter test settles it. +4. ~~**VRAM headroom.**~~ Settled for the current shape by reading + `estimate_table_vram_bytes` (`prover.rs:622-633`): BITWISE ≈ 1.19 GiB, and + `VramGate` (`prover.rs:635-647`) admits an oversized table alone so nothing + deadlocks. ✓ VERIFIED by reading. **Reopens under lever 1**: `KECCAK_RND` at + 2^18 × 1,480 prices at ~6–7 GiB, the first LFM table where admission matters. +5. **How many wrap proves a real recursion campaign needs**, and whether the + production-shaped epoch (350.6 GiB projected, `lfm-agent-status.log:198`) is ever + provable — GPU speed is moot if the shape does not fit at all. +6. **Whether the precomputed-tree cache survives the LFM proving schedule.** It is + process-wide and root-keyed (✓ VERIFIED), so it should — but if LFM proving runs + one process per proof, the precomputed trees are rebuilt every time (on GPU, but + still a full leaf hash plus a node D2H at `lde.rs:785`). + +--- + +## 7. Provenance + +- Split-tree preprocessed GPU path introduced by **`d83b4d9e` — "perf(prover): halve + GPU continuation proving time (#863)"** (found via `git log -S + try_expand_split_trees_row_major_keep origin/main`). Its own message says: + *"Precomputed-column Merkle trees are cached process-wide keyed by their commitment + root, so preprocessed tables (DECODE/BITWISE/range) stop rebuilding identical trees + on every prove; only the multiplicity columns are recommitted."* ✓ VERIFIED +- Device-resident rounds 2-4 introduced by **`5749a956` — "perf(prover): + device-resident rounds 2-4 and fused NTT for GPU continuations (#875)"**. ✓ VERIFIED +- Both predate this worktree's merge and postdate the earlier LFM-GPU finding, which + is why that finding's map no longer holds. diff --git a/thoughts/shared/lfm-real-hash/A6R-signoff.md b/thoughts/shared/lfm-real-hash/A6R-signoff.md new file mode 100644 index 000000000..7421c72f2 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/A6R-signoff.md @@ -0,0 +1,234 @@ +# A6R — the 6-round BLAKE3 assumption: decision sheet + +**For:** the user, to sign or decline. **From:** Phase 1. **Date:** 2026-08-10. +**One question:** *is a named, unratified, non-interoperable assumption worth ~5.5% of the epoch column?* +**Recommendation: no — instantiate 7 rounds. Keep 6 as a measured variant behind an explicit signature.** + +--- + +## 1. The assumption, stated precisely + +**A6R:** *the 6-round internal variant of the BLAKE3 compression function is +collision resistant.* + +It is named and recorded as unratified at `prover/src/lfm/blake3.rs:40-42` +(✓ VERIFIED), which says outright that the module "exists to price the AIR, not +to endorse the hash". It originates in PR #903's `IMPLEMENTATION.md`. + +**Today A6R costs nothing.** `LFM_BLAKE3` is unregistered and unreachable. +✓ VERIFIED independently, not inherited from the plan: `grep -rln blake3 +prover/src crypto executor` returns only `lfm/blake3.rs`, `lfm/blake3_chip.rs`, +`lfm/blake3_probe.rs` and `lfm/mod.rs` (the module declarations), and grepping +`LFM_BLAKE3|Blake3` across `airs.rs`, `instr.rs`, `compiler.rs`, `executor.rs` +and `trace.rs` returns **nothing**. **The moment BLAKE3 becomes a +selectable hasher with a registry entry, A6R becomes a live soundness surface +for every program that selects it.** That is what needs a signature — not the +code, the exposure. + +### 1.1 What the spec actually says — quoted, not paraphrased + +⚠ **Correction to `PLAN.md` §7.** The plan renders the external-review note as +ending *"variants below 6 rounds are out of scope and MUST NOT be +instantiated."* That is a strengthening of the source. The actual text +(`git show 783c5a95:spec/blake3.typ`, ✓ VERIFIED by reading) is: + +> *External review (2026-08).* The round-count choice was reviewed with external +> symmetric-cryptography experts consulted by the project: removing *one* round +> (7 → 6) was judged comfortable; removing *two* (7 → 5) was explicitly not. +> Accordingly, 6 rounds is the endorsed floor. Variants below 6 rounds are not +> formally ruled out, but they are not available on the project's own authority: +> adopting one would require the external experts to study the reduced-round +> margin specifically — a dedicated cryptanalytic review, not an engineering or +> configuration decision. + +"Not available on the project's own authority" is a procedural bar, not a +prohibition. The distinction matters for how §6's third record item is worded. + +The spec also supplies the context that argues *for* A6R, and it belongs on a +fair decision sheet: + +> (Precedent: KangarooTwelve's reduced-round Keccak. Best public cryptanalysis of +> BLAKE3 reaches far fewer rounds; the margin removed here is one round of seven.) + +and the scope of what the assumption covers, which is wider than the compress +socket alone: + +> *A6R.* The BLAKE3 compression function restricted to 6 rounds is +> collision-resistant and suitable as a 2-to-1 compression for Merkle hashing +> **and as a PRF for Fiat–Shamir**, in the same sense the full 7-round function +> is believed to be. + +> Any use of BLAKE3 as a Merkle or transcript hash *invokes this assumption*. The +> z3 gate proves the chip computes 6-round BLAKE3 correctly; it neither proves +> nor addresses whether 6 rounds are secure. + +So A6R is not reckless. It is *reviewed but unratified*, non-interoperable by +construction, and it covers the transcript sponge as well as the Merkle compress. + +### 1.2 ⚠ This sheet's recommendation reverses the spec's recorded default + +State this plainly rather than letting it pass. `spec/blake3.typ` records: + +> The 6-round variant is the primary internal target per the review above; the +> 7-round variant is the interoperability / zero-assumption fallback. If both are +> instantiated they are distinct chips with distinct ECALL numbers. + +§5 below recommends the opposite ordering — 7 primary, 6 as the measured +variant. That is a deliberate disagreement, argued on the reference chain rather +than on cryptanalysis, and **if it is accepted the spec section must be updated +to match**, or the tree will carry two contradictory statements of intent. The +plan (§7) reaches the same recommendation; neither it nor this sheet is a +cryptographic re-assessment of the 6-round margin. + +## 2. What 7 rounds buys + +Setting the round count to 7 makes the primitive **bit-identical to published +BLAKE3**. Concretely, and this is the argument: + +- **The reference problem dissolves.** Today the chain is: official crate + vectors pin the oracle at 7 rounds → the oracle at 6 rounds emitted ten + vectors → those vectors pin the Rust port. `blake3.rs:33-38` describes its own + anchor as "one step removed… weaker than a direct KAT and is recorded as + such" (✓ VERIFIED). At 7 rounds there is no step removed: the `blake3` crate + *is* the KAT. +- **The 2-to-1 socket becomes a library call too.** Phase 1 specified the socket + so that `compress(a, b) = blake3::hash(a ‖ b ‖ "LFMC")[0..16]` + (`thoughts/blake3/socket-kats/SOCKET.md`). At 7 rounds that identity is + checkable in one line against the crate — **already executed** against + upstream BLAKE3's C, which passes the official vectors in all three modes. At + 6 rounds the socket vectors can only ever come from our own two sources. +- **Nothing to ratify, re-litigate, or disclose at audit.** No assumption in + `SOUNDNESS.md`, no caveat on the registry entry, no "MUST NOT go below 6" rule + to enforce in perpetuity. +- **Interoperability.** 7-round parent merges are bit-compatible with published + BLAKE3, so an external verifier can recompute a tree. 6-round merges are + computed by nothing else in the world. + +## 3. What 6 rounds buys: the cost delta + +✓ MEASURED, at 6 rounds, standalone prove+verify against the production +`BITWISE` table (`prover/src/lfm/blake3_probe.rs:327-356`, re-read and +confirmed): + +``` +main columns 3,056 + 3 × aux 630 = 4,946 base-field-equivalent cells / compression +interactions = 11 + 832 + 384 + 32 = 1,259 ; aux = ceil(1259/2) = 630 +``` + +? INFERRED for 7 rounds — arithmetic over the chip's own parameterised formulas, +shown so it can be rechecked. `NUM_G = BLAKE3_ROUNDS * 8` goes 48 → 56: + +``` +main columns 3,056 + 8 G-blocks × 60 cells = 3,536 +BITWISE XOR (56×4 + 16) × 4 (was (48×4+16)×4 = 832) = 960 +shift halfwords 56 × 2 × 4 (was 384) = 448 +message bytes unchanged = 32 +LfmMem tokens unchanged = 11 +interactions 960 + 448 + 32 + 11 (was 1,259) = 1,451 +aux ceil(1451 / 2) (was 630) = 726 +base-equiv 3,536 + 3 × 726 (was 4,946) = 5,714 (+15.5%) + +epoch column 2.752 B − 967 M + (195,593 × 5,714 = 1.118 B) = 2.903 B (+5.5%) + vs keccak 11.166 B = 3.85× (was 4.06×) +``` + +**Cross-check against the spec's independent figure.** `spec/blake3.typ` states +7-round costs "roughly 10–12% more per merge end-to-end", and its cost section +gives "≈7,194 committed cell-equivalents per compression end-to-end (≈5,316 +table-only)". So ≈1,878 cell-equivalents per merge are *not* table cells and do +not grow with the round count. Applying +15.5% to the table part alone: +`5,316 × 1.155 = 6,140`, so end-to-end goes `7,194 → 8,018`, i.e. **+11.5%** — +inside the spec's 10–12%. ? INFERRED but it is two independent routes agreeing, +which is real evidence for both. + +**So A6R buys ≈ 5.5% of the epoch column.** For scale, the plan's own §6.1 notes +that the felt-absorbing variant is a ~2.5× lever on the same column — an order +of magnitude more leverage than the round count. + +## 4. The switch is a constant, not a redesign + +✓ VERIFIED, and this materially changes the cost of choosing 7 — the chip is +**already round-parameterised**: + +- `BLAKE3_ROUNDS = 6` (`blake3.rs:56`); the primitive's loop permutes the + schedule when `r < BLAKE3_ROUNDS - 1` (`blake3.rs:106-125`), so setting it to + 7 yields standard BLAKE3's `f` with no other edit. +- `NUM_G: usize = BLAKE3_ROUNDS * 8` (`blake3_chip.rs:98`); the column layout + derives from `NUM_G` (`blake3_chip.rs:157`), the dataflow loops + `for r in 0..BLAKE3_ROUNDS` (`blake3_chip.rs:280`), and + `NUM_CONSTRAINTS = 16 × NUM_G + 1` (`blake3_chip.rs:1042`). + +So the plan's "build round-parameterised" recommendation is already satisfied. +Flipping the constant re-derives the layout, the constraints and the census. + +**Not literally one line, and the difference matters:** the 6-round *expected +values* are baked into tests as literals — `blake3_probe.rs` asserts `2_880`, +`1_259`, `4_946` and `769`, and `blake3_probe.rs:403-421` checks the chip's `OUT` +columns against `CANONICAL_VECTORS`, which are 6-round-specific. A 7-round +instantiation needs those expectations regenerated (to `3,360` / `1,451` / +`5,714` / `897`) and needs 7-round vectors, which — unlike the 6-round ones — +come straight from the crate. ? INFERRED for the four projected constants; they +are compile-time consts and a build phase can confirm them in one `cargo test`. + +## 5. Recommendation + +**Instantiate 7 rounds. Do not sign A6R.** + +In order of weight: + +1. The reference problem dissolves — the crate becomes a direct external KAT for + both the primitive and the socket framing, satisfying standing rule 9 in its + intended form rather than one step removed. +2. No assumption to sign, ratify, or defend at audit; no floor rule to police. +3. Bit-compatibility with published BLAKE3 is worth something on its own. +4. 5.5% is inside the noise of the decisions still open above it. +5. The switch costs a constant plus regenerated test expectations (§4), and the + 6-round path stays available as a measured variant. + +Keep 6-round behind `BLAKE3_ROUNDS` as the performance variant, and switch to it +**if and when** the 5.5% matters — at which point it needs the signature below +and not before. + +## 6. If you sign A6R anyway + +The record needs all four of these, and the first is the one usually missed: + +1. The assumption named in `prover/src/lfm/SOUNDNESS.md`, not only in + `blake3.rs`'s module header — a soundness surface belongs in the soundness + document. +2. The registry entry's doc comment stating which hash its `program_id` rests on. +3. A note that sub-6-round variants are **not available on the project's own + authority** and would need a dedicated external cryptanalytic review — + quoting §1.1's actual wording, not the plan's stronger paraphrase. +4. The 6-round socket vectors (`thoughts/blake3/socket-kats/socket_kats.json`, + `rounds.6`) as the only reference that will ever exist for the socket, with + `SOCKET.md` §6's deferred-crate-check row struck through as unachievable + rather than pending. + +--- + +## Sign-off + +> **A6R** — the 6-round BLAKE3 internal variant is collision resistant. Reviewed +> by external symmetric-cryptography experts as "comfortable" at one round +> removed; unratified; non-interoperable. Buys ≈ 5.5% of the epoch column. + +- [x] **Start with 7 rounds** (recommended path) — decision recorded from the user + in-session, 2026-08-10: *"6 or 7 rounds is the same, we have the greenlight from + symmetric cryptographers to use 6. But we can start with 7, as long as it works + I don't care."* +- [ ] **Sign A6R — instantiate 6 rounds**, and complete §6's four record items + +Signed: ______________________ Date: ____________ + +### Decision record — 2026-08-10 + +The user confirmed the round-parameterised build (`BLAKE3_ROUNDS` knob, both counts +compiled and swept) with **7 rounds as the instantiated baseline**. The external +greenlight for 6 rounds is acknowledged and consistent with §1.1's quoted review +note; the 6-round variant stays available behind the knob. This is **not** a +signature on A6R: per §5, the signature (and §6's four record items) become due +if and when the default switches to 6 — not before. + +Follow-up when Phase 2 lands: update `spec/blake3.typ`'s primary/fallback ordering +(§1.2) so the spec and this sheet agree — 7 primary, 6 as the measured variant. diff --git a/thoughts/shared/lfm-real-hash/ORCHESTRATION.md b/thoughts/shared/lfm-real-hash/ORCHESTRATION.md new file mode 100644 index 000000000..4dc1ee644 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/ORCHESTRATION.md @@ -0,0 +1,82 @@ +# BLAKE3 real-hash implementation — orchestration tracker + +Goal: make the LFM machine's role-2 hash (LFM_HASH) cryptographically real with BLAKE3, +round-parameterized (7-round baseline / 6-round perf), bound into the digest/registry. +Full plan: PLAN.md. Decisions locked: Route A (behind LFM_HASH, no digest move); +Phase-4 mapping = **Option A** (truncate 256→128 into the 1-cell digest + domain separation). + +Worktree: /Users/maurofab/workspace/lambda_vm-blake3-impl (branch `blake3-real-hash` off pr915). +Agents work there; heavy builds SERIALIZE (one cargo build at a time). Agents checkpoint to +this dir; lead reviews verdicts, commits per phase, does not read full file dumps. + +## Dependency order & waves +- WAVE 1 (parallel — only one builder): + - [x] P1 Phase 1 DONE (green): reference now TWO-source (python oracle + upstream BLAKE3 portable C, + round-parameterized) — all 10 6r vectors reproduced byte-for-byte, 7r matches official vectors, + neg controls pass. Socket KATs (Option A) generated. Rust cross-check deferred to a build phase. + - [x] P3 Phase 3 DONE + COMMITTED (2d236786): hasher bound into program_id/registry/lfm_verify. + SOUNDNESS PROPERTY HOLDS + tested (different hasher => distinct program_id). Registry regenerated + (6 program_ids moved, no root moved). lint/fmt green. 19 lfm:: failures = pre-existing stub-ELF. + - [ ] DOC Phase 6 A6R sign-off doc (folded into P1). NO build. + - [ ] ORACLE (spawned ahead, no build): build the human-owned z3 gate ORACLE for BLAKE3 BEFORE the chip — + reference f (6/7r), the Option-A socket reference + KATs, the COLUMN-ROLE MAP (which is ALSO the + Phase-2 chip spec), the chip-contract library, and the z3 gate framework with mandatory negative + controls + width audit. Output: thoughts/shared/lfm-real-hash/gate-oracle/. FEEDS: Phase 2 (chip + conforms to the column-role map) + the z3 gate (plug the real chip constraints into the seam). +- WAVE 2 (after P1 + P3, verified): + - [ ] P2 Phase 2: HasherKind::Blake3, round-parameterized (BLAKE3_ROUNDS knob), Route A behind LFM_HASH, + wire executor/trace/AIR, hasher-dependent bus_interactions, measure at 6/7. BUILD. THE BULK. +- WAVE 3 (after P2): + - [ ] P5 Phase 5: prove+verify a wrap under BLAKE3 (swap TestPermutation). BUILD. +- VERIFY GATES: adversarial check after P3 (binding sound?), after P2 (proves+verifies+KAT match?), after P5. +- Z3 FORMAL GATE (after P2, since it needs the round-parameterized chip + socket): extend the existing + blake3 z3/QF-BV gate (thoughts/blake3/blake3-chip/z3_blake_verify.py, restored by P1) to cover the chip + at BOTH 6 and 7 rounds + the Option-A socket (2-to-1 compress + truncate 256->128 + domain tag + byte + enforcement). Mandatory rigor: negative controls (drop an AreBytes/BITWISE contract -> SAT; wrong + truncation window -> SAT; missing domain tag -> SAT) + a non-vacuity positive control + the width audit + (every field-lifted width cites a real BITWISE/AreBytes contract). z3/QF-BV fits because BLAKE3 is a + byte computation; native-field (Poseidon) would need cvc5-FF/Lean and is out of scope. NOT a z3 target: + the Phase-3 hasher binding (domain-separation argument -> pinned by the "distinct program_id" test). + +## Status log +- 2026-08-10: Phase-1 anchor pre-confirmed GREEN by lead (oracle vs official vectors + Plonky3, 6r derivative). PLAN.md written. +- WAVE 1 spawned. +- 2026-08-10 (late): ORACLE complete (ORACLE.md + gate board PASS; chip_model.py = the Phase-2 spec). + A6R decision recorded by user: 7-round instantiated baseline, 6 behind the blake3-6round feature. + O5 RATIFIED by user: future leaf hashing uses the reserved "LFML" tag. +- 2026-08-11: **P2 DONE + COMMITTED `b693eece`** (+ O5 docs `cece4a0b`): Blake3 arm behind LFM_HASH, + compress-only (MODE_P=0 pinned; permute = task #8 and it BLOCKS P5's full wrap), 7r default, + KATs 15/15 both counts + direct crate anchor, 4,741/5,509 cell-equiv (6r/7r). Adversarial review + closed both directions: 0 soundness, 0 regression (phase2-verify.md); executor compress_out fix + is the one cross-hasher touch (latent inlining bug, defaults preserve Test/Poseidon). + **Z3 CHIP GATE DONE: CHIP-GATE.md VERDICT PASS 75/75 on the chip AS BUILT** (seam transcription, + argued algebra ledger AR1-AR4, census == built chip to the unit at both round counts); + artifact_pin v2 --check confirms the verdict applies to the committed file (semantic regions + + resolved framing unchanged; drift is comments/docs only). Remaining oracle-side: D7b (ORACLE.md + §3.2 census refresh), D10 (chip_model.py:152-156 docstring), D6 durable note, commit-SHA pin update. +- 2026-08-11: oracle closed all four doc items + re-ran the board (PASS, 23:06). Reviewer's exit audit + flagged O5's safety claim as unaudited; lead verified in code: "no leaf-hashing path" was FALSE + (FriToyV0 compresses raw rows into leaves under the LFMC tag, programs.rs:577/585/625); safety rests + on fixed-depth static circuits ALONE. Corrected in commit 2957c3f9 + ORACLE.md §7 + CHIP-GATE.md. + Phase-2 agents shut down after independently re-verifying the commit hashes. Branch head: 2957c3f9. + NEXT: task #8 permute-socket spec (Phase-5 blocker; mapping decision goes to the user first). +- 2026-08-11: permute-socket options paper delivered (A: LFMP socket / B: compress-based sponge / + C: mixed Poseidon). Corrected map inside: WRAP IS HASH-NEUTRAL (epoch emits no Instr::Hash) — only + TrivialV0/FriToyV0 gate on this; TrivialV0 calls b.permute directly. **USER RATIFIED OPTION B (B1): + compress-based FS chain for all hashers, no permute socket ever, MODE_P=0 permanent, no assumption + beyond A6R.** Oracle assigned the transcript spec (TAG_LFMT, reference+KATs incl. end-to-end + FriToyV0-preamble vector, gate two-tag framing, TrivialV0-fate rec) → transcript-spec/. Build agent + spawns on the spec, not before. +- 2026-08-11: oracle re-gate on post-B1 chip: CHIP-GATE PASS 79/79, re-pinned (pin extended: TAG_LFMT + resolved + tags-distinct check), census unit-exact incl. program totals; M8 standing control added + (idx-4-alone forgery SAT / one-hot UNSAT / both real tags reachable); TRANSCRIPT.md §3.3 + both §2.2 + framing rows corrected BEFORE transcription; two own-instrument bugs caught (§4.6.3). + Leaf-convention options note delivered (leaf-convention-options.md): NEW option C found — in-socket + felt mode reusing O1's lane machinery + Z/GINV canonicity (p−1 = 0xFFFFFFFF_00000000 ⇒ "hi maximal ⇒ + lo zero"; 2 cols + 4 constraints/felt, no new sends). FriToyV0 @7r: C = 502,047 (+36%) vs A + (felt_be_halves) = 585,039 (+58.5%, a floor) vs B = off-BLAKE3 forever. + **USER RATIFIED OPTION C + LFML** (leaves hash under the ratified LFML tag via MODE_L): cheapest by + 14%, reuses the machine's own canonicity idiom, retires O5's fixed-depth crutch; cost = one more + re-gate (new pin + canonicity width-audit pair; M8 extends unchanged). §6 open point resolved by + decision: MODE_L implies felt-input (contiguous one-hot span). SEQUENCE: commit B1 first (b1-verify + pending), then MODE_L spec-first (oracle), then build. diff --git a/thoughts/shared/lfm-real-hash/PLAN.md b/thoughts/shared/lfm-real-hash/PLAN.md new file mode 100644 index 000000000..14ca92b8d --- /dev/null +++ b/thoughts/shared/lfm-real-hash/PLAN.md @@ -0,0 +1,671 @@ +# Proving the LFM machine with a real hash — BLAKE3 + +**Status:** plan, not implementation. Nothing here has been built. +**Date:** 2026-08-10. **Target branch:** `pr915` (worktree `/Users/maurofab/workspace/lambda_vm-pr915`). +**Author's ground rules:** every factual claim about the tree is marked ✓ VERIFIED (I read +the code and cite `file:line`), ? INFERRED (derived, arithmetic shown), or ESTIMATE +(labelled, with its basis). Line numbers are as of the `pr915` worktree read on 2026-08-10. + +--- + +## 0. The short version + +Three things need saying before the phases, because two of them change what the goal *is*. + +**(a) The 6-round reference problem is much closer to solved than the brief assumes.** +The oracle, the official-crate cross-check, the recorded 6-round vectors and the z3 gate +all exist — in **git**, on `feat/blake3-accelerator` / `spike/blake3-recovered`, not in +the working tree (which has decayed to `__pycache__` and a venv). And the ten canonical +6-round vectors are already transcribed into `prover/src/lfm/blake3.rs:151-342` **with a +negative control that breaks one convention at a time** (`blake3.rs:463-514`), and the +chip's own `OUT` columns are asserted against them (`blake3_probe.rs:403-421`). Phase 1 +is therefore *restoration plus a second independent source*, not construction. Effort: S. + +**(b) "BLAKE3 as the machine's real hash" names two different sockets, and the cost +numbers in the PR body belong to the one the brief is not asking about.** There are +three hash roles in this system; `HasherKind`/`TestPermutation` is role 2, the measured +2.75 B / 1.1 B epoch figures are role 1. Swapping `TestPermutation` for BLAKE3 changes +the wrap's cost by **exactly zero**, because the assembled epoch verifier emits no +`Instr::Hash` at all. §1.1 and §6 work this through. This is not a reason to abandon the +goal — it is a reason to state the goal as "make socket 2 cryptographically real and +bound", which is achievable now, rather than "make the wrap cost 2.75 B", which is a +production-side migration. + +**(c) The A6R assumption buys about 5% of the epoch bill, and standard 7-round BLAKE3 +removes it entirely.** Working the chip's own column and interaction budget forward from +6 to 7 rounds (§7, arithmetic shown) gives ≈ +15.5% per compression and ≈ +5.5% on the +whole epoch column. Against that, 7-round is bit-compatible with published BLAKE3, so +the `blake3` crate becomes a *direct external KAT* and A6R disappears. **My +recommendation is to build the chip round-parameterised and instantiate 7-round first**, +keeping 6-round as the measured performance variant behind an explicit signed assumption. +The user's "as long as it works with Blake 6r, or blake, it's fine" permits this, and it +is the cheaper path to a defensible result. + +**Start here:** §9. + +--- + +## 1. Ground truth — what exists today + +### 1.1 There are three hash roles, and they are not interchangeable + +This taxonomy is the single most important thing in this document. The scoping report +already found two of them and says so in its headline: *"The machine already has a hash +swap surface, and it is NOT the socket keccak is plugged into"* +(`others/lfm-hash-matrix-scope.md:14-56`, ✓ VERIFIED by reading). + +| | role 1 — the **inner** hash | role 2 — the **program** hash | role 3 — the **outer** hash | +|---|---|---|---| +| What it is | the hash the *proof being verified* was committed under | the hash an LFM *program* calls via `Instr::Hash` | the hash the LFM prover commits its own traces under | +| Today | keccak (production RV64 proofs) | `TestPermutation` behind `LFM_HASH` | keccak (the `stark` framework) | +| In-machine chip | `LFM_KECCAK` + hosted `KECCAK_RND`/`KECCAK_RC`/`BITWISE` (`airs.rs:51-66`) | `LFM_HASH`, chip slot 5 (`airs.rs:427-435`) | none — it is outside the machine | +| Gadgets | `edsl::keccak_merkle_walk`, `edsl::keccak256` | `edsl::merkle_walk`, `edsl::SpongeVar` (`edsl.rs:16-79`) | — | +| Digest | 2 machine cells / 8 felts / 32 bytes | 1 machine cell / 4 felts (`word.rs:1-9`) | `Commitment = [u8; 32]` | +| Selected by | the inner proof's own construction | `HasherKind` (`hash.rs:101-109`) | the framework, not swappable here | +| Cost measured | 11.17 B cells / epoch verify | **0 permutations in the wrap** | n/a | + +`edsl.rs:137-143` states the non-interchangeability outright: *"`merkle_walk` compresses +with `LFM_HASH`/`TestPermutation`, the deliberately non-cryptographic Milestone-C +placeholder, so it can only ever authenticate the Milestone-C fixture tree. Production +trees are keccak throughout."* ✓ VERIFIED. + +**The wrap emits no `Instr::Hash`.** ✓ VERIFIED independently of the review: grepping +`b.permute(` / `b.compress(` across `builder.rs`, `edsl.rs`, `programs.rs`, `epoch.rs`, +`epoch_verify.rs`, `fri.rs`, `sub_proof.rs`, `transcript_replay.rs`, `statement_replay.rs` +returns callers in `edsl.rs:30,41,76` (the library) and `programs.rs:44-46` and +`programs.rs:576-623` (`trivial_program`, `fri_toy_program`) — and **nothing in the epoch +verifier's own modules**. `wrap_tests.rs:26-28` says the same thing from the other side. + +So role 2's only current consumers are two toy programs, both of which *are* in the +registry (`registry.rs:27-42`: `TrivialV0`, `FriToyV0`). That is exactly the F3.4 +disclosure: `FriToyV0` is billed as "the Milestone-C FRI-opening verifier" while its +Merkle authentication and its Fiat–Shamir sponge are both cryptographically vacuous. +**Making role 2 real is what retires that disclosure**, and it is a well-sized, +self-contained project. It is not what makes the wrap cheaper. + +### 1.2 What `LFM_BLAKE3` is today, and why it is unregistered + +✓ VERIFIED by reading `blake3.rs`, `blake3_chip.rs`, `blake3_probe.rs` and `mod.rs`: + +- `lfm/blake3.rs` is the **primitive**: `blake3_compress_6round(h, m, t, block_len, flags) + -> [u32; 16]`, a byte-for-byte vendoring of #903 at head `89aeeb8c` + (`blake3.rs:1-13`), plus the ten `CANONICAL_VECTORS` and four convention tests. + `BLAKE3_ROUNDS = 6` (`blake3.rs:56`); the loop permutes the schedule when + `r < ROUNDS - 1` (`blake3.rs:119`), which means **setting that constant to 7 yields + exactly standard BLAKE3's compression function `f`** — no other edit. ? INFERRED from + reading the loop against the BLAKE3 spec; it is the property the whole 7-round fallback + rests on and should be pinned by a test, not assumed (Phase 1, step 4). +- `lfm/blake3_chip.rs` is the **chip**: 3,072 columns of which 16 are preprocessed + (`blake3_chip.rs:148,154-159,221`), 1,259 bus interactions, one row per compression, + 769 constraints at degree 3. Its I/O side was re-expressed on `LfmMem` word tokens + (7 reads + 4 writes) in place of #903's syscall `Ecall`/`Memw` shape, and the header + argues each dropped range check (`blake3_chip.rs:34-61`). +- `lfm/blake3_probe.rs` proves and verifies it standalone against the **production** + `BITWISE` table, at 4,946 base-field-equivalent cells per compression + (`blake3_probe.rs:327-356`), with five tamper-rejection tests. +- **Registration status:** `blake3` and `blake3_chip` appear at `mod.rs:19-20` and + `blake3_probe` at `mod.rs:74`, and `grep -rn blake3 prover/src crypto executor` returns + those four files and nothing else. ✓ VERIFIED — they are absent from `LFM_CHIP_NAMES`, + `LfmAirs`, `LfmTraces`, `Instr`, the compiler and the executor. + +**Why unregistered, precisely.** `NUM_LFM_CHIPS = 14` (`airs.rs:50`) and `lfm_program_id` +iterates `0..NUM_LFM_CHIPS` folding each slot's root and log-height into a keccak preimage +(`statement.rs:49-53`). Adding a 15th chip class changes the loop bound and therefore +**every registered program's digest**, which invalidates all six registry entries and +every attestation that folded one. That is the "registration moves every program digest" +consequence, stated exactly (`blake3_chip.rs:72-77`). It is a re-blessing, not a bug — +but it is a decision with a blast radius, and §3 shows it is **avoidable**. + +### 1.3 The `HasherKind` swap surface as it stands + +✓ VERIFIED: + +- `HasherKind` has exactly two variants, `Test` (the `#[default]`) and `Poseidon` + (`hash.rs:101-109`). **There is no `Blake3` variant.** `lfm_prove_with_hasher(..., + HasherKind::Blake3)` does not compile today. +- The contract behind it is `LfmHasher` (`hash.rs:27-44`): `permute([FE; 12]) -> [FE; 12]`, + `compress_iv() -> LfmWord`, and a defaulted `compress(a, b)` that permutes `a ‖ b ‖ IV` + and truncates to the first cell. `HASH_STATE_FELTS = 12`, `HASH_DIGEST_FELTS = 4` + (`hash.rs:19-21`). +- One `hasher` value reaches the executor, the trace filler and the AIR set through a + single function, `lfm_prove_with_hasher` (`proof.rs:61-78`) — that is the agreement + mechanism the brief refers to. +- `hash::num_columns(kind)` and `HashConstraints::num_constraints(kind)` are two-arm + matches (`chips.rs:583-588`, `chips.rs:652-657`). `hash::bus_interactions()` takes **no + hasher argument** and returns 6 `LfmMem` interactions (3 receivers over `IN_ADDR0..2`, + 3 senders over `OUT_ADDR0..2`, `chips.rs:590-623`). +- The shared value prefix `IN0..11`, `S8..11`, `OUT0..11` is 28 columns and is frozen at + fixed offsets in **every** layout, "which is why they keep their offsets in EVERY layout + — a candidate appends its witness columns after them rather than reflowing the prefix" + (`chips.rs:487-491`). Poseidon appends 584 columns after it, reaching 612 + 11. +- `layout::hash::PREP_WIDTH = 11` (`layout.rs:81-94`), and because it is 11 under both + hashers **the preprocessed roots and hence `lfm_program_id` are hasher-independent by + construction** (`airs.rs:375-380`). + +**What is missing to make BLAKE3 selectable end to end,** enumerated against those facts: +a `HasherKind::Blake3` variant and its `LfmHasher` impl; an arm in `num_columns` and +`num_constraints`; a BLAKE3 arm in `HashConstraints::eval`; a witness-filling arm in +`trace.rs` (which currently special-cases `hasher == HasherKind::Poseidon` at +`trace.rs:222`); and — the one structural change — `hash::bus_interactions()` must become +**hasher-dependent**, because BLAKE3 needs 1,248 `BITWISE` lookups per permutation that +Poseidon and `TestPermutation` do not. That is a signature change with three call sites +(`airs.rs:189`, `airs.rs:429`, and the census). See §3. + +### 1.4 The binding gap (the brief calls it F3-2; the findings file numbers it **F3.3**) + +✓ VERIFIED, and worth restating precisely because it is the soundness-gating item: + +- `LfmRegistryEntry` has fields `kind`, `blowup_factor`, `roots`, `log_heights`, + `keccak_rnd_chunks`, `program_id` — **no hasher** (`registry.rs:52-60`). +- `lfm_program_id`'s preimage is the tag, the machine version, the preset tag, then per + slot `(index, root, log_height)`, then the chunk count — **no hasher** + (`statement.rs:40-56`). +- `lfm_verify` resolves the registry entry and calls `verify_against` + (`proof.rs:135-150`), which hardwires `HasherKind::default()` (`proof.rs:172-181`). + +So today the *only* thing separating a Poseidon-proved trace from a Test-built AIR set is +that `hash::num_columns` differs (11 + 28 = 39 vs 11 + 612 = 623), which the framework rejects as a width +mismatch. That is a coincidence of layout, not a binding — and a BLAKE3 arm is exactly the +kind of third candidate that could collide with an existing width. **Fix before, not +after, adding the third arm.** + +### 1.5 The 6-round reference material — where it actually is + +✓ VERIFIED by `git log --all --diff-filter=A -- 'thoughts/blake3/*'`: + +| artifact | added in | what it is | +|---|---|---| +| `thoughts/blake3/blake3-oracle/blake3_ref.py` | `3b9b8137` | the round-parameterised Python oracle | +| `thoughts/blake3/blake3-oracle/ORACLE.md`, `test_oracle.py` | `3b9b8137` | its documentation and tests | +| `thoughts/blake3/blake3-chip/DESIGN.md`, `z3_blake_verify.py` | `3b9b8137` | the gate-proved chip design and its z3 gate | +| `thoughts/blake3/blake3-oracle/official_test_vectors.json` | `19ed761b` | the **official crate** vectors — the external anchor | +| `thoughts/blake3/blake3-oracle/canonical_6round_vectors.json` | `19ed761b` | the ten 6-round vectors | +| `thoughts/blake3/ground-truth/{Cargo.toml,src/main.rs}` | `19ed761b` | a Rust project that links the real `blake3` crate | +| `thoughts/blake3/{TRANSCRIPTION,GATE-TRANSCRIPTION}-AUDIT.md` | `8fec369e` | two transcription audits and the corrections they forced | +| `thoughts/blake3/blake3-chip/IMPLEMENTATION.md` | `35038501` | #903's implementation notes, where A6R is named | +| `spec/blake3.typ` | `2e0f0b41`, `a7a8bdd5`, `783c5a95` | the chip spec page and the A6R section | + +These live on `feat/blake3-accelerator` (and `origin/feat/blake3-accelerator`), **not on +`main` and not on `pr915`.** ✓ VERIFIED: `git ls-files thoughts/blake3` on `main` returns +nothing, and the working-tree directory now contains only +`blake3-oracle/__pycache__/blake3_ref.cpython-314.pyc`, +`blake3-chip/__pycache__/z3_blake_verify.cpython-314.pyc`, a venv, and +`ground-truth/target/`. The `.pyc` files are the compiled form of the two deleted +sources — recoverable, but `git show` is the honest route. + +The provenance chain the primitive currently rests on (`blake3.rs:15-38`, ✓ VERIFIED as an +accurate self-description): official crate vectors pin the oracle **at 7 rounds**, so the +G-function, message schedule, counter split and feed-forward are externally validated; +only the round count is varied; the oracle at `rounds = 6` emitted the ten vectors. The +module says outright that this is "weaker than a direct KAT and is recorded as such." + +--- + +## 2. Phase 1 — a trustworthy 6-round reference, and the KATs it pins + +**Goal:** a re-runnable, two-source derivation of the 6-round vectors, plus a KAT layer +for the *socket instantiation* that `CANONICAL_VECTORS` does not cover. +**Effort:** S (1–2 days). **Risk:** LOW. **Blocks:** everything else. + +### 2.1 What is already pinned, and what is not + +✓ VERIFIED — do not redo this work: + +- The primitive reproduces all ten vectors (`blake3.rs:431-440`). +- A *parameterised* control at canonical parameters equals the port + (`blake3.rs:445-454`), so the negative controls differ in exactly one convention. +- Four conventions each break the vectors when perturbed alone (`blake3.rs:462-514`): + `rotr12 → rotr13`, `rotr16 ↔ rotr8`, message schedule transposed, and **7 rounds**. + The last one is the round-count discriminator, and it is already there. +- The counter halves are not interchangeable (`blake3.rs:519-538`). +- **The chip is pinned to the vectors, not merely to the primitive.** + `the_hosted_chip_proves_and_verifies` asserts `expected == CANONICAL_VECTORS[row].out` + and then checks every one of the 64 `OUT` byte columns against it + (`blake3_probe.rs:403-421`). This closes the obvious "the chip is only checked against + the same Rust that produced it" worry. + +**Not pinned, and this is the real gap:** `CANONICAL_VECTORS` pins `f(h, m, t, block_len, +flags)`. It says nothing about *how the socket calls it* — which flags, where the two +input digest cells land in `m`, what `t` is, how the 16-word output becomes one digest +cell. Every one of those is a fresh way to be wrong, and rule 9's whole point +(`others/lfm-standing-decisions.md:121-136`) is that a right constant plus a wrong framing +is the normal failure. §5 fixes the framing; this phase must pin it. + +### 2.2 Steps + +1. **Restore the artifacts into the working tree.** `git show 3b9b8137:` and + `git show 19ed761b:` for the eight files in §1.5's table, into + `thoughts/blake3/`. Do not resurrect them from the `.pyc` files — the git blobs are + authoritative and the audits in `8fec369e` apply to them. +2. **Re-run the first link.** `test_oracle.py` against `official_test_vectors.json` at + `rounds = 7`. This is the only external anchor in the chain; if it does not run green + the chain is broken and nothing downstream means anything. +3. **Add a second, independently derived 6-round source.** The Python oracle is in-repo + and was itself recovered from transcripts, so one source is thin. The best second + source is **upstream BLAKE3's own `reference_impl/reference_impl.rs`** with its round + loop parameterised — external code, a minimal and reviewable diff, a different author + and a different language from the Python oracle. Vendor it under + `thoughts/blake3/reference-impl/` with the diff visible. Run it at 7 rounds against the + official vectors (proving the parameterisation is inert), then at 6. + **Acceptance:** both sources, at `rounds = 6`, reproduce all ten + `CANONICAL_VECTORS` byte for byte. If they disagree, stop — the vectors in + `blake3.rs:151-342` are wrong and everything built on them is wrong. +4. **Pin the 7-round claim as a test, not a comment.** Add a test that instantiates the + parameterised control (`blake3.rs:375-428`) at `rounds = 7` and checks it against the + **`blake3` crate**, via `blake3::hash()` of a ≤ 64-byte message with + `h = IV, t = 0, block_len = len, flags = CHUNK_START|CHUNK_END|ROOT`. This is a direct + external KAT of `f` with no oracle in the middle, and it is what makes the 7-round + fallback assumption-free. ? INFERRED that the public `hash()` API suffices for a + single-chunk message — verify against the crate's docs before writing the test rather + than assuming the flag values. +5. **Pin the socket framing** (depends on §5's decision): once `compress(a, b)` is defined + in terms of `(h, m, t, block_len, flags)`, add vectors for **that function**, generated + by both sources, plus a negative control per framing degree of freedom (swap `a`/`b`, + change the flags byte, move the truncation window). At 7 rounds this KAT can be + `blake3::hash(a ‖ b)` from the crate directly — a further reason to prefer 7. +6. **Wire the gate into CI or delete the claim.** Today nothing re-derives the vectors. + Either add a job that runs steps 2–4, or state plainly in `blake3.rs` that the chain is + a one-time historical derivation. Rule 8 (`a search that ERRORS looks exactly like a + search that found nothing`) argues for the job. + +### 2.3 The 6r-vs-full tradeoff, stated once + +| | 6-round | 7-round (standard) | +|---|---|---| +| Reference for the primitive | oracle + reference-impl at `rounds = 6`; **no library, no published vector** | the `blake3` crate directly; published vectors | +| Reference for the 2-to-1 socket | must be generated by the same two sources | `blake3::hash(a ‖ b)` — a library call | +| Security | assumption **A6R**, named and unratified (`blake3.rs:40-42`) | standard BLAKE3; no new assumption | +| Interop | none — nothing else computes it | bit-compatible with BLAKE3 parent merges | +| Cost per compression | 4,946 base-equiv ✓ MEASURED | ≈ 5,714 ? INFERRED (§7) | +| Cost on the epoch column | 2.752 B ✓ MEASURED | ≈ 2.902 B ? INFERRED (+5.5%) | + +--- + +## 3. Phase 2 — make BLAKE3 a first-class selectable hasher + +**Goal:** `lfm_prove_with_hasher(program, artifacts, arenas, options, HasherKind::Blake3)` +proves, and the matching verify accepts. **Effort:** L (the chip re-expression is the +bulk). **Risk:** MEDIUM. **Depends on:** Phase 1 and §5's mapping decision. + +### 3.1 Two routes, and the recommendation + +**Route A — host BLAKE3 *behind* the frozen `LFM_HASH` socket. ★ RECOMMENDED.** + +Add `HasherKind::Blake3` and give `LFM_HASH` a BLAKE3 layout the same way Poseidon got +one: keep `PREP_WIDTH = 11` and the frozen 28-column shared value prefix, append the +BLAKE3 witness columns after it, and reuse `blake3_chip`'s mixing core. + +What this costs: +- `hash::bus_interactions()` gains a `hasher` parameter (3 call sites: `airs.rs:189`, + `airs.rs:429`, and `lfm_chip_census_with_hasher`). Under `Blake3` it returns the 6 + `LfmMem` tuples **plus** the 1,248 `BITWISE` lookups. Aux columns go from 3 to ≈ 627. +- `chips::hash` grows a `blake3_cols` module and an `eval_blake3` arm, sharing + `blake3_chip::run_flow`/`WireFlow`/`ValueFlow` so the single-dataflow rule survives + (`blake3_chip.rs:63-70` — this property is why the sender list and the witness cannot + drift, and it is worth preserving on sight). +- The `LFM_HASH` chip's constraint count and degree change; degree stays 3 (the BLAKE3 + chip is already degree 3, `blake3_probe.rs:361`). + +What this **buys**, and it is the decisive argument: +- `NUM_LFM_CHIPS` stays 14. `PREP_WIDTH` stays 11. **No root moves and no program digest + moves** (`airs.rs:375-380`, `statement.rs:49-53`). The six registry entries survive + untouched; only programs that actually opt into `HasherKind::Blake3` get a different + identity, and after §4 they get it *deliberately*. +- The frozen `LFM_HASH` bus contract — 2 cells in, 1 cell out — is honoured, so every + existing `edsl::merkle_walk` / `SpongeVar` caller works unchanged. That is exactly what + `hash.rs:1-8` promises the swap surface is for. +- `blake3_chip.rs`/`blake3_probe.rs` stay as the measurement probe and the standalone + falsification harness. Their 4,946 number remains a real, separately-proved datum. + +**Route B — register `LFM_BLAKE3` as chip class 15.** + +`NUM_LFM_CHIPS` 14 → 15, a new `LFM_CHIP_NAMES` entry, a new `LfmAirs` field, a new +`LfmTraces` field, a new instruction and a compiler lowering. Every registered program's +digest moves; all six entries must be regenerated and re-blessed; every attestation that +folded an old `program_id` is invalidated. Every LFM proof — including programs with no +BLAKE3 at all — carries a padded `LFM_BLAKE3` instance, which is the fixed-machine +principle working as designed (`airs.rs:34-49`) but is still 4 rows × 3,056 columns of +nothing. + +Route B is only necessary if a single proof must use BLAKE3 **and** a different +`LFM_HASH` permutation simultaneously. Nothing in the roadmap wants that. + +**Recommendation: Route A.** Route B's only advantage is that `blake3_chip.rs` could be +registered nearly as-is; Route A's I/O re-expression (from 7-in/4-out syscall-shaped words +to `LFM_HASH`'s 3-in/3-out) is real work, but it is the *same kind* of work +`blake3_chip.rs` already did once when it re-expressed #903's `Ecall`/`Memw` side onto +`LfmMem` — and the header documenting that swap (`blake3_chip.rs:13-32`) is a ready-made +template for doing it again. + +### 3.2 The `LfmHasher` contract problem — read this before writing code + +This is the sharpest technical issue in the whole plan and it is easy to miss. + +`LfmHasher::permute` is typed `[FE; 12] -> [FE; 12]` — **arbitrary** Goldilocks elements. +BLAKE3 operates on 32-bit words. A Goldilocks felt is up to ~64 bits. So a BLAKE3 +`permute` cannot honour that signature on its whole domain without deciding how a 64-bit +felt becomes BLAKE3 input, and the naive answer is unsound: `Σ byteₖ·256ᵏ = v` over the +field does **not** pin the byte string, because `v` and `v + p` both satisfy it — so +without a `< p` argument the prover chooses what gets absorbed and Fiat–Shamir breaks. +That is not my analysis; it is recorded verbatim at +`others/lfm-hash-matrix-scope.md:1440-1452`, and it is why `felt_be_halves` routes through +`bit_dec`, whose contract enforces canonicity (`transcript_replay.rs:735-736`). ✓ VERIFIED. + +Two ways out: + +- **(i) u32 lanes — restrict the domain. ★ RECOMMENDED.** Every state felt carries a + `u32`. The state is 12 × 32 = 384 bits = 48 bytes, which fits **one** 64-byte BLAKE3 + block with room to spare (`m[0..12] = state`, `m[12..16] = 0`) — so one permutation is + one compression, and a digest cell is 4 × `u32` = 128 bits, exactly the machine's + declared "128-bit target" (`word.rs:1-9`). The map `u32 → FE` is injective with no + canonicity argument at all, and the chip's existing byte decomposition already + range-checks each lane. This is the same four-`u32`-lanes-per-machine-word convention + `LFM_KECCAK` and `blake3_chip` already use (`layout.rs:96-102`, `blake3_chip.rs:8-11`). + **Cost:** a program that wants to absorb an arbitrary felt must split it into two `u32` + halves first — the byteswap gadget, in-machine. +- **(ii) Felt-absorbing with an in-chip canonicity gate.** The chip receives full 64-bit + felts and decomposes them to bytes inside its own constraints, with a borrow-chain + `< p` gate per absorbed felt (ESTIMATE ≈ 20 base-equiv per felt, so ≈ 240 per + permutation). This keeps `permute` total on `[FE; 12]` and deletes the byteswap gadget + from callers. **Cost:** the 12-felt state is 96 bytes, which does **not** fit one + 64-byte block — either two compressions per permutation or a restructured state. + +Route (i) is simpler, sounder-by-construction, and one-compression-per-permutation. Take +it. Then **the trait's contract must be made explicit**: `LfmHasher::permute` becomes +documented as partial for lane-restricted hashers, and the `HasherKind::Blake3` impl must +reject (not silently reduce) an out-of-range lane, so the host and the chip agree on the +domain. Silently reducing is the bug that would make a host-side `assert` pass while the +chip proves something else. + +### 3.3 Steps + +1. Decide §5 first — the mapping is an input to the layout, not an output. +2. `hash.rs`: add `HasherKind::Blake3`; implement `LfmHasher` for it via a + `Blake3Permutation` struct; make the partial-domain contract explicit in the trait doc + and enforce it in the impl. +3. `chips.rs`: `blake3_cols` module appended after `SHARED_VALUE_COLUMNS`; `eval_blake3` + sharing `blake3_chip`'s `run_flow`; arms in `num_columns` and `num_constraints`; + `bus_interactions(hasher)`. +4. `airs.rs`, `trace.rs`: thread the hasher through the three call sites and add the + witness-filling arm beside the Poseidon one (`trace.rs:76-113`, `trace.rs:222`). +5. `executor.rs`: nothing structural — `Instr::Hash` already dispatches through + `&impl LfmHasher` (`executor.rs:363-399`). +6. Tests, in this order: primitive KAT (Phase 1) → a `blake3_chip_tests`-style constraint + test mirroring `poseidon_chip_tests.rs` → prove+verify of `TrivialV0` under + `HasherKind::Blake3` → the five tamper-rejection analogues from `blake3_probe.rs` + (rule 2: an execute-only test proves nothing about the chip). + +--- + +## 4. Phase 3 — bind the hasher into `lfm_program_id` and the registry + +**Goal:** a BLAKE3-backed machine has a distinct, pinned program digest, and `lfm_verify` +reads the hasher from the registry entry instead of defaulting. +**Effort:** S. **Risk:** LOW mechanically, but this is the **soundness-gating** step. +**Do it BEFORE Phase 2 lands**, not after — see §1.4. + +Steps: + +1. Give `HasherKind` a stable `u8` discriminant with an explicit `as_tag()`, so the wire + value never follows enum declaration order. +2. `statement.rs`: fold the tag into `lfm_program_id`'s preimage — after + `LFM_PRESET_TAG`, before the per-slot loop (`statement.rs:45-55`). This moves all six + existing digests **once**, which is a deliberate re-blessing and must be called out in + the PR body. +3. `registry.rs`: add `hasher: HasherKind` to `LfmRegistryEntry` and `LfmArtifacts`; thread + it through `build_artifacts`; regenerate via `compute_lfm_registry`. +4. `proof.rs`: `lfm_verify` passes `entry.hasher` to `verify_against_with_hasher` instead + of `verify_against`'s `HasherKind::default()` (`proof.rs:141-149`, `172-181`). +5. Add the test that makes it real: a proof produced under one hasher must be **rejected** + when verified under an entry naming another — and, per the honest-control rule, an + accompanying test that the matched pair still **verifies**. A rejection test alone + passes just as well if the fix rejects everything. +6. While in `compute_lfm_registry`: it never calls `validate()` (finding F3.2, + `compute_lfm_registry.rs:28-66`). Same file, same PR, one line — take it. + +**Note on Route A's interaction with this phase.** Under Route A the roots do *not* move +with the hasher, so after step 2 the hasher tag is the *only* thing distinguishing a +BLAKE3 machine's digest from a Test machine's. That makes step 2 load-bearing rather than +belt-and-braces, and it is the reason it cannot be deferred. + +--- + +## 5. Phase 4 — the digest→felt mapping decision + +**Goal:** one written, signed decision. **Effort:** XS to write, but it gates Phases 1.5 +and 2. **Risk:** HIGH if got wrong, and it is not recoverable by testing. + +The problem, stated exactly as the measurement report leaves it +(`others/lfm-hash-matrix-scope.md:1454-1460`, ✓ VERIFIED): *"A blake output word is 32 +bits, so a felt built from 8 output bytes is a 64-bit value reduced mod `p` and the map is +not injective. How a blake digest becomes felts — truncate to four `u32`s, reduce, +domain-separate — changes the security argument, the digest width, and the token count."* + +There are two directions and they are **not** symmetric: + +**Input side (felt → BLAKE3).** Covered by §3.2. Under option (i) it is the identity on +`u32`-valued lanes and needs no argument. Under option (ii) it needs a per-felt `< p` +gate, and omitting that gate breaks Fiat–Shamir. + +**Output side (BLAKE3 → felt).** BLAKE3 emits 8 `u32` words of chaining value. The +options: + +| option | digest | injective? | notes | +|---|---|---|---| +| **A. Truncate to 4 `u32`s, one per lane** ★ | 128 bits, 1 cell | yes, on the truncated image | matches `word.rs`'s declared 128-bit target and `HASH_DIGEST_FELTS = 4`; no reduction anywhere; collision resistance is 64 bits | +| B. All 8 `u32`s, 2 cells | 256 bits, 2 cells | yes | breaks the frozen `LFM_HASH` contract (1 cell out) and doubles the token count; this is keccak's shape, i.e. socket 1's | +| C. Pack 8 output bytes into one felt, reduce mod `p` | 256 bits, 4 cells | **no** | the non-injective case the report flags; needs a rejection or canonicalisation argument; avoid | +| D. 4 `u32`s + domain separation in `flags` | 128 bits, 1 cell | yes | A, plus a distinct flag byte per use (leaf / parent / sponge) | + +**Recommendation: D — option A with domain separation.** It preserves the frozen 1-cell +digest contract, is injective without any reduction argument, matches the machine's own +stated security target, and the domain byte costs nothing (it is a constant in the +constraints, not a column). The 64-bit collision bound is the honest consequence of a +128-bit digest and must be written down next to the decision; if the ecosystem target is +128-bit *collision* resistance rather than 128-bit *security level*, option B and a +2-cell digest is the answer and the `LFM_HASH` contract has to be reopened. **That is the +question to put to the user, and it is the only one in this phase.** + +--- + +## 6. Phase 5 — the end goal, and the cost reconciliation + +**This is where the brief's framing needs correcting, so the reconciliation comes first.** + +### 6.1 Reconciling 2.75 B, 1.1 B, and 4,946 + +All three numbers are correct and they are about different things. ✓ VERIFIED against +`blake3_probe.rs:327-356` and `blake3_probe.rs:549-733`, and the derivation in +`others/lfm-hash-matrix-scope.md:1400-1460`. + +- **4,946 base-field-equivalent cells per compression** — MEASURED, standalone, via a real + prove+verify against the production `BITWISE` table. `MAIN_COLUMNS + 3 × aux = + 3,056 + 3 × 630`. This is a property of the chip and is solid. +- **2.752 B cells per epoch verify** — the **role 1** column: what the epoch verifier would + cost *if the inner proofs it verifies were BLAKE3-committed instead of keccak-committed*. + It is `hash + residue + BITWISE` at P = 192,000 permutations, where the hash term is + 195,593 × 4,946 ≈ 967 M and the rest is the measured keccak-shaped residue. Against + keccak's measured 11.166 B that is 4.06×. +- **≈ 1.097 B** — the same column for an **unbuilt** felt-absorbing variant. The delta is + almost entirely the **byteswap gadget**: the felt→byte serialization that exists only + because the chip consumes `u32` lanes and `felt_be_halves` is what produces them. That + gadget is 95.84% of the residue in padding-aware cells, and it is **upstream** of the + chip's input format, so hosting the chip cannot delete it + (`others/lfm-agent-status.log:223`, ✓ VERIFIED as the recorded finding). The variant + adds ≈ 156 cells/compression (an ESTIMATE: 8 absorbed felts × ~20 for the canonicity + gate) and deletes the gadget, landing at 1.097 B / 10.2×. +- **What none of these measure: role 2.** `TestPermutation` costs 37 base-equiv per + permutation (`others/lfm-hash-matrix-scope.md:106-121`), and **the wrap performs zero of + them** (§1.1). Replacing it with BLAKE3 multiplies zero by ~135. + +**So: swapping `TestPermutation → BLAKE3` does not move the wrap's cost at all.** The +wrap's 11.17 B is keccak, hosted, verifying keccak-committed inner proofs, and it stays +11.17 B. Anyone reading "we proved the wrap with a real hash" should understand that the +wrap's *own* hashing work was, and remains, keccak. + +### 6.2 What the end goal should be instead — three rungs + +**E1 — socket 2 becomes real and bound.** Phases 1–4 complete. `TrivialV0` and `FriToyV0` +prove and verify under `HasherKind::Blake3`, with distinct registry entries and distinct +program digests. The F3.4 disclosure is retired: `FriToyV0`'s Merkle authentication and +Fiat–Shamir sponge become cryptographically meaningful. **This is achievable now and is +what I would ship.** + +**E1.5 — a BLAKE3 Merkle fixture wrap.** Replace the Milestone-C fixture tree that +`edsl::merkle_walk` authenticates with a real BLAKE3 tree, and prove+verify the resulting +program. This exercises the whole path end to end — executor, trace, AIR, registry, and +the §5 mapping — under a real hash, with no production migration. **This is the honest +"prove a wrap with a real hash" milestone**, and it is a much better demo than E1. + +**E2 — the epoch verifier over BLAKE3-committed inner proofs.** This is where 2.752 B (or +1.097 B) lives. It requires the *production* RV64 prover's Merkle and Fiat–Shamir hash to +be BLAKE3 — i.e. the ecosystem hash decision, landed, in `stark/`. Out of LFM's control +and far larger than everything above combined. Scope it separately; do not fold it into +this plan's deliverable. + +### 6.3 What changes in the wrap tests + +- `wrap_tests.rs:26-28` already says the module "cannot see the hash". After E1 that + remains true and should be **strengthened**, not deleted: add an assertion that the + epoch program's `LFM_HASH` group is empty, so the fact is enforced rather than narrated. + If a future epoch-verifier change starts emitting `Instr::Hash`, that assertion is what + makes it visible. +- `the_wrap_census_at_blowup_8` and `the_blake_column_and_the_residue_split` are both + `#[ignore]`d and stay so. +- For E1.5, a new `blake3_wrap_tests` module mirroring `wrap_tests`' structure. + +### 6.4 Memory and box requirements + +✓ VERIFIED from the PR body and the scoping report: a production-shaped wrap (73 queries) +needs 290–350 GiB under keccak and fits a single 124 GiB box under any candidate hash +(blake ≈ 71 GiB, Poseidon ≈ 7 GiB). **Those figures are role 1.** E1 and E1.5 are +ordinary LFM proves at blowup 2 and run wherever the existing wrap tests run; they need no +special box. E2 does, and by then the hash change is what makes it fit. + +--- + +## 7. Phase 6 — the A6R decision + +**A6R** is the named, unratified assumption that the 6-round BLAKE3 internal variant is +collision resistant (`blake3.rs:40-42`, from #903's `IMPLEMENTATION.md`). Today it costs +nothing, because `LFM_BLAKE3` is unregistered and unreachable. **The moment BLAKE3 becomes +a selectable hasher with a registry entry, A6R becomes a live soundness surface for every +program that selects it.** That is a protocol decision, not an engineering one, and it +needs a signature. + +**What is already on the record** (✓ VERIFIED, `git show a7a8bdd5`, `783c5a95`): + +> *External review (2026-08).* The round-count choice was reviewed with external +> symmetric-cryptography experts consulted by the project: removing *one* round +> (7 → 6) was judged comfortable; removing *two* (7 → 5) was explicitly not. +> Accordingly, 6 rounds is the endorsed floor. Variants below 6 rounds are not +> formally ruled out, but they are not available on the project's own authority: +> adopting one would require the external experts to study the reduced-round +> margin specifically — a dedicated cryptanalytic review, not an engineering or +> configuration decision. + +(Corrected 2026-08-10: an earlier revision of this plan paraphrased the last sentence as +"variants below 6 rounds are out of scope and MUST NOT be instantiated" — a strengthening +of the source. The text above is the actual wording; see A6R-signoff.md §1.1.) + +and, in the same commit, the alternative: + +> *The assumption-free alternative.* The chip design is round-parameterised; a 7-round +> instantiation (standard BLAKE3 compression, bit-compatible with official parent-node +> merges) costs roughly 10–12% more per merge end-to-end and requires no assumption beyond +> standard BLAKE3. + +**What that costs in this machine.** ? INFERRED — arithmetic over the chip's own verified +budget (`blake3_probe.rs:336-352`), shown so it can be rechecked. Going 6 → 7 rounds means +`NUM_G` 48 → 56: + +``` +main columns 3,056 + 8 G-blocks x 60 cells = 3,536 +BITWISE XOR 56 G x 16 + 64 feed-forward (was 48x16+64=832) = 960 +shift halfwords 56 G x 8 (was 48x8 =384) = 448 +message bytes (unchanged) = 32 +LfmMem tokens (unchanged) = 11 +interactions 960 + 448 + 32 + 11 (was 1,259) = 1,451 +aux ceil(1451 / 2) (was 630) = 726 +base-equiv 3,536 + 3 x 726 (was 4,946) = 5,714 (+15.5%) + +epoch column 2.752 B - 967 M + (195,593 x 5,714 = 1.118 B) = 2.902 B (+5.5%) + vs keccak 11.166 B = 3.85x (was 4.06x) +``` + +The +15.5%-per-compression figure is consistent with the spec's "10–12% per merge +end-to-end" once the non-hash terms are included, which is a useful cross-check on both. + +**The decision to sign, in one line:** *A6R buys roughly 5.5% of the epoch column. Is a +named, unratified, non-interoperable assumption worth 5.5%?* + +**My recommendation: no — build round-parameterised, instantiate 7-round first.** Reasons, +in order of weight: (1) the reference problem dissolves — the `blake3` crate becomes a +direct external KAT for both the primitive *and* the 2-to-1 socket framing, satisfying +rule 9 in its intended form rather than one step removed; (2) no assumption to sign, +ratify, or re-litigate at audit; (3) bit-compatibility with published BLAKE3 parent merges +is worth something on its own; (4) 5.5% is inside the noise of the decisions still open +above it (the felt-absorbing variant is a 2.5× lever on the same column — an order of +magnitude more leverage than the round count). Keep 6-round as a measured variant behind +`BLAKE3_ROUNDS`, gated on an explicit signed A6R, and switch to it if and when the 5.5% +matters. The user's stated tolerance — "as long as it works with Blake 6r, or blake, it's +fine" — permits this. + +**If the user signs A6R anyway**, the record needs: the assumption named in +`SOUNDNESS.md` (not only in `blake3.rs`), the registry entry's doc comment saying which +hash it rests on, and a note that sub-6-round variants are **not available on the +project's own authority** and would need a dedicated external cryptanalytic review +(the spec's actual wording — see §7's quote). + +--- + +## 8. Phases, effort, risk, order + +| # | Phase | Effort | Risk | Depends on | +|---|---|---|---|---| +| 4 | §5 digest→felt mapping decision | XS (a decision) | **HIGH if wrong** | user sign-off | +| 6 | §7 A6R decision / round count | XS (a decision) | **HIGH if wrong** | user sign-off | +| 1 | §2 reference + KATs | S, 1–2 d | LOW | git restore | +| 3 | §4 bind hasher into digest + registry | S, 1–2 d | LOW mechanically, soundness-gating | — | +| 2 | §3 `HasherKind::Blake3`, Route A | **L, 1–2 w** | MEDIUM | 1, 3, 4, 6 | +| 5a | §6.2 E1 — registered BLAKE3 toy programs | S | LOW | 2 | +| 5b | §6.2 E1.5 — BLAKE3 Merkle fixture wrap | M | MEDIUM | 5a | +| — | §6.2 E2 — BLAKE3-committed inner proofs | **XL** | — | the ecosystem hash decision | + +**Ordering constraint worth flagging:** Phase 3 (binding) is listed after Phase 1 but must +**land before** Phase 2, because Route A deliberately keeps the roots hasher-independent — +which makes the digest tag the only separator between a BLAKE3 machine and a Test machine. +Adding the third hasher first would create exactly the collision F3.3 warns about. + +**Two decisions block the largest phase.** Phases 4 and 6 are one paragraph of writing +each and gate ~2 weeks of work. Get them signed before starting Phase 2. + +--- + +## 9. Start here + +**Step 1, today, ~30 minutes, no decisions required:** + +``` +git show 3b9b8137:thoughts/blake3/blake3-oracle/blake3_ref.py +git show 3b9b8137:thoughts/blake3/blake3-oracle/ORACLE.md +git show 3b9b8137:thoughts/blake3/blake3-oracle/test_oracle.py +git show 3b9b8137:thoughts/blake3/blake3-chip/DESIGN.md +git show 3b9b8137:thoughts/blake3/blake3-chip/z3_blake_verify.py +git show 19ed761b:thoughts/blake3/blake3-oracle/official_test_vectors.json +git show 19ed761b:thoughts/blake3/blake3-oracle/canonical_6round_vectors.json +git show 19ed761b:thoughts/blake3/ground-truth/Cargo.toml +git show 19ed761b:thoughts/blake3/ground-truth/src/main.rs +``` + +into `thoughts/blake3/`, then run `test_oracle.py` against `official_test_vectors.json` at +`rounds = 7`. + +That single run either confirms or breaks the only external anchor the entire 6-round +chain hangs from. Everything else in this plan — the chip, the 4,946 measurement, the +2.752 B column, A6R itself — is downstream of it. If it does not go green, nothing below +it is worth starting. + +**Step 2, in parallel, requires no code:** put §5's mapping question and §7's round-count +question to the user as two yes/no decisions. They gate the largest phase and each is one +paragraph. diff --git a/thoughts/shared/lfm-real-hash/b1-verify.md b/thoughts/shared/lfm-real-hash/b1-verify.md new file mode 100644 index 000000000..d0b67d7f1 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/b1-verify.md @@ -0,0 +1,460 @@ +# B1 adversarial verification — the compress-chain transcript + +**Ground:** worktree `lambda_vm-blake3-impl`, branch `blake3-real-hash`, parent +`2957c3f9`, uncommitted (19 modified `.rs` under `prover/src/lfm/` + `SOCKET.md` ++ 2 new files). **Date:** 2026-08-11. **Method:** read the code and the diff, +then execute. Nothing here is inherited from the implementer's report; where the +report is quoted it is because I checked the claim. + +Claims are ✓ EXECUTED (ran it, output quoted), ✓ VERIFIED (read the code, cited) +or ✗ UNVERIFIABLE. + +**Bottom line: no soundness defect found in the change.** Seven defects, all +outside the soundness core: one MEDIUM (a live cost-model constant whose +derivation B1 deleted, missed by the report), one inherited HIGH-if-unrebased +framework gap that B1's central argument leans on, five LOW. + +--- + +## 0. Verdict table + +| # | target | verdict | +|---|---|---| +| 1 | selector/layout change (PREP_WIDTH 11→12, MODE_T@8, MULT 9..11) | **CONFIRMED-SOUND** | +| 2 | M5/M6 closure on the verify path | **CONFIRMED-SOUND** — with inherited caveat **D5** | +| 3 | trait additions (`transcript`/`transcript_out`) | **CONFIRMED-SOUND** | +| 4 | trace/AIR/bus coherence of the domain tag | **CONFIRMED-SOUND** — design note **D4** | +| 5 | SpongeVar/HostSponge lockstep | **CONFIRMED-SOUND** (by test, not by construction; the test exists) — latent **D6** | +| 6 | `TrivialV0` public-output shape | **CONFIRMED-SOUND** (verified by search, not by suite-greenness) | +| 7 | O1 tripwire | **CONFIRMED** — all four legs present, passes at both round counts | +| 8 | claim verification by execution | **CONFIRMED** — every number reproduces exactly, zero deviation | +| 9 | hygiene | **CONFIRMED** for the panics, the re-bless and debug leftovers; **D2/D3/D7** on docs | + +--- + +## 1. The selector/layout change — CONFIRMED-SOUND + +### (a) The #915 admission bound follows the shift + +`validator.rs:400-405` — the multiplicity columns are named symbolically, so +`3638b825`'s bound moved with the layout automatically: + +```rust +("LFM_HASH", &g.hash, vec![hash::MULT0, hash::MULT1, hash::MULT2]), +``` + +`layout::hash::MULT0/1/2` are now 9/10/11 (`layout.rs:113-115`). ✓ VERIFIED: the +negative-multiplicity `Compress` forgery stays closed; **MULT2 is bounded**, and +`MODE_T` is *not* in `mult_columns` (it is a selector, bounded by `one_hot` +instead). There is no hard-coded `8`, `9` or `10` anywhere in the hash paths — I +grepped every `layout::hash::` consumer across `prover/src`. + +The one place that *did* hard-code positions was `compiler.rs`'s +`hash_rows.push(vec![…])`, an 11-element positional literal. It was rewritten to +write by name (`compiler.rs:332-350`): + +```rust +let mut row = vec![FE::zero(); layout::hash::PREP_WIDTH]; +row[layout::hash::IN_ADDR0] = fe(ins[0].0); +… +row[match mode { Compress => MODE_C, Transcript => MODE_T, Permute => MODE_P }] = FE::one(); +row[layout::hash::MULT0] = fe(mults[0]); +``` + +Left positional, the shift would have written `mults[0]` into `MODE_T` and +dropped `MULT2` entirely. **This rewrite is what makes the layout move safe**, +and it is the highest-value line in the diff. + +### (b) The one-hot span + +`validator.rs:304-308` now reads `layout::hash::NUM_SELECTORS = 3` from +`layout::hash::MODE_C = 6`, i.e. columns 6, 7, 8 = MODE_C, MODE_P, MODE_T — +contiguous and complete. ✓ VERIFIED. Report §7.2's account of why `MODE_T` sits +at 8 rather than 11 is accurate: at 11 it would have been outside this span. + +### (c) Does the one-hot check run for every row of every admitted program? + +Yes, and — importantly — `validate` is on the real admission path, not only in +tests. `prover/src/bin/compute_lfm_registry.rs:51` calls it for each of the six +programs before emitting the entry: + +```rust +validate(program).unwrap_or_else(|v| panic!("{kind} is not admissible: {v:?}")); +``` + +Coverage of the hash group is total: `one_hot` walks `0..group.real_rows` +(`validator.rs:468`), and `DirtyPadding` walks `real_rows..padded_rows` +(`validator.rs:328-334`) requiring every column zero. No row escapes. + +--- + +## 2. The M5/M6 closure on the VERIFY path — CONFIRMED-SOUND, one inherited caveat + +I traced the verify path, not the prove path. + +1. `proof.rs:150-166` `lfm_verify` resolves the registry entry and passes + `entry.roots` — a hard-coded constant table — to `verify_against`. There is no + path that reads roots off the proof. +2. `airs.rs:429-436` builds the `LFM_HASH` AIR as + `build_air(…, roots[5], layout::hash::PREP_WIDTH)` → + `.with_preprocessed(root, 12)` (`airs.rs:340-349`). +3. `crypto/stark/src/verifier.rs:1183-1202`: if `air.is_preprocessed()`, the + proof's precomputed Merkle root must equal `air.precomputed_commitment()` or + verification returns `false`; a missing root also returns `false`. The + transcript then absorbs the **expected** (hard-coded) root, not the proof's. +4. `verifier.rs:548-556`: each query's precomputed opening is Merkle-authenticated + against that same hard-coded root, leaf-hashed over the whole opening. + +So `MODE_C`, `MODE_P`, `MODE_T` are values the prover supplies but cannot +*choose*: any deviation breaks step 3 or 4. The fractional split M5/M6 exhibits +is unreachable through the registry verify path. ✓ VERIFIED. + +Mechanism 2 (the registrar) is likewise real, per §1(c) above. + +### D5 — INHERITED, HIGH if this branch merges unrebased + +The verifier **never consults `air.num_precomputed_columns()`**. The +precomputed/main split is taken from the proof's own opening lengths: + +```rust +// crypto/stark/src/verifier.rs:949 +let num_precomputed = lde_trace_precomputed_evaluations.len(); +let num_base = num_precomputed + lde_trace_main_evaluations.len(); +``` + +The fix — `6949ceb9` *"fix(verifier): pin each trace-opening column width to the +AIR, not just their sum (#909)"*, with the precomputed-split PoC at `03870867` — +exists on other branches but is **not in this branch's ancestry** +(`git merge-base --is-ancestor 6949ceb9 HEAD` → *NOT in ancestry*, and +`verifier.rs` here contains no `num_precomputed_columns` call). + +**Not introduced by B1.** But B1's report states mechanism (1) — "the mode +columns are preprocessed, so a prover supplies none of them" — as unconditional, +and on *this* branch its enforcement rests on the precomputed leaf hash alone, +with the width unpinned. The class is the one already recorded in +`opening-width-unpinned-splits`. **Action: rebase onto a main containing #909 +before merging, and re-run the M5/M6 control.** Severity is about what happens if +that step is skipped, not about the diff. + +--- + +## 3. The trait additions — CONFIRMED-SOUND + +**(a) Defaults preserve Test/Poseidon exactly.** `hash.rs:85-93`: +`transcript_out` defaults to `compress_out`, `transcript` truncates it. Neither +`TestPermutation` nor `PoseidonGoldilocks` overrides them; `HasherKind`'s +dispatch (`hash.rs:236-256`) forwards to the concrete type, which then takes the +default. ✓ EXECUTED: `transcript_tests::the_transcript_proves_and_verifies_under_every_hasher` +proves and verifies the preamble under Test, Poseidon and BLAKE3 — passes. + +**(b) No other call site moves Test/Poseidon semantics.** I enumerated every +`HashMode::` branch in non-test code (7 sites: `blake3_socket.rs:295-297,387`, +`compiler.rs:343-345`, `executor.rs:402-411`, `builder.rs:286,299,323`, +`instr.rs:75`); all are exhaustive. On a `Compress` or `Permute` row every +`MODE_T` term added to the Test and Poseidon arms is multiplied by zero: + +- capacity `S_i = MODE_P·IN_i + (MODE_C + MODE_T)·IV_i` (`chips.rs:730-737`) +- round-constant scale `m = MODE_C + MODE_T + MODE_P` (`chips.rs:758-759`, `:799`) +- mode-sum booleanity (`chips.rs:771`, `:818`) +- the `LfmMem` receive gate `selector_sum(MODE_C, 3)` (`chips.rs:620`) + +The only behavioural movement for those hashers is `PREP_WIDTH` 11→12 shifting +their value columns by one, which is exactly what the re-bless captures. ✓ VERIFIED. + +**(c) Executor routing, no cross-wiring.** `executor.rs:395-412`: + +```rust +HashMode::Compress | HashMode::Transcript => { + … + if *mode == HashMode::Compress { hasher.compress_out(&a, &b) } + else { hasher.transcript_out(&a, &b) } +} +HashMode::Permute => hasher.permute(state), +``` + +Compress→`compress_out`, Transcript→`transcript_out`, Permute→`permute`. ✓ VERIFIED. + +--- + +## 4. Trace/AIR/bus coherence of the domain tag — CONFIRMED-SOUND + +**The tag cannot disagree with the row's mode, by two independent arguments.** + +*Custody, prover side.* `program.instrs` is the single source. The compiler sets +the mode column from it (`compiler.rs:340-346`); the trace filler re-derives +`hash_modes` by filtering the **same** `program.instrs` in the same order +(`trace.rs:132-139`); the executor pushes `records.hash` once per `Instr::Hash` +in the same order. Row *i* of the hash group ↔ `hash_modes[i]` ↔ +`records.hash[i]` by construction. ✓ VERIFIED. + +*Enforcement, verifier side.* The AIR reads the tag from the **preprocessed** +columns, never from the witness. `TAG_SELECTOR` +(`blake3_socket.rs:518-522`) is `[(MODE_C, TAG_LFMC), (MODE_T, TAG_LFMT)]`; +`message_word_ref(8)` returns `WordRef::ModeSelected(TAG_SELECTOR)` +(`blake3_socket.rs:536`); `word_expr`'s new arm (`blake3_chip.rs:1041-1047`) +emits `Σ main(col)·tag`. That expression enters the mu-gated `add3` sum identity +(`blake3_socket.rs:1000-1013`), so a witness computed under the wrong tag +violates a constraint. ✓ EXECUTED — M1 and M2 do exactly this in both directions +and both are rejected, each with an honest control that the same row in its own +domain passes. + +`m[8]` reaches only `add3` — never `byte()` and never `rotr_bytes()` — which is +what makes the `unreachable!()`s in §7.4 unreachable *structurally*, not by luck: +BLAKE3's G uses message words solely in `a = a + b + m`. Empirically confirmed +too, since `socket_wires()` runs at every AIR construction and 52 socket/transcript +tests build it without panicking. + +**BITWISE accounting is identical for the two domains.** `bitwise_ops_for` +(`blake3_socket.rs:757-765`) takes `(a, b, tag)` per hash record and runs the same +`ValueFlow` for both modes — same number of XOR and `AreBytes` ops, different +values. The senders are gated by `Multiplicity::Sum(MODE_C, MODE_T)` +(`blake3_socket.rs:711`), which is 1 on every real row of either mode and 0 on +padding. ✓ VERIFIED. + +### D4 — LOW: the trace filler takes the tag as an argument, not from the row + +`trace.rs:251-255` calls `fill_socket_witness(out, tag_for_mode(hash_modes[row]))`. +But `chip_trace` copies the group into the leading columns of **every** row +(`trace.rs:67-70`) *before* calling `fill` (`trace.rs:71-73`), so `out[MODE_C]` +(=6) and `out[MODE_T]` (=8) are already populated when the filler runs. Two +functions below, the Poseidon filler makes the opposite choice deliberately, and +says why (`trace.rs:80-86`): + +> The permutation input is read back out of the row's own `IN`/`S` columns — the +> exact cells round 0's constraints read — rather than from the executor record, +> so the witness cannot describe a different input than the one the AIR constrains. + +Same file, opposite discipline. Not exploitable (the AIR constrains the tag; a +mismatch is a failed proof, not a forged one), but it is an invariant held by +caller convention where it could be held by construction — the same shape as the +positional-`vec!` hazard the implementer correctly removed from `compiler.rs`. +The `bitwise_ops_for` feed at `trace.rs:186-197` is the same pattern. + +--- + +## 5. SpongeVar / HostSponge lockstep — CONFIRMED-SOUND + +Agreement is **by test, not by construction** — they are separate code — and the +tests that would catch a divergence exist and pass. + +Checked for an input shape that diverges, found none: + +- **operand order**: machine `transcript_step(state.as_digest(), c.as_digest())` + (`edsl.rs:96-99`); host `hasher.transcript(&self.state, c)` (`fixture.rs:105`); + reference `transcript_digest_rounds(state, operand, …)`. All `(state, operand)`. ✓ +- **absorb2 ordering**: both are literally `absorb(c0); absorb(c1)` + (`edsl.rs:104-107`, `fixture.rs:108-111`). ✓ +- **squeeze counter timing**: both output the pre-advance state, then advance with + `SQ(i)`, then increment (`edsl.rs:113-131`, `fixture.rs:115-122`). ✓ +- **zero-init**: machine `b.felt_const(FE::zero()).as_cell()` → the word + `[0,0,0,0]`; host `[FE::zero(); 4]`. ✓ + +**Interning.** `builder.rs:123-136` keys the constant pool on the canonical +four-lane value, so one `LFM_CONST` row per distinct `SQ(i)` within a program and +distinct `i` are distinct rows. A user constant that happened to equal +`[SQZ0, i, 0, 0]` would *share* the row — harmless, because the separation +`SQUEEZE_MARK` provides is explicitly defence-in-depth: the load-bearing argument +is that the operation sequence is a compile-time constant bound by `program_id`, +which `edsl.rs:16-20` states correctly. ✓ VERIFIED. + +Caught by `the_machine_and_the_host_chain_agree_under_every_hasher` and +`the_machine_reproduces_the_end_to_end_vector` if either side moves. + +### D6 — LOW, latent, not introduced here + +`HostSponge` became hasher-parameterised; `HostTree::build` (`fixture.rs:141-148`) +still hard-codes `TestPermutation.compress`, and `fixture_prove_columns` +(`fixture.rs:216`) calls `HostSponge::new()` (default = Test). Harmless today +because `FriToyV0` cannot run under BLAKE3 at all. When O1 closes, the fixture's +Merkle tree and the machine's `edsl::merkle_walk` will hash with different +functions and the authentication paths will not verify — a completeness trap +waiting at exactly the milestone this work is aimed at. + +--- + +## 6. `TrivialV0`'s public output — CONFIRMED-SOUND, by search + +The shape moved from `[d1, permuted_cell, m]` to `[d1, d2, m]` +(`programs.rs:57-68`). I searched rather than relying on the suite: + +- `.rs` across `prover/`, `executor/`, `crypto/`: every `TrivialV0` / + `trivial_program` use is shape-agnostic. `machine_tests.rs:36-142` proves, + verifies, tampers `claimed[0]`, cross-claims, and pins the registry entry — + none reads slot 1 or asserts a count. `blake3_socket_tests.rs:772-786` asserts + only "no permute". `transcript_tests.rs:648-653` asserts row counts `(3, 0)`. +- `.md` / `.typ` / `.py` across `thoughts/` and `spec/`: no hit on the old shape + (`permuted_cell`, `st[0]`, "one permuted cell"). +- Registry metadata carries roots/heights/id, not output shape. + +✓ VERIFIED — the report's inference is correct, and now it is a search result +rather than an inference. + +--- + +## 7. The O1 tripwire — CONFIRMED + +`blake3_socket_tests.rs:1558-1597`. All four legs the report claims are present +and each is a real assertion: + +| leg | line | form | +|---|---|---| +| no permute remains | 1563-1568 | `!instrs.any(mode == Permute)` | +| fixture not u32-laned | 1571-1580 | counts values failing `lanes_of`, asserts `> 0`, with a doc note that a zero count means the test must be replaced | +| refusal is *specifically* O1 | 1584-1591 | `Err(HasherRejected(msg)) if msg.contains("O1")` | +| honest control under `Test` | 1596 | same program, same arenas, `.expect(…)` | + +✓ EXECUTED, passes at both round counts. + +--- + +## 8. Claim verification by execution — every number reproduces, zero deviation + +| claim | executed result | +|---|---| +| full `lfm::` suite @7r | **290 passed; 19 failed; 7 ignored** (202.0s) ✓ exact | +| full `lfm::` suite @6r (`--features blake3-6round`) | **290 passed; 19 failed; 7 ignored** (180.7s) ✓ exact | +| the 19 are the pre-existing `fibonacci.elf` set, byte-identical at both round counts | ✓ — the two failure lists are identical, name for name | +| `transcript_tests` + `blake3_socket_tests` @7r | **52 passed; 0 failed** = 17 + 35 ✓ exact | +| the three-hasher transcript test | `the_transcript_proves_and_verifies_under_every_hasher` … ok | +| the two cost-exactness tests | `the_programs_cost_what_option_b_priced_them_at` … ok; `the_preamble_costs_eleven_transcript_steps` … ok | +| `make lint` (fmt + 4 feature combos) | **clean** | +| `cargo clippy --features blake3-6round -D warnings` | **clean** | + +**No deviation from the report's numbers.** The failure set at 6 rounds: +7 × `epoch_tests`, 6 × `epoch_verify_tests`, 1 × `logup_tests`, 5 × +`machine_tests` — identical to 7 rounds. + +--- + +## 9. Hygiene — confirmed, with three documentation defects + +- **`WordRef::ModeSelected` panics** (`blake3_chip.rs:378-407`): unreachable from + library callers, structurally (see §4) and empirically (every AIR construction + walks `socket_wires()`). +- **Registry re-bless completeness**: six drift tests, one per entry + (`machine_tests.rs:118, 230, 511, 754, 1417, 2281`), each pinning **roots, + log_heights, keccak_rnd_chunks, hasher and program_id**. `FriToyV0`'s + `LFM_CONST` group move 4→5 is inside `log_heights` and therefore pinned. All six + pass. ✓ +- **No debug leftovers**: the only `println!`s under `prover/src/lfm/` are in + `blake3_probe.rs`, untouched by this diff. No `dbg!`, `TODO`, `FIXME`. +- **Diff scope exactly as claimed**: 19 modified `.rs` all under + `prover/src/lfm/`, plus `thoughts/blake3/socket-kats/SOCKET.md`, plus 2 new + files (`transcript_kats.rs`, `transcript_tests.rs`). Nothing in `crypto/`, + `executor/`, `prover/src/tables/`. ✓ + +--- + +## Defects + +### D1 — MEDIUM. `LFM_HASH_RATE_FELTS` is derived from the construction B1 deleted. **Missed by the report.** + +`prover/src/lfm/epoch_verify.rs:428-436`: + +```rust +/// Felts an `LFM_HASH` permutation absorbs — the sponge's rate is 2 of its 3 +/// state cells (`edsl::SpongeVar`: "state = 3 cells (rate 2, capacity 1)") and a +/// cell is [`super::hash::HASH_DIGEST_FELTS`] felts. +/// +/// **This is 2.125× WORSE than keccak's 17** … +pub const LFM_HASH_RATE_FELTS: usize = 8; +``` + +The value `8` is `2 cells × 4 felts`, taken directly from the three-cell duplex +that `edsl.rs` no longer contains. **Under B1 the chain absorbs one cell per +step, so the rate is 4, not 8.** + +This is a live constant, not a comment. It drives the epoch verifier's +permutation-axis projection: + +- `epoch_verify.rs:465-471` `leaf_permutations_at_rate` +- `epoch_verify.rs:485-492` `query_permutations_at_rate` +- `epoch_verify_tests.rs:641-706` — the "HASH MATRIX — the PERMUTATION axis" + block, whose printed ratio and whose `assert_eq!(cand_p, leaf_c + path_and_fri)` + are the numbers the hash decision cites. + +**Concrete consequence.** At the true rate the leaf term roughly doubles, so the +projected candidate/keccak permutation ratio is currently understated. Worse, the +model's *rate-invariance* premise breaks: `epoch_verify_tests.rs:650-655` asserts + +```rust +s.fri.num_committed() == 0 || 6 <= LFM_HASH_RATE_FELTS, +"a FRI layer leaf must fit one block at the candidate's rate" +``` + +which holds at 8 and **fails at 4** — a 6-felt FRI-layer leaf no longer fits one +block, so `epoch_verify.rs:483-484`'s "a FRI layer leaf … fits any rate ≥ 6" and +the "only the leaf term may move with the rate" decomposition both stop being +true. The decision paper rests on the same number +(`others/lfm-hash-matrix-scope.md:128, 228`). + +The enclosing test is `the_assembled_epoch_verifier_runs`, one of the 19 +currently blocked on `fibonacci.elf` — so this is *unexercised in this +environment* and will surface in CI where the ELF exists. + +This is precisely the "fixing the mechanism ≠ restoring the invariant" class. The +report's §6 statement that "the entire diff is 19 files under `prover/src/lfm/`" +is true of the *diff* but was never checked against **semantic dependents of the +deleted sponge**, and this is one. + +### D2 — LOW. Two stale `PREP_WIDTH = 11` claims survive. **Missed by the report.** + +- `prover/src/lfm/statement.rs:43` — *"`LFM_HASH`'s preprocessed width is 11 + under every candidate"*. `statement.rs` is not in the diff at all. +- `prover/src/lfm/poseidon_chip_tests.rs:546` — *"`PREP_WIDTH` is 11 in both + layouts"*. The same file's assertion at line 163-165 **was** updated to 12; the + prose 380 lines later was not. + +Doc-only; the conclusions still hold. But `statement.rs`'s comment is the +justification for why `lfm_program_id` folds the hasher tag in, so it is load- +bearing prose in the one file that explains program identity. + +### D3 — LOW. Report §6's SOCKET.md claim is now false (a race, not an error). + +The report says its `SOCKET.md` §2.2 edit was "backed out … the oracle's pass is +byte-for-byte intact", and separately lists §2.2's `m[8]` row as +"⚠ REPORTED, not edited — one genuine staleness". + +`git diff thoughts/blake3/socket-kats/SOCKET.md` **does** now contain the §2.2 +row rewritten to `MODE_C·TAG_LFMC + MODE_T·TAG_LFMT` plus a "⚠ UPDATED FOR B1" +note block and a §2.3 rewrite. Timestamps say this is a race, not a +misstatement: `SOCKET.md` mtime `01:00:45`, report mtime `00:55:40`. The oracle +did its re-transcription pass five minutes after the report was written +(`ORACLE.md` `01:00`, `chip_model.py` `01:01`, `gate.py` `01:03`, +`artifact_pin.*` `01:04`, `CHIP-GATE.md` `01:13`). + +Net: the report's §8 open items *"the two stale `m[8]` framing rows"* are closed; +the report should be amended rather than the files. I verified the new §2.2 text +matches `TAG_SELECTOR` exactly. + +### D4 — LOW. Trace filler takes the tag by argument. See §4. + +### D5 — INHERITED, HIGH if merged unrebased. #909 width pin absent from ancestry. See §2. + +### D6 — LOW, latent. `HostTree` vs `HostSponge` hasher asymmetry. See §5. + +### D7 — LOW. Residual contradiction inside the oracle's updated `ORACLE.md` §2.2. + +The word-level table row now reads *"on the built chip, a mode-selected linear +form"*, but the sentence immediately below the table still reads: + +> Everything except `a` and `b` is a compile-time constant. + +`SOCKET.md` got the corresponding sentence fixed (*"Everything in that table +except `a` and `b` **and `m[8]`**…"*); `ORACLE.md` did not. Since §2.2 is the +table the gate transcribes into `chip_model.py`, the contradiction sits in the +one place a transcriber reads. Reported, not edited — `gate-oracle/` is the +oracle's instrument. + +--- + +## What I did not verify + +- **`FriToyV0` under BLAKE3.** Blocked by O1, correctly and with a tripwire; not + re-derived here. +- **The gate extension** (`chip_model.py` `MODE_T` role, `gate.py` B0a/B0b + widening). The oracle's files moved during this review (mtimes `01:01`–`01:13`); + I did not read or run them, per instruction. The chip exposes everything the + report says it does — `cols::MODE_T`, `cols::MU_COLUMNS`, `TAG_SELECTOR`, + `tag_for_mode`, constraint indices 0–5 unchanged — ✓ VERIFIED against the source. +- **Squeeze-run entropy analysis.** A spec claim, not a code claim. diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/CHIP-GATE.md b/thoughts/shared/lfm-real-hash/gate-oracle/CHIP-GATE.md new file mode 100644 index 000000000..686ea1905 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/CHIP-GATE.md @@ -0,0 +1,605 @@ +# Gating the BLAKE3 socket chip AS BUILT + +> ## ⟳ RE-GATED FOR THE D1 FIX — 2026-08-11. **VERDICT: PASS, 86/86.** +> +> Fourth re-gate. **The pin caught this one against an explicit assertion that it +> was out of scope** — the D1 fix was believed to be `chips.rs`-only, but the fix +> is a *shared* `emit_unread_input_pins` and the BLAKE3 arm calls it too, so +> `eval` and `framing_consts` both drifted and the 84/84 verdict did not carry. +> A verdict nearly shipped on a stale board; the instrument is what stopped it. +> +> **What moved:** the unread-input pins went from 4 (one cell) to **8 (both +> cells)**, `LEAF_IDX` shifted 30 → **34**, `CORE_IDX` → 50, `NUM_CONSTRAINTS` +> 942 → **946** @7r. **No cell counts moved** — 5,509 @7r / 4,741 @6r unchanged, +> so `program_id`s stay byte-identical, as the build reported. +> +> | region | sha256 | | +> |---|---|---| +> | `eval` | `240619f1580493b3998ac9fbc86aec61352132219ce5ca87c642e6ec2099a6be` | moved | +> | `framing_consts` | `21ab1892612cfd3b815c3d3c983a4baedd842de9f4d1b9268bdbc78843a26c24` | moved | +> | `bitwise_interactions` | `c880036158518796d36e097f3a676a197c5a707aec94ea1284d196dca3a46fa4` | unchanged | +> | `cols` | `f370814ae32795fe6366dbba7956f4e38bfb33681d810c0000bd3e5c799edf44` | unchanged | +> +> whole file `9d358f7bb3e2457065a473d478542aa7d218826e6ce81d96c2652d846f7a4cf6`. +> +> **Two new rows — §4.8.** One is an audit; the other is a *blindness*, and the +> blindness is the more important of the two. + + +**Status:** the seam is closed. The gate now certifies the chip that exists, not +the pre-Phase-2 model. **Date:** 2026-08-10. **VERDICT: PASS, 75/75.** + +Companion to `ORACLE.md`, which describes the oracle itself (reference `f`, the +Option-A socket, the contract library, the column-role map). This document +records what changed when the real constraint bodies were transcribed in, and +what the board says about them. + +No cargo was run. Everything here is python + z3 4.15.4. + +--- + +## 1. WHICH ARTIFACT WAS GATED + +### 1.1 ⚠ My first pin had a fail-open, and it fired + +`artifact_pin.py` v1 hashed three regions (`eval`, `bitwise_interactions`, +`cols`) and recorded constants as their **expression text**. Asked to re-verify +after the implementer's wave, it answered **"artifact matches the pin"** — a +**FALSE PASS**, for two reasons: + +1. `SOCKET_ROUNDS` changed definition (it is now an alias of `BLAKE3_ROUNDS`). + v1 recorded `NUM_G = "SOCKET_ROUNDS * 8"`, which is stable under exactly that + change. **Hashing an expression is not hashing a value.** +2. Worse: `SOCKET_ROUNDS`, `TAG_LFMC`, `FLAGS_LFMC`, `BLOCK_LEN_LFMC`, + `COUNTER_LFMC` and `OUT_WINDOW` are top-level constants living in **none** of + the three hashed regions. They are precisely the framing degrees of freedom + the negative-control board tests. **A change of `FLAGS_LFMC` from `0x0B` to + anything else — a live control (`flags_parent`) — would have passed silently.** + +The change that exposed this was benign. The hole was not. A drift detector that +answers PASS without looking at the thing that matters is worse than none, +because it is trusted. This is the same fail-open class the whole gate is built +to prevent, and I had reproduced it in my own instrument. + +**Fixed in v2**, which now (a) hashes a fourth region — the top-level constant +block — and (b) **resolves the framing constants and checks them against +`socket_ref.py`'s specification**, so the pin answers *"does the chip still +compute the socket the oracle specifies?"* rather than *"has this text changed?"*. +Every extraction is mandatory: a constant it cannot find is a hard failure, never +a silent skip. `artifact_pin.py` refuses to pin at all if conformance fails. + +### 1.2 What the file actually is, and when + +**COMMIT ANCHORS.** Phase 2 is now committed to `blake3-real-hash`: + +| commit | what | +|---|---| +| **`b693eece`** | `feat(lfm): BLAKE3 as a first-class LFM_HASH hasher (compress socket, 7-round default)` — **the gated code** | +| **`cece4a0b`** | `docs(lfm): record the O5 decision — future leaf hashing uses the LFML tag` — docs only | + +✓ EXECUTED **2026-08-11**: `python3 artifact_pin.py --check` against the +committed file → *"artifact matches the pin AND its framing still equals the +oracle spec; the gate verdict applies"*, noting the whole-file hash moved +`9d91954d…` → `b3d2755d…` **outside the four hashed regions only**. So the +board's **PASS 75/75 covers the committed artifact**, not merely the worktree +copy I transcribed from. + +The narrative below is retained because it is how the pin came to be trusted; +it describes the same content, before it was committed. Base commit at +transcription time was `65025095`, file `prover/src/lfm/blake3_socket.rs`. + +**Timeline, checked rather than assumed.** The file's mtime is `21:09:50`; I +pinned at `21:18` and ran the board at `21:25`. The wave the lead flagged — the +O5 module-doc note and the `SOCKET_ROUNDS` alias — is therefore the +`a03211d9… → 9d91954d…` change that landed **before** the pin and which the +gated transcription already reflects. I re-read the constraint bodies from the +file as it now is, per the instruction, and re-derived the pin from scratch. + +**The file then moved a third time, during the board run** — +`9d91954d…` → `fd19f4c5…`. The v2 pin's verdict on that: + +``` +artifact matches the pin AND its framing still equals the oracle spec; +the gate verdict applies + (whole-file hash moved 9d91954dd243 -> fd19f4c55d4b, but only outside + the four hashed regions -- i.e. in comments/docs) +``` + +That is the instrument working as intended, and a live demonstration of why v2 +was needed: it distinguishes *"the prose moved"* from *"the semantics moved"* +and says which. v1 would have answered PASS here too — but for the wrong reason, +having never looked at the framing constants at all. Concurrent editing of an +uncommitted file is the normal condition for this task, not an anomaly, so the +pin has to be the thing that carries the claim. + +| region (normalized: comments + whitespace stripped) | sha256 | +|---|---| +| `pub fn eval` | `0441de9b71229ef5000c4a19f7d50273eb9abb826f38e7e8b3a22ec3ca1a2650` | +| `pub fn bitwise_interactions()` | `b49ff66c374161b7acd3742d03ba2fc969f1fa2f26efdc9659b4e8d7a81ba94a` | +| `pub mod cols` | `fac9bd6634cfc29e607b37a4c1f49e9b89c556431214c2afd6710e32b0f56b70` | +| **framing constants (v2, new)** | `64e9babf31afc7e2a9850324d4fa0a680a0d53adf6fc978412fe12ee2aa972b9` | + +### 1.3 Framing conformance — resolved values vs the oracle spec + +✓ EXECUTED. The chip's constants are resolved from source and compared against +`socket_ref.py`. This is the check v1 lacked entirely: + +| chip constant | resolved | oracle spec | | +|---|---|---|:--:| +| `TAG_LFMC` | `"LFMC"` = `0x434D464C` | `0x434D464C` | ✓ | +| `FLAGS_LFMC` | `0x0B` | `CHUNK_START\|CHUNK_END\|ROOT` | ✓ | +| `BLOCK_LEN_LFMC` | `36` | `36` | ✓ | +| `COUNTER_LFMC` | `0` | `0` | ✓ | +| `OUT_WINDOW` | `HASH_DIGEST_FELTS` | low 4 of 16 words | ✓ | +| `NUM_LANES` | `8` | 2 cells × 4 lanes | ✓ | +| `G_SIZE` | `60` | the gated per-G cell count | ✓ | +| `FLOW.full_output` | `false` | requirement R3 | ✓ | +| `NUM_G` | `SOCKET_ROUNDS * 8` | 8 G-calls per round | ✓ | +| `SOCKET_ROUNDS` | `BLAKE3_ROUNDS` → **7** default, **6** under `blake3-6round` | the gated pair {6,7} | ✓ | + +The round-count alias is the one semantic item in the wave. It is benign **for +this gate specifically** because the board covers *both* reachable values; the +chip compiles to exactly one. The pin now fails if that pair ever stops being +`{6, 7}`, since the board would then be certifying a round count the chip does +not use. + +> **If any region hash or framing value changes, this verdict does not carry +> over.** `python3 artifact_pin.py --check` is the one-command test. + +## 2. What changed in the seam + +### 2.1 `emit_add2` — the deviation, closed + +The pre-Phase-2 model witnessed the add2 carry as a **column** and constrained +it twice (sum identity, degree 2; booleanity, degree 3). The chip derives it as +an **expression**, `carry := (A + B − s)·2^{−32}`, and emits **one** constraint: + +``` +MU · carry · (1 − carry) = 0 blake3_socket.rs, add2 loop +``` + +The model now does the same. It is the same statement — the model's pair asserts +*∃ carry ∈ {0,1} with A + B = s + 2^32·carry*, the chip eliminates an existential +whose witness is determined — but the gate must certify **the chip that exists, +not a stronger cousin**, so the model follows the chip. + +**Modelling note, and the reason WA7 exists.** `2^{−32}` is a *field* inverse +with no faithful BV counterpart. The BV domain therefore encodes the +*post-audit* statement — the difference lies in `{0, 2^32}` — and the side +condition that those are the only reachable roots is discharged in the field by +WA7. Encoding that disjunction in BV *without* the audit would be assuming +precisely the thing that makes the form sound. + +### 2.2 BLOCK 0 — the four framing constraints the model did not cover + +All four are over felts and mode selectors, not bytes. **Decision: they go to the +FIELD/structural ledger, not BV** — a BV model has no faithful representation of +a Goldilocks mode selector, and pretending otherwise would be the fail-open this +whole split exists to prevent. All four are now *checked* in the field, both +ways, rather than merely asserted: + +| chip idx | constraint | where it is checked | +|---|---|---| +| 0–3 | `S_k − (MODE_P·IN_{8+k} + MODE_C·IV_k)` | **B0a**, field | +| 4 | `mode_sum·(1 − mode_sum)`, `mode_sum = MODE_C + MODE_P` | **B0b**, field | +| 5 | `MODE_P = 0` | **B0a/B0b**, field | +| 14–21 | `OUT_{4+j} = 0`, j ∈ 0..8 | **B0c**, field | + +All four are **ungated** (no `MU` factor), which is correct: they must hold on +padding rows too, and padding is all-zero. + +**The finding worth having.** idx 0–3 pin nothing on their own — they only +constrain the capacity prefix because **idx 5 kills the `MODE_P` term**. Drop +idx 5 and `MODE_P` is free, so the prefix becomes a prover-chosen copy of +`IN_{8+k}`. ✓ EXECUTED both ways (B0a): with the pin → UNSAT, without → SAT. +`MODE_P` being *preprocessed* is the deeper defence, but idx 5 is what makes the +capacity family mean anything, and the two should not be confused. + +Also: the model previously deferred MU booleanity to "structural, not a BV +theorem". The chip emits it as a real constraint, so it is now checked (B0b) — +`ORACLE.md`'s claim that it is not checkable is superseded. + +### 2.3 Census prefix + +13 → **28**, the frozen shared prefix as built (12 `IN` + 4 `S` + 12 `OUT`). +`MU = MODE_C` is *preprocessed*, so it is outside the main-column census +entirely — and, more importantly, a prover cannot choose it. + +--- + +## 3. Census reconciliation — exact, to the unit + +✓ EXECUTED. The model's census is derived from the gated constraints, so this is +a real cross-check, not a restatement: + +| | model | built chip | | +|---|---:|---:|:--:| +| main columns, 6r | 2,956 | 2,956 | ✓ | +| main columns, 7r | 3,436 | 3,436 | ✓ | +| sends, 6r | 1,190 | 1,190 | ✓ | +| sends, 7r | 1,382 | 1,382 | ✓ | +| cell-equiv, 6r | 4,741 | 4,741 | ✓ | +| cell-equiv, 7r | 5,509 | 5,509 | ✓ | + +Both deltas the report predicted are accounted for exactly: **−96 (6r) / −112 +(7r)** from dropping the add2 carry column (2 per G × `NUM_G`), and **+15** from +the 28-column prefix replacing the model's 13. Per-G block is now **60 cells**, +matching `cols::G_SIZE = 60`. + +7r blocks: `rotr_shift` 1,344 · `xor_out` 912 · `add3` 672 · `add2` 448 · +`lane_bytes` 32 · `frozen_socket_prefix` 28. + +--- + +## 4. THE BOARD — 75 checks, VERDICT PASS + +`python3 gate.py`, ~13 min. Full output in `run-chip-gate.log`. + +| section | checks | all as wanted | +|---|---:|:--:| +| main theorems (symbolic BV) + per-theorem discrimination controls | 12 | ✓ | +| T4 full pipeline, concrete, vs anchored KATs (6r and 7r, both directions) | 4 | ✓ | +| negative controls (**both round counts**) | 36 | ✓ | +| documented BV blindness | 1 | ✓ | +| optional tail truncation | 2 | ✓ | +| non-vacuity | 1 | ✓ | +| width audit (field) | 13 | ✓ | +| BLOCK-0 framing audit (field) | 6 | ✓ | + +### 4.1 Negative controls — re-measured against the transcribed bodies + +**A transcription that accidentally strengthens is as wrong as one that weakens, +and only the controls can tell the difference.** Every control was therefore +re-run against the new bodies, at **both** round counts (the chip ships 7r by +default and 6r behind `blake3-6round`), against the **full concrete pipeline**: + +`swap_a_b`, `tag_changed`, `tag_omitted`, `truncate_high_half`, `flags_parent`, +`flags_no_root`, `block_len_64`, `block_len_32`, `counter_one`, `cv_zero`, +`lanes_big_endian`, `tag_slot_moved`, `msg_perm_swapped`, round-count confusion +(`rounds_6_not_7` at 7r, `rounds_7_not_6` at 6r), `drop_ff_xor`, +`swap_g_operand`, `drop_add2_carry` — **all SAT at both round counts**, plus the +two symbolic G-level controls and nine per-theorem discrimination controls. + +**New control for the new form: `drop_add2_carry`** — removes the add2 +constraint outright. Under the expression-carry form there is no carry column +left to un-boolean, so the whole constraint *is* the booleanity, and unlike the +add3 case it is **BV-visible**. SAT at both round counts. Without this control +the new `emit_add2` would have had no test that its single constraint is +load-bearing at all. + +### 4.2 The documented BV blindness, now sharper + +`drop_carry_bool` (which un-booleans add3's carry **columns**) remains **UNSAT in +BV** — correct, and recorded. In BV a carry column is an 8-bit variable, so +removing its booleanity leaves it bounded and `s` stays pinned; the same bug is a +live forgery in the field (WA4 → SAT). + +The distinction now matters more than before, because the two adds are +constrained differently: **add3's carries are columns (field-only bug), add2's +carry is an expression (BV-visible bug)**. Same chip, two bug classes, two +domains. A gate running only BV would report the add3 class as absent. + +### 4.3 Width audit — 13 items, including the new WA7 + +| item | present | dropped | +|---|---|---| +| WA1 lane decomposition (obligation O1) | UNSAT | **SAT** | +| WA2 lane `< 2^32` forced | UNSAT | **SAT** | +| WA3 shift `SLL` tight bound | UNSAT | **SAT** | +| WA4 add3 carry booleanity | UNSAT | **SAT** | +| WA5 tail case: word value pinned / bytes not | UNSAT | **SAT** | +| WA6 no-wrap side condition (worst `2^34 ≪ p`) | ok | — | +| **WA7 add2 expression-carry pins `s`** | **UNSAT** | **SAT** | + +**WA7** is the companion the expression-carry form needs. With `A`, `B`, `s` +byte-bounded below `2^32`, are `0` and `2^32` the only reachable roots — can a +*negative* difference alias `2^32 mod p`? + +It cannot. If `A + B − s ≥ 0` it lies in `[0, 2^33)` and `2^33 ≪ p`, so the only +residues are the honest two. If `A + B − s < 0` it lies in `(−2^32, 0)`, i.e. the +field element sits in `(p − 2^32, p)`; that equals `0` only for a zero difference, +and equals `2^32` only if the difference were `2^32 − p ≈ −2^64`, far below +`−2^32`. Hence `s` is pinned to `(A + B) mod 2^32`. Dropping the byte bound on +`s` makes it a free field element and the add forgeable — **SAT**, the same class +as WA4 and equally invisible to BV. + +### 4.4 The argued ledger — new, and deliberately visible + +z3 4.15.4 has **no finite-field sort** (✓ VERIFIED — `FiniteFieldSort` does not +exist in this build), and the `Int`+`mod` encodings of the quadratic field facts +are nonlinear and intractable: the first attempt at WA7 and B0b hung the solver. + +So four steps are discharged by **algebra, not by a solver**, and the board now +prints them rather than baking them in silently — an unstated assumption is +exactly how a fail-open happens: + +| | fact | relied on by | +|---|---|---| +| **AR1** | `F_p` has no zero divisors, so `x·(1−x) = 0` has root set exactly `{0,1}` | WA4, B0b | +| **AR2** | `2^{−32}` is a unit, so `d·2^{−32} ∈ {0,1}` iff `d ∈ {0, 2^32}` | WA7 | +| **AR3** | `2^16` is invertible mod `p` | WA3 | +| **AR4** | every field-lifted expression stays below `2^34 ≪ p` | WA6 | + +This is the posture the audit already took for WA4 (whose "present" case encodes +booleanity as a root set rather than asking z3 to derive it). Making it explicit +is the change; the solver is left on the questions it can actually decide. + +--- + +## 4.5 ⚠ STANDING NOTE (D6) — the last-round diagonal-G `Y` columns are +underconstrained-but-unread. **Harmless as built. Do not "tighten" without re-gating.** + +Independently found by F9 in review; it is the same surface as WA5 and O-TAIL, +recorded here because it is a **live constraint on future edits**, not a defect. + +**What it is.** The chip does *not* take the tail-truncation option: the last +round emits all eight G-calls in full, including `X4` and the `rotr7` that +produces `B2` = `v[b]`. In the last round nothing reads those `v[b]` values — +the feed-forward reads `v[0..4]` and `v[8..12]`, and the diagonal group's +`b`-positions are `{5,6,7,4}`. So `B2`'s four `Y` byte columns are: + +* **constrained** as a word — the two recombine identities pin + `Y0 + 256·Y1` and `Y2 + 256·Y3`, hence `Σ Yₖ·2^{8k}`; +* **not** pinned per byte — nothing forces the split between `Y0` and `Y1` + (an extra 8 bits of prover freedom per halfword); +* **read by nothing.** + +**Why it is harmless.** An unread column cannot influence the digest. ✓ EXECUTED +in the field, both directions (WA5): the rotation's **word value** is still +pinned (UNSAT) while its **individual bytes** are not (SAT). Those two results +are exactly this note. + +**The constraint on future work.** The safety rests on *unread*, not on +*constrained*. Two ways a later PR breaks it, both plausible-looking cleanups: + +1. **Giving the columns a reader.** Any consumer that reads `Y`'s bytes + individually — a byte relabel, a `ByteAlu` operand, any sub-combination + rather than the full linear form — is **unsound** without an added + `AreBytes`. The surviving reader in the non-last rounds (`add3`) is safe only + because it reads `Σ Yₖ·2^{8k}`, which regroups exactly into the two + constrained halfword sums. +2. **Deleting them as dead** (the tail-truncation optimisation). Legal, and + ✓ EXECUTED as correct — but worth only **112 cell-equiv of 5,509 (2.0%)**, + and it must drop `X4` **only** for the last round's *diagonal* group + (`gi ≥ 4`). A column G's `v[b]` is consumed by the diagonal group that + follows it in the same round; dropping that one is a bug, not an + optimisation. My own first draft of the option made exactly that mistake and + it was caught only because the option was exercised rather than described. + +**If you change this surface, re-run `gate.py` and `artifact_pin.py --check`.** +Neither the region hashes nor the framing values would catch a *reader* being +added to a previously-unread column, because it is a change inside `eval` — the +region hash moves, which is the signal to re-transcribe and re-gate. + +--- + +## 4.6 POST-B1 — the new audits, and the claim of mine they falsified + +### 4.6.1 `m[8]` is no longer a constant, and transcribing it as one was the trap + +The post-B1 chip computes `m[8] = MODE_C·TAG_LFMC + MODE_T·TAG_LFMT` +(`WordRef::ModeSelected`, evaluated `Σ col·tag`). Two documents I own — +`ORACLE.md` §2.1/§2.2 and `SOCKET.md` §2.2 — still described it as the constant +`0x434D464C`, and `ORACLE.md` justified its zero cost *because* it was constant. + +**Those framing tables are exactly what this gate transcribes.** Transcribed as +written, the z3 model would carry a constant where the chip has a linear form — +a model that no longer checks the chip **and still reports PASS**. That is the +same fail-open class as the pin's v1, on the transcription side instead of the +identification side, and it would have been mine. Both rows are now corrected, +with the reason spelled out: `m[8]` is still free and still prover-unchosen, but +because the selectors are **preprocessed**, not because the value is constant. + +Caught by the builder and relayed; recorded here so the lesson has a home. + +### 4.6.2 ⚠ M8 — idx 4 does NOT make the tag one-hot. My spec said it did. + +`TRANSCRIPT.md` §3.3 asserted *"idx 4 forces the mode sum to a bit, so at most +one tag is selected."* **The clause after "so" does not follow**, and the +consequence is not academic: a refactor trusting that sentence could delete the +registrar's one-hot check as redundant, and every constraint would still pass. + +Over a prime field `mode_sum ∈ {0,1}` pins the SUM, not the selectors: +`MODE_C = x`, `MODE_T = 1 − x` satisfies idx 4 for any `x`, and since the tags +differ, `x = (T − TAG_LFMT)/(TAG_LFMC − TAG_LFMT)` reaches **any** target `T`. + +✓ EXECUTED independently, twice — the builder's Rust M5/M6 run forges the tag +`"XXXX"` by a fractional split with zero constraint violations, and this board +reproduces it in the field model: + +| | check | result | +|---|---|---| +| M8 | forged tag reachable with **idx 4 alone** | **SAT** — `MODE_C = 4387334679741772800`, `MODE_T = 14059409389672811522`, sum ≡ 1, `m[8] = 0x58585858` | +| M8 | forged tag **excluded** once one-hot is present | UNSAT | +| M8 | honest leg: `TAG_LFMC` still reachable | SAT | +| M8 | honest leg: `TAG_LFMT` still reachable | SAT | + +Both honest legs are there deliberately: a "fix" that rejected everything would +pass the attack leg on its own. + +**What actually closes it:** the selectors being **preprocessed** (the prover +cannot choose them at all), plus the **registrar's exactly-one-of check**. Idx 4 +buys only the exclusion of the both-set case. `TRANSCRIPT.md` §3.3 is corrected +and M8 is now a standing control so the mistake cannot be re-made silently. + +Related, and ✓ VERIFIED from the layout: this is also why `MODE_T` sits at index +**8**, inside the selector run, rather than after the multiplicities — the +admission validator reads the selectors as a contiguous span, so a selector +parked past the mults would sit outside the one-hot check and be silently +unchecked. + +### 4.6.3 Two modelling bugs the board caught in its own audits + +Recorded because they are the argument for keeping two-sided controls on +everything, including the audits themselves. + +1. **B0b went vacuous.** My first widened version added the registrar's one-hot + unconditionally, which forces `MU = 1` outright — so the `dropped` leg came + back UNSAT and the audit was testing nothing. Removed: idx 4 *does* give MU + booleanity (MU **is** the mode sum), and that is what B0b checks; one-hotness + is M8's job. The division of labour is sharper than the original claim. +2. **A residue bug.** `MU` is a *sum* of felts, so as a z3 `Int` it can exceed + `p`; comparing the raw value against 0/1 let `mu = p + 1` count as "not 1" + and reported SAT for a sound chip. Now compared by residue. + +Neither would have been visible without the `present`/`dropped` pair on each +audit. An audit with only one leg is an audit that can quietly stop testing. + +--- + +## 4.7 POST-MODE_L — the gating split, and the audit it demanded + +### 4.7.1 ⚠ WA9 — O1 is TWO obligations and only ONE of them narrowed + +The review target worth the attention it was given. When `MODE_L` landed, the +lane identity `idx 6-13` was **narrowed to the digest modes** +(`DIGEST_MODE_COLUMNS = MODE_C + MODE_T`). That is correct and necessary: on a +leaf row the eight lanes are four felts' *halves*, so `IN_lane` and `m[lane]` are +deliberately different field elements, and gating on the full `MU` would make +every leaf row unprovable. + +**But O1 was never one obligation.** It is a *lane identity* plus an *AreBytes +range bound*, and the leaf block depends on the second, not the first: +canonicity **assumes** `lo, hi < 2^32` and does not establish it. + +✓ VERIFIED from the source, and this is what makes the design sound: the lane +`AreBytes` sends carry `Multiplicity::Sum3(MODE_C, MODE_T, MODE_L)` — the **full** +mu — so all 32 lane byte columns stay bounded on leaf rows. **The identity +narrowed; the range bound did not.** + +WA9 turns that from an observation into a control, because the plausible future +refactor is *"tidy the multiplicities so they match"*: + +| | check | result | +|---|---|---| +| WA9 | `AreBytes` still covers leaf rows (as built) → felt→halves map injective | **UNSAT** | +| WA9 | `AreBytes` **narrowed** to the digest modes → a felt gets a second half-pair | **SAT** | + +The second row is the finding: with the bound gone, `lo` and `hi` become full +field elements, a second encoding of the same felt satisfies binding *and* +canonicity, and **the canonicity gate is still there but VACUOUS**. That is +precisely the trap `LEAF.md` §2.2 warned about, now executable. + +### 4.7.2 WA8 — leaf canonicity + +| | check | result | +|---|---|---| +| WA8 | canonicity present → a non-canonical half-pair is unprovable | **UNSAT** | +| WA8 | canonicity dropped → a felt acquires a second half-pair | **SAT** | + +`p − 1 = 0xFFFFFFFF_00000000`, so every pair with `hi` maximal and `lo ≥ 1` +encodes a field element that *also* has an ordinary encoding — one felt, two leaf +digests, which is the collision a Merkle tree must not have. The honest leg is +covered Rust-side by the build's own tests; the "dropped ⇒ SAT" leg needs the +gate, since it means editing the constraint set. + +### 4.7.3 M8 over four selectors + +A third tag does not weaken the M8 finding and does not strengthen idx 4: the +mode sum is still only a *sum*, so a fractional split still reaches any target +tag. Verified with `MODE_L` in the span — forged target **SAT** under idx 4 +alone, **UNSAT** under the four-way one-hot, and all three real tags still +reachable (the honest legs). + +--- + +## 4.8 THE D1 FIX — one audit, and one documented blindness + +### 4.8.1 No honest row is over-constrained + +`emit_unread_input_pins` derives slot `k`'s selector as the sum of modes with +`num_input_cells() <= k`. ✓ VERIFIED against `instr.rs:104-110` +(Compress/Transcript 2, Leaf 1, Permute 3), the resulting pin matrix is: + +| mode | reads | slot 1 (`IN4..8`) | slot 2 (`IN8..12`) | +|---|---:|---|---| +| Leaf | 1 cell | **pinned** | **pinned** | +| Compress | 2 cells | free | **pinned** | +| Transcript | 2 cells | free | **pinned** | +| Permute | 3 cells | free | free | + +**No mode is ever pinned on a cell it reads** — UNSAT, and that is the property +a soundness fix most easily breaks, because over-constraining makes honest rows +unprovable rather than making dishonest ones provable, so the tests that would +catch it are the *honest-path* ones. + +### 4.8.2 ⚠ DOCUMENTED BLINDNESS — the pins are inert on BLAKE3 + +**This gate cannot show the D1 pins are necessary, and it is important that this +is written down rather than inferred from a green board.** + +On the BLAKE3 arm the two unread cells are read by nothing: cell 1 is read only +by the lane identity `idx 6-13`, which is gated on the digest modes and therefore +zero on the one row (leaf) where cell 1 is unread; cell 2 is read only through +`idx 0-3`'s `MODE_P · IN` term, and `idx 5` pins `MODE_P` to zero permanently. +So dropping the BLAKE3 pins cannot change a BLAKE3 digest, and any "wrong +output" question this gate asks about them returns UNSAT. + +| | what | verdict | +|---|---|---| +| **this gate certifies** | the pins are **inert** on BLAKE3 (hygiene) | UNSAT | +| **this gate cannot show** | their **necessity** on `Test`/`Poseidon`, where those cells *are* read — D1's actual defect | out of model | +| **what carries that instead** | the builder's Rust junk-rejection controls, in the WA9 shape (drop the pins → SAT) | Rust side | + +**"The gate said UNSAT" is exactly how a fix gets dropped as redundant**, which +is why this is a labelled blindness row on the board and not an omission. It is +the same discipline as the `drop_carry_bool` BV blindness in §4.2. + +The general lesson, worth keeping: **hygiene in one arm was soundness in +another.** The BLAKE3 arm pinned its unread cell and called it hygiene — correctly +— and the identical omission in `eval_test`/`eval_poseidon` was a HIGH soundness +defect. A property's importance is not a property of the constraint; it is a +property of the constraint *plus the arm it sits in*. + +--- + +## 5. Conformance verdict, row by row + +Against the Phase-2 report's §3 table, re-derived from the source rather than +taken on trust. All rows conformant. The one flagged deviation (row 6, `add2`) is +**closed** — the model now matches the chip. Specifically re-verified by reading +`eval`: the framing indices (0–3, 4, 5, 6–13, 14–21, 22–25), `word_expr` as +`Σ byte·2^{8k}` little-endian, `half_expr` as `b0 + 256·b1`, add3 as sum identity +plus two booleanities, the rotation's four identities, and the send shapes +(`ByteAlu[XOR]` 4 per XOR word; `AreBytes` 4 per rotation; `AreBytes` 2 per lane +× 8 lanes = 16), every send `Multiplicity::Column(MU)`. + +**Max degree 3**, unchanged: reached by the µ-gated carry booleanities, including +add2's, whose expression carry is a linear form so the product stays degree 3. + +--- + +## 6. What this gate does and does not establish + +| claim | status | +|---|---| +| the transcribed bodies compute the anchored socket reference at 6r and 7r | ✓ EXECUTED (T1–T4) | +| every framing/wiring bug class is still caught after transcription, both round counts | ✓ EXECUTED (36 controls) | +| the add2 expression-carry form pins `s`, and its bound is necessary | ✓ EXECUTED (WA7) | +| the four BLOCK-0 framing constraints do what they claim, and idx 5 is load-bearing for idx 0–3 | ✓ EXECUTED (B0a–B0c) | +| model census == built chip census, both round counts | ✓ EXECUTED (exact) | +| the gated artifact is the one on disk | ✓ EXECUTED (`artifact_pin.py --check`, v2) | +| the chip's framing constants EQUAL the oracle spec (tag/flags/block_len/counter/window/rounds) | ✓ EXECUTED (§1.3) -- **new in v2; v1 could not see this** | +| AR1–AR4 | ✗ argued, not solved — stated in §4.4 | +| monolithic symbolic `rounds = 1,2` UNSAT | ✗ bonus, still not completed (see `ORACLE.md` §8) | +| the socket identity against the Rust `blake3` crate | ✗ deferred — needs cargo | +| **the `permute` socket** | ✗ **OPEN** — not specified, not built; chip pins `MODE_P = 0` so a program using it is unprovable rather than silently wrong. Good failure mode, still a gap | +| **O5 leaf/parent domain separation** | ✓ **DECIDED 2026-08-10 — no longer open.** Ratified by the user: any future leaf-hashing path MUST use the reserved `"LFML"` tag (the RFC 6962 leaf/parent split expressed in the tag scheme, keeping both domains direct `blake3::hash` KATs). Recorded in `ORACLE.md` §7 and in the chip's module docs (`cece4a0b`, justification corrected in `2957c3f9`). Nothing implements `"LFML"` yet, and the safety argument is **fixed depth alone** — NOT absence of leaf hashing: FriToyV0 already compresses raw data rows into leaf digests under the same `"LFMC"` tag (`programs.rs:577/585/625`), safe only because every current tree is a fixed-depth static circuit (eDSL shape fixed at build time; hints supply values, never structure). **The obligation binds review, not code**: a change adding variable-depth trees, or leaf hashing meant to coexist with them, without `"LFML"` is rejected on O5 | + +Two things I did **not** do, deliberately: I did not run cargo (the reviewer is +using the worktree), so the chip's own Rust tests are not part of this verdict; +and I did not re-derive the reference or the KATs, which are unchanged from +`ORACLE.md` and remain externally anchored. + +--- + +## 7. Files touched + +| file | change | +|---|---| +| `chip_model.py` | `emit_add2` → expression-carry (no carry column, 4 cells); BLOCK-0 documented as built with its four constraints; census prefix 13 → 28; `drop_add2_carry` bug hook | +| `gate.py` | WA7 + B0a/B0b/B0c field audits; argued ledger AR1–AR4; control sweep at both round counts; blindness row sharpened | +| `artifact_pin.py`, `artifact_pin.json` | **v2** — four hashed regions (the fourth being the framing constants v1 missed) plus resolved-value conformance against `socket_ref.py`; refuses to pin if the chip's framing diverges from the oracle. v1's fail-open is described in §1.1 | +| `run-chip-gate.log` | **the board of record** — 75 checks, both round counts | +| `run-gate.log` → `run-gate-PRE-PHASE2-SUPERSEDED.log` | renamed with a DO-NOT-CITE header; its census figures are the superseded model's and contradict §3 | +| `ORACLE.md` | unchanged; superseded only where noted in §2.2 (MU booleanity) and §2.1 (add2) | diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/ORACLE.md b/thoughts/shared/lfm-real-hash/gate-oracle/ORACLE.md new file mode 100644 index 000000000..87cdf452c --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/ORACLE.md @@ -0,0 +1,702 @@ +# The BLAKE3 real-hash gate ORACLE + +**Status:** built and executed, ahead of the chip. **Date:** 2026-08-10. +**Scope:** the human-owned oracle for the `LFM_HASH`-hosting-BLAKE3 z3 gate — +reference `f`, the Option-A socket definition, the column-role map (which is +also the Phase-2 spec), the chip-contract library, and a runnable gate with its +negative-control board and width audit. + +Nothing here needs the chip to exist. That is the point: the oracle is +**pre-committed**, so when Phase 2 lands it is a genuinely independent anchor +rather than a restatement of whatever got written. + +Claims are marked ✓ EXECUTED (a command was run and its result is recorded), +✓ VERIFIED (read the code / derived it), ? INFERRED, or ✗ OPEN. + +> **⚠ SUPERSEDED IN TWO PLACES BY `CHIP-GATE.md` (2026-08-10, after Phase 2).** +> The chip now exists and the seam is closed, so the gate certifies the built +> chip rather than this model. Two claims below no longer hold as written: +> **(1)** `emit_add2` here witnesses the carry as a COLUMN; the chip derives it +> as an EXPRESSION and emits one constraint. The model was changed to match, and +> the census figures in §3.2 moved accordingly (see `CHIP-GATE.md` §2.1, §3). +> **(2)** §3.1/§5 call MU booleanity "structural, not a BV theorem"; the chip +> emits it as a real constraint and it is now checked in the field +> (`CHIP-GATE.md` §2.2, audit B0b). Everything else here — the reference, the +> socket, the contracts, the column-role map, O1–O5 — stands unchanged. + +--- + +## 0. The board + +`python3 gate.py` — **49 checks, ~2.5 min, VERDICT: PASS**. ✓ EXECUTED. +(Log: `run-gate-PRE-PHASE2-SUPERSEDED.log`. **This board is the +pre-Phase-2 model's.** The board of record is now `run-chip-gate.log` — +75 checks against the committed chip; see `CHIP-GATE.md` §4.) + +| layer | check | result | +|---|---|---| +| **1 reference** | official BLAKE3 vectors @7 rounds: 35 hash + 35 keyed + 35 derive_key + 115 random × 3 modes + 2 known | PASS | +| | differential vs the independently-written in-repo `blake3_ref.py`, 200 compressions × {6,7} rounds | PASS | +| | 4 single-convention perturbations each break the anchor | PASS | +| **2 socket** | 10 inputs × 2 round counts, byte-level and word-level routes agree | PASS | +| | @7 rounds the socket equals standard `BLAKE3(a‖b‖"LFMC")[..16]` | PASS | +| | all 14 framing degrees of freedom discriminated by ≥1 vector | PASS | +| | 20 peer vectors reproduced exactly from a second implementation | PASS | +| **5 gate** | T1–T4 main theorems (see §5) | PASS | +| | **27 negative controls, all SAT** | PASS | +| | 1 documented BV blindness, correctly UNSAT | PASS | +| | 11 width-audit items | PASS | + +Run order: `python3 anchor_check.py` → `python3 socket_kats.py --write` → +`python3 gate.py`. Files are plain `python3` + `z3` (4.15.4), no venv needed. + +--- + +## 1. Layer 1 — the reference `f`, and why it is trustworthy + +`blake3_oracle.py`. Round-parameterised BLAKE3: the compression function, the +chunk state, the CV stack, the output node, and the XOF. Written from the +specification, not transcribed from any in-repo file — `blake3_ref.py` and +`prover/src/lfm/blake3.rs` are used as cross-checks, never as sources. + +The provenance chain, strongest first: + +1. **At `rounds = 7` this is standard BLAKE3**, so it is anchored on + `official_test_vectors.json` in all three modes — hash, keyed_hash, + derive_key — across 35 input lengths and the extended-output window, plus a + 115-case randomised block. ✓ EXECUTED — **452 vector comparisons, all exact**. + + Provenance of that file, stated precisely because it matters: its `cases` + block carries **upstream BLAKE3's published test-vector values** under + upstream's own conventions (the `i % 251` input pattern, the key + `"whats the Elvish word for friend"`, the context string + `"BLAKE3 2019-12-27 16:29:52 test vectors context"`); its `random` block was + generated by the real Rust `blake3` crate over a self-contained xorshift64* + stream that `anchor_check.py` re-implements independently rather than shares. + Two entries are additionally checkable against constants published outside + any test file — `BLAKE3("") = af1349b9…f3262` and + `BLAKE3("abc") = 6437b3ac…d9d85` — and both match. ✓ EXECUTED. The anchor is + therefore external to this repo and to this project. +2. **Differential** against `thoughts/blake3/blake3-oracle/blake3_ref.py`, an + independently written reference: 200 random compressions at **both** round + counts, exact agreement. ✓ EXECUTED. Two agreeing sources. +3. **Anchor sensitivity.** An anchor nobody can fail is not an anchor. Four + perturbations — wrong round count, swapped message-permutation entry, flipped + IV bit, swapped counter halves — each break the official vectors. ✓ EXECUTED. + +The round loop permutes the schedule when `r < rounds - 1`, so `rounds = 7` is +bit-for-bit standard BLAKE3 **with no other edit**. That single-integer +parameterisation is deliberate and load-bearing: it is what lets the 7-round +anchor certify the *code path* that the 6-round variant then instantiates. + +> **At `rounds = 6` there is no external anchor and there cannot be one.** No +> library computes it; no published vector contains it. The 6-round values in +> this directory are the *definition* of the variant, defensible only as "the +> anchored code path with the loop bound changed". That is assumption **A6R**, +> and it is the honest reason to prefer 7 rounds as the baseline. + +--- + +## 2. Layer 2 — the Option-A socket reference + +`socket_ref.py`, vectors in `socket_kats.json` (`socket_kats.py --write`). + +Between "we have a correct `f`" and "we have a correct 2-to-1 compress" sit +**seven independent choices**, each a way to be wrong while every primitive test +stays green. Six are the usual framing; the seventh is new to Route A. + +### 2.1 The specification + +**Byte level (normative).** Deliberately expressible as a library call: + +``` +msg = LE32(a0)‖LE32(a1)‖LE32(a2)‖LE32(a3) (16 bytes) + ‖ LE32(b0)‖LE32(b1)‖LE32(b2)‖LE32(b3) (16 bytes) + ‖ "LFMC" ( 4 bytes) = 36 bytes + +digest_bytes = BLAKE3(msg)[0..16] +c_i = LE32⁻¹(digest_bytes[4i .. 4i+4]) i in 0..4 +``` + +**Word level (what the chip proves).** 36 bytes is one block, so this is exactly +one compression: + +| input to `f` | value | +|---|---| +| `h` | `IV[0..8]` — all eight words (the unkeyed default) | +| `m[0..4]` | `a` | +| `m[4..8]` | `b` | +| `m[8]` | **on the built chip, a mode-selected linear form** — `MODE_C·TAG_LFMC + MODE_T·TAG_LFMT`, which is `0x434D464C` on a Merkle row (`MODE_C = 1`). See the note under §2.2. | +| `m[9..16]` | `0` | +| `t` | `0` | +| `block_len` | `36` | +| `flags` | `0x0B` = `CHUNK_START | CHUNK_END | ROOT` | +| digest | `out[0..4]` — the **low four** of the 16 output words | + +Everything except `a`, `b` **and `m[8]`** is a compile-time constant — `m[8]` is +the mode-selected linear form noted in the table and detailed under §2.2. +✓ EXECUTED: the two routes are computed by separate code paths and agree on all +20 vector-instances. + +### 2.2 Why the domain tag is in the message, not in `flags` + +Independently re-derived; agrees with the parallel agent's `SOCKET.md`. + +A tag in `flags` (or `t`, or `h`) makes the socket a **nonstandard** invocation +of `f` that no library computes — so its KATs could only ever come from our own +oracle, at 6 **and** at 7 rounds, and the external anchor is thrown away for +nothing. Putting the tag in the message keeps the socket a standard BLAKE3 hash +of a domain-separated byte string, which is precisely what lets §2.1 be a +library call. The separation is just as real: the message is fixed-length with +the tag at a fixed offset and `block_len` is itself an input to `f`, so the +encoding is unambiguous and distinct tags give distinct messages. + +Cost: 36 bytes rather than 32 — same single block, and **zero extra columns**. + +> **⚠ UPDATED FOR B1 — the conclusion holds, the REASON changed.** Before B1 +> `m[8]` was a compile-time *constant*, and "constant" was the reason it cost +> nothing. On the built chip it is a **linear form over two preprocessed mode +> columns**, `MODE_C·TAG_LFMC + MODE_T·TAG_LFMT` +> (`WordRef::ModeSelected`, evaluated as `Σ col·tag`). It still costs zero main +> columns and zero sends — it is only ever an `add3` operand, read as a whole +> word value and never byte-decomposed — and it is still prover-unchosen, but +> now because the selectors are **preprocessed**, not because it is constant. +> +> **This distinction is load-bearing for anyone transcribing this table into a +> model.** Transcribed as a constant, the z3 model would describe something the +> chip does not do — and would still report PASS. That is the fail-open class +> this gate exists to prevent; it is the same lesson as the pin's v1. Degree is +> unaffected (0 → 1 inside a body that was already degree 1; × MU = 2 ≤ 3). +> Full mechanism: `transcript-spec/TRANSCRIPT.md` §3. + +### 2.3 Tag allocation + +| tag | u32 (LE) | use | status | +|---|---|---|---| +| `"LFMC"` | `0x434D464C` | **this socket** — 2-to-1 compress / Merkle parent | built | +| `"LFMT"` | `0x544D464C` | **transcript step** — the compress-chain Fiat–Shamir transcript | specified: `transcript-spec/TRANSCRIPT.md` | +| `"LFMP"` | `0x504D464C` | ~~`permute` socket~~ | **RETIRED UNUSED** — see below | +| `"LFML"` | `0x4C4D464C` | **leaf domain** — felt-input leaf rows (`MODE_L`) | **LIVE** — built; `leaf-spec/LEAF.md` | + +All three live tags are **pairwise distinct** and `artifact_pin.py` enforces +that — one clash is one collapsed domain. A tag is never reused for a second +purpose. **`"LFMP"` is retired rather than deleted**, and that distinction is load-bearing: the user ratified option B1 +(`permute-socket-options.md`), so no `permute` socket will ever be built and the +value is permanently unused — but deleting the row would let a future allocation +reuse `0x504D464C` and silently create a domain nobody analysed. + +### 2.4 The vectors + +10 written-out inputs (5 structural, 5 formula-derived) × 2 round counts, each +with the full framing-control table. Applicability is derived **structurally**, +not hand-listed: a control is inapplicable on an input exactly when its +*effective trace* — initial state, per-round schedule, output window — is +identical to the honest one, which is a sound criterion because identical traces +force identical digests. This caught a real gap: `msg_perm_swapped` exchanges +`m[2]` and `m[6]`, a no-op whenever `a[2] == b[2]`, so three structural vectors +cannot see it. That is why the formula vectors exist. + +**Cross-check.** All 20 vectors of +`thoughts/blake3/socket-kats/socket_kats.json`, produced by a different agent +from a different reference, are reproduced **exactly** by this code, spec fields +included. ✓ EXECUTED. Two independent implementations of both the primitive and +the framing agree. + +### 2.5 The external cross-check the build phase must run + +At `rounds = 7`, `socket_digest(a, b)` **is** `blake3::hash(a‖b‖"LFMC")[..16]` +re-read as four little-endian u32s. That is a one-line assertion against the +Rust `blake3` crate and it should be written, because it is the version of the +check that survives this directory being deleted. ✗ OPEN — needs cargo, which +this task must not run. + +--- + +## 3. Layer 3 — THE COLUMN-ROLE MAP (this is the Phase-2 spec) + +`chip_model.py`. Every committed column appears as a free variable and every +constraint as an equation. **A chip that conforms to this file is one the gate +proves correct; a chip that does not conform is one the gate says nothing +about.** Phase 2 should implement against this and flag any deviation. + +One row = one `compress` call, fully unrolled. + +### BLOCK 0 — socket I/O (shared with the `LFM_HASH` host) + +| role | count | note | +|---|---|---| +| `MU` | 1 | multiplicity / gate flag; 1 on a real row, 0 on padding | +| `IN_A[0..4]`, `IN_B[0..4]` | 8 felts | the two input digest cells | +| `OUT_C[0..4]` | 4 felts | the one output digest cell | + +**REQUIREMENT R1.** These are the host's **existing** cell columns — the frozen +2-cells-in / 1-cell-out bus contract. The BLAKE3 arm must **reuse** them, not +commit a second copy linked by an equality constraint: that is 12 wasted columns +and one more way to be wrong. + +### BLOCK 1 — the lane boundary ⚠ THE new soundness surface for Route A + +| role | count | +|---|---| +| `MB[j][k]`, j in 0..8 lanes, k in 0..4 bytes | 32 byte columns | + +**CHIP CONSTRAINT** (per lane `j`), µ-gated, degree 2: +``` +MU · ( LANE_j − (MB[j][0] + 2^8·MB[j][1] + 2^16·MB[j][2] + 2^24·MB[j][3]) ) = 0 +``` +**CHIP SENDS** (per lane `j`): `AreBytes(MB[j][0], MB[j][1])`, +`AreBytes(MB[j][2], MB[j][3])` — 16 sends total. + +**BOTH ARE REQUIRED. This pair is obligation O1 and it is the single most +important line in this document.** `edsl::merkle_walk` feeds `compress` +*arena-hinted* — i.e. prover-chosen — sibling cells. A lane is a Goldilocks +felt, ranging over `[0, p)` with `p ≈ 2^64`. Without the `AreBytes`, the byte +columns are full field elements, one linear equation in four unknowns leaves +three of them free, and **the prover chooses the message that gets hashed** — +every load authenticated through `compress` becomes forgeable. Without the +identity, the bytes are simply unrelated to the lane. + +✓ EXECUTED both ways in the field model: WA1/WA2 in §6. + +Note this is **not** visible in the bit-vector domain, where a byte *is* eight +bits. It is proved in the field, mod `p`. See §6. + +**O1 is nearly free, and that is worth knowing before anyone tries to optimise +it away.** The message enters `f` only through `add3` — it is never XORed — so +those 32 bytes needed an explicit `AreBytes` regardless. The lane boundary +reuses *the same 16 sends*; O1's marginal cost over a chip that merely +range-checked its message is **8 linear constraints and nothing else**. There is +no performance argument for dropping it. + +### BLOCK 2 — message words + +`m[0..4] = a`, `m[4..8] = b` (from BLOCK 1's bytes), +`m[8] = MODE_C·TAG_LFMC + MODE_T·TAG_LFMT` (post-B1; `0x434D464C` on a Merkle +row), `m[9..16] = 0`. + +**REQUIREMENT R2.** `m[8..16]` carry **no columns and no range checks**, so the +4-byte domain tag is free. `m[9..16]` are compile-time constants; **`m[8]` is +not** — it is a linear form over the two **preprocessed** mode columns, which +costs no columns and no sends either (it is only ever an `add3` operand, read as +a whole word and never byte-decomposed) and is prover-unchosen because the +selectors are preprocessed. Same conclusion, different reason — see §2.2. + +### BLOCK 3 — initial state: **all sixteen words are compile-time constants** + +``` +v[0..8] = IV[0..8] v[8..12] = IV[0..4] +v[12] = t_lo = 0 v[13] = t_hi = 0 v[14] = 36 v[15] = 0x0B +``` + +Because `h = IV`, the *entire* initial state is constant, so the socket costs +**zero** input-state columns where a syscall-shaped BLAKE3 chip pays 112 bytes. +? INFERRED consequence worth stating: round 0 could therefore be partially +constant-folded. That is **permitted but must be re-gated** — a folded round 0 +no longer matches this model, so the gate's T1-plus-composition argument would +not cover it. + +### BLOCK 4 — per-G SSA logic (8 G-calls × R rounds) + +Per G-call — 56 byte cells + 4 carry cells = **60 cells**, matching the built +chip's `cols::G_SIZE = 60`: + +| sub-op | SSA output | bytes | carry bits | +|---|---|---:|---:| +| `add3` v[a] += v[b] + mx | `A1` | 4 | 2 | +| `xor` v[d]^v[a] → rotr16 **free** | `X1` | 4 | – | +| `add2` v[c] + v[d] | `C1` | 4 | – | +| `xor` v[b]^v[c] | `X2` | 4 | – | +| `rotr12`(X2) | `SLL_lo,SLLC_lo,SLL_hi,SLLC_hi,B1` | 12 | – | +| `add3` v[a] += v[b] + my | `A2` | 4 | 2 | +| `xor` v[d]^v[a] → rotr8 **free** | `X3` | 4 | – | +| `add2` v[c] + v[d] | `C2` | 4 | – | +| `xor` v[b]^v[c] | `X4` | 4 | – | +| `rotr7`(X4) | (as above) `B2` | 12 | – | + +Constraint families, all µ-gated: + +- **`add2`**: the carry is an **expression**, not a column — + `carry := (wval(A)+wval(B) − wval(s))·2^{−32}` — and there is exactly ONE + constraint, `MU·carry·(1−carry) = 0` (deg 3), which says + `wval(A)+wval(B) − wval(s) ∈ {0, 2^32}`: the sum identity and the booleanity + together. (Earlier revisions of this document specified a witnessed carry + column and two constraints; the chip does it this way and the model follows + the chip — `CHIP-GATE.md` §2.1, and WA7 is the field audit it needs.) +- **`add3`**: `MU·(wval(A)+wval(B)+wval(M) − wval(s) − 2^32·(c1+c2)) = 0` (deg 2), + plus booleanity on `c1` and `c2` (deg 3). **Two summed carry bits, NOT a + single ternary carry** — `k(k−1)(k−2)` is degree 3 ungated and µ-gating pushes + it to 4, over the hard budget. This is the tightest coupling in the design. +- **`rotr12`/`rotr7`**: inner `rotl r` with `r = 4` / `r = 9`. + `MU·(x_hw·2^r − SLLC·2^16 − SLL) = 0` per halfword, then + `MU·(Ylo − SLL_hi − SLLC_lo) = 0`, `MU·(Yhi − SLL_lo − SLLC_hi) = 0`. + `AreBytes` over the 8 bytes of `SLL_lo/SLLC_lo/SLL_hi/SLLC_hi` = 4 sends. +- **`rotr16`/`rotr8`**: free byte relabels `[b2,b3,b0,b1]` / `[b1,b2,b3,b0]`. + No columns, no lookups, no constraints. +- **`xor`**: 4 `ByteAlu[XOR]` sends; no eval constraint. The lookup pins the + output *and* byte-range-checks both operands, which is why nearly every word + in the design needs no explicit `AreBytes`. + +### BLOCK 5 — feed-forward, truncation window, output recomposition + +``` +OUTW[i] = v[i] XOR v[i+8] for i in 0..4 ONLY +MU · ( OUT_C[i] − Σₖ OUTW[i][k]·2^{8k} ) = 0 +``` + +**REQUIREMENT R3.** The socket produces **four** of the sixteen output words. +`out[i+8] = v[i+8] ⊕ h[i]` is never computed — `h` is the constant IV and those +words are not in the digest. This is where most of the saving over a +syscall-shaped BLAKE3 chip comes from: 12 words × 4 bytes of columns and the +same number of XOR sends, never built. + +No range check is needed on `OUTW`: its bytes are `ByteAlu[XOR]` outputs. The +sum is `< 2^32 ≪ p`, so `OUT_C[i]` is forced to the honest u32 — **and therefore +the socket's output always satisfies O1**, which is why only leaf digests and +prover-hinted siblings need the input check. + +### 3.1 Degree ledger + +| constraint | body | ×µ | ≤3? | +|---|---:|---:|:--:| +| lane decomposition | 1 | 2 | ✅ | +| add2 sum / add3 sum | 1 | 2 | ✅ | +| carry booleanity | 2 | 3 | ✅ | +| shift identity | 1 | 2 | ✅ | +| recombine | 1 | 2 | ✅ | +| digest recomposition | 1 | 2 | ✅ | +| *(rejected)* ternary carry | 3 | **4** | ❌ | + +Worst legal constraint = **3**, matching `LFM_HASH`'s existing degree budget. + +### 3.2 Cost census + +> **The census of record is `CHIP-GATE.md` §3**, which reconciles the model +> against the **built chip** to the unit. The figures below are the pre-Phase-2 +> model's and are **superseded**; they are kept only for the reconciliation +> against the standalone chip, which is still the useful comparison. + +Current, matching the built chip (✓ EXECUTED, both round counts): + +| | 7-round | 6-round | +|---|---:|---:| +| main columns | 3,436 | 2,956 | +| bus sends | 1,382 | 1,190 | +| aux (`3·⌈N/2⌉`) | 2,073 | 1,785 | +| **cell-equiv** | **5,509** | **4,741** | + +Breakdown (7-round): `rotr_shift` 1,344 · `xor_out` 912 · `add3` 672 · +`add2` **448** · `lane_bytes` 32 · `frozen_socket_prefix` **28**. + +The two figures that moved from this document's original numbers, and why: +`add2` 560 → 448 and main 3,533 → 3,436 (7r) / 3,037 → 2,956 (6r), because the +chip derives the add2 carry as an EXPRESSION rather than witnessing it as a +column (−1 cell per add2, 2 per G); and I/O 13 → 28, the frozen shared prefix as +built, with `MU` preprocessed and therefore outside the main-column census. +`CHIP-GATE.md` §2.1 and §3 carry the derivation. + +**Reconciliation against the standalone syscall-shaped chip** in +`thoughts/blake3/blake3-chip/DESIGN.md` (≈3,155 main / ≈1,250 sends / ≈5,030 +cell-equiv at 6 rounds). Per-G logic differs only by the add2 carry column +(60 cells here vs 62 there), so the rest of the difference is I/O: + +| | main columns | sends | +|---|---:|---:| +| standalone (6r) | 3,155 | 1,250 | +| − `h[0..8]` input words (IV is constant here) | −32 | — | +| − `t_lo,t_hi,block_len,flags` (all constant here) | −16 | — | +| − half the message (`m[8..16]` constant) | −32 | −16 | +| − 12 of 16 output words (R3, the truncation window) | −48 | −48 | +| − add2 carry columns (expression carry, 2 per G × 48) | −96 | — | +| + frozen socket prefix (28) and `LfmMem` tuples | +25 | +6 | +| **socket (6r), as built** | **2,956** | **1,190** | + +✓ VERIFIED: the deltas sum exactly to the census the gate emits, which in turn +equals the built chip's to the unit. The mixing core dominates and is untouched, +so the saving stays modest in relative terms. + +### 3.3 ⚠ OPTIONAL and **NOT recommended**: last-round tail truncation + +In the last round, only `v[0..4]` and `v[8..12]` are read by the feed-forward. +The diagonal group's `b`-positions are `{5,6,7,4}`, so for those **four** G-calls +`X4` and `B2` produce nothing anyone reads and could be omitted. + +**Measured saving: 112 cell-equiv of 5,509 — 2.0%.** ✓ EXECUTED (the truncated +pipeline still reproduces the anchored KAT and still excludes a wrong digest). + +It carries an obligation, and 2% does not buy it: + +> **O-TAIL.** Dropping `X4` removes `B1`'s downstream XOR, hence its **per-byte** +> range check. This is sound *only because* the sole surviving consumer +> (`A2`'s `add3`) reads `B1` as the full linear form `Σ B1[k]·2^{8k}`, which +> regroups exactly into the two constrained halfword sums. ✓ EXECUTED, both +> sides: the **word value** is still pinned (WA5, UNSAT) but the **individual +> bytes are not** (WA5, SAT). Any consumer that reads `B1`'s bytes — a relabel, a +> byte lookup, any sub-combination — is **unsound** without an explicit +> `AreBytes`. + +Recommendation: **keep the uniform G**. Take R3 (the output truncation), which +is large and unconditional; skip this one. + +A first draft of this optimisation skipped the *column* group too, which is a +real bug — a column G's `v[b]` is consumed by the diagonal group that follows it +in the same round. It was caught only because the option was exercised rather +than merely described. + +--- + +## 4. Layer 4 — the chip-contract library + +`contracts.py`. Assume-guarantee: the gate proves the compression/framing layer +**given** these; it does not re-prove `prover/src/tables/bitwise.rs`, which is an +existing, separately-audited chip. Same assumption the keccak gate makes. + +What is *not* optional is writing them down — an unstated contract is how a +fail-open gate happens: the model quietly assumes a bound the chip never +enforces, every theorem returns UNSAT, and the gate certifies nothing. + +| contract | guarantee | obligation on the chip | width it licenses | +|---|---|---|---| +| `AreBytes[x,y]` | `x,y ∈ [0,256)` | one send per **pair** of bytes, `Multiplicity::Column(MU)` | treating a column as 8-bit inside a field-lifted linear form | +| `ByteAlu[XOR](x,y)→z` | `x,y,z ∈ [0,256)` **and** `z = x⊕y` | one send per output **byte** | range-checks both operands *and* the output for free — the reason most words need no explicit `AreBytes`; operands may be linear combos while each stays ≤255, which is what makes a free byte relabel legal in place | +| `LaneDecomposition` | `lane ∈ [0,2^32)` and the bytes are its unique LE decomposition | one µ-gated identity **and** two `AreBytes` sends — **neither alone suffices** | the load-bearing width: `Σ bₖ·2^{8k} < 2^32 ≪ p`, so the identity cannot wrap and `lane` is forced `< 2^32`. This is all of O1 | +| `CarryBit` | `c ∈ {0,1}` | one µ-gated degree-3 constraint per carry column | treating a carry as a bit; dropping it is a **field-level** forgery invisible to BV | +| `ShiftRemainderBound` | `SLL ∈ [0,2^16)` | `AreBytes` on `SLL`'s byte pair | the **tight** remainder bound; with `2^16` invertible mod `p` it pins `SLL = (x·2^r) mod 2^16`. The quotient `SLLC` needs only a loose bound | +| `NoWrapSideCondition` | `expr ≡ 0 (mod p)` ⟹ `expr = 0` over ℤ | a static bound argument per identity — §6, **not** a solver run | the bridge between the BV model and the field | + +Two modelling domains, because they see different bugs — and getting the split +wrong is the classic fail-open (model a dropped range check in BV, observe +UNSAT, conclude the check is unnecessary): + +- **BV** (bytes as 8-bit bitvectors) sees logic and wiring bugs. It **cannot** + see bound-necessity, because the bound is baked into the variable's width. +- **FIELD** (`Int` mod `p`) sees exactly those. A column with no range check is a + full field element, and `2^16`/`2^32` are invertible mod `p` while being zero + divisors mod `2^n`. + +--- + +## 5. Layer 5 — the gate, and THE SEAM + +`gate.py`. Every committed column is a free variable; every lookup (under its +contract) and every eval constraint is an equation; then + +``` +assert chip_output != reference_f(input) +UNSAT -> for every satisfying assignment the output equals the reference + (correctly AND tightly constrained) +SAT -> the constraints admit a wrong output +``` + +### 5.1 The theorems and the argument they compose into + +| | theorem | result | +|---|---|---| +| **T1** | one G quarter-round vs the reference G, **free inputs** | UNSAT (13.9s) | +| **T2** | the message schedule fed to **all 7 rounds**, free inputs — placement, tag word, tag slot, lane byte order, permutation | UNSAT | +| **T3** | framing at `rounds = 0` — constant initial state, feed-forward, truncation window | UNSAT | +| **T4** | the **full pipeline, concrete**, vs the anchored KATs at 6 and 7 rounds; and the same pipeline **excludes** a wrong digest | SAT / UNSAT (≈8s / ≈2s) | + +**The argument.** A round is a *fixed* composition of eight G-calls on fixed +indices and the round count is a compile-time constant. T1 gives a correct G on +arbitrary inputs; T2 gives a correct schedule on arbitrary inputs; T3 gives the +wrapper. Hence the full N-round socket is correct for both round counts. T4 then +runs the whole thing concretely against externally-anchored vectors, so the +composition argument has an **executed end-to-end witness** rather than only a +proof sketch — including the `EXCLUDES a wrong digest` direction, which is a +concrete tightness check at that input. + +Monolithic symbolic runs at `rounds = 1, 2` are available behind `--full` as +bonus confirmation. **They were attempted and did not complete** — `rounds = 1` +was still running after 25 minutes and was stopped. This is expected rather than +alarming: the prior chip gate put the same class of check behind `--full` with +30-minute timeouts for the same reason. They are **not** required for the +verdict, which rests on T1–T4 and the control board. Recorded here because a +check that was tried and abandoned should not silently look like a check that +was never needed. + +**Every theorem has controls of its own.** A theorem with no control may be +vacuous, so T2 and T3 each carry controls proving they discriminate *at their own +layer*, not merely end-to-end. Where a layer genuinely cannot see a control it is +said so rather than papered over: at `rounds = 0` the counter, `block_len` and +`flags` words sit at `v[12..16]` and reach the digest only *through* the rounds, +so they are invisible to T3 and are covered by T4 instead. Listing them under T3 +would be a false claim of coverage. + +### 5.2 ⇒ THE SEAM — how the real chip plugs in after Phase 2 + +The gate touches `chip_model.py` **only** through `SocketChip`'s public surface: + +```python +chip = SocketChip(tag, framing) # allocate columns +chip.build() # emit every constraint +chip.in_lane_bytes # 8 × [4 byte columns] — the two input cells +chip.digest_words # 4 × [4 byte columns] — the one output cell +chip.assertions # the constraint system +``` + +To validate the real chip, **replace the bodies of the `emit_*` methods with a +transcription of the corresponding arms of `HashConstraints::eval`** (the BLAKE3 +arm Phase 2 adds), keeping the same signatures. Nothing else changes. Each +`emit_*` carries a `CHIP CONSTRAINT` comment naming the exact constraint the +Rust body must contain; **those comments are the conformance checklist.** + +The model is written in the primitives the Rust body will use — byte columns, +µ-gated linear identities, `ByteAlu`/`AreBytes` sends — and not in 32-bit +arithmetic. A word-level model would be easy to make UNSAT and would prove +nothing about the chip that exists. + +--- + +## 6. The negative-control board and the width audit + +**Negative controls are mandatory.** Without them "UNSAT = verified" is +meaningless. All must be SAT; a control that returns UNSAT means the gate is +**blind** to that bug class, which is the finding that matters. + +### 6.1 Controls — all ✓ EXECUTED, all SAT + +*Logic, symbolic at G level:* `rot_wrong_amount`, `swap_g_operand`. + +*Framing and wiring, against the **full 7-round pipeline**, concrete:* +`swap_a_b`, `tag_changed`, `tag_omitted`, `truncate_high_half`, `flags_parent`, +`flags_no_root`, `block_len_64`, `block_len_32`, `counter_one`, `cv_zero`, +`lanes_big_endian`, `tag_slot_moved`, `msg_perm_swapped`, `rounds_6_not_7`, +`drop_ff_xor`, `swap_g_operand`. + +*Per-theorem discrimination:* six at T2's layer, three at T3's. + +The four the brief named specifically: **dropped `AreBytes`/`BITWISE` → SAT** +(WA1/WA2/WA3), **wrong truncation window → SAT** (`truncate_high_half`), +**missing/altered domain tag → SAT** (`tag_omitted`, `tag_changed`), +**7-vs-6-round confusion → SAT** (`rounds_6_not_7`). + +*Non-vacuity:* the honest system is satisfiable at `rounds = 7` (SAT), and T4 +pins it to the anchored value. + +### 6.2 One documented BV blindness — and why it is in the board + +`drop_carry_bool` is **UNSAT in BV** ✓ EXECUTED. That is correct, not a failure: +in BV a carry column is an 8-bit variable, so removing its booleanity leaves it +bounded and `s` stays pinned. The same bug is a live forgery in the field +(WA4 → SAT). A gate that ran only the BV domain would report this class as +absent. Recording it makes the split auditable instead of implicit. + +### 6.3 Width audit — every field-lifted width, its contract, its bound + +**The rule:** every field-lifted byte/word width must cite a real range-check +contract **and** a non-overflow side condition, or a field-level attacker walks +out of the bit-vector model. + +| identity | max magnitude | backing contract | necessity ✓ EXECUTED | +|---|---:|---|---| +| `lane == Σ bₖ·2^{8k}` | `2^32` | `AreBytes` on `MB[j][0..4]` | **WA1** present → UNSAT, dropped → **SAT** | +| `lane < 2^32` forced | `2^32` | same | **WA2** present → UNSAT, dropped → **SAT** | +| `A+B == s + 2^32·c` | `2^33` | `ByteAlu[XOR]` on operands + `CarryBit` | WA4 | +| `A+B+M == s + 2^32·(c1+c2)` | `2^34` | as above ×2 | **WA4** present → UNSAT, dropped → **SAT** | +| `hw·2^r == SLLC·2^16 + SLL` | `2^32` | `AreBytes` on `SLL`/`SLLC` bytes | **WA3** present → UNSAT, dropped → **SAT** | +| `Ylo == SLL_hi + SLLC_lo` | `2^17` | `AreBytes` on `SLL`/`SLLC` bytes | WA5 (both sides) | +| `OUT_C[i] == Σ OUTWₖ·2^{8k}` | `2^32` | `ByteAlu[XOR]` output bytes | — (outputs are bytes by construction) | + +**WA6 — no-wrap side condition.** Worst magnitude across all identities is +`2^34 ≪ p ≈ 2^64`. ✓ EXECUTED (a static check, not a solver run — as it must be: +no solver can discharge a side condition about the model's own faithfulness). + +--- + +## 7. Obligations and requirements on the eventual chip + +Phase 2 is expected to conform to §3. Where I had to **assume** something about a +layout that does not exist yet, it is stated as a requirement, not a fact. + +- **O1 — input lanes MUST be range-checked to 32 bits.** BLOCK 1. Soundness, not + hygiene: `merkle_walk`'s siblings are prover-chosen. The host-side `LfmHasher` + impl must **reject** an out-of-range lane, not silently reduce, or host and + chip disagree about what was proved. +- **O2 — the socket is closed on its own output.** `c_i` is a u32 by + construction (BLOCK 5), so a digest this socket produced always satisfies O1. + Only leaf digests and prover-hinted siblings can violate it. +- **O3 — `compress_iv()` does not participate.** The BLAKE3 arm overrides + `compress` entirely; the IV enters through `h`, all eight words, not through + state lanes 8–11. The override must be honoured through `HasherKind::compress`'s + explicit delegation. +- **O4 — byte order is the `keccak_host` convention:** one felt = one u32 = four + little-endian bytes. **Not** `word::pack_digest`, which serialises a lane as + eight bytes. The two are different serialisations of a cell and must not be + confused. `lanes_big_endian` is a live control precisely because this is easy + to get wrong. +- **O5 — ✓ DECIDED 2026-08-10: the `"LFML"` leaf tag is the answer.** This socket + has one tag, so it separates LFM compressions from other BLAKE3 uses but **not** + leaves from parents within the tree. If leaves enter the tree as raw cells rather + than through a distinct domain, a variable-depth tree admits the classic Merkle + second-preimage confusion. BLAKE3's own `PARENT` flag cannot be reused without + leaving the standard-hash framing of §2.2. **RATIFIED by the user, 2026-08-10 + ("reserve a second tag"):** any future leaf-hashing path MUST use the + reserved `"LFML"` tag (§2.3) — the RFC 6962 leaf/parent split expressed in the tag + scheme, keeping both domains directly KAT-able against the `blake3` crate. + Nothing implements `"LFML"` yet, and — **correction, 2026-08-11, lead-verified + in code after the Phase-2 reviewer flagged the claim as unaudited** — what makes + that safe is **fixed depth alone**, not any absence of leaf hashing: FriToyV0 + already forms leaf digests by compressing raw data rows under the same `"LFMC"` + tag (`programs.rs:577/585/625`, `leaf = compress(row_even, row_odd)` feeding + `merkle_walk`), so leaves and parents are NOT domain-separated today. That is + sound only because every current tree is a fixed-depth static circuit — the eDSL + builder fixes program shape at build time; hints supply values, never structure. + Consequence for review: a future PR adding variable-depth trees, or a + leaf-hashing API meant to coexist with them, without `"LFML"` is REJECTED on + this obligation. Mechanism over policy — the PAGE lesson. +- **R1/R2/R3** — reuse the host's cell columns; no columns for `m[8..16]`; build + only the four in-window output words. §3. +- **Constant-folding round 0 is permitted but must be re-gated.** §3, BLOCK 3. + +### Security consequence, stated plainly + +The digest is **128 bits**, so this socket offers **64-bit collision resistance** +by the birthday bound. That follows from `HASH_DIGEST_FELTS = 4` and the +machine's declared 128-bit target — it is not introduced by BLAKE3 or by the +truncation. **This is the open question Plan §5 puts to the user.** If the target +is 128-bit *collision* resistance, the digest must be two cells and the frozen +1-cell `LFM_HASH` output contract has to be reopened; nothing else in this +document changes if it does, only the digest width. Preimage resistance of the +truncated digest is 128 bits (? INFERRED — standard for a truncated random +oracle, not an assumption specific to this design). + +--- + +## 8. What is executed, and what closes only after Phase 2 + +| claim | status | +|---|---| +| reference `f` @7 rounds == official BLAKE3, three modes | ✓ EXECUTED | +| reference agrees with a second independent implementation, both round counts | ✓ EXECUTED | +| socket byte-level and word-level routes agree, all vectors, both round counts | ✓ EXECUTED | +| socket @7 rounds == standard `BLAKE3(a‖b‖"LFMC")[..16]` | ✓ EXECUTED | +| socket vectors match a second agent's independent table | ✓ EXECUTED | +| all 14 framing degrees of freedom discriminated | ✓ EXECUTED | +| G quarter-round tight and correct, all inputs | ✓ EXECUTED (UNSAT) | +| message schedule correct, all 7 rounds, all inputs | ✓ EXECUTED (UNSAT) | +| framing/feed-forward/window correct | ✓ EXECUTED (UNSAT) | +| full pipeline reproduces the anchored KATs and excludes wrong digests | ✓ EXECUTED | +| 27 negative controls all SAT; 11 width-audit items | ✓ EXECUTED | +| the tail optimisation is correct and worth only 2.0% | ✓ EXECUTED | +| monolithic symbolic `rounds = 1, 2` UNSAT | ✗ **NOT RUN TO COMPLETION** — attempted, exceeded the time budget on this machine (>25 min at `rounds = 1`) and was stopped. Bonus only; the verdict does not rest on it | +| the same 7-round identity against the Rust **`blake3` crate** | ✗ DEFERRED — needs cargo | +| **the REAL chip's constraints satisfy T1–T4** | ✗ **OPEN — this is what the seam is for** | +| the real chip's `OUT` columns match `socket_kats.json` | ✗ OPEN — no chip arm exists | +| O5 (leaf/parent domain separation) decided | ✗ **OPEN — needs a decision** | + +--- + +## 9. Files + +| file | what | +|---|---| +| `blake3_oracle.py` | Layer 1 — round-parameterised BLAKE3 reference (compression, chunks, tree, XOF) | +| `anchor_check.py` | Layer 1 anchors: official vectors, differential, anchor sensitivity | +| `socket_ref.py` | Layer 2 — the Option-A socket, `Framing`, the control catalogue | +| `socket_kats.py`, `socket_kats.json` | Layer 2 vectors + discrimination + peer cross-check | +| `contracts.py` | Layer 3 — the chip-contract library, BV and FIELD domains | +| `chip_model.py` | Layer 4 — **the column-role map / Phase-2 spec**, and the seam | +| `gate.py` | Layer 5 — theorems, negative controls, width audit, cost census | +| `run-anchor.log`, `run-kats.log` | captured output of layers 1–2 | +| `run-gate-PRE-PHASE2-SUPERSEDED.log` | the 49-check pre-Phase-2 board — **superseded**, do not cite | +| `run-chip-gate.log` | **the board of record**: 75 checks vs the committed chip | diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/anchor_check.py b/thoughts/shared/lfm-real-hash/gate-oracle/anchor_check.py new file mode 100644 index 000000000..e3d279853 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/anchor_check.py @@ -0,0 +1,238 @@ +""" +LAYER 1 ANCHOR: certify `blake3_oracle.py` at rounds = 7 against external truth. + +Three independent anchors, in decreasing order of strength: + + A1 OFFICIAL BLAKE3 test vectors (upstream `test_vectors.json`), all three + modes (hash / keyed_hash / derive_key), 35 input lengths, extended output. + This is external to this repo and to this project. + A2 The upstream-published `known` digests for the empty string and "abc". + A3 Differential against the independently-written in-repo reference + `thoughts/blake3/blake3-oracle/blake3_ref.py` (if reachable), on random + compression inputs at BOTH round counts. + +A1/A2 certify the 7-round code path. A3 additionally certifies that the SAME +code path at rounds = 6 agrees with a second implementation -- which is all that +can be said for 6 rounds, since no external anchor for it exists (assumption A6R). + +Run: python3 anchor_check.py +""" + +from __future__ import annotations + +import json +import os +import random +import sys + +import blake3_oracle as ora + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# Search paths for the restored phase-1 artifacts (worktree first, then repo). +_CANDIDATE_ROOTS = [ + "/Users/maurofab/workspace/lambda_vm-blake3-impl/thoughts/blake3", + os.path.join(HERE, "..", "..", "..", "blake3"), +] + + +def _find(rel: str): + for root in _CANDIDATE_ROOTS: + p = os.path.join(root, rel) + if os.path.exists(p): + return p + return None + + +def official_input(length: int) -> bytes: + """Upstream's test input: the repeating byte pattern i % 251.""" + return bytes((i % 251) for i in range(length)) + + +class _Xorshift64Star: + """The `random` block's inputs are not the 251-pattern -- they come from a + self-contained xorshift64* stream (`ground-truth/src/main.rs`, struct Rng), + deliberately re-implemented here rather than shared, so the Python and Rust + sides agree only if both are right.""" + + M64 = (1 << 64) - 1 + + def __init__(self, seed: int): + self.x = seed & self.M64 + + def next_u64(self) -> int: + x = self.x + x ^= x >> 12 + x = (x ^ (x << 25)) & self.M64 + x ^= x >> 27 + self.x = x + return (x * 0x2545F4914F6CDD1D) & self.M64 + + def byte(self) -> int: + return (self.next_u64() >> 33) & 0xFF + + def bytes_(self, n: int) -> bytes: + return bytes(self.byte() for _ in range(n)) + + +def anchor_official_vectors() -> tuple[bool, str]: + path = _find("blake3-oracle/official_test_vectors.json") + if path is None: + return False, "official_test_vectors.json NOT FOUND -- anchor A1 CANNOT RUN" + with open(path) as f: + vec = json.load(f) + + key = vec["key"].encode("utf-8") + assert len(key) == 32, "official key must be 32 bytes" + ctx = vec["context_string"] + + n_hash = n_keyed = n_derive = 0 + for case in vec["cases"]: + data = official_input(case["input_len"]) + want_hash = bytes.fromhex(case["hash"]) + got = ora.hash_bytes(data, len(want_hash)) + if got != want_hash: + return False, (f"HASH mismatch at input_len={case['input_len']}: " + f"got {got.hex()[:64]} want {want_hash.hex()[:64]}") + n_hash += 1 + + want_keyed = bytes.fromhex(case["keyed_hash"]) + got = ora.Hasher.new_keyed(key).update(data).finalize(len(want_keyed)) + if got != want_keyed: + return False, f"KEYED mismatch at input_len={case['input_len']}" + n_keyed += 1 + + want_derive = bytes.fromhex(case["derive_key"]) + got = ora.Hasher.new_derive_key(ctx).update(data).finalize(len(want_derive)) + if got != want_derive: + return False, f"DERIVE_KEY mismatch at input_len={case['input_len']}" + n_derive += 1 + + # The `random` block: independent seeds, short XOF windows, all three modes. + n_rand = 0 + for case in vec.get("random", []): + data = _Xorshift64Star(case["seed"]).bytes_(case["len"]) + xof = case["xof"] + if bytes.fromhex(case["hash"]) != ora.hash_bytes(data, xof): + return False, f"random HASH mismatch seed={case['seed']} len={case['len']}" + k = bytes.fromhex(case["key"]) + if bytes.fromhex(case["keyed"]) != ora.Hasher.new_keyed(k).update(data).finalize(xof): + return False, f"random KEYED mismatch seed={case['seed']}" + if bytes.fromhex(case["derive"]) != ( + ora.Hasher.new_derive_key(case["ctx"]).update(data).finalize(xof)): + return False, f"random DERIVE mismatch seed={case['seed']}" + n_rand += 1 + + known = vec.get("known", {}) + for name, want in known.items(): + data = b"" if name == "empty" else name.encode() + if ora.hash_bytes(data, len(want) // 2).hex() != want: + return False, f"known-digest mismatch: {name}" + + return True, (f"A1 PASS: {n_hash} hash + {n_keyed} keyed + {n_derive} derive_key " + f"cases, {n_rand} random cases, {len(known)} known digests") + + +def anchor_differential(trials: int = 200) -> tuple[bool, str]: + path = _find("blake3-oracle/blake3_ref.py") + if path is None: + return False, "blake3_ref.py NOT FOUND -- anchor A3 CANNOT RUN" + sys.path.insert(0, os.path.dirname(path)) + try: + import blake3_ref as other # type: ignore + except Exception as exc: # pragma: no cover + return False, f"blake3_ref.py import failed: {exc}" + + if not hasattr(other, "compress"): + return False, "blake3_ref.py has no `compress` -- differential CANNOT RUN" + + rng = random.Random(0xB3_0A_11) + for rounds in (6, 7): + for _ in range(trials): + h = [rng.randrange(1 << 32) for _ in range(8)] + m = [rng.randrange(1 << 32) for _ in range(16)] + t = rng.randrange(1 << 64) + bl = rng.randrange(65) + fl = rng.randrange(128) + mine = ora.compress(h, m, t, bl, fl, rounds=rounds) + theirs = other.compress(h, m, t, bl, fl, rounds=rounds) + if list(mine) != list(theirs): + return False, (f"DIFFERENTIAL mismatch at rounds={rounds}\n" + f" mine ={[hex(x) for x in mine]}\n" + f" theirs={[hex(x) for x in theirs]}") + return True, (f"A3 PASS: {trials} random compressions x rounds in (6,7) agree " + f"with {os.path.relpath(path, HERE)}") + + +def negative_control_anchor() -> tuple[bool, str]: + """A1 is only meaningful if a perturbed oracle FAILS it. Four perturbations, + each breaking exactly one convention, must each break the official vectors.""" + data = official_input(1024 + 5) + good = ora.hash_bytes(data) + + fails = [] + + # (i) wrong round count. + if ora.hash_bytes(data, rounds=6) != good: + fails.append("rounds=6") + + # (ii) message permutation perturbed. + saved = list(ora.MSG_PERMUTATION) + ora.MSG_PERMUTATION[0], ora.MSG_PERMUTATION[1] = saved[1], saved[0] + try: + if ora.hash_bytes(data) != good: + fails.append("msg_permutation_swapped") + finally: + ora.MSG_PERMUTATION[:] = saved + + # (iii) IV perturbed. + saved_iv = list(ora.IV) + ora.IV[0] ^= 1 + try: + if ora.hash_bytes(data) != good: + fails.append("iv_bit_flipped") + finally: + ora.IV[:] = saved_iv + + # (iv) counter halves swapped (only observable with >1 chunk, hence the size). + saved_compress = ora.compress + + def swapped(cv, bw, counter, bl, fl, rounds=ora.STANDARD_ROUNDS): + c = ((counter & ora.MASK32) << 32) | ((counter >> 32) & ora.MASK32) + return saved_compress(cv, bw, c, bl, fl, rounds) + + ora.compress = swapped + try: + # Rebuild the tree path through the patched compress. + if ora.Hasher().update(data).finalize() != good: + fails.append("counter_halves_swapped") + finally: + ora.compress = saved_compress + + want = {"rounds=6", "msg_permutation_swapped", "iv_bit_flipped", + "counter_halves_swapped"} + missing = want - set(fails) + if missing: + return False, f"NEGATIVE CONTROL FAILED -- these perturbations went undetected: {sorted(missing)}" + return True, f"NC PASS: all 4 single-convention perturbations break the anchor" + + +def main() -> int: + print("=" * 74) + print("LAYER 1 ANCHOR CHECK -- blake3_oracle.py") + print("=" * 74) + results = [] + for name, fn in (("A1 official vectors", anchor_official_vectors), + ("A3 differential", anchor_differential), + ("NC anchor sensitivity", negative_control_anchor)): + ok, msg = fn() + results.append(ok) + print(f"[{'PASS' if ok else 'FAIL'}] {name}: {msg}") + ok = all(results) + print("-" * 74) + print(f"LAYER 1: {'ANCHORED' if ok else 'NOT ANCHORED -- do not build on this'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.json b/thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.json new file mode 100644 index 000000000..df3cd0b94 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.json @@ -0,0 +1,41 @@ +{ + "path": "/Users/maurofab/workspace/lambda_vm-blake3-impl/prover/src/lfm/blake3_socket.rs", + "file_sha256": "9d358f7bb3e2457065a473d478542aa7d218826e6ce81d96c2652d846f7a4cf6", + "regions": { + "eval": { + "sha256": "240619f1580493b3998ac9fbc86aec61352132219ce5ca87c642e6ec2099a6be", + "normalized_len": 4200 + }, + "bitwise_interactions": { + "sha256": "c880036158518796d36e097f3a676a197c5a707aec94ea1284d196dca3a46fa4", + "normalized_len": 958 + }, + "cols": { + "sha256": "f370814ae32795fe6366dbba7956f4e38bfb33681d810c0000bd3e5c799edf44", + "normalized_len": 1350 + }, + "framing_consts": { + "sha256": "21ab1892612cfd3b815c3d3c983a4baedd842de9f4d1b9268bdbc78843a26c24", + "normalized_len": 1435 + } + }, + "framing": { + "tag_ascii": "LFMC", + "tag_word": 1129137740, + "flags": 11, + "block_len": 36, + "counter": 0, + "out_window_expr": "HASH_DIGEST_FELTS", + "num_g_expr": "SOCKET_ROUNDS * 8", + "g_size": 60, + "num_lanes": 8, + "tag_t_ascii": "LFMT", + "tag_t_word": 1414350412, + "tag_l_ascii": "LFML", + "tag_l_word": 1280132684, + "socket_rounds_expr": "BLAKE3_ROUNDS", + "full_output": "false", + "rounds_default": 7, + "rounds_under_blake3_6round": 6 + } +} \ No newline at end of file diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.py b/thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.py new file mode 100644 index 000000000..ed2b8fcfc --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.py @@ -0,0 +1,317 @@ +""" +WHICH ARTIFACT WAS GATED -- a re-checkable pin, AND a framing-conformance check. + +The chip is UNCOMMITTED and is being edited by a concurrent reviewer, so "the +file at path X" is not an identification and line numbers are not either. + +WHAT THIS DOES + 1. hashes the NORMALIZED content (comments and whitespace stripped) of the + four constraint- and framing-bearing regions, so the hash tracks semantics + and is stable under prose edits and line drift; + 2. RESOLVES the chip's framing constants and checks them against + `socket_ref.py`'s specification -- so the pin answers "does the chip still + compute the socket the oracle specifies?", not merely "has this text + changed?". + +## Why (2) exists: this file's first version had a fail-open, and it fired + +v1 hashed three regions -- `eval`, `bitwise_interactions`, `cols` -- and recorded +constants as their EXPRESSION TEXT. When the implementer's second wave landed it +reported "artifact matches the pin". That was a FALSE PASS, for two reasons, and +both are the exact failure mode this whole gate is built to prevent: + + * `SOCKET_ROUNDS` changed definition (to an alias of `BLAKE3_ROUNDS`). v1 + recorded `NUM_G = "SOCKET_ROUNDS * 8"`, which is stable under that change, + so the pin could not see it. Hashing an expression is not hashing a value. + * worse: `SOCKET_ROUNDS`, `TAG_LFMC`, `FLAGS_LFMC`, `BLOCK_LEN_LFMC`, + `COUNTER_LFMC` and `OUT_WINDOW` are top-level constants that live in NONE of + the three hashed regions. They are precisely the framing degrees of freedom + the negative-control board tests. A change of `FLAGS_LFMC` from `0x0B` to + anything else -- a live control, `flags_parent` -- would have passed silently. + +The change that exposed it was benign. The hole was not. A drift detector that +answers PASS without looking at the thing that matters is worse than no detector, +because it is trusted. + +Run: python3 artifact_pin.py # record + python3 artifact_pin.py --check # verify against the record +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sys + +import socket_ref as sk + +CHIP = ("/Users/maurofab/workspace/lambda_vm-blake3-impl/" + "prover/src/lfm/blake3_socket.rs") +PRIMITIVE = ("/Users/maurofab/workspace/lambda_vm-blake3-impl/" + "prover/src/lfm/blake3.rs") + +HERE = os.path.dirname(os.path.abspath(__file__)) +PIN_FILE = os.path.join(HERE, "artifact_pin.json") + +BRACE_REGIONS = { + "eval": "pub fn eval str: + i = src.index(start_pat) + j = src.index("{", i) + depth, k = 0, j + while True: + if src[k] == "{": + depth += 1 + elif src[k] == "}": + depth -= 1 + if depth == 0: + break + k += 1 + return src[i:k + 1] + + +def _normalize(s: str) -> str: + """Hash the SEMANTICS, not the prose: strip line comments (including `//!` + module docs and `///` item docs) and collapse whitespace. A doc rewrite must + not invalidate the gate; a changed constraint must.""" + s = re.sub(r"//[^\n]*", "", s) + s = re.sub(r"\s+", " ", s) + return s.strip() + + +def _const_region(src: str) -> str: + """REGION 4, added after the v1 fail-open: every top-level const declaration. + This is where the framing constants live -- outside `eval`, outside `cols`, + and therefore outside v1's coverage entirely.""" + lines = [ln for ln in src.splitlines() + if re.match(r"\s*(pub(\([^)]*\))?\s+)?const\s+[A-Z_0-9]+\s*:", ln) + or re.match(r"\s*(pub(\([^)]*\))?\s+)?const\s+_\s*:", ln)] + return _normalize("\n".join(lines)) + + +# --------------------------------------------------------------------------- +# Resolve the framing constants and check them against the ORACLE's spec. +# Every extraction is MANDATORY: a constant we cannot find is a FAILURE, never +# a silent skip. (Silently skipping is how v1 passed.) +# --------------------------------------------------------------------------- + +def _find(src: str, pattern: str, name: str) -> str: + m = re.search(pattern, src) + if not m: + raise LookupError(f"could not resolve `{name}` -- the pin cannot vouch " + f"for a constant it cannot find") + return m.group(1).strip() + + +def resolve_framing(chip_src: str, prim_src: str) -> dict: + def _tag(name: str) -> tuple[str, int]: + expr = _find(chip_src, rf"pub const {name}:\s*u32\s*=\s*([^;]+);", name) + mm = re.match(r'u32::from_le_bytes\(\*b"(\w{4})"\)', expr) + if not mm: + raise LookupError(f"{name} has an unexpected form: {expr!r}") + return mm.group(1), int.from_bytes(mm.group(1).encode(), "little") + + m_ascii, tag_val = _tag("TAG_LFMC") + class m: # keep the existing .group(1) call site working + @staticmethod + def group(_): + return m_ascii + # POST-B1: the transcript tag is part of the chip's framing too, so the pin + # must resolve and check it. v1's lesson was that a framing value living + # outside the hashed regions passes silently; a SECOND tag that the pin does + # not know about is the same hole one tag over. + t_ascii, tag_t_val = _tag("TAG_LFMT") + l_ascii, tag_l_val = _tag("TAG_LFML") + + flags = int(_find(chip_src, r"pub const FLAGS_LFMC:\s*u32\s*=\s*([^;]+);", + "FLAGS_LFMC"), 0) + blen = int(_find(chip_src, r"pub const BLOCK_LEN_LFMC:\s*u32\s*=\s*([^;]+);", + "BLOCK_LEN_LFMC"), 0) + counter = int(_find(chip_src, r"pub const COUNTER_LFMC:\s*u64\s*=\s*([^;]+);", + "COUNTER_LFMC"), 0) + out_window = _find(chip_src, r"pub const OUT_WINDOW:\s*usize\s*=\s*([^;]+);", + "OUT_WINDOW") + num_g = _find(chip_src, r"pub const NUM_G:\s*usize\s*=\s*([^;]+);", "NUM_G") + g_size = int(_find(chip_src, r"pub const G_SIZE:\s*usize\s*=\s*([^;]+);", + "G_SIZE"), 0) + num_lanes = int(_find(chip_src, r"pub const NUM_LANES:\s*usize\s*=\s*([^;]+);", + "NUM_LANES"), 0) + socket_rounds = _find(chip_src, r"pub const SOCKET_ROUNDS:\s*usize\s*=\s*([^;]+);", + "SOCKET_ROUNDS") + full_output = _find(chip_src, r"full_output:\s*(\w+)", "FLOW.full_output") + + # SOCKET_ROUNDS resolves through BLAKE3_ROUNDS, whose value is cfg-dependent. + # Record BOTH arms: the chip compiles to exactly one, the gate covers both. + std = _find(prim_src, r'#\[cfg\(not\(feature = "blake3-6round"\)\)\]\s*' + r'pub const BLAKE3_ROUNDS:\s*usize\s*=\s*([^;]+);', + "BLAKE3_ROUNDS (default arm)") + six = _find(prim_src, r'#\[cfg\(feature = "blake3-6round"\)\]\s*' + r'pub const BLAKE3_ROUNDS:\s*usize\s*=\s*([^;]+);', + "BLAKE3_ROUNDS (6round arm)") + std_v = int(_find(prim_src, r"pub const BLAKE3_STANDARD_ROUNDS:\s*usize\s*=\s*([^;]+);", + "BLAKE3_STANDARD_ROUNDS"), 0) + six_v = int(_find(prim_src, r"pub const BLAKE3_SIX_ROUNDS:\s*usize\s*=\s*([^;]+);", + "BLAKE3_SIX_ROUNDS"), 0) + rounds_default = std_v if "STANDARD" in std else six_v + rounds_feature = six_v if "SIX" in six else std_v + + return { + "tag_ascii": m.group(1), + "tag_word": tag_val, + "flags": flags, + "block_len": blen, + "counter": counter, + "out_window_expr": out_window, + "num_g_expr": num_g, + "g_size": g_size, + "num_lanes": num_lanes, + "tag_t_ascii": t_ascii, + "tag_t_word": tag_t_val, + "tag_l_ascii": l_ascii, + "tag_l_word": tag_l_val, + "socket_rounds_expr": socket_rounds, + "full_output": full_output, + "rounds_default": rounds_default, + "rounds_under_blake3_6round": rounds_feature, + } + + +def check_against_oracle(fr: dict) -> list[str]: + """The pin's real job: does the chip's framing EQUAL the oracle's spec?""" + bad = [] + if fr["tag_word"] != sk.TAG_LFMC: + bad.append(f"TAG_LFMC {fr['tag_word']:#x} != oracle {sk.TAG_LFMC:#x}") + if fr["tag_ascii"].encode() != sk.TAG_LFMC_ASCII: + bad.append(f"tag ascii {fr['tag_ascii']!r} != oracle " + f"{sk.TAG_LFMC_ASCII.decode()!r}") + if fr["tag_t_ascii"] != "LFMT" or fr["tag_t_word"] != 0x544D464C: + bad.append(f"TAG_LFMT {fr['tag_t_ascii']!r}/{fr['tag_t_word']:#x} != " + f"'LFMT'/0x544D464C (transcript-spec/TRANSCRIPT.md §2)") + if fr["tag_l_ascii"] != "LFML" or fr["tag_l_word"] != 0x4C4D464C: + bad.append(f"TAG_LFML {fr['tag_l_ascii']!r}/{fr['tag_l_word']:#x} != " + f"'LFML'/0x4C4D464C (leaf-spec/LEAF.md §1)") + # PAIRWISE distinct across all three -- one clash is one collapsed domain. + tags = {"LFMC": fr["tag_word"], "LFMT": fr["tag_t_word"], + "LFML": fr["tag_l_word"]} + for x in tags: + for y in tags: + if x < y and tags[x] == tags[y]: + bad.append(f"TAG_{x} == TAG_{y} -- that domain separation is gone") + if fr["flags"] != sk.FLAGS_LFMC: + bad.append(f"FLAGS_LFMC {fr['flags']:#x} != oracle {sk.FLAGS_LFMC:#x}") + if fr["block_len"] != sk.BLOCK_LEN_LFMC: + bad.append(f"BLOCK_LEN_LFMC {fr['block_len']} != oracle {sk.BLOCK_LEN_LFMC}") + if fr["counter"] != sk.HONEST_7.counter: + bad.append(f"COUNTER_LFMC {fr['counter']} != oracle {sk.HONEST_7.counter}") + if fr["num_lanes"] != 2 * sk.DIGEST_LANES: + bad.append(f"NUM_LANES {fr['num_lanes']} != 2 cells x {sk.DIGEST_LANES} lanes") + if fr["g_size"] != 60: + bad.append(f"G_SIZE {fr['g_size']} != 60 (the gated per-G cell count)") + if fr["full_output"] != "false": + bad.append(f"FLOW.full_output = {fr['full_output']}, expected false " + f"(requirement R3: only the window's 4 words are built)") + if fr["num_g_expr"].replace(" ", "") != "SOCKET_ROUNDS*8": + bad.append(f"NUM_G = {fr['num_g_expr']!r}, expected SOCKET_ROUNDS * 8") + gated = {6, 7} + got = {fr["rounds_default"], fr["rounds_under_blake3_6round"]} + if got != gated: + bad.append(f"round counts {sorted(got)} are not the gated pair " + f"{sorted(gated)} -- the board covers only 6 and 7") + return bad + + +def compute() -> dict: + with open(CHIP) as f: + chip_src = f.read() + with open(PRIMITIVE) as f: + prim_src = f.read() + with open(CHIP, "rb") as f: + raw = f.read() + + regions = {} + for name, pat in BRACE_REGIONS.items(): + body = _normalize(_brace_region(chip_src, pat)) + regions[name] = {"sha256": hashlib.sha256(body.encode()).hexdigest(), + "normalized_len": len(body)} + cb = _const_region(chip_src) + regions["framing_consts"] = {"sha256": hashlib.sha256(cb.encode()).hexdigest(), + "normalized_len": len(cb)} + + return { + "path": CHIP, + "file_sha256": hashlib.sha256(raw).hexdigest(), + "regions": regions, + "framing": resolve_framing(chip_src, prim_src), + } + + +def main() -> int: + try: + cur = compute() + except LookupError as exc: + print(f"PIN FAILED: {exc}") + return 1 + + conformance = check_against_oracle(cur["framing"]) + + if "--check" in sys.argv: + if not os.path.exists(PIN_FILE): + print("no pin recorded; run without --check first") + return 1 + with open(PIN_FILE) as f: + old = json.load(f) + drift = [n for n, v in cur["regions"].items() + if old["regions"].get(n, {}).get("sha256") != v["sha256"]] + fdrift = {k: (old["framing"].get(k), v) + for k, v in cur["framing"].items() + if old["framing"].get(k) != v} + ok = True + if drift: + print(f"REGION DRIFT: {drift}") + print(" The gate verdict does NOT carry over. Re-transcribe the " + "changed region into chip_model.py and re-run gate.py.") + ok = False + if fdrift: + print(f"FRAMING CONSTANT DRIFT: {fdrift}") + ok = False + if conformance: + print("FRAMING NO LONGER MATCHES THE ORACLE SPEC:") + for b in conformance: + print(f" - {b}") + ok = False + if ok: + print("artifact matches the pin AND its framing still equals the " + "oracle spec; the gate verdict applies") + if old["file_sha256"] != cur["file_sha256"]: + print(f" (whole-file hash moved {old['file_sha256'][:12]} -> " + f"{cur['file_sha256'][:12]}, but only outside the four " + f"hashed regions -- i.e. in comments/docs)") + return 0 if ok else 1 + + if conformance: + print("REFUSING TO PIN -- the chip's framing does not match the oracle:") + for b in conformance: + print(f" - {b}") + return 1 + + with open(PIN_FILE, "w") as f: + json.dump(cur, f, indent=1) + print(f"pinned -> {PIN_FILE}") + print(f" file {cur['file_sha256']}") + for name, v in cur["regions"].items(): + print(f" {name:22s} {v['sha256']}") + print(" framing (resolved values, checked against socket_ref.py):") + for k, v in cur["framing"].items(): + print(f" {k:28s} {v}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/blake3_oracle.py b/thoughts/shared/lfm-real-hash/gate-oracle/blake3_oracle.py new file mode 100644 index 000000000..259e125ef --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/blake3_oracle.py @@ -0,0 +1,264 @@ +""" +BLAKE3 reference, round-parameterised. THE ORACLE'S LAYER 1. + +Written from the BLAKE3 specification (the `reference_impl` algorithm: compression +function, chunk state, CV stack, output node, XOF), NOT transcribed from any +in-repo file. Independence is the point: `thoughts/blake3/blake3-oracle/blake3_ref.py` +and `prover/src/lfm/blake3.rs` are cross-checks, not sources. + +WHAT MAKES THIS TRUSTWORTHY (the provenance chain, in order of strength): + + 1. At `rounds = 7` this is standard BLAKE3, so it is checked against the + OFFICIAL BLAKE3 test vectors (`official_test_vectors.json`, upstream's + `test_vectors.json`) in all three modes -- hash, keyed_hash, derive_key -- + across 35 input lengths and the full extended-output (XOF) window. That + anchor is external to this repo and to this project. + 2. Differentially checked against the independently-written in-repo reference + `thoughts/blake3/blake3-oracle/blake3_ref.py` (two agreeing sources). + 3. At `rounds = 6` NO external anchor exists -- no library computes it and no + published vector contains it. 6-round values in this file are therefore + the *definition* of the 6-round variant, defensible only as "the same code + path with the loop bound changed". That is assumption A6R and it is why + the parameterisation is a single integer with no other edit: the 7-round + anchor is what certifies the code path, and the 6-round instantiation + inherits nothing but the code. + +The round loop permutes the message schedule when `r < rounds - 1`, so +`rounds = 7` is bit-for-bit standard BLAKE3 with no other change. +""" + +from __future__ import annotations + +MASK32 = 0xFFFFFFFF + +IV = [ + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19, +] + +MSG_PERMUTATION = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + +# Flag bits. +CHUNK_START = 1 << 0 +CHUNK_END = 1 << 1 +PARENT = 1 << 2 +ROOT = 1 << 3 +KEYED_HASH = 1 << 4 +DERIVE_KEY_CONTEXT = 1 << 5 +DERIVE_KEY_MATERIAL = 1 << 6 + +BLOCK_LEN = 64 +CHUNK_LEN = 1024 + +STANDARD_ROUNDS = 7 + +# The eight G-calls of one round: (a, b, c, d, mx_index, my_index). +G_CALLS = [ + (0, 4, 8, 12, 0, 1), + (1, 5, 9, 13, 2, 3), + (2, 6, 10, 14, 4, 5), + (3, 7, 11, 15, 6, 7), + (0, 5, 10, 15, 8, 9), + (1, 6, 11, 12, 10, 11), + (2, 7, 8, 13, 12, 13), + (3, 4, 9, 14, 14, 15), +] + + +def rotr32(x: int, n: int) -> int: + x &= MASK32 + return ((x >> n) | (x << (32 - n))) & MASK32 + + +def g(v: list[int], a: int, b: int, c: int, d: int, mx: int, my: int) -> None: + v[a] = (v[a] + v[b] + mx) & MASK32 + v[d] = rotr32(v[d] ^ v[a], 16) + v[c] = (v[c] + v[d]) & MASK32 + v[b] = rotr32(v[b] ^ v[c], 12) + v[a] = (v[a] + v[b] + my) & MASK32 + v[d] = rotr32(v[d] ^ v[a], 8) + v[c] = (v[c] + v[d]) & MASK32 + v[b] = rotr32(v[b] ^ v[c], 7) + + +def round_fn(v: list[int], m: list[int]) -> None: + for (a, b, c, d, ix, iy) in G_CALLS: + g(v, a, b, c, d, m[ix], m[iy]) + + +def permute(m: list[int]) -> list[int]: + return [m[MSG_PERMUTATION[i]] for i in range(16)] + + +def compress( + chaining_value: list[int], + block_words: list[int], + counter: int, + block_len: int, + flags: int, + rounds: int = STANDARD_ROUNDS, +) -> list[int]: + """The compression function f. Returns all 16 output words. + + `rounds = 7` is standard BLAKE3. Any other value is the LFM variant and has + no external anchor (see the module docstring, assumption A6R). + """ + assert len(chaining_value) == 8 and len(block_words) == 16 + state = [ + chaining_value[0], chaining_value[1], chaining_value[2], chaining_value[3], + chaining_value[4], chaining_value[5], chaining_value[6], chaining_value[7], + IV[0], IV[1], IV[2], IV[3], + counter & MASK32, + (counter >> 32) & MASK32, + block_len & MASK32, + flags & MASK32, + ] + schedule = list(block_words) + for r in range(rounds): + round_fn(state, schedule) + if r < rounds - 1: + schedule = permute(schedule) + + out = [0] * 16 + for i in range(8): + out[i] = state[i] ^ state[i + 8] + out[i + 8] = state[i + 8] ^ chaining_value[i] + return out + + +# --------------------------------------------------------------------------- +# Tree hashing -- needed ONLY so the official vectors can anchor `compress`. +# The LFM socket never uses more than one block, but the anchor does. +# --------------------------------------------------------------------------- + +def words_from_le_bytes(b: bytes) -> list[int]: + assert len(b) % 4 == 0 + return [int.from_bytes(b[i:i + 4], "little") for i in range(0, len(b), 4)] + + +def le_bytes_from_words(w: list[int]) -> bytes: + return b"".join(int(x & MASK32).to_bytes(4, "little") for x in w) + + +class _Output: + """A not-yet-finalised node: the inputs to one last compression.""" + + __slots__ = ("cv", "block_words", "counter", "block_len", "flags", "rounds") + + def __init__(self, cv, block_words, counter, block_len, flags, rounds): + self.cv = cv + self.block_words = block_words + self.counter = counter + self.block_len = block_len + self.flags = flags + self.rounds = rounds + + def chaining_value(self) -> list[int]: + return compress(self.cv, self.block_words, self.counter, + self.block_len, self.flags, self.rounds)[:8] + + def root_output_bytes(self, length: int) -> bytes: + out = bytearray() + block_counter = 0 + while len(out) < length: + words = compress(self.cv, self.block_words, block_counter, + self.block_len, self.flags | ROOT, self.rounds) + out += le_bytes_from_words(words) + block_counter += 1 + return bytes(out[:length]) + + +class _ChunkState: + def __init__(self, key_words, chunk_counter, flags, rounds): + self.cv = list(key_words) + self.chunk_counter = chunk_counter + self.block = bytearray() + self.blocks_compressed = 0 + self.flags = flags + self.rounds = rounds + + def length(self) -> int: + return BLOCK_LEN * self.blocks_compressed + len(self.block) + + def start_flag(self) -> int: + return CHUNK_START if self.blocks_compressed == 0 else 0 + + def update(self, data: bytes) -> None: + while data: + if len(self.block) == BLOCK_LEN: + block_words = words_from_le_bytes(bytes(self.block)) + self.cv = compress(self.cv, block_words, self.chunk_counter, + BLOCK_LEN, self.flags | self.start_flag(), + self.rounds)[:8] + self.blocks_compressed += 1 + self.block = bytearray() + take = min(BLOCK_LEN - len(self.block), len(data)) + self.block += data[:take] + data = data[take:] + + def output(self) -> _Output: + padded = bytes(self.block) + b"\x00" * (BLOCK_LEN - len(self.block)) + return _Output(self.cv, words_from_le_bytes(padded), self.chunk_counter, + len(self.block), self.flags | self.start_flag() | CHUNK_END, + self.rounds) + + +def _parent_output(left_cv, right_cv, key_words, flags, rounds) -> _Output: + return _Output(list(key_words), left_cv + right_cv, 0, BLOCK_LEN, + PARENT | flags, rounds) + + +class Hasher: + """Full BLAKE3 tree hasher. Exists to run the official-vector anchor.""" + + def __init__(self, key_words=None, flags=0, rounds: int = STANDARD_ROUNDS): + self.key_words = list(key_words) if key_words is not None else list(IV) + self.flags = flags + self.rounds = rounds + self.chunk_state = _ChunkState(self.key_words, 0, flags, rounds) + self.cv_stack: list[list[int]] = [] + + @classmethod + def new_keyed(cls, key: bytes, rounds: int = STANDARD_ROUNDS) -> "Hasher": + assert len(key) == 32 + return cls(words_from_le_bytes(key), KEYED_HASH, rounds) + + @classmethod + def new_derive_key(cls, context: str, rounds: int = STANDARD_ROUNDS) -> "Hasher": + ctx = cls(list(IV), DERIVE_KEY_CONTEXT, rounds) + ctx.update(context.encode("utf-8")) + ctx_key = ctx.finalize(32) + return cls(words_from_le_bytes(ctx_key), DERIVE_KEY_MATERIAL, rounds) + + def _add_chunk_cv(self, new_cv: list[int], total_chunks: int) -> None: + while total_chunks & 1 == 0: + left = self.cv_stack.pop() + new_cv = _parent_output(left, new_cv, self.key_words, + self.flags, self.rounds).chaining_value() + total_chunks >>= 1 + self.cv_stack.append(new_cv) + + def update(self, data: bytes) -> "Hasher": + while data: + if self.chunk_state.length() == CHUNK_LEN: + cv = self.chunk_state.output().chaining_value() + counter = self.chunk_state.chunk_counter + self._add_chunk_cv(cv, counter + 1) + self.chunk_state = _ChunkState(self.key_words, counter + 1, + self.flags, self.rounds) + take = min(CHUNK_LEN - self.chunk_state.length(), len(data)) + self.chunk_state.update(data[:take]) + data = data[take:] + return self + + def finalize(self, length: int = 32) -> bytes: + output = self.chunk_state.output() + for cv in reversed(self.cv_stack): + output = _parent_output(cv, output.chaining_value(), self.key_words, + self.flags, self.rounds) + return output.root_output_bytes(length) + + +def hash_bytes(data: bytes, length: int = 32, + rounds: int = STANDARD_ROUNDS) -> bytes: + return Hasher(rounds=rounds).update(data).finalize(length) diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/chip_model.py b/thoughts/shared/lfm-real-hash/gate-oracle/chip_model.py new file mode 100644 index 000000000..24438f8bb --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/chip_model.py @@ -0,0 +1,546 @@ +""" +LAYER 4: the COLUMN-ROLE MAP, executable -- and THE SEAM. + +============================================================================ +THIS FILE IS THE PHASE-2 SPECIFICATION. +============================================================================ +Every committed column of the BLAKE3 arm of `LFM_HASH` appears here as a free +variable, and every constraint the chip must impose appears here as an equation +over those variables. A chip that conforms to this file is one the gate proves +correct; a chip that does not conform is one the gate says nothing about. + +============================================================================ +THE SEAM -- how the real chip plugs in after Phase 2 +============================================================================ +The gate (`gate.py`) touches this module ONLY through `SocketChip`'s public +surface: + + chip = SocketChip(tag, framing) # allocate columns + chip.build() # emit every constraint + chip.in_lane_bytes -> 8 x [4 byte columns] (the socket's two input cells) + chip.digest_words -> 4 x [4 byte columns] (the socket's one output cell) + chip.assertions -> the constraint system + +To validate the REAL chip, replace the bodies of the `emit_*` methods with a +transcription of the corresponding arms of `HashConstraints::eval` (the BLAKE3 +arm added in Phase 2), keeping the same method signatures. Nothing else in the +gate changes. Each `emit_*` carries a `CHIP CONSTRAINT` comment naming the +exact constraint the Rust body must contain; that comment is the conformance +checklist. + +Deliberately, the model is written in terms of the same primitives the Rust body +will use -- byte columns, mu-gated linear identities, ByteAlu/AreBytes sends -- +rather than in terms of 32-bit arithmetic. A model written at word level would +be easy to make UNSAT and would prove nothing about the chip that exists. + +============================================================================ +MU-GATING +============================================================================ +Every eval constraint in the real chip is multiplied by the MU column (1 on a +real compression row, 0 on padding) and every bus send carries +`Multiplicity::Column(MU)`; padding rows are all-zero. The gate models a REAL +row, so mu = 1 and drops out. MU's own obligations -- booleanity, and the +all-zero-padding property -- are NOT BV theorems; they are checked structurally +and recorded in ORACLE.md's degree ledger. +""" + +from __future__ import annotations + +from z3 import BitVecVal + +import blake3_oracle as ora +import socket_ref as sk +from contracts import WIDE, BvContracts + +# The eight G-calls of a round, as (a, b, c, d, mx_index, my_index). +G_CALLS = ora.G_CALLS + + +class ColumnCensus: + """Cost accounting, kept in lockstep with the model so the numbers in + ORACLE.md cannot drift from the constraints that were actually gated. + + Sends are NOT counted here: they are counted once, at the point of issue, by + `BvContracts` (`byte_xor` and `are_bytes`), because a send is a lookup and a + lookup only exists where a contract is invoked. A second counter incremented + by hand at the call sites is how a census silently double-counts one family + and drops another.""" + + def __init__(self, contracts): + self.main = 0 # committed main columns (cells) + self.by_block: dict[str, int] = {} + self._c = contracts + self.io_sends = 0 # the host socket's LfmMem tuples + + def add(self, block: str, n: int): + self.main += n + self.by_block[block] = self.by_block.get(block, 0) + n + + @property + def sends(self) -> int: + return self._c.sends + self.io_sends + + def aux_cells(self) -> int: + # LogUp aux width: 3 extension columns per pair of sends (the verified + # Tier-2 cost model: a send costs ~1.5 base cells of aux). + return 3 * ((self.sends + 1) // 2) + + def cell_equiv(self) -> int: + return self.main + self.aux_cells() + + +class SocketChip: + """The BLAKE3 arm of `LFM_HASH`: one row = one 2-to-1 compress.""" + + def __init__(self, tag: str, framing: sk.Framing = sk.HONEST_7, + bug: str | None = None, tail_truncate: bool = False): + self.fr = framing + self.bug = bug + self.tail_truncate = tail_truncate + self.c = BvContracts(tag) + self.census = ColumnCensus(self.c) + self.in_lane_bytes: list[list] = [] + self.digest_words: list[list] = [] + self._built = False + + # -- convenience ------------------------------------------------------ + @property + def assertions(self): + return self.c.assertions + + def _bug(self, name: str, flag: bool = True) -> bool: + return self.bug == name and flag + + # ===================================================================== + # BLOCK 0 -- socket I/O (shared with the LFM_HASH host) + # ===================================================================== + # AS BUILT: the frozen 28-column shared prefix -- 12 `IN` + 4 `S` + 12 `OUT` + # (of which 4 `IN` lanes and 8 `OUT` lanes are unused on a Compress row). + # `MU = MODE_C` is a PREPROCESSED column, so it is outside the main-column + # census entirely AND a prover cannot choose it. R1 is satisfied: the arm + # re-exports `cols::{IN0, OUT0, S8}` rather than committing a second copy. + # + # These are felts, not bytes, so BLOCK 0 is not modelled in the BV domain. + # The chip emits FOUR framing constraint families here that the pre-Phase-2 + # model did not cover; all four are over felts and mode selectors, so they + # go to the FIELD/structural ledger and are checked in gate.py's + # `audit_block0_*`, NOT in BV: + # + # idx 0-3 S_k - (MODE_P*IN_{8+k} + MU*IV_k) capacity prefix; MU is the + # FULL three-way sum -- a leaf row is a compress in framing too + # idx 4 mode_sum*(1 - mode_sum), mode_sum = MODE_C+MODE_T+MODE_P + # idx 5 MODE_P = 0 no permute socket, PERMANENT + # idx 14-21 OUT_{4+j} = 0, j in 0..8 digest is ONE cell + # idx 22-25 digest recomposition + # idx 26-33 UNREAD INPUT PINS -- 8, both unread cells (was 4, one cell, + # before the D1 fix). Shared helper `emit_unread_input_pins`, + # derived ONCE from HashMode::num_input_cells: + # slot 1 (IN4..8): modes with <=1 input cell -> MODE_L + # slot 2 (IN8..12): modes with <=2 input cells -> MODE_L+MODE_C+MODE_T + # idx 34-49 the LEAF block (LEAF_IDX = UNREAD_IDX + NUM_UNREAD_INPUT_PINS) + # idx 50+ the mixing core (CORE_IDX) + # NUM_CONSTRAINTS = 26 + 8 + 16 + 16*NUM_G = 946 @7r (was 942). + # + # AS BUILT POST-MODE_L (layout::hash): PREP_WIDTH = 13, MODE_C = 6, + # MODE_P = 7, MODE_T = 8, MODE_L = 9, MULT0..2 = 10..12, NUM_SELECTORS = 4. + # Every selector sits INSIDE the contiguous run read from MODE_C, because the + # admission validator's one-hot check reads that span -- a selector parked + # past the mults would be outside the check and silently unchecked. + # + # ⚠ TWO DIFFERENT MULTIPLICITIES, and the distinction is load-bearing: + # MU_COLUMNS = MODE_C + MODE_T + MODE_L (the is-real gate; also + # the multiplicity on EVERY BITWISE send) + # DIGEST_MODE_COLUMNS = MODE_C + MODE_T (gates idx 6-13 only) + # + # O1 IS TWO OBLIGATIONS AND ONLY ONE OF THEM NARROWED: + # * the LANE IDENTITY (IN_lane == m[lane]) narrowed to the digest modes. + # Correct: on a leaf row the eight lanes are four felts' HALVES, so + # IN_lane and m[lane] are deliberately different field elements, and + # gating this on the full mu would make every leaf row unprovable. + # * the AreBytes RANGE BOUND did NOT narrow -- ✓ VERIFIED the lane sends + # carry `Multiplicity::Sum3(MODE_C, MODE_T, MODE_L)`, so all 32 lane byte + # columns are bounded on leaf rows too. + # + # That second point is what makes the leaf block sound: canonicity ASSUMES + # lo, hi < 2^32 and does not establish it. Had the range bound narrowed with + # the identity, leaf halves would be unbounded field elements and the whole + # canonicity gate would be vacuous. Audited as WA9. + # + # idx 0-3, 4, 5 and 14-21 are all UNGATED (no MU factor), which is correct: + # they must hold on padding rows too, and padding is all-zero. + # + # The dependency worth executing, and the reason these are not merely + # "structural": idx 0-3 only PIN anything because idx 5 kills the MODE_P + # term. Without idx 5 the capacity prefix is a prover-chosen copy of + # IN_{8+k}. That is checked, both ways, by `audit_block0_capacity`. + + # ===================================================================== + # BLOCK 1 -- the lane boundary (THE new soundness surface for Route A) + # ===================================================================== + def emit_lane_bytes(self): + """Columns: MB[j][k], j in 0..8 lanes, k in 0..4 bytes = 32 byte columns. + + CHIP CONSTRAINT (per lane j), mu-gated, degree 2: + MU * ( LANE_j - (MB[j][0] + 2^8*MB[j][1] + 2^16*MB[j][2] + 2^24*MB[j][3]) ) = 0 + CHIP SENDS (per lane j): AreBytes(MB[j][0], MB[j][1]), AreBytes(MB[j][2], MB[j][3]) + + WHY THE SENDS ARE REQUIRED (the verified argument -- see the CORRECTION + below before citing any older wording). + + The 16 lane `AreBytes` are `m[0..8]`'s ONLY range check. The message + reaches the mixing core through `add3` alone and is never an XOR operand + -- ✓ VERIFIED: `message_word_ref` appears in `blake3_socket.rs` solely as + an add3 `m` operand, and `blake3_chip.rs`'s header had already recorded + the same property of `m`. Every OTHER committed word in this design gets + its bytes range-checked for free by a downstream `ByteAlu[XOR]`; the + message has no such consumer, so if these sends go, nothing bounds it. + + What breaks without them: `m` becomes a free field element instead of a + u32. Round 0's `add3` has CONSTANT `a` and `b` (BLOCK 3 -- the entire + initial state is compile-time constant) and a byte-bounded `s`, so a + prover solves `m = s + 2^32*(c1+c2) - a - b` for ANY chosen `s` -- put + the whole value in `MB[0]` and zero the other three bytes -- and owns + the compression from the first add onward. The chip then computes + something that is not BLAKE3 of any 36-byte string, which is exactly the + freedom a forged Merkle path needs. + + CORRECTION (D10). Earlier revisions of this docstring justified the sends + with a `v` / `v + 2^32` collision -- "two lanes that hash alike". That + attack is UNCONSTRUCTIBLE against this chip and the claim was wrong: the + mixing core reads the SAME linear form the decomposition identity pins + (`message_word_ref` is `Sum MB[j][k]*2^{8k}`), so `IN_lane` and the + message word are one field element by construction and there is no + reduction step for two felts to alias through. The identity is what makes + them the same element; the sends are what make that element a u32. Both + are still required -- for the reason above, not that one. + + In the BV domain a byte IS 8 bits, so the necessity of the sends is NOT + visible here; it is proved in gate.py's FIELD width audit (WA2 is the + executable form: without the sends the lane is not forced below 2^32). + """ + for _j in range(8): + word = self.c.fresh_word() + self.c.are_bytes(*word) # 2 sends per lane + self.in_lane_bytes.append(word) + self.census.add("lane_bytes(MB)", 32) + + # ===================================================================== + # BLOCK 2 -- message words. Only m[0..8] are columns; m[8..16] are constants. + # ===================================================================== + def message_words(self) -> list: + """m[a_slot..+4] = a, m[b_slot..+4] = b, m[tag_slot] = tag, rest = 0. + + REQUIREMENT: m[8..16] carry NO columns and NO range checks. They are + compile-time constants, which is what makes the 4-byte domain tag free. + """ + fr = self.fr + m = [self.c.const_word(0) for _ in range(16)] + a = self.in_lane_bytes[0:4] + b = self.in_lane_bytes[4:8] + if not fr.lane_le: + # Control: a big-endian lane serialisation. BV-observable, because + # the message WORD changes even though the columns do not. + a = [list(reversed(w)) for w in a] + b = [list(reversed(w)) for w in b] + for i in range(4): + m[fr.a_slot + i] = a[i] + m[fr.b_slot + i] = b[i] + # m[8] AS BUILT: NOT a constant -- a linear form over the two + # PREPROCESSED mode columns, `MODE_C*TAG_LFMC + MODE_T*TAG_LFMT` + # (`WordRef::ModeSelected`, evaluated `sum col*tag`). On a real row + # exactly one selector is 1, so the value equals that row's tag; the + # model therefore carries the SELECTED tag, and the mechanism that makes + # the selection trustworthy -- preprocessed-ness plus the registrar's + # one-hot check, NOT idx 4 -- is audited in the FIELD domain + # (gate.audit_block0_tag_selection / M8). Modelling it as a bare + # constant here would describe something the chip does not do and would + # still report PASS: the fail-open this gate exists to prevent. + m[fr.tag_slot] = self.c.const_word(fr.tag_word) + return m + + # ===================================================================== + # BLOCK 3 -- initial state. ALL SIXTEEN WORDS ARE COMPILE-TIME CONSTANTS. + # ===================================================================== + def init_state(self) -> list: + """v[0..8] = h = IV[0..8]; v[8..12] = IV[0..4]; v[12] = t_lo = 0; + v[13] = t_hi = 0; v[14] = block_len = 36; v[15] = flags = 0x0B. + + Note the consequence of h = IV: the ENTIRE initial state is constant, so + the socket costs zero input-state columns (a syscall-shaped chip pays 112 + bytes here). It also means constant-folding round 0 is possible -- see + ORACLE.md; it is permitted but must be re-gated, because a folded round 0 + no longer matches this model. + """ + fr = self.fr + cv = list(fr.cv) + return [self.c.const_word(cv[i]) for i in range(8)] + \ + [self.c.const_word(ora.IV[i]) for i in range(4)] + \ + [self.c.const_word(fr.counter & 0xFFFFFFFF), + self.c.const_word((fr.counter >> 32) & 0xFFFFFFFF), + self.c.const_word(fr.block_len), + self.c.const_word(fr.flags)] + + # ===================================================================== + # BLOCK 4 -- per-G SSA logic + # ===================================================================== + def emit_xor(self, A: list, B: list) -> list: + """CHIP SENDS: 4 x ByteAlu[XOR]. No eval constraint. + The lookup pins the output AND byte-range-checks both operands -- which + is why nearly every word in this design needs no explicit AreBytes.""" + out = [self.c.byte_xor(A[i], B[i]) for i in range(4)] + self.census.add("xor_out", 4) + return out + + @staticmethod + def rotr16(A: list) -> list: + """FREE byte relabel [b0,b1,b2,b3] -> [b2,b3,b0,b1]. No columns.""" + return [A[2], A[3], A[0], A[1]] + + @staticmethod + def rotr8(A: list) -> list: + """FREE byte relabel -> [b1,b2,b3,b0]. No columns.""" + return [A[1], A[2], A[3], A[0]] + + def emit_add2(self, A: list, B: list, drop_carry_bool: bool = False) -> list: + """s = (A + B) mod 2^32, in the implementation's EXPRESSION-CARRY form. + + CHIP COLUMNS: s[0..4] bytes. **NO carry column.** + CHIP CONSTRAINT (mu-gated), the only one — `blake3_socket.rs:826-834`: + carry := (wval(A) + wval(B) - wval(s)) * 2^{-32} (a linear form) + MU * carry * (1 - carry) = 0 (degree 3) + + The carry is *derived*, not witnessed, so the sum identity and the + booleanity collapse into one constraint: `carry in {0,1}` is exactly + `wval(A) + wval(B) - wval(s) in {0, 2^32}`. + + MODELLING NOTE, and it is the whole reason WA7 exists. `2^{-32}` is a + FIELD inverse; there is no faithful BV counterpart, so the BV domain + models the post-audit statement -- the difference lies in {0, 2^32} -- + and the side condition that those are the ONLY reachable roots (in + particular that a negative difference cannot alias 2^32 mod p) is + discharged in the field by WA7. Encoding the disjunction here without + that audit would be assuming the very thing that makes the form sound. + + This deviates from the pre-Phase-2 model, which witnessed the carry as a + column and constrained it twice. The two are equivalent -- the model's + pair asserts `exists carry in {0,1}` where the implementation eliminates + an existential whose witness is determined -- but the gate must certify + the chip that EXISTS, not a stronger cousin, so the model follows the + chip. Saves 1 column per add2: 2 per G, 96 (6r) / 112 (7r) overall.""" + from z3 import Or as _Or + s = self.c.fresh_word() + lhs = self.c.wval(A) + self.c.wval(B) + rhs = self.c.wval(s) + if drop_carry_bool: + pass # control: the difference is unconstrained + else: + self.c.assertions.append( + _Or(lhs == rhs, lhs == rhs + BitVecVal(1 << 32, WIDE))) + self.census.add("add2", 4) + return s + + def emit_add3(self, A: list, B: list, M: list, + drop_carry_bool: bool = False) -> list: + """s = (A + B + M) mod 2^32, carry in {0,1,2} as TWO summed carry bits. + CHIP COLUMNS: s[0..4] bytes + 2 carry columns. + CHIP CONSTRAINTS (mu-gated): + MU * ( wval(A)+wval(B)+wval(M) - wval(s) - 2^32*(c1+c2) ) = 0 (deg 2) + MU * c1 * (1 - c1) = 0 ; MU * c2 * (1 - c2) = 0 (deg 3) + + NOT a single ternary carry k(k-1)(k-2)=0: that body is degree 3 already + and mu-gating pushes it to 4, over the hard budget. This coupling between + mu-gating and the 3-operand add is the tightest in the design.""" + s = self.c.fresh_word() + c1 = self.c.carry_bit(enforce=not drop_carry_bool) + c2 = self.c.carry_bit(enforce=not drop_carry_bool) + csum = self.c.wide(c1) + self.c.wide(c2) + self.c.assertions.append( + self.c.wval(A) + self.c.wval(B) + self.c.wval(M) + == self.c.wval(s) + csum * BitVecVal(1 << 32, WIDE)) + self.census.add("add3", 6) + return s + + def emit_rotr(self, A: list, n: int, wrong_amount: bool = False) -> list: + """rotr12 / rotr7, inlined as the mu-gated linear shift identity. + + rotr12 = rotl20 = rotl16 . rotl4 (inner r = 4) + rotr7 = rotl25 = rotl16 . rotl9 (inner r = 9) + + CHIP COLUMNS: SLL_lo(2B), SLLC_lo(2B), SLL_hi(2B), SLLC_hi(2B), Y[0..4](4B). + CHIP CONSTRAINTS (mu-gated, all linear bodies): + MU * ( xlo*2^r - SLLC_lo*2^16 - SLL_lo ) = 0 + MU * ( xhi*2^r - SLLC_hi*2^16 - SLL_hi ) = 0 + MU * ( Ylo - SLL_hi - SLLC_lo ) = 0 + MU * ( Yhi - SLL_lo - SLLC_hi ) = 0 + CHIP SENDS: AreBytes over the 8 bytes of SLL_lo/SLLC_lo/SLL_hi/SLLC_hi + = 4 sends. THE SLL BOUND IS TIGHT AND LOAD-BEARING. + + Y is range-checked free by the XOR that consumes it. Soundness needs 2^16 + invertible mod p -- a BV model cannot see that, so it is audited in the + FIELD domain.""" + r = {12: 4, 7: 9}[n] + if wrong_amount: + r += 1 + xlo = self.c.hwval(A[0], A[1]) + xhi = self.c.hwval(A[2], A[3]) + sll_lo = [self.c.fresh_byte(), self.c.fresh_byte()] + sllc_lo = [self.c.fresh_byte(), self.c.fresh_byte()] + sll_hi = [self.c.fresh_byte(), self.c.fresh_byte()] + sllc_hi = [self.c.fresh_byte(), self.c.fresh_byte()] + self.c.are_bytes(*sll_lo, *sllc_lo, *sll_hi, *sllc_hi) # 4 sends + SLL_lo = self.c.hwval(*sll_lo) + SLLC_lo = self.c.hwval(*sllc_lo) + SLL_hi = self.c.hwval(*sll_hi) + SLLC_hi = self.c.hwval(*sllc_hi) + two_r = BitVecVal(1 << r, WIDE) + two16 = BitVecVal(1 << 16, WIDE) + self.c.assertions.append(xlo * two_r == SLLC_lo * two16 + SLL_lo) + self.c.assertions.append(xhi * two_r == SLLC_hi * two16 + SLL_hi) + Y = self.c.fresh_word() + self.c.assertions.append(self.c.hwval(Y[0], Y[1]) == SLL_hi + SLLC_lo) + self.c.assertions.append(self.c.hwval(Y[2], Y[3]) == SLL_lo + SLLC_hi) + self.census.add("rotr_shift", 12) + return Y + + def emit_g(self, v: list, a: int, b: int, c: int, d: int, + mx: list, my: list, gflag: bool, skip_tail: bool = False): + """One G quarter-round, in SSA. 56 byte cells + 6 carry cells. + + `skip_tail` omits the final XOR + rotr7, which produce v[b] only. That is + legal in the LAST round for the four G-calls whose b-position is outside + the truncation window -- and it carries an obligation, spelled out in + ORACLE.md, about how the surviving consumer reads B1.""" + b_first = c if self._bug("swap_g_operand", gflag) else b + v[a] = self.emit_add3(v[a], v[b_first], mx) + v[d] = self.rotr16(self.emit_xor(v[d], v[a])) + v[c] = self.emit_add2(v[c], v[d], + drop_carry_bool=self._bug("drop_add2_carry", gflag)) + v[b] = self.emit_rotr(self.emit_xor(v[b], v[c]), 12, + wrong_amount=self._bug("rot_wrong_amount", gflag)) + v[a] = self.emit_add3(v[a], v[b], my, + drop_carry_bool=self._bug("drop_carry_bool", gflag)) + v[d] = self.rotr8(self.emit_xor(v[d], v[a])) + v[c] = self.emit_add2(v[c], v[d], + drop_carry_bool=self._bug("drop_add2_carry", gflag)) + if not skip_tail: + v[b] = self.emit_rotr(self.emit_xor(v[b], v[c]), 7) + + def emit_rounds(self, v: list, m: list): + """R rounds of 8 G-calls; the schedule is permuted between rounds by the + compile-time MSG_PERMUTATION, so a round references the ORIGINAL message + columns under permute^r with zero runtime handoff.""" + fr = self.fr + window = set(range(fr.out_window, fr.out_window + 4)) + needed = window | {i + 8 for i in window} + schedule = list(m) + for r in range(fr.rounds): + last = (r == fr.rounds - 1) + for gi, (a, b, c, d, ix, iy) in enumerate(G_CALLS): + gflag = (gi == 0 and r == 0) + # The tail is droppable only when NOTHING later reads v[b]. In + # the last round that means the DIAGONAL group only (gi >= 4): + # a column G's v[b] is consumed by the diagonal group that + # follows it in the same round, so dropping its tail is a bug, + # not an optimisation. + skip = (self.tail_truncate and last and gi >= 4 + and b not in needed) + self.emit_g(v, a, b, c, d, schedule[ix], schedule[iy], + gflag, skip_tail=skip) + if not last: + schedule = [schedule[fr.msg_permutation[i]] for i in range(16)] + + # ===================================================================== + # BLOCK 5 -- feed-forward, truncation window, output recomposition + # ===================================================================== + def emit_feedforward(self, v: list, h: list): + """CHIP CONSTRAINT: out[i] = v[i] XOR v[i+8], for i in the truncation + window ONLY. + + The socket produces FOUR of the sixteen output words. out[i+8] = + v[i+8] XOR h[i] is never computed: h is the constant IV and those words + are not part of the digest. That is where most of the saving over a + syscall-shaped BLAKE3 chip comes from -- 12 words x 4 bytes of columns + and the same number of XOR sends, never built.""" + fr = self.fr + for i in range(fr.out_window, fr.out_window + 4): + w = self.emit_xor(v[i], v[i + 8]) + if self._bug("drop_ff_xor", i == fr.out_window): + w = self.c.fresh_word() # control: output left free + self.digest_words.append(w) + + def digest_lane_values(self): + """CHIP CONSTRAINT (per output lane i), mu-gated, degree 2: + MU * ( OUT_C[i] - (OUTW[i][0] + 2^8*OUTW[i][1] + + 2^16*OUTW[i][2] + 2^24*OUTW[i][3]) ) = 0 + No range check needed: OUTW's bytes are ByteAlu[XOR] outputs, hence + already bytes. The sum is < 2^32 << p, so OUT_C is forced to the honest + u32 -- and therefore the socket's OUTPUT always satisfies O1, which is + why only leaf digests and prover-hinted siblings need the input check.""" + return [self.c.wval(w) for w in self.digest_words] + + # ===================================================================== + def build(self) -> "SocketChip": + if self._built: + return self + self.emit_lane_bytes() + m = self.message_words() + v = self.init_state() + h = list(v[0:8]) + self.emit_rounds(v, m) + self.emit_feedforward(v, h) + # AS BUILT: the frozen shared prefix is 28 value columns (12 IN + 4 S + + # 12 OUT). MU is preprocessed, so it is NOT a main column. + # PREP_WIDTH 12 -> 13 (MODE_L) is PREPROCESSED and so does NOT enter the + # main-column census; the frozen VALUE prefix is unchanged at 28. + self.census.add("frozen_socket_prefix(IN/S/OUT)", 28) + self.census.io_sends = 6 # the LfmMem tuples of the host socket + self._built = True + return self + + +# --------------------------------------------------------------------------- +# The reference, expressed over the SAME symbolic lane values, so the gate +# compares like with like. +# --------------------------------------------------------------------------- + +def reference_digest_bv(chip: SocketChip, fr: sk.Framing): + """Word-level BLAKE3 over 32-bit BVs -- structurally independent of the + chip's byte-level XOR / halfword-shift wiring, exactly as the keccak gate + keeps zref_round independent of the byte circuit.""" + from z3 import Concat, RotateRight + + def w32(word): + return Concat(word[3], word[2], word[1], word[0]) + + def ref_g(v, a, b, c, d, mx, my): + v[a] = v[a] + v[b] + mx + v[d] = RotateRight(v[d] ^ v[a], 16) + v[c] = v[c] + v[d] + v[b] = RotateRight(v[b] ^ v[c], 12) + v[a] = v[a] + v[b] + my + v[d] = RotateRight(v[d] ^ v[a], 8) + v[c] = v[c] + v[d] + v[b] = RotateRight(v[b] ^ v[c], 7) + + lanes = [w32(w) for w in chip.in_lane_bytes] + if not fr.lane_le: + lanes = [w32(list(reversed(w))) for w in chip.in_lane_bytes] + m = [BitVecVal(0, 32) for _ in range(16)] + for i in range(4): + m[fr.a_slot + i] = lanes[i] + m[fr.b_slot + i] = lanes[4 + i] + m[fr.tag_slot] = BitVecVal(fr.tag_word, 32) + + v = [BitVecVal(x, 32) for x in fr.cv] + \ + [BitVecVal(ora.IV[i], 32) for i in range(4)] + \ + [BitVecVal(fr.counter & 0xFFFFFFFF, 32), + BitVecVal((fr.counter >> 32) & 0xFFFFFFFF, 32), + BitVecVal(fr.block_len, 32), BitVecVal(fr.flags, 32)] + + schedule = list(m) + for r in range(fr.rounds): + for (a, b, c, d, ix, iy) in G_CALLS: + ref_g(v, a, b, c, d, schedule[ix], schedule[iy]) + if r < fr.rounds - 1: + schedule = [schedule[fr.msg_permutation[i]] for i in range(16)] + return [v[i] ^ v[i + 8] for i in range(fr.out_window, fr.out_window + 4)] diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/contracts.py b/thoughts/shared/lfm-real-hash/gate-oracle/contracts.py new file mode 100644 index 000000000..5d492fb30 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/contracts.py @@ -0,0 +1,232 @@ +""" +LAYER 3: the CHIP-CONTRACT LIBRARY. + +Assume-guarantee. The gate proves the compression/framing layer *given* these +contracts; it does not re-prove the tables that supply them. Those tables +(`prover/src/tables/bitwise.rs`) are existing, separately-audited chips, and +this is the same assumption the keccak gate makes. What is NOT optional is +writing the contracts down: an unstated contract is how a fail-OPEN gate +happens -- the model quietly assumes a bound the chip never enforces, every +theorem comes back UNSAT, and the gate certifies nothing. + +Each contract below records, in one place: + * the GUARANTEE the gate is allowed to assume, + * the OBLIGATION the chip must discharge to earn it (a real bus send), + * the WIDTH it licenses -- which is the entry the width audit cites. + +Two modelling domains, because they see different bugs: + + BV (QF_BV, bytes as 8-bit bitvectors) -- sees logic/wiring bugs. It CANNOT + see bound-necessity bugs, because in a bounded BV model the bound is + baked into the variable's width: dropping a range check is unrepresentable. + FIELD (Int mod p, Goldilocks) -- sees exactly those. A committed column with + no range check is a full field element, and 2^16 / 2^32 are invertible + mod p while being zero divisors mod 2^n. Every field-lifted width in the + design must be justified HERE, not in BV. + +Getting that split wrong is the classic fail-open: model a dropped range check +in BV, observe UNSAT, and conclude the range check is unnecessary. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from z3 import BitVec, BitVecVal, Int, Or, ZeroExt + +# Goldilocks. +P = 2**64 - 2**32 + 1 + +# Wide BV width used for the add/shift identities. Honest field expressions in +# this design stay < 2^35; 48 bits is comfortably above that and below 64, so a +# BV overflow inside the model would be a modelling bug, not a masked forgery. +WIDE = 48 + + +@dataclass(frozen=True) +class Contract: + name: str + guarantee: str + obligation: str + width: str + + +CONTRACTS: dict[str, Contract] = { + "AreBytes": Contract( + name="AreBytes[x, y]", + guarantee="x, y are integers in [0, 256).", + obligation="one send to the precomputed BITWISE AreBytes receiver " + "(`bitwise.rs`, AreBytes), multiplicity Column(MU), per PAIR " + "of bytes.", + width="licenses treating a committed column as an 8-bit value in a " + "field-lifted linear form.", + ), + "ByteAlu_XOR": Contract( + name="ByteAlu[XOR](x, y) -> z", + guarantee="x, y, z in [0, 256) AND z = x XOR y, exactly.", + obligation="one send to the precomputed BITWISE ByteAlu receiver with " + "op = XOR, multiplicity Column(MU), per output BYTE.", + width="range-checks BOTH operands and the output for free -- this is why " + "most words in the design need no explicit AreBytes: they are " + "consumed by a later XOR. Operands may be linear combinations " + "provided each stays <= 255 (which is what makes a free byte " + "relabel legal in place).", + ), + "LaneDecomposition": Contract( + name="lane = b0 + 2^8*b1 + 2^16*b2 + 2^24*b3, with AreBytes on b0..b4", + guarantee="the felt `lane` is in [0, 2^32) and b0..b4 are its unique " + "little-endian byte decomposition.", + obligation="ONE mu-gated eval constraint (the linear identity) AND TWO " + "AreBytes sends. NEITHER ALONE SUFFICES -- see the width audit: " + "without AreBytes the bytes are free field elements and the " + "identity is satisfiable for arbitrary byte strings; without " + "the identity the bytes are unrelated to the lane.", + width="THE load-bearing width of this design. Sum of four bytes weighted " + "by 2^{8k} is < 2^32 << p, so the identity cannot wrap, so `lane` " + "is forced < 2^32. This is what makes felt -> u32 injective and is " + "the whole content of obligation O1.", + ), + "CarryBit": Contract( + name="mu * c * (1 - c) = 0", + guarantee="c in {0, 1} as a field element.", + obligation="one mu-gated degree-3 eval constraint per carry column.", + width="licenses treating a carry column as a bit in the add identities. " + "Dropping it is a FIELD-level forgery invisible to BV.", + ), + "ShiftRemainderBound": Contract( + name="AreBytes on the two bytes of SLL", + guarantee="SLL in [0, 2^16).", + obligation="AreBytes sends on SLL's byte pair (per halfword, per rotation).", + width="the TIGHT remainder bound. With 2^16 invertible mod p it pins " + "SLL = (x * 2^r) mod 2^16 uniquely. The quotient SLLC needs only a " + "loose 16-bit bound. Dropping the SLL bound makes the rotation " + "forgeable -- demonstrable ONLY in the field model.", + ), + "NoWrapSideCondition": Contract( + name="every field-lifted expression < 2^35 << p", + guarantee="`expr == 0 mod p` implies `expr == 0` over the integers, so " + "the BV model's arithmetic is faithful to the field's.", + obligation="a static bound argument on each identity, discharged by the " + "width audit table in ORACLE.md -- NOT by any solver run.", + width="the bridge between the BV model and the field. If any identity " + "could reach p, the BV theorems say nothing about the real chip.", + ), +} + + +# --------------------------------------------------------------------------- +# BV domain +# --------------------------------------------------------------------------- + +class BvContracts: + """Contracts as BV constructions. A byte IS an 8-bit BitVec: that is the + AreBytes guarantee, structurally enforced and therefore un-droppable here. + That structural enforcement is exactly why bound-necessity must be argued in + the FIELD domain instead.""" + + def __init__(self, tag: str): + self.tag = tag + self.assertions: list = [] + self._n = 0 + self.sends = 0 # bus-send accounting, for the cost model + + def fresh(self, width: int = 8): + v = BitVec(f"{self.tag}_{self._n}", width) + self._n += 1 + return v + + def fresh_byte(self): + return self.fresh(8) + + def fresh_word(self) -> list: + """A 32-bit word as 4 little-endian byte columns.""" + return [self.fresh_byte() for _ in range(4)] + + @staticmethod + def const_word(val: int) -> list: + return [BitVecVal((val >> (8 * i)) & 0xFF, 8) for i in range(4)] + + # -- value lifts ------------------------------------------------------ + @staticmethod + def wide(x): + return ZeroExt(WIDE - x.size(), x) + + def wval(self, word: list): + """The field-lifted word value: sum of bytes * 2^{8k}. < 2^32.""" + acc = BitVecVal(0, WIDE) + for i in range(4): + acc = acc + self.wide(word[i]) * BitVecVal(1 << (8 * i), WIDE) + return acc + + def hwval(self, blo, bhi): + """Field-lifted halfword value. < 2^16.""" + return self.wide(blo) + self.wide(bhi) * BitVecVal(256, WIDE) + + # -- contracts -------------------------------------------------------- + def are_bytes(self, *bytes_): + """AreBytes. Structural in BV (8-bit width). Counted for the cost model: + one send per PAIR.""" + self.sends += (len(bytes_) + 1) // 2 + + def byte_xor(self, x, y): + """ByteAlu[XOR]: fresh output byte, pinned to x ^ y.""" + z = self.fresh_byte() + self.assertions.append(z == (x ^ y)) + self.sends += 1 + return z + + def carry_bit(self, enforce: bool = True): + """A carry column with (or, for a control, without) its booleanity.""" + c = self.fresh(8) + if enforce: + self.assertions.append(Or(c == 0, c == 1)) + return c + + +# --------------------------------------------------------------------------- +# FIELD domain +# --------------------------------------------------------------------------- + +class FieldContracts: + """Contracts as mod-p Int constraints. Here a column is a FULL field element + unless a contract bounds it, so dropping a contract is expressible -- which + is the entire point of having this second domain.""" + + def __init__(self, solver): + self.s = solver + self._n = 0 + + def fresh_felt(self, name: str | None = None): + v = Int(name or f"felt_{self._n}") + self._n += 1 + self.s.add(v >= 0, v < P) # a committed column: any field element + return v + + def are_bytes(self, *vals): + for v in vals: + self.s.add(v >= 0, v < 256) + + def bounded(self, v, bound: int): + self.s.add(v >= 0, v < bound) + + def carry_bit(self, v): + self.s.add(Or(v == 0, v == 1)) + + @staticmethod + def lane_from_bytes(b: list): + return b[0] + 256 * b[1] + 65536 * b[2] + 16777216 * b[3] + + +def contract_table_md() -> str: + lines = ["| contract | guarantee | obligation on the chip | width it licenses |", + "|---|---|---|---|"] + for c in CONTRACTS.values(): + g = c.guarantee.replace("\n", " ") + o = c.obligation.replace("\n", " ") + w = c.width.replace("\n", " ") + lines.append(f"| `{c.name}` | {g} | {o} | {w} |") + return "\n".join(lines) + + +if __name__ == "__main__": + print(contract_table_md()) diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/gate.py b/thoughts/shared/lfm-real-hash/gate-oracle/gate.py new file mode 100644 index 000000000..52a735568 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/gate.py @@ -0,0 +1,957 @@ +""" +LAYER 5: THE z3 GATE. + +Method. Every committed column of the chip is a FREE variable; every lookup +(under its contract) and every eval constraint becomes an equation over those +variables; the chip's OUTPUT is whatever the constraints force. Then: + + assert chip_output != reference_f(input) and ask z3. + + UNSAT -> for EVERY constraint-satisfying assignment the output equals the + reference: the chip is correctly AND tightly constrained. + SAT -> the constraints admit a wrong output: under-constrained or mis-wired. + +FAIL-OPEN IS THE ONLY DANGEROUS MODE. A gate that returns UNSAT because the +model quietly assumed something the chip never enforces certifies nothing. Two +defences, both mandatory and both run below: + + * NEGATIVE CONTROLS -- inject a bug, demand SAT. A control that comes back + UNSAT means the gate cannot see that class of bug at all. + * THE WIDTH AUDIT -- every field-lifted byte/word width must cite a real + range-check contract AND a non-overflow side condition. Bound-necessity is + invisible in BV (a byte IS 8 bits there), so those controls run in the FIELD + domain, mod p. This is where a field-level attacker who escapes the + bit-vector model gets caught. + +Run: python3 gate.py # fast board (symbolic core + all controls + audit) + python3 gate.py --full # + concrete full 6- and 7-round pipeline runs +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from dataclasses import replace + +from z3 import (And, BitVecVal, Int, Or, Solver, sat, unsat) + +import blake3_oracle as ora +import chip_model as cm +import socket_ref as sk +from contracts import P, WIDE, FieldContracts + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +class Board: + def __init__(self): + self.rows: list[tuple[str, str, str, str, bool, float]] = [] + + def add(self, section, name, got, want, elapsed=0.0): + ok = (str(got) == want) + self.rows.append((section, name, str(got), want, ok, elapsed)) + mark = "PASS" if ok else "**FAIL**" + print(f" [{mark:8s}] {name:44s} -> {str(got):6s} (want {want})" + f"{f' {elapsed:.1f}s' if elapsed > 0.3 else ''}") + return ok + + def ok(self): + return all(r[4] for r in self.rows) + + +def _solve(assertions, goal, timeout_ms=0): + s = Solver() + if timeout_ms: + s.set("timeout", timeout_ms) + s.add(And(*assertions)) + s.add(goal) + t0 = time.time() + res = s.check() + return res, time.time() - t0 + + +# =========================================================================== +# SYMBOLIC THEOREMS (BV) -- the chip's logic and framing +# =========================================================================== + +def theorem_g(bug=None, timeout_ms=0): + """One G quarter-round against the reference G, free inputs. + + A round is a FIXED composition of eight G-calls on fixed indices, and the + message schedule is a compile-time permutation, so a G that is correct on + arbitrary inputs gives a correct round, hence a correct N-round core for + BOTH round counts. That chaining argument is what makes the fast board + sufficient and the monolithic multi-round runs a bonus.""" + from z3 import Concat, RotateRight + + chip = cm.SocketChip("G" + (f"_{bug}" if bug else ""), bug=bug) + va, vb, vc, vd = (chip.c.fresh_word(), chip.c.fresh_word(), + chip.c.fresh_word(), chip.c.fresh_word()) + mx, my = chip.c.fresh_word(), chip.c.fresh_word() + v = [va, vb, vc, vd] + chip.emit_g(v, 0, 1, 2, 3, mx, my, gflag=True) + + def w32(w): + return Concat(w[3], w[2], w[1], w[0]) + + rv = [w32(va), w32(vb), w32(vc), w32(vd)] + rmx, rmy = w32(mx), w32(my) + rv[0] = rv[0] + rv[1] + rmx + rv[3] = RotateRight(rv[3] ^ rv[0], 16) + rv[2] = rv[2] + rv[3] + rv[1] = RotateRight(rv[1] ^ rv[2], 12) + rv[0] = rv[0] + rv[1] + rmy + rv[3] = RotateRight(rv[3] ^ rv[0], 8) + rv[2] = rv[2] + rv[3] + rv[1] = RotateRight(rv[1] ^ rv[2], 7) + + goal = Or(*[w32(v[i]) != rv[i] for i in range(4)]) + return _solve(chip.assertions, goal, timeout_ms) + + +def theorem_socket(rounds: int, chip_framing: sk.Framing | None = None, + bug=None, ref_framing: sk.Framing | None = None, + timeout_ms=0, tail_truncate=False): + """The SOCKET layer: lane bytes -> message placement -> constant initial + state -> R rounds -> feed-forward -> truncation window -> digest lanes, + against the reference framing. + + The chip is built with `chip_framing` (perturbed for a control); the + reference always uses `ref_framing` (honest). Symbolic in the eight input + lanes.""" + cf = chip_framing or sk.honest(rounds) + rf = ref_framing or sk.honest(rounds) + tag = f"S{rounds}" + (f"_{bug}" if bug else "") + f"_{id(cf) & 0xFFFF:x}" + chip = cm.SocketChip(tag, framing=cf, bug=bug, + tail_truncate=tail_truncate).build() + ref = cm.reference_digest_bv(chip, rf) + got = chip.digest_lane_values() + # Compare as WIDE values so the byte->word lift is part of what is checked. + from z3 import ZeroExt + goal = Or(*[got[i] != ZeroExt(WIDE - 32, ref[i]) for i in range(4)]) + return _solve(chip.assertions, goal, timeout_ms) + + +def theorem_schedule(rounds: int, chip_framing: sk.Framing | None = None, + ref_framing: sk.Framing | None = None, timeout_ms=0): + """MESSAGE LAYER: the schedule the chip feeds to every round, against the + reference schedule, symbolic in the eight input lanes. + + Covers framing choices 1 (where a and b land), 5 (the tag word and its slot) + and 7 (the lane byte order), plus the compile-time message permutation -- + all of it without paying for a single G. Isolating the layer this way is not + a shortcut: a G that is correct on ARBITRARY inputs (T1) composed with a + schedule that is correct on ARBITRARY inputs (this) is a correct round, and + the composition is fixed at compile time.""" + from z3 import Concat, ZeroExt + cf = chip_framing or sk.honest(rounds) + rf = ref_framing or sk.honest(rounds) + chip = cm.SocketChip(f"MSG{rounds}_{id(cf) & 0xFFFF:x}", framing=cf) + chip.emit_lane_bytes() + + chip_sched = chip.message_words() + lanes = [Concat(w[3], w[2], w[1], w[0]) for w in chip.in_lane_bytes] + if not rf.lane_le: + lanes = [Concat(w[0], w[1], w[2], w[3]) for w in chip.in_lane_bytes] + ref_sched = [BitVecVal(0, 32) for _ in range(16)] + for i in range(4): + ref_sched[rf.a_slot + i] = lanes[i] + ref_sched[rf.b_slot + i] = lanes[4 + i] + ref_sched[rf.tag_slot] = BitVecVal(rf.tag_word, 32) + + diffs = [] + cs, rs = list(chip_sched), list(ref_sched) + for r in range(max(rounds, 1)): + for i in range(16): + diffs.append(chip.c.wval(cs[i]) != ZeroExt(WIDE - 32, rs[i])) + if r < rounds - 1: + cs = [cs[cf.msg_permutation[i]] for i in range(16)] + rs = [rs[rf.msg_permutation[i]] for i in range(16)] + return _solve(chip.assertions, Or(*diffs), timeout_ms) + + +def concrete_pipeline(rounds: int, a, b, expect_digest, negate=False, + timeout_ms=0, tail_truncate=False, + chip_framing: sk.Framing | None = None, bug=None): + """Non-vacuity + external anchor: pin the eight input lanes to a KAT input + and the four digest lanes to the KAT output. + + negate=False -> expect SAT: the full byte-level pipeline reproduces the + externally-anchored vector. + negate=True -> pin the digest to a WRONG value and expect UNSAT: at this + concrete input the system is FUNCTIONAL, i.e. tight. + + Cheap (a pinned input propagates), end-to-end, and anchored -- so this, not + a monolithic symbolic run, is what every negative control is measured + against below.""" + fr = chip_framing or sk.honest(rounds) + chip = cm.SocketChip(f"C{rounds}_{'n' if negate else 'p'}_{id(fr) & 0xFFFF:x}", + framing=fr, bug=bug, + tail_truncate=tail_truncate).build() + extra = [] + lanes = list(a) + list(b) + for word, val in zip(chip.in_lane_bytes, lanes): + for k in range(4): + extra.append(word[k] == BitVecVal((val >> (8 * k)) & 0xFF, 8)) + want = list(expect_digest) + if negate: + want[0] ^= 1 + for expr, val in zip(chip.digest_lane_values(), want): + extra.append(expr == BitVecVal(val, WIDE)) + return _solve(list(chip.assertions) + extra, And(True), timeout_ms) + + +def concrete_control(rounds: int, a, b, honest_digest, chip_framing=None, + bug=None, timeout_ms=0): + """A negative control, run against the FULL pipeline at a concrete input. + + Build the perturbed chip, pin the input lanes, and ask whether the digest can + differ from the honest anchored value. SAT = the gate sees this bug class. + UNSAT = the gate is BLIND to it, which is the finding that matters.""" + fr = chip_framing or sk.honest(rounds) + chip = cm.SocketChip(f"NC{rounds}_{bug or ''}_{id(fr) & 0xFFFF:x}", + framing=fr, bug=bug).build() + extra = [] + for word, val in zip(chip.in_lane_bytes, list(a) + list(b)): + for k in range(4): + extra.append(word[k] == BitVecVal((val >> (8 * k)) & 0xFF, 8)) + goal = Or(*[expr != BitVecVal(val, WIDE) + for expr, val in zip(chip.digest_lane_values(), honest_digest)]) + return _solve(list(chip.assertions) + extra, goal, timeout_ms) + + +# =========================================================================== +# WIDTH AUDIT (FIELD, mod p) -- bound necessity. BV provably cannot show these. +# =========================================================================== + +def audit_lane_decomposition(drop_arebytes: bool): + """The Route-A lane boundary, obligation O1 -- part 1 of 2. + + The chip decomposes each input lane into four byte columns with ONE linear + identity. Question: does that identity alone pin the bytes? + + with AreBytes -> UNSAT: the bytes are the unique LE decomposition. + without -> SAT: one linear equation in four unknowns leaves three + free, so the byte columns are unpinned. + + WHAT THIS DOES AND DOES NOT SHOW (corrected, D10). It shows the identity is + not self-sufficient. It does NOT show a `v` / `v + 2^32` collision -- that + attack is unconstructible here, because the mixing core reads the SAME linear + form the identity pins, so the lane and the message word are one field + element by construction. The consequence that matters is WA2's: without the + sends the message word is not forced to be a u32, and since `m` reaches the + core through `add3` only (never an XOR), these 16 sends are its ONLY range + check. See `chip_model.emit_lane_bytes` for the full argument.""" + honest_lane = 0x89ABCDEF + hb = [(honest_lane >> (8 * k)) & 0xFF for k in range(4)] + + s = Solver() + fc = FieldContracts(s) + b = [fc.fresh_felt(f"mb{k}") for k in range(4)] + if not drop_arebytes: + fc.are_bytes(*b) # the AreBytes sends + # the mu-gated lane-decomposition identity, in the field + s.add((honest_lane - (b[0] + 256 * b[1] + 65536 * b[2] + 16777216 * b[3])) % P == 0) + # the attacker names the three low bytes; only the top byte is left to absorb + s.add(b[0] == (hb[0] ^ 0x5A), b[1] == (hb[1] ^ 0x3C), b[2] == (hb[2] ^ 0xF0)) + return str(s.check()) + + +def audit_lane_upper_range(drop_arebytes: bool): + """The Route-A lane boundary, obligation O1 -- part 2 of 2, and THE one that + carries the soundness argument. Can a felt >= 2^32 pass the identity? + + with AreBytes -> UNSAT: the sum of four bytes is < 2^32, so `lane` -- which + IS the message word, the same linear form -- is + forced below 2^32. The compression therefore denotes + BLAKE3 of an actual 36-byte string. + without -> SAT: the message word ranges over the whole field. Round + 0's add3 has constant a, b and byte-bounded s, so a + prover solves `m = s + 2^32*(c1+c2) - a - b` for any + chosen s and owns the compression from the first add + onward -- and what the chip computes is no longer + BLAKE3 of any message.""" + s = Solver() + fc = FieldContracts(s) + lane = fc.fresh_felt("lane") + b = [fc.fresh_felt(f"ub{k}") for k in range(4)] + if not drop_arebytes: + fc.are_bytes(*b) + s.add((lane - (b[0] + 256 * b[1] + 65536 * b[2] + 16777216 * b[3])) % P == 0) + s.add(lane >= 2**32) + return str(s.check()) + + +def audit_shift_bound(r: int, in_hw: int, drop_sll_bound: bool): + """hw*2^r == SLLC*2^16 + SLL (mod p). SLL is the TIGHT remainder (AreBytes on + its two bytes); SLLC is the quotient and a loose 16-bit bound suffices. + Soundness needs 2^16 invertible mod p -- true in Goldilocks, false mod 2^n, + which is exactly why this cannot be a BV check.""" + s = Solver() + fc = FieldContracts(s) + if drop_sll_bound: + SLL = fc.fresh_felt("SLL") # unbounded column + else: + lo, hi = fc.fresh_felt("sll_lo"), fc.fresh_felt("sll_hi") + fc.are_bytes(lo, hi) + SLL = lo + 256 * hi + SLLC = fc.fresh_felt("SLLC") + fc.bounded(SLLC, 2**16) + s.add((in_hw * (2 ** r) - SLLC * (2 ** 16) - SLL) % P == 0) + s.add(SLL != (in_hw * (2 ** r)) % (2 ** 16)) + return str(s.check()) + + +def audit_add_carry(a, b, m, drop_bool: bool): + """3-operand add: a+b+m == s + 2^32*(c1+c2) (mod p), s in [0,2^32) from its + byte columns. Dropping the carry booleanity turns c into a full field element + and s becomes forgeable -- again field-only.""" + s = Solver() + fc = FieldContracts(s) + S = fc.fresh_felt("S") + fc.bounded(S, 2**32) + if drop_bool: + c1 = fc.fresh_felt("c1") + csum = c1 + else: + c1, c2 = fc.fresh_felt("c1"), fc.fresh_felt("c2") + fc.carry_bit(c1) + fc.carry_bit(c2) + csum = c1 + c2 + s.add((a + b + m - S - (2**32) * csum) % P == 0) + s.add(S != (a + b + m) % (2**32)) + return str(s.check()) + + +def audit_add2_expression_carry(drop_s_bound: bool): + """WA7 -- THE audit item the expression-carry form needs, and the reason the + BV model may encode it as a two-way disjunction. + + The chip emits ONE constraint, `MU * carry * (1 - carry) = 0`, where + `carry := (A + B - s) * 2^{-32}` is a linear form over existing columns. + Over the field that says `A + B - s in {0, 2^32}`. The question BV cannot + answer: given A, B, s byte-bounded below 2^32, are 0 and 2^32 the ONLY + reachable roots -- in particular, can a NEGATIVE difference alias 2^32 mod p? + + It cannot. If `A + B - s >= 0` it lies in [0, 2^33) and 2^33 << p, so the + only residues are the honest two. If `A + B - s < 0` it lies in (-2^32, 0), + i.e. the field element sits in (p - 2^32, p); that equals 0 only for a zero + difference, and equals 2^32 only if the difference were 2^32 - p, which is + about -2^64 and far below -2^32. Hence s is pinned to (A + B) mod 2^32. + + present -> UNSAT: s is pinned. + drop the byte bound on s -> SAT: s becomes a free field element, the + difference can be steered onto a root, and the add is forgeable. This is + the same class as WA4 and equally invisible to BV. + + ENCODING: `carry in {0,1}` is encoded as its root set `d in {0, 2^32}` -- + two LINEAR congruences -- rather than as the quadratic `carry*(1-carry) = 0` + with a nested inverse, which is intractable for z3's integer arithmetic. The + step from one to the other is AR2 in the argued ledger (`2^{-32}` is a unit, + so multiplying by it is a bijection and maps the root set exactly). This is + the same posture WA4 already takes for the add3 carry, and it keeps the + solver on the question it can actually decide: whether s is pinned.""" + s_ = Solver() + fc = FieldContracts(s_) + A, Bv = 0xFFFF_FFF0, 0xFFFF_FFF5 # a case that genuinely carries + S = fc.fresh_felt("S2") + if not drop_s_bound: + fc.bounded(S, 2**32) + d = A + Bv - S + s_.add(Or(d % P == 0, (d - 2**32) % P == 0)) + s_.add(S != (A + Bv) % (2**32)) + return str(s_.check()) + + +def audit_block0_capacity(drop_mode_p_pin: bool): + """BLOCK-0 idx 0-3 with idx 5. `S_k - (MODE_P*IN_{8+k} + MODE_C*IV_k) = 0`. + + with `MODE_P = 0` pinned -> UNSAT: S_k is forced to MODE_C * IV_k. + without it -> SAT: MODE_P is free, so the capacity prefix becomes a + prover-chosen copy of IN_{8+k}. idx 0-3 pin nothing on their own; idx 5 + is what gives them meaning.""" + s_ = Solver() + fc = FieldContracts(s_) + IV0 = ora.IV[0] + mode_c, mode_p = fc.fresh_felt("mode_c"), fc.fresh_felt("mode_p") + in_8 = fc.fresh_felt("in_8") + S = fc.fresh_felt("S_cap") + ms = mode_c + mode_p # widened below once mode_t exists + s_.add((ms * (1 - ms)) % P == 0) # idx 4 + if not drop_mode_p_pin: + s_.add(mode_p == 0) # idx 5 + mode_t = fc.fresh_felt("mode_t") + s_.add((S - (mode_p * in_8 + (mode_c + mode_t) * IV0)) % P == 0) # idx 0-3 + s_.add(S != ((mode_c + mode_t) * IV0) % P) + return str(s_.check()) + + +def audit_block0_mu_boolean(drop_mode_sum_bool: bool): + """BLOCK-0 idx 4 + idx 5 give MU booleanity. With MODE_P = 0, + mode_sum = MODE_C = MU, so `mode_sum*(1-mode_sum)=0` IS `MU in {0,1}`. + + The pre-Phase-2 model deferred MU booleanity to "structural, not a BV + theorem". The chip emits it as a real constraint, so it is checkable -- and + checked here in the field. + + ENCODING: the emitted polynomial's root set is `{0,1}` by AR1 (a prime field + has no zero divisors), so the contract is encoded as that root set. What the + solver decides is the consequence: with MODE_P pinned, does mode_sum being a + bit force MU to be a bit -- and what happens when the constraint is absent. + + present -> UNSAT (MU is a bit); dropped -> SAT (MU is any felt, and a + non-boolean MU scales every gated constraint and every send multiplicity).""" + s_ = Solver() + fc = FieldContracts(s_) + mode_c, mode_t = fc.fresh_felt("mc2"), fc.fresh_felt("mt2") + mode_p = fc.fresh_felt("mp2") + s_.add(mode_p == 0) # idx 5 + ms = mode_c + mode_t + mode_p # AS BUILT + if not drop_mode_sum_bool: + s_.add(Or(ms % P == 0, ms % P == 1)) # idx 4, via AR1 + # MU = MODE_C + MODE_T + MODE_L as built, which IS the mode sum once + # MODE_P = 0. (This audit is written over the two-selector case; the + # four-way form is M8's, and adding MODE_L here changes nothing about what + # idx 4 buys -- it bounds the sum either way.) + # + # DO NOT add the registrar's one-hot here: it would force MU = 1 outright and + # make this audit vacuous (an earlier draft did exactly that and the control + # caught it -- `dropped` came back UNSAT). The division of labour is the + # point, and it is sharper than the spec's original claim: + # * idx 4 DOES give MU booleanity -- MU is the sum, so bounding the sum to + # a bit bounds MU. That is what this audit checks. + # * idx 4 does NOT give one-hotness -- which tag `m[8]` selects is the + # registrar's preprocessed check. That is M8's job. + # + # NB: MU is a SUM of felts, so as a z3 Int it can exceed p. It must be + # compared by RESIDUE, not raw value -- otherwise `mu = p + 1` counts as + # "not 1" and the audit reports SAT for a chip that is fine. (A draft did + # exactly that; the `present` leg caught it.) + mu = (mode_c + mode_t + mode_p) % P + s_.add(mu != 0, mu != 1) + return str(s_.check()) + + +def audit_block0_tag_selection(with_one_hot: bool, target_tag: int | None = None): + """M8 model-side — WHAT ACTUALLY MAKES `m[8]` TRUSTWORTHY. + + `m[8] = MODE_C*TAG_LFMC + MODE_T*TAG_LFMT`. The question is what forces it to + be ONE of the two tags rather than a blend. + + IT IS NOT idx 4. Over a prime field `mode_sum in {0,1}` pins the SUM, not the + selectors: `MODE_C = x`, `MODE_T = 1 - x` satisfies it for ANY x, and since + the tags differ, `x = (T - TAG_T)/(TAG_C - TAG_T)` reaches ANY target tag T. + + with_one_hot=False -> SAT: a forged tag is reachable (idx 4 is not enough). + with_one_hot=True -> UNSAT for a forged target, SAT for either real tag + (the honest-path leg: a fix that rejected everything + would pass the attack leg alone). + + ✓ Reproduces the builder's Rust M5/M6 finding independently. The real closure + is (i) MODE_* being PREPROCESSED and (ii) the registrar's one-hot check.""" + TAG_C, TAG_T = 0x434D464C, 0x544D464C # "LFMC", "LFMT" + target = TAG_C if target_tag is None else target_tag + s_ = Solver() + fc = FieldContracts(s_) + mc, mt = fc.fresh_felt("mc8"), fc.fresh_felt("mt8") + ms = mc + mt + s_.add(Or(ms % P == 0, ms % P == 1)) # idx 4 + if with_one_hot: # the registrar's check + s_.add(Or(And(mc == 1, mt == 0), And(mc == 0, mt == 1))) + s_.add((mc * TAG_C + mt * TAG_T - target) % P == 0) + return str(s_.check()) + + +MAX_HALF = 0xFFFFFFFF +TAG_C, TAG_T, TAG_L = 0x434D464C, 0x544D464C, 0x4C4D464C + + +def audit_leaf_canonicity(drop_canon: bool): + """WA8 — the leaf mode's canonicity gate (obligation O1 on a leaf row). + + A leaf row binds `v = lo + 2^32*hi` with lo, hi bounded to u32 by AreBytes. + That is a decomposition, NOT a canonical one: `p - 1 = 0xFFFFFFFF_00000000`, + so every pair with `hi` maximal and `lo >= 1` encodes a field element that + ALSO has an ordinary encoding -- one felt, two half-pairs, two leaf digests, + which is precisely the collision a Merkle tree must not have. + + present -> UNSAT: no non-canonical pair satisfies the constraints. + dropped -> SAT: a second encoding of an already-encodable felt exists.""" + s_ = Solver() + fc = FieldContracts(s_) + lo, hi = fc.fresh_felt("lo"), fc.fresh_felt("hi") + fc.bounded(lo, 2**32) # from AreBytes + the lane bytes + fc.bounded(hi, 2**32) + z, ginv = fc.fresh_felt("z"), fc.fresh_felt("ginv") + g = MAX_HALF - hi + if not drop_canon: + s_.add((z * g) % P == 0) # canon-a + s_.add((1 - z - g * ginv) % P == 0) # canon-b + s_.add((z * lo) % P == 0) # canon-c + # the attack: a NON-canonical pair, i.e. one encoding a value >= p + s_.add(hi == MAX_HALF, lo >= 1) + return str(s_.check()) + + +def audit_leaf_range_dependency(narrow_arebytes_to_digest: bool): + """WA9 — ⚠ THE HAZARD THE GATING SPLIT CREATES, and the reason it is safe. + + `idx 6-13` (the lane identity) narrowed to the DIGEST modes when MODE_L + landed; the AreBytes range bound did NOT (its sends carry + `Sum3(MODE_C, MODE_T, MODE_L)`). This audit asks what would happen if a + future change narrowed the RANGE bound too -- the plausible "tidy up the + multiplicities to match" refactor. + + bound present (as built) -> UNSAT: with lo, hi < 2^32 the canonicity block + admits only canonical pairs, so the felt->halves map is injective. + bound narrowed away -> SAT: lo and hi become full field elements, and + a felt acquires a second half-pair that still satisfies binding AND + canonicity -- the gate is intact but VACUOUS. Canonicity assumes the + u32 bound; it does not establish it.""" + s_ = Solver() + fc = FieldContracts(s_) + lo, hi = fc.fresh_felt("lo9"), fc.fresh_felt("hi9") + if not narrow_arebytes_to_digest: + fc.bounded(lo, 2**32) + fc.bounded(hi, 2**32) + z, ginv = fc.fresh_felt("z9"), fc.fresh_felt("ginv9") + g = MAX_HALF - hi + s_.add((z * g) % P == 0) + s_.add((1 - z - g * ginv) % P == 0) + s_.add((z * lo) % P == 0) + # a SECOND encoding of the felt v = 1: binding says v == lo + 2^32*hi + v = 1 + s_.add((v - lo - (2**32) * hi) % P == 0) + s_.add(Or(lo != 1, hi != 0)) # anything other than the honest pair + return str(s_.check()) + + +def audit_tag_selection_4way(with_one_hot: bool, target_tag: int): + """M8 over the FOUR-way one-hot: m[8] = MODE_C*TAG_C + MODE_T*TAG_T + + MODE_L*TAG_L. A third tag does not change the finding -- idx 4 still pins + only the SUM, so a fractional split still reaches any target.""" + s_ = Solver() + fc = FieldContracts(s_) + mc, mt, ml = (fc.fresh_felt("mc4"), fc.fresh_felt("mt4"), fc.fresh_felt("ml4")) + ms = mc + mt + ml + s_.add(Or(ms % P == 0, ms % P == 1)) # idx 4 + if with_one_hot: + s_.add(Or(And(mc == 1, mt == 0, ml == 0), + And(mc == 0, mt == 1, ml == 0), + And(mc == 0, mt == 0, ml == 1))) + s_.add((mc * TAG_C + mt * TAG_T + ml * TAG_L - target_tag) % P == 0) + return str(s_.check()) + + +# HashMode arities, ✓ VERIFIED instr.rs:104-110. +MODE_ARITY = {"Compress": 2, "Transcript": 2, "Leaf": 1, "Permute": 3} + + +def audit_unread_pin_selectors(): + """D1's shared unread-input pins: does any HONEST row get OVER-constrained? + + `emit_unread_input_pins` derives the selector for input slot `k` as the sum + of the modes with `num_input_cells() <= k`. A pin fires on a row iff that + row's mode is in the sum. The obligation is that a mode is NEVER pinned on a + cell it actually READS -- otherwise honest rows become unprovable, which is + the failure mode a soundness fix most easily introduces. + + UNSAT = no mode is pinned on a cell it reads.""" + s_ = Solver() + bad = [] + for mode, arity in MODE_ARITY.items(): + for slot in (1, 2): + pinned = arity <= slot # the helper's filter + reads = slot < arity # this mode reads that cell + if pinned and reads: + bad.append(f"{mode} pinned on slot {slot} which it READS") + s_.add(Int("dummy") == (1 if bad else 0), Int("dummy") == 1) + return ("sat" if bad else "unsat"), bad + + +def audit_unread_pins_inert_on_blake3(): + """⚠ DOCUMENTED GATE BLINDNESS, in the `drop_carry_bool` shape. + + On the BLAKE3 arm the two unread cells are read by NOTHING: + * cell 1 (IN4..8) is read by the lane identity idx 6-13, which is gated on + the DIGEST modes -- and on a leaf row, the only row where cell 1 is + unread, that gate is zero; + * cell 2 (IN8..12) is read only through idx 0-3's `MODE_P * IN` term, and + idx 5 pins MODE_P to zero PERMANENTLY (option B1). + + So dropping the BLAKE3 unread pins cannot change a BLAKE3 digest, and this + gate would report UNSAT for any "wrong output" question about them. That is + a TRUE statement about the BLAKE3 arm and NOT evidence the pins are + unnecessary: D1 was a defect in `eval_test` / `eval_poseidon`, where those + cells ARE read, and those arms are outside this QF-BV model entirely. + + WHAT THIS GATE CERTIFIES : the pins are inert on BLAKE3 (hygiene). + WHAT IT CANNOT : their necessity on Test/Poseidon. + WHAT CARRIES THAT INSTEAD: the builder's Rust junk-rejection controls. + + Recorded rather than left implicit, because "the gate said UNSAT" is exactly + how a fix gets dropped as redundant.""" + return "unsat" + + +def audit_block0_upper_out(drop_pins: bool): + """BLOCK-0 idx 14-21: `OUT_{4+j} = 0`. The digest is ONE cell, so the upper + eight OUT lanes must carry nothing. present -> UNSAT; dropped -> SAT.""" + s_ = Solver() + fc = FieldContracts(s_) + outs = [fc.fresh_felt(f"outhi{j}") for j in range(8)] + if not drop_pins: + for o in outs: + s_.add(o == 0) + s_.add(Or(*[o != 0 for o in outs])) + return str(s_.check()) + + +def audit_recombine_pins(target: str): + """The TAIL-TRUNCATION obligation, both sides. + + The rotation's output word Y is constrained only by two halfword identities: + Ylo == SLL_hi + SLLC_lo, Yhi == SLL_lo + SLLC_hi + where Ylo = Y0 + 256*Y1 and Yhi = Y2 + 256*Y3. If Y's downstream XOR is + removed (the last-round tail optimisation), Y's BYTES lose their range check. + + target='word' -> UNSAT: the WORD VALUE sum(Y_k * 2^{8k}) is still pinned, + because it regroups exactly into the two constrained + halfword sums. So a consumer that reads Y as the full + linear form (the add3) is safe. + target='byte' -> SAT: the individual BYTES are NOT pinned. So a consumer + that reads Y's bytes -- a relabel, a byte lookup, any + sub-combination -- is UNSOUND without an explicit AreBytes. + """ + SLL_hi, SLLC_lo, SLL_lo, SLLC_hi = 0x1230, 0x0004, 0x5670, 0x0008 + ylo, yhi = SLL_hi + SLLC_lo, SLL_lo + SLLC_hi + s = Solver() + fc = FieldContracts(s) + Y = [fc.fresh_felt(f"Y{k}") for k in range(4)] # NO AreBytes: tail case + s.add((Y[0] + 256 * Y[1] - ylo) % P == 0) + s.add((Y[2] + 256 * Y[3] - yhi) % P == 0) + if target == "word": + word = Y[0] + 256 * Y[1] + 65536 * Y[2] + 16777216 * Y[3] + s.add(word % P != (ylo + 65536 * yhi) % P) + else: + s.add(Y[0] != ylo & 0xFF) + return str(s.check()) + + +# --------------------------------------------------------------------------- +# The non-overflow side condition: a static bound argument, not a solver run. +# --------------------------------------------------------------------------- + +WIDTH_AUDIT_TABLE = [ + # (identity, max |LHS| and |RHS| given the contracts, backing contract) + ("lane decomposition lane == sum b_k*2^{8k}", + 2**32, "AreBytes on MB[j][0..4] (LaneDecomposition)"), + ("add2 sum A+B == s + 2^32*c", + 2**33, "ByteAlu[XOR] on operands + CarryBit"), + ("add3 sum A+B+M == s + 2^32*(c1+c2)", + 2**34, "ByteAlu[XOR]/AreBytes on operands + CarryBit x2"), + ("shift identity hw*2^r == SLLC*2^16 + SLL", + 2**32, "AreBytes on SLL/SLLC bytes (ShiftRemainderBound)"), + ("recombine Ylo == SLL_hi + SLLC_lo", + 2**17, "AreBytes on SLL/SLLC bytes"), + ("digest recomposition OUT_C[i] == sum OUTW_k*2^{8k}", + 2**32, "ByteAlu[XOR] output bytes"), +] + + +def audit_no_wrap() -> tuple[bool, int]: + worst = max(m for (_, m, _) in WIDTH_AUDIT_TABLE) + return worst < P, worst + + +# --------------------------------------------------------------------------- +# THE ARGUED LEDGER -- steps discharged by algebra, not by a solver. +# +# Recording them is the point. Each is a one-line field fact that some encoding +# above relies on; a gate that silently baked them in would be asserting exactly +# the kind of unstated assumption that makes a fail-open possible. z3 4.15.4 has +# no finite-field sort, and the Int+mod encodings of these are nonlinear and +# intractable, so they are argued -- and SAID to be argued -- rather than solved. +# --------------------------------------------------------------------------- + +ARGUED_LEDGER = [ + ("AR1", "F_p has no zero divisors (p prime), so `x*(1-x) = 0` has root set " + "exactly {0,1}.", + "add3 carry booleanity (WA4); mode-sum booleanity (B0b)"), + ("AR2", "2^{-32} is a unit in F_p, so `d * 2^{-32} in {0,1}` iff " + "`d in {0, 2^32}` -- multiplication by a unit is a bijection and " + "maps the root set exactly.", + "add2 expression-carry (WA7)"), + ("AR3", "2^16 is invertible mod p, which is what makes the tight AreBytes " + "bound on SLL pin the shift remainder uniquely.", + "rotation shift identity (WA3)"), + ("AR4", "every field-lifted expression stays below 2^34 << p, so `expr = 0 " + "mod p` implies `expr = 0` over the integers.", + "all identities (WA6)"), +] + + +# =========================================================================== +def load_kats(): + path = os.path.join(HERE, "socket_kats.json") + if not os.path.exists(path): + return None + with open(path) as f: + return json.load(f) + + +def main() -> int: + full = "--full" in sys.argv + B = Board() + print("=" * 78) + print("BLAKE3-behind-LFM_HASH -- z3 GATE (Option A socket)") + print("=" * 78) + + kats = load_kats() + if kats is None: + print(" socket_kats.json missing -- run `python3 socket_kats.py --write` first") + return 1 + VEC = {r: next(e for e in kats["rounds"][str(r)] if e["name"] == "formula_1") + for r in (6, 7)} + + # ---------------------------------------------------------------- core + # The argument the board rests on, stated once: + # T1 a G-call is correct on ARBITRARY inputs; + # T2 the schedule fed to every round is correct on ARBITRARY inputs; + # T3 the constant initial state, the feed-forward, the truncation window + # and the two felt<->byte recompositions are correctly wired; + # a round is a FIXED composition of eight G-calls on fixed indices, and the + # round count is a compile-time constant. + # Hence the full N-round socket is correct, for BOTH round counts. T4 then + # runs the whole pipeline concretely against externally-anchored vectors, so + # the composition argument has an executed end-to-end witness rather than + # only a proof sketch. + print("\n--- MAIN THEOREMS (symbolic, BV) -- want UNSAT ---") + r, t = theorem_g() + B.add("core", "T1 G quarter-round, free inputs (covers every G)", r, "unsat", t) + r, t = theorem_schedule(7) + B.add("core", "T2 message schedule, all 7 rounds (placement/tag/LE/perm)", + r, "unsat", t) + r, t = theorem_socket(0) + B.add("core", "T3 framing @rounds=0 (init state/feed-forward/window)", + r, "unsat", t) + + # A theorem with no control of its own is a theorem that may be vacuous, so + # each of T2 and T3 gets controls proving it discriminates -- against ITS OWN + # layer, not only against the end-to-end pipeline. + print(" per-theorem discrimination controls -- want SAT:") + T2_LAYER = ["swap_a_b", "tag_changed", "tag_omitted", "tag_slot_moved", + "lanes_big_endian", "msg_perm_swapped"] + for name in T2_LAYER: + cfr = replace(sk.CONTROLS[name], rounds=7) + r, t = theorem_schedule(7, chip_framing=cfr, ref_framing=sk.honest(7)) + B.add("neg", f" T2-ctl {name}", r, "sat", t) + # T3 sees only what reaches the window with ZERO rounds: v[0..12] and the + # feed-forward wiring. The counter/block_len/flags words sit at v[12..16] and + # reach the digest only THROUGH the rounds, so they are genuinely invisible + # here and are covered by T4 instead. Listing them as T3 controls would be a + # false claim of coverage. + for name in ["cv_zero", "truncate_high_half"]: + cfr = replace(sk.CONTROLS[name], rounds=0) + r, t = theorem_socket(0, chip_framing=cfr, ref_framing=sk.honest(0)) + B.add("neg", f" T3-ctl {name}", r, "sat", t) + r, t = theorem_socket(0, bug="drop_ff_xor") + B.add("neg", " T3-ctl drop_ff_xor", r, "sat", t) + if full: + for rr in (1, 2): + r, t = theorem_socket(rr, timeout_ms=5_400_000) + B.add("core", f"T5 monolithic symbolic socket @rounds={rr} (bonus)", + r, "unsat", t) + + print("\n--- T4 FULL PIPELINE, CONCRETE, vs the anchored KATs ---") + for rounds in (7, 6): + v = VEC[rounds] + r, t = concrete_pipeline(rounds, v["a"], v["b"], v["digest"], + timeout_ms=900_000) + B.add("core", f"T4 full {rounds}-round pipeline == anchored KAT", + r, "sat", t) + r, t = concrete_pipeline(rounds, v["a"], v["b"], v["digest"], negate=True, + timeout_ms=900_000) + B.add("core", f"T4 full {rounds}-round pipeline EXCLUDES a wrong digest", + r, "unsat", t) + + # ------------------------------------------------- negative controls + print("\n--- NEGATIVE CONTROLS -- want SAT (an UNSAT here means the gate is BLIND) ---") + print(" logic bugs, symbolic at G level:") + for bug in ("rot_wrong_amount", "swap_g_operand"): + r, t = theorem_g(bug=bug) + B.add("neg", f"NC {bug}", r, "sat", t) + + # The transcribed bodies must be re-measured against every control: a + # transcription that accidentally STRENGTHENS is as wrong as one that + # weakens, and only the controls can tell the difference. Run at BOTH round + # counts, because the chip ships both (7r default, 6r behind `blake3-6round`). + for rounds in (7, 6): + print(f" framing + wiring bugs, FULL {rounds}-round pipeline, concrete:") + vv = VEC[rounds] + for name, cfr in sk.CONTROLS.items(): + if name == "rounds_6_not_7": + # at 6 rounds this control IS the honest framing; the meaningful + # form is the opposite confusion, checked below + cr, cfr2 = (6, cfr) if rounds == 7 else (7, replace(cfr, rounds=7)) + label = "rounds_6_not_7" if rounds == 7 else "rounds_7_not_6" + r, t = concrete_control(cr, vv["a"], vv["b"], vv["digest"], + chip_framing=cfr2, timeout_ms=900_000) + B.add("neg", f"NC {label} @{rounds}", r, "sat", t) + continue + r, t = concrete_control(rounds, vv["a"], vv["b"], vv["digest"], + chip_framing=replace(cfr, rounds=rounds), + timeout_ms=900_000) + B.add("neg", f"NC {name} @{rounds}", r, "sat", t) + for bug in ("drop_ff_xor", "swap_g_operand", "drop_add2_carry"): + r, t = concrete_control(rounds, vv["a"], vv["b"], vv["digest"], + bug=bug, timeout_ms=900_000) + # `drop_add2_carry` removes the add2 constraint outright. Under the + # expression-carry form there is no carry column left to un-boolean, + # so the whole constraint IS the booleanity -- and unlike the add3 + # case it is therefore BV-visible. (`drop_carry_bool`, which + # un-booleans add3's carry COLUMNS, stays BV-blind; see below.) + B.add("neg", f"NC {bug} @{rounds}", r, "sat", t) + + # ------------------------------------------- documented BV blindness + # Not a failure: a demonstration of WHY the field audit is mandatory. In BV + # a carry column is an 8-bit variable, so removing its booleanity leaves it + # bounded and `s` is still pinned -> UNSAT. The same bug is a live forgery in + # the field (WA4 below). A gate that ran only the BV domain would report this + # class of bug as absent. That is the fail-open this split exists to prevent. + print("\n--- DOCUMENTED BV BLINDNESS (why the FIELD domain is mandatory) ---") + r, t = theorem_g(bug="drop_carry_bool") + B.add("blind", "BV drop_carry_bool (add3 carry COLUMNS) invisible in BV " + "-> WA4 has the field verdict", r, "unsat", t) + + # ------------------------------------------- the tail optimisation + # Documented in ORACLE.md as OPTIONAL and NOT recommended; checked anyway, + # because an optimisation described but never exercised is an unverified + # claim. Its first draft skipped the column group too -- caught here. + print("\n--- OPTIONAL TAIL TRUNCATION (last-round diagonal X4/B2 omitted) ---") + v7 = VEC[7] + r, t = concrete_pipeline(7, v7["a"], v7["b"], v7["digest"], + tail_truncate=True, timeout_ms=900_000) + B.add("core", "TT tail-truncated 7-round pipeline == anchored KAT", + r, "sat", t) + r, t = concrete_pipeline(7, v7["a"], v7["b"], v7["digest"], negate=True, + tail_truncate=True, timeout_ms=900_000) + B.add("core", "TT tail-truncated pipeline EXCLUDES a wrong digest", + r, "unsat", t) + tt = cm.SocketChip("tt7", framing=sk.honest(7), tail_truncate=True).build() + base = cm.SocketChip("base7", framing=sk.honest(7)).build() + print(f" saves {base.census.cell_equiv() - tt.census.cell_equiv()} " + f"cell-equiv of {base.census.cell_equiv()} " + f"({100*(base.census.cell_equiv()-tt.census.cell_equiv())/base.census.cell_equiv():.1f}%)") + + # ---------------------------------------------- non-vacuity + print("\n--- NON-VACUITY ---") + chip = cm.SocketChip("nv", framing=sk.honest(7)).build() + r, t = _solve(chip.assertions, And(True)) + B.add("pos", "NV honest system satisfiable @rounds=7 (not vacuous)", + r, "sat", t) + + # --------------------------------------------------------- width audit + print("\n--- WIDTH AUDIT (FIELD, mod p) -- bound necessity; BV cannot see these ---") + B.add("audit", "WA1 lane decomposition, AreBytes PRESENT", + audit_lane_decomposition(False), "unsat") + B.add("audit", "WA1 lane decomposition, AreBytes DROPPED", + audit_lane_decomposition(True), "sat") + B.add("audit", "WA2 lane < 2^32 forced, AreBytes PRESENT", + audit_lane_upper_range(False), "unsat") + B.add("audit", "WA2 lane < 2^32 forced, AreBytes DROPPED", + audit_lane_upper_range(True), "sat") + B.add("audit", "WA3 shift SLL bound PRESENT (r=9)", + audit_shift_bound(9, 0x9C3A, False), "unsat") + B.add("audit", "WA3 shift SLL bound DROPPED (r=9)", + audit_shift_bound(9, 0x9C3A, True), "sat") + B.add("audit", "WA4 add3 carry booleanity PRESENT", + audit_add_carry(0xF0000000, 0xF0000000, 0xF0000000, False), "unsat") + B.add("audit", "WA4 add3 carry booleanity DROPPED", + audit_add_carry(0xF0000000, 0xF0000000, 0xF0000000, True), "sat") + B.add("audit", "WA5 tail case: rotation WORD value still pinned", + audit_recombine_pins("word"), "unsat") + B.add("audit", "WA5 tail case: rotation BYTES not pinned (hazard is real)", + audit_recombine_pins("byte"), "sat") + nowrap, worst = audit_no_wrap() + B.add("audit", f"WA6 no-wrap side condition (worst 2^{worst.bit_length()-1} < p)", + "ok" if nowrap else "OVERFLOW", "ok") + B.add("audit", "WA7 add2 expression-carry pins s (s byte-bound)", + audit_add2_expression_carry(False), "unsat") + B.add("audit", "WA7 add2 expression-carry, s bound DROPPED", + audit_add2_expression_carry(True), "sat") + + print("\n--- BLOCK-0 FRAMING AUDIT (FIELD) -- the four constraints BV cannot reach ---") + B.add("audit", "B0a capacity prefix pinned, MODE_P=0 PRESENT (idx 0-3,5)", + audit_block0_capacity(False), "unsat") + B.add("audit", "B0a capacity prefix, MODE_P pin DROPPED (idx 5 gone)", + audit_block0_capacity(True), "sat") + B.add("audit", "B0b MU booleanity from mode-sum (idx 4,5) PRESENT", + audit_block0_mu_boolean(False), "unsat") + B.add("audit", "B0b MU booleanity DROPPED", + audit_block0_mu_boolean(True), "sat") + B.add("audit", "B0c upper OUT lanes pinned to 0 (idx 14-21) PRESENT", + audit_block0_upper_out(False), "unsat") + B.add("audit", "B0c upper OUT lane pins DROPPED", + audit_block0_upper_out(True), "sat") + + print("\n--- M8 (FOUR-way one-hot): what makes the mode-selected m[8] trustworthy ---") + FORGED = 0x58585858 # "XXXX" + B.add("audit", "M8 forged tag reachable with idx 4 ALONE (no one-hot)", + audit_tag_selection_4way(False, FORGED), "sat") + B.add("audit", "M8 forged tag EXCLUDED once four-way one-hot is present", + audit_tag_selection_4way(True, FORGED), "unsat") + for nm, tg in (("LFMC", TAG_C), ("LFMT", TAG_T), ("LFML", TAG_L)): + B.add("audit", f"M8 honest leg: TAG_{nm} still reachable under one-hot", + audit_tag_selection_4way(True, tg), "sat") + + print("\n--- D1 UNREAD-INPUT PINS (8, both cells) ---") + r_sel, bad_sel = audit_unread_pin_selectors() + B.add("audit", "D1 no honest row over-constrained (pin selectors vs arities)", + r_sel, "unsat") + if bad_sel: + for x in bad_sel: + print(f" OVER-CONSTRAINED: {x}") + B.add("blind", "D1 pins are INERT on BLAKE3 -- necessity rests on the Rust " + "controls (see the docstring)", + audit_unread_pins_inert_on_blake3(), "unsat") + + print("\n--- LEAF MODE (MODE_L): canonicity, and the gating-split hazard ---") + B.add("audit", "WA8 leaf canonicity PRESENT -> non-canonical pair unprovable", + audit_leaf_canonicity(False), "unsat") + B.add("audit", "WA8 leaf canonicity DROPPED -> a felt gets a 2nd half-pair", + audit_leaf_canonicity(True), "sat") + B.add("audit", "WA9 AreBytes still covers leaf rows (as built) -> map injective", + audit_leaf_range_dependency(False), "unsat") + B.add("audit", "WA9 AreBytes NARROWED to digest modes -> canonicity goes VACUOUS", + audit_leaf_range_dependency(True), "sat") + + print("\n--- ARGUED LEDGER (algebra, not solver output -- stated, not hidden) ---") + for tag, fact, used_by in ARGUED_LEDGER: + print(f" {tag}: {fact}") + print(f" relied on by: {used_by}") + + # --------------------------------------------------------------- census + chip = cm.SocketChip("census7", framing=sk.honest(7)).build() + chip6 = cm.SocketChip("census6", framing=sk.honest(6)).build() + print("\n--- COST CENSUS (derived from the gated model, not hand-counted) ---") + for label, ch in (("7-round", chip), ("6-round", chip6)): + print(f" {label}: main={ch.census.main:5d} sends={ch.census.sends:5d} " + f"aux={ch.census.aux_cells():5d} cell-equiv={ch.census.cell_equiv():5d}") + for blk, n in sorted(chip.census.by_block.items(), key=lambda x: -x[1]): + print(f" {blk:24s} {n:5d}") + + print("\n" + "=" * 78) + print(f"GATE VERDICT: {'PASS' if B.ok() else 'FAIL -- investigate above'}") + print("=" * 78) + fails = [r for r in B.rows if not r[4]] + if fails: + for f in fails: + print(f" FAILED: {f[1]} -> {f[2]} (wanted {f[3]})") + return 0 if B.ok() else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/run-anchor.log b/thoughts/shared/lfm-real-hash/gate-oracle/run-anchor.log new file mode 100644 index 000000000..33763bdb8 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/run-anchor.log @@ -0,0 +1,8 @@ +========================================================================== +LAYER 1 ANCHOR CHECK -- blake3_oracle.py +========================================================================== +[PASS] A1 official vectors: A1 PASS: 35 hash + 35 keyed + 35 derive_key cases, 115 random cases, 2 known digests +[PASS] A3 differential: A3 PASS: 200 random compressions x rounds in (6,7) agree with ../../../../../lambda_vm-blake3-impl/thoughts/blake3/blake3-oracle/blake3_ref.py +[PASS] NC anchor sensitivity: NC PASS: all 4 single-convention perturbations break the anchor +-------------------------------------------------------------------------- +LAYER 1: ANCHORED diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/run-chip-gate.log b/thoughts/shared/lfm-real-hash/gate-oracle/run-chip-gate.log new file mode 100644 index 000000000..5bbbe9e01 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/run-chip-gate.log @@ -0,0 +1,141 @@ +============================================================================== +BLAKE3-behind-LFM_HASH -- z3 GATE (Option A socket) +============================================================================== + +--- MAIN THEOREMS (symbolic, BV) -- want UNSAT --- + [PASS ] T1 G quarter-round, free inputs (covers every G) -> unsat (want unsat) 20.3s + [PASS ] T2 message schedule, all 7 rounds (placement/tag/LE/perm) -> unsat (want unsat) + [PASS ] T3 framing @rounds=0 (init state/feed-forward/window) -> unsat (want unsat) + per-theorem discrimination controls -- want SAT: + [PASS ] T2-ctl swap_a_b -> sat (want sat) + [PASS ] T2-ctl tag_changed -> sat (want sat) + [PASS ] T2-ctl tag_omitted -> sat (want sat) + [PASS ] T2-ctl tag_slot_moved -> sat (want sat) + [PASS ] T2-ctl lanes_big_endian -> sat (want sat) + [PASS ] T2-ctl msg_perm_swapped -> sat (want sat) + [PASS ] T3-ctl cv_zero -> sat (want sat) + [PASS ] T3-ctl truncate_high_half -> sat (want sat) + [PASS ] T3-ctl drop_ff_xor -> sat (want sat) + +--- T4 FULL PIPELINE, CONCRETE, vs the anchored KATs --- + [PASS ] T4 full 7-round pipeline == anchored KAT -> sat (want sat) 6.8s + [PASS ] T4 full 7-round pipeline EXCLUDES a wrong digest -> unsat (want unsat) 7.1s + [PASS ] T4 full 6-round pipeline == anchored KAT -> sat (want sat) 5.9s + [PASS ] T4 full 6-round pipeline EXCLUDES a wrong digest -> unsat (want unsat) 6.1s + +--- NEGATIVE CONTROLS -- want SAT (an UNSAT here means the gate is BLIND) --- + logic bugs, symbolic at G level: + [PASS ] NC rot_wrong_amount -> sat (want sat) + [PASS ] NC swap_g_operand -> sat (want sat) + framing + wiring bugs, FULL 7-round pipeline, concrete: + [PASS ] NC swap_a_b @7 -> sat (want sat) 6.7s + [PASS ] NC tag_changed @7 -> sat (want sat) 7.0s + [PASS ] NC tag_omitted @7 -> sat (want sat) 5.9s + [PASS ] NC truncate_high_half @7 -> sat (want sat) 5.8s + [PASS ] NC flags_parent @7 -> sat (want sat) 7.1s + [PASS ] NC flags_no_root @7 -> sat (want sat) 7.4s + [PASS ] NC block_len_64 @7 -> sat (want sat) 5.2s + [PASS ] NC block_len_32 @7 -> sat (want sat) 4.8s + [PASS ] NC counter_one @7 -> sat (want sat) 5.1s + [PASS ] NC cv_zero @7 -> sat (want sat) 5.0s + [PASS ] NC lanes_big_endian @7 -> sat (want sat) 4.3s + [PASS ] NC tag_slot_moved @7 -> sat (want sat) 5.1s + [PASS ] NC msg_perm_swapped @7 -> sat (want sat) 5.1s + [PASS ] NC rounds_6_not_7 @7 -> sat (want sat) 4.2s + [PASS ] NC drop_ff_xor @7 -> sat (want sat) 5.0s + [PASS ] NC swap_g_operand @7 -> sat (want sat) 4.6s + [PASS ] NC drop_add2_carry @7 -> sat (want sat) 95.4s + framing + wiring bugs, FULL 6-round pipeline, concrete: + [PASS ] NC swap_a_b @6 -> sat (want sat) 3.7s + [PASS ] NC tag_changed @6 -> sat (want sat) 4.0s + [PASS ] NC tag_omitted @6 -> sat (want sat) 3.8s + [PASS ] NC truncate_high_half @6 -> sat (want sat) 4.3s + [PASS ] NC flags_parent @6 -> sat (want sat) 4.2s + [PASS ] NC flags_no_root @6 -> sat (want sat) 4.1s + [PASS ] NC block_len_64 @6 -> sat (want sat) 3.8s + [PASS ] NC block_len_32 @6 -> sat (want sat) 3.8s + [PASS ] NC counter_one @6 -> sat (want sat) 4.2s + [PASS ] NC cv_zero @6 -> sat (want sat) 4.1s + [PASS ] NC lanes_big_endian @6 -> sat (want sat) 4.1s + [PASS ] NC tag_slot_moved @6 -> sat (want sat) 4.1s + [PASS ] NC msg_perm_swapped @6 -> sat (want sat) 3.7s + [PASS ] NC rounds_7_not_6 @6 -> sat (want sat) 4.5s + [PASS ] NC drop_ff_xor @6 -> sat (want sat) 4.3s + [PASS ] NC swap_g_operand @6 -> sat (want sat) 3.6s + [PASS ] NC drop_add2_carry @6 -> sat (want sat) 11.2s + +--- DOCUMENTED BV BLINDNESS (why the FIELD domain is mandatory) --- + [PASS ] BV drop_carry_bool (add3 carry COLUMNS) invisible in BV -> WA4 has the field verdict -> unsat (want unsat) 36.6s + +--- OPTIONAL TAIL TRUNCATION (last-round diagonal X4/B2 omitted) --- + [PASS ] TT tail-truncated 7-round pipeline == anchored KAT -> sat (want sat) 4.5s + [PASS ] TT tail-truncated pipeline EXCLUDES a wrong digest -> unsat (want unsat) 4.9s + saves 112 cell-equiv of 5509 (2.0%) + +--- NON-VACUITY --- + [PASS ] NV honest system satisfiable @rounds=7 (not vacuous) -> sat (want sat) 4.4s + +--- WIDTH AUDIT (FIELD, mod p) -- bound necessity; BV cannot see these --- + [PASS ] WA1 lane decomposition, AreBytes PRESENT -> unsat (want unsat) + [PASS ] WA1 lane decomposition, AreBytes DROPPED -> sat (want sat) + [PASS ] WA2 lane < 2^32 forced, AreBytes PRESENT -> unsat (want unsat) + [PASS ] WA2 lane < 2^32 forced, AreBytes DROPPED -> sat (want sat) + [PASS ] WA3 shift SLL bound PRESENT (r=9) -> unsat (want unsat) + [PASS ] WA3 shift SLL bound DROPPED (r=9) -> sat (want sat) + [PASS ] WA4 add3 carry booleanity PRESENT -> unsat (want unsat) + [PASS ] WA4 add3 carry booleanity DROPPED -> sat (want sat) + [PASS ] WA5 tail case: rotation WORD value still pinned -> unsat (want unsat) + [PASS ] WA5 tail case: rotation BYTES not pinned (hazard is real) -> sat (want sat) + [PASS ] WA6 no-wrap side condition (worst 2^34 < p) -> ok (want ok) + [PASS ] WA7 add2 expression-carry pins s (s byte-bound) -> unsat (want unsat) + [PASS ] WA7 add2 expression-carry, s bound DROPPED -> sat (want sat) + +--- BLOCK-0 FRAMING AUDIT (FIELD) -- the four constraints BV cannot reach --- + [PASS ] B0a capacity prefix pinned, MODE_P=0 PRESENT (idx 0-3,5) -> unsat (want unsat) + [PASS ] B0a capacity prefix, MODE_P pin DROPPED (idx 5 gone) -> sat (want sat) + [PASS ] B0b MU booleanity from mode-sum (idx 4,5) PRESENT -> unsat (want unsat) + [PASS ] B0b MU booleanity DROPPED -> sat (want sat) + [PASS ] B0c upper OUT lanes pinned to 0 (idx 14-21) PRESENT -> unsat (want unsat) + [PASS ] B0c upper OUT lane pins DROPPED -> sat (want sat) + +--- M8 (FOUR-way one-hot): what makes the mode-selected m[8] trustworthy --- + [PASS ] M8 forged tag reachable with idx 4 ALONE (no one-hot) -> sat (want sat) + [PASS ] M8 forged tag EXCLUDED once four-way one-hot is present -> unsat (want unsat) + [PASS ] M8 honest leg: TAG_LFMC still reachable under one-hot -> sat (want sat) + [PASS ] M8 honest leg: TAG_LFMT still reachable under one-hot -> sat (want sat) + [PASS ] M8 honest leg: TAG_LFML still reachable under one-hot -> sat (want sat) + +--- D1 UNREAD-INPUT PINS (8, both cells) --- + [PASS ] D1 no honest row over-constrained (pin selectors vs arities) -> unsat (want unsat) + [PASS ] D1 pins are INERT on BLAKE3 -- necessity rests on the Rust controls (see the docstring) -> unsat (want unsat) + +--- LEAF MODE (MODE_L): canonicity, and the gating-split hazard --- + [PASS ] WA8 leaf canonicity PRESENT -> non-canonical pair unprovable -> unsat (want unsat) + [PASS ] WA8 leaf canonicity DROPPED -> a felt gets a 2nd half-pair -> sat (want sat) + [PASS ] WA9 AreBytes still covers leaf rows (as built) -> map injective -> unsat (want unsat) + [PASS ] WA9 AreBytes NARROWED to digest modes -> canonicity goes VACUOUS -> sat (want sat) + +--- ARGUED LEDGER (algebra, not solver output -- stated, not hidden) --- + AR1: F_p has no zero divisors (p prime), so `x*(1-x) = 0` has root set exactly {0,1}. + relied on by: add3 carry booleanity (WA4); mode-sum booleanity (B0b) + AR2: 2^{-32} is a unit in F_p, so `d * 2^{-32} in {0,1}` iff `d in {0, 2^32}` -- multiplication by a unit is a bijection and maps the root set exactly. + relied on by: add2 expression-carry (WA7) + AR3: 2^16 is invertible mod p, which is what makes the tight AreBytes bound on SLL pin the shift remainder uniquely. + relied on by: rotation shift identity (WA3) + AR4: every field-lifted expression stays below 2^34 << p, so `expr = 0 mod p` implies `expr = 0` over the integers. + relied on by: all identities (WA6) + +--- COST CENSUS (derived from the gated model, not hand-counted) --- + 7-round: main= 3436 sends= 1382 aux= 2073 cell-equiv= 5509 + 6-round: main= 2956 sends= 1190 aux= 1785 cell-equiv= 4741 + rotr_shift 1344 + xor_out 912 + add3 672 + add2 448 + lane_bytes(MB) 32 + frozen_socket_prefix(IN/S/OUT) 28 + +============================================================================== +GATE VERDICT: PASS +============================================================================== +EXIT=0 diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/run-gate-PRE-PHASE2-SUPERSEDED.log b/thoughts/shared/lfm-real-hash/gate-oracle/run-gate-PRE-PHASE2-SUPERSEDED.log new file mode 100644 index 000000000..4a659ea3c --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/run-gate-PRE-PHASE2-SUPERSEDED.log @@ -0,0 +1,99 @@ +################################################################################ +# SUPERSEDED -- DO NOT CITE THIS BOARD +# +# This is the PRE-PHASE-2 board (49 checks), run against chip_model.py when it +# still modelled a WITNESSED add2 carry column and a 13-cell socket prefix. Its +# census figures (main 3,533 / 3,037, cell-equiv 5,606 / 4,822) are therefore +# NOT the built chip's and contradict the current numbers. +# +# CURRENT BOARD OF RECORD: run-chip-gate.log (75 checks, both round counts, +# transcribed from the committed chip b693eece) +# Census of record: CHIP-GATE.md section 3 +# +# Kept only as the evidence behind ORACLE.md section 0's 49-check claim. +################################################################################ + +============================================================================== +BLAKE3-behind-LFM_HASH -- z3 GATE (Option A socket) +============================================================================== + +--- MAIN THEOREMS (symbolic, BV) -- want UNSAT --- + [PASS ] T1 G quarter-round, free inputs (covers every G) -> unsat (want unsat) 400.1s + [PASS ] T2 message schedule, all 7 rounds (placement/tag/LE/perm) -> unsat (want unsat) + [PASS ] T3 framing @rounds=0 (init state/feed-forward/window) -> unsat (want unsat) + per-theorem discrimination controls -- want SAT: + [PASS ] T2-ctl swap_a_b -> sat (want sat) + [PASS ] T2-ctl tag_changed -> sat (want sat) + [PASS ] T2-ctl tag_omitted -> sat (want sat) + [PASS ] T2-ctl tag_slot_moved -> sat (want sat) + [PASS ] T2-ctl lanes_big_endian -> sat (want sat) + [PASS ] T2-ctl msg_perm_swapped -> sat (want sat) + [PASS ] T3-ctl cv_zero -> sat (want sat) + [PASS ] T3-ctl truncate_high_half -> sat (want sat) + [PASS ] T3-ctl drop_ff_xor -> sat (want sat) + +--- T4 FULL PIPELINE, CONCRETE, vs the anchored KATs --- + [PASS ] T4 full 7-round pipeline == anchored KAT -> sat (want sat) 12.1s + [PASS ] T4 full 7-round pipeline EXCLUDES a wrong digest -> unsat (want unsat) 2.8s + [PASS ] T4 full 6-round pipeline == anchored KAT -> sat (want sat) 9.6s + [PASS ] T4 full 6-round pipeline EXCLUDES a wrong digest -> unsat (want unsat) 2.4s + +--- NEGATIVE CONTROLS -- want SAT (an UNSAT here means the gate is BLIND) --- + logic bugs, symbolic at G level: + [PASS ] NC rot_wrong_amount -> sat (want sat) + [PASS ] NC swap_g_operand -> sat (want sat) + framing + wiring bugs, against the FULL 7-round pipeline, concrete: + [PASS ] NC swap_a_b -> sat (want sat) 12.0s + [PASS ] NC tag_changed -> sat (want sat) 2.8s + [PASS ] NC tag_omitted -> sat (want sat) 12.3s + [PASS ] NC truncate_high_half -> sat (want sat) 2.9s + [PASS ] NC flags_parent -> sat (want sat) 13.0s + [PASS ] NC flags_no_root -> sat (want sat) 2.8s + [PASS ] NC block_len_64 -> sat (want sat) 13.0s + [PASS ] NC block_len_32 -> sat (want sat) 2.8s + [PASS ] NC counter_one -> sat (want sat) 12.7s + [PASS ] NC cv_zero -> sat (want sat) 2.9s + [PASS ] NC lanes_big_endian -> sat (want sat) 13.2s + [PASS ] NC tag_slot_moved -> sat (want sat) 3.0s + [PASS ] NC msg_perm_swapped -> sat (want sat) 12.7s + [PASS ] NC rounds_6_not_7 -> sat (want sat) 2.4s + [PASS ] NC drop_ff_xor -> sat (want sat) 12.8s + [PASS ] NC swap_g_operand (full pipeline) -> sat (want sat) 2.9s + +--- DOCUMENTED BV BLINDNESS (why the FIELD domain is mandatory) --- + [PASS ] BV drop_carry_bool is INVISIBLE in BV (see WA4 for the field verdict) -> unsat (want unsat) 19.6s + +--- OPTIONAL TAIL TRUNCATION (last-round diagonal X4/B2 omitted) --- + [PASS ] TT tail-truncated 7-round pipeline == anchored KAT -> sat (want sat) 13.8s + [PASS ] TT tail-truncated pipeline EXCLUDES a wrong digest -> unsat (want unsat) 2.9s + saves 112 cell-equiv of 5606 (2.0%) + +--- NON-VACUITY --- + [PASS ] NV honest system satisfiable @rounds=7 (not vacuous) -> sat (want sat) 24.2s + +--- WIDTH AUDIT (FIELD, mod p) -- bound necessity; BV cannot see these --- + [PASS ] WA1 lane decomposition, AreBytes PRESENT -> unsat (want unsat) + [PASS ] WA1 lane decomposition, AreBytes DROPPED -> sat (want sat) + [PASS ] WA2 lane < 2^32 forced, AreBytes PRESENT -> unsat (want unsat) + [PASS ] WA2 lane < 2^32 forced, AreBytes DROPPED -> sat (want sat) + [PASS ] WA3 shift SLL bound PRESENT (r=9) -> unsat (want unsat) + [PASS ] WA3 shift SLL bound DROPPED (r=9) -> sat (want sat) + [PASS ] WA4 add3 carry booleanity PRESENT -> unsat (want unsat) + [PASS ] WA4 add3 carry booleanity DROPPED -> sat (want sat) + [PASS ] WA5 tail case: rotation WORD value still pinned -> unsat (want unsat) + [PASS ] WA5 tail case: rotation BYTES not pinned (hazard is real) -> sat (want sat) + [PASS ] WA6 no-wrap side condition (worst 2^34 < p) -> ok (want ok) + +--- COST CENSUS (derived from the gated model, not hand-counted) --- + 7-round: main= 3533 sends= 1382 aux= 2073 cell-equiv= 5606 + 6-round: main= 3037 sends= 1190 aux= 1785 cell-equiv= 4822 + rotr_shift 1344 + xor_out 912 + add3 672 + add2 560 + lane_bytes(MB) 32 + digest_out_felts+in_felts+MU 13 + +============================================================================== +GATE VERDICT: PASS +============================================================================== diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/run-kats.log b/thoughts/shared/lfm-real-hash/gate-oracle/run-kats.log new file mode 100644 index 000000000..bc9640178 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/run-kats.log @@ -0,0 +1,25 @@ +========================================================================== +LAYER 2 -- socket KATs +========================================================================== + vectors : 20 (10 inputs x 2 round counts) + framing controls evaluated : 254 + framing degrees of freedom : 14 + swap_a_b discriminated by 16 vector-instances + tag_changed discriminated by 20 vector-instances + tag_omitted discriminated by 20 vector-instances + truncate_high_half discriminated by 20 vector-instances + flags_parent discriminated by 20 vector-instances + flags_no_root discriminated by 20 vector-instances + block_len_64 discriminated by 20 vector-instances + block_len_32 discriminated by 20 vector-instances + counter_one discriminated by 20 vector-instances + cv_zero discriminated by 20 vector-instances + lanes_big_endian discriminated by 12 vector-instances + tag_slot_moved discriminated by 20 vector-instances + msg_perm_swapped discriminated by 16 vector-instances + rounds_6_not_7 discriminated by 10 vector-instances + [PASS] cross-check PASS: 20 peer vectors reproduced exactly (spec fields agree too) + + wrote /Users/maurofab/workspace/lambda_vm/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.json +-------------------------------------------------------------------------- +LAYER 2: PASS diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.json b/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.json new file mode 100644 index 000000000..a717355f9 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.json @@ -0,0 +1,1162 @@ +{ + "socket": "LFM_HASH 2-to-1 BLAKE3 compress (Option A + domain tag)", + "spec": { + "digest_lanes": 4, + "digest_bits": 128, + "domain_tag_ascii": "LFMC", + "domain_tag_word": 1129137740, + "chaining_value_in": "BLAKE3 IV[0..8]", + "counter": 0, + "block_len": 36, + "flags": 11, + "flags_meaning": "CHUNK_START|CHUNK_END|ROOT", + "message_layout": "m[0..4]=a, m[4..8]=b, m[8]=tag, m[9..16]=0", + "truncation_window": "out[0..4] (the LOW four of 16 output words)", + "lane_serialisation": "one felt = one u32 = four little-endian bytes (keccak_host convention, NOT word::pack_digest)" + }, + "rounds": { + "6": [ + { + "name": "zeros", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 2809853715, + 2395900105, + 421057723, + 4135460974 + ], + "digest_lanes_hex": "a77af7138ece88c91918d4bbf67e206e", + "digest_bytes_hex": "13f77aa7c988ce8ebbd418196e207ef6", + "full_blake3_32B_hex": "13f77aa7c988ce8ebbd418196e207ef6a3b9cefd6055504eb6de0f527873cc75", + "negative_controls": { + "tag_changed": "4e3b06b26312fb30c1dd90d7b84d4af7", + "tag_omitted": "352128092537bd88c388742c16735c3e", + "truncate_high_half": "fdceb9a34e505560520fdeb675cc7378", + "flags_parent": "146216bdb493af6cc6926d8ce2793fa6", + "flags_no_root": "0b49fde18d8e680cf489f7f9e5a1101d", + "block_len_64": "2c70504d328661d1708608167472be04", + "block_len_32": "b1e72e7905309c0451532588a2e164dd", + "counter_one": "94a9d96181b42b61b22cdd0190b19a77", + "cv_zero": "6ce66e5e8c35ff8034cee1136dc5dd88", + "tag_slot_moved": "68eb023f1dbe8b1196d78eaec9642cd7" + }, + "controls_inapplicable": [ + "swap_a_b", + "lanes_big_endian", + "msg_perm_swapped", + "rounds_6_not_7" + ] + }, + { + "name": "unit_a", + "a": [ + 1, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "01000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 3558314982, + 1135936504, + 1880898970, + 561388701 + ], + "digest_lanes_hex": "d41793e643b503f8701c3d9a21761c9d", + "digest_bytes_hex": "e69317d4f803b5439a3d1c709d1c7621", + "full_blake3_32B_hex": "e69317d4f803b5439a3d1c709d1c762185faa4d14a4cdfb0a1b577eda9d21518", + "negative_controls": { + "swap_a_b": "9204d33ac3ce7c22023d183839247c70", + "tag_changed": "c855a93c1148cbd234c3622c672e8bf9", + "tag_omitted": "c1fe0fc6573abe94ec8f503e603e39aa", + "truncate_high_half": "d1a4fa85b0df4c4aed77b5a11815d2a9", + "flags_parent": "4f27dea055f1ba715d5fdbab86b2ccbd", + "flags_no_root": "97d8a406c8c3b1aeda690a388fba32e7", + "block_len_64": "ce03ba6b92f2aee40b239d564dfa5735", + "block_len_32": "b452db8ca1c9ac6d77c58f121c56cc08", + "counter_one": "0591bd434c3d1d6bdaaf09de5e719037", + "cv_zero": "0e2047a1e8dfd09008cae73ab2da9fca", + "lanes_big_endian": "39ba5d0ec334e72cc8a20f7c6d46b2a6", + "tag_slot_moved": "c3073aca57879bf534db8ad756cbf4c9" + }, + "controls_inapplicable": [ + "msg_perm_swapped", + "rounds_6_not_7" + ] + }, + { + "name": "unit_b", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 1, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000010000000000000000000000000000004c464d43", + "digest": [ + 2449789754, + 3285089314, + 37558328, + 958692464 + ], + "digest_lanes_hex": "9204d33ac3ce7c22023d183839247c70", + "digest_bytes_hex": "3ad30492227ccec338183d02707c2439", + "full_blake3_32B_hex": "3ad30492227ccec338183d02707c2439ad24af458d05428457fe5921792060f5", + "negative_controls": { + "swap_a_b": "d41793e643b503f8701c3d9a21761c9d", + "tag_changed": "71640fd4ef1bea0785b59ae8b52cc2cd", + "tag_omitted": "4d1f531355bbbc37b5f96f631e2a51f0", + "truncate_high_half": "45af24ad8442058d2159fe57f5602079", + "flags_parent": "be444621fda85db79f5498e9cff3af46", + "flags_no_root": "1053ae795356dd8a215a0be2960584f0", + "block_len_64": "150ce35bf1bb857eb95577e36991e3bc", + "block_len_32": "8e062dcfe814767995cedb6a6773d2ff", + "counter_one": "4c426206df625fd24cbe7104383a8b8f", + "cv_zero": "2480cbfaa15e407d7380de10d580b329", + "lanes_big_endian": "15a44481c0ca23a9a56f9cf43d8af6a0", + "tag_slot_moved": "5bfeb4fe43fff40ee000caa9ee2557dd", + "msg_perm_swapped": "3ce1bd1ed6fea1eedcad8c87e8fc6ea4" + }, + "controls_inapplicable": [ + "rounds_6_not_7" + ] + }, + { + "name": "all_ones", + "a": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "b": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "message_bytes_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4c464d43", + "digest": [ + 481531477, + 1181580457, + 2833532478, + 1295194463 + ], + "digest_lanes_hex": "1cb39655466d7ca9a8e4463e4d33195f", + "digest_bytes_hex": "5596b31ca97c6d463e46e4a85f19334d", + "full_blake3_32B_hex": "5596b31ca97c6d463e46e4a85f19334d7f6c9c4d7425904412031d12ec8b3bc1", + "negative_controls": { + "tag_changed": "f2c42a1e2b6ce9bbe1f398cb8152eade", + "tag_omitted": "78334cb81c61ead22da2aa2058ecd193", + "truncate_high_half": "4d9c6c7f44902574121d0312c13b8bec", + "flags_parent": "46bdc476750e6ce1ddd5aa6dd0aba788", + "flags_no_root": "5f6110aa1219601e9494813c4554168c", + "block_len_64": "838fcf10b069399ea5ce9b714467d0f0", + "block_len_32": "d26ddc80ed45f6f2928d2d5d2520dfe2", + "counter_one": "0d3fbfe4ed5a93772bb0b8b0aa43ca22", + "cv_zero": "026537571b5285c339ea4c0f456393cd", + "tag_slot_moved": "f325a27cf7642cb8b1ec1e7b3c39b551", + "msg_perm_swapped": "f33fa56bbc3e738fa7c035fbffe0d2e3" + }, + "controls_inapplicable": [ + "swap_a_b", + "lanes_big_endian", + "rounds_6_not_7" + ] + }, + { + "name": "nibble_ramp", + "a": [ + 0, + 286331153, + 572662306, + 858993459 + ], + "b": [ + 1145324612, + 1431655765, + 1717986918, + 2004318071 + ], + "message_bytes_hex": "00000000111111112222222233333333444444445555555566666666777777774c464d43", + "digest": [ + 788131140, + 1263186933, + 1810255302, + 3669948337 + ], + "digest_lanes_hex": "2ef9ed444b4ab3f56be64dc6dabef7b1", + "digest_bytes_hex": "44edf92ef5b34a4bc64de66bb1f7beda", + "full_blake3_32B_hex": "44edf92ef5b34a4bc64de66bb1f7beda7fcc336c120bacd6f6abfed12df8d48e", + "negative_controls": { + "swap_a_b": "0e05578a5482f609247efb751b6b44ea", + "tag_changed": "3c3104ba2baff75dc4ffa27f513dfef6", + "tag_omitted": "5c4b9579d95579c5b3a590abb9d7c1d2", + "truncate_high_half": "6c33cc7fd6ac0b12d1feabf68ed4f82d", + "flags_parent": "9318e6cd872e8b40cec83d211d819ddf", + "flags_no_root": "37e885a641ec1bf1ffc6cd33bf17783e", + "block_len_64": "00de032d969e9c3ca246092b4197f3d3", + "block_len_32": "b74cfedb2f9894faaac7e010ae6c50ee", + "counter_one": "79cad6ec590585a11e9eabe9f20095a6", + "cv_zero": "1b538dbf0afab1f59b573e5719a3a680", + "tag_slot_moved": "e5b39417a93d09f74d4fd3d00b3c34ef", + "msg_perm_swapped": "52ade08777289fa8b78b5059146982b0" + }, + "controls_inapplicable": [ + "lanes_big_endian", + "rounds_6_not_7" + ] + }, + { + "name": "max_min", + "a": [ + 4294967295, + 0, + 4294967295, + 0 + ], + "b": [ + 0, + 4294967295, + 0, + 4294967295 + ], + "message_bytes_hex": "ffffffff00000000ffffffff0000000000000000ffffffff00000000ffffffff4c464d43", + "digest": [ + 1129923381, + 3615082472, + 1078087193, + 2637432116 + ], + "digest_lanes_hex": "43594335d779c7e840424e199d340534", + "digest_bytes_hex": "35435943e8c779d7194e42403405349d", + "full_blake3_32B_hex": "35435943e8c779d7194e42403405349daf2f4a72c32f9282e2fa8378749e38e0", + "negative_controls": { + "swap_a_b": "5eadd60626c9f8140c1b7c0cbcc5bddf", + "tag_changed": "0c42d90de048f4db7e25489965e9ea8e", + "tag_omitted": "77dfb62c566f015de8f8e9fad46d7447", + "truncate_high_half": "724a2faf82922fc37883fae2e0389e74", + "flags_parent": "0ead17cfe64130e91a4a136de3f7975e", + "flags_no_root": "575a035d10b9786ee3af8bf8389d4866", + "block_len_64": "b4beb2d5e90867bed9bd36c53fe70546", + "block_len_32": "cdfefa544f3d4a204e91d56d784217b5", + "counter_one": "4aa795e8b32ad9469771c0cb9e090f33", + "cv_zero": "d3f032cbe91edf5a74a33edcd30075c1", + "tag_slot_moved": "c9127d3dfd17ae424a89952c3191efbb", + "msg_perm_swapped": "230b3d9903b3caa91cea34ca21fb1122" + }, + "controls_inapplicable": [ + "lanes_big_endian", + "rounds_6_not_7" + ] + }, + { + "name": "formula_1", + "a": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "b": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "message_bytes_hex": "04030201080706050c0b0a09100f0e0d14131211181716151c1b1a19201f1e1d4c464d43", + "digest": [ + 3123320983, + 1967507865, + 3072761050, + 2317957007 + ], + "digest_lanes_hex": "ba2a18977545c999b7269cda8a29378f", + "digest_bytes_hex": "97182aba99c94575da9c26b78f37298a", + "full_blake3_32B_hex": "97182aba99c94575da9c26b78f37298a9d5fefe278deb8a2f943a66f9345ff74", + "negative_controls": { + "swap_a_b": "578f2505eb526f2c792421ca92bc775e", + "tag_changed": "c3f58f7414d41179c70e9dfdecc85a1d", + "tag_omitted": "fb8f942c5528af636bf39d7a7b3c7f3a", + "truncate_high_half": "e2ef5f9da2b8de786fa643f974ff4593", + "flags_parent": "954f4102a49bc06e17d79f6f3542f1de", + "flags_no_root": "a32f300afba15452f66bc4af0e189b25", + "block_len_64": "562371b3a4a583456bf838a44d834c45", + "block_len_32": "055e64e4fd8d02fafa9c636cb2d1b351", + "counter_one": "f7df1092e1ace8d743dbcd1419a72a05", + "cv_zero": "6cd3cb0f115b972068069c31ccef009c", + "lanes_big_endian": "23aa16d2400a7c0a7f09bb317a0bb4a7", + "tag_slot_moved": "8e039279af18f86b8e3a8ed2e536f343", + "msg_perm_swapped": "702e7b934aa653dc67b2050e3c139c4e" + }, + "controls_inapplicable": [ + "rounds_6_not_7" + ] + }, + { + "name": "formula_2", + "a": [ + 3735928559, + 3405691582, + 2343432205, + 4277009102 + ], + "b": [ + 195936478, + 3512640997, + 3237998080, + 3131746989 + ], + "message_bytes_hex": "efbeaddebebafeca0df0ad8bcefaedfedec0ad0be5a55ed100eeffc0adaaaaba4c464d43", + "digest": [ + 695838104, + 2007981278, + 1468357915, + 815839415 + ], + "digest_lanes_hex": "2979a59877af5cde57855d1b30a0b8b7", + "digest_bytes_hex": "98a57929de5caf771b5d8557b7b8a030", + "full_blake3_32B_hex": "98a57929de5caf771b5d8557b7b8a030f3638996507cf2a832f15db1dfa39977", + "negative_controls": { + "swap_a_b": "dc9fb22a568178acfab5e4b49ca053b6", + "tag_changed": "b44e628153fbb86ddd336b13f29fa2b1", + "tag_omitted": "da9131e0ba343b266909d31340fdbe67", + "truncate_high_half": "968963f3a8f27c50b15df1327799a3df", + "flags_parent": "19a8b3d15d9e4161cd6a728366eac123", + "flags_no_root": "79981f0b4ecaf1175a30bc45541c6fdc", + "block_len_64": "f9dfaff2ad332ecc3605daa3771fc0e3", + "block_len_32": "59ba68ad23969456cf20595f2dc891af", + "counter_one": "f811c1dac391e8d9de85271215e78c6f", + "cv_zero": "ef5e45a880b3c5c8caa360bf2deb827d", + "lanes_big_endian": "04ad0496f299795d3324a232b8de6d56", + "tag_slot_moved": "8e2d1e118b7de60d4c77cf3ca4aa3b3a", + "msg_perm_swapped": "47dc6684bb25e97e4eaa01d233566f17" + }, + "controls_inapplicable": [ + "rounds_6_not_7" + ] + }, + { + "name": "formula_3", + "a": [ + 2139095041, + 2, + 2147483648, + 2147483647 + ], + "b": [ + 16711935, + 4278255360, + 252645135, + 4042322160 + ], + "message_bytes_hex": "0100807f0200000000000080ffffff7fff00ff0000ff00ff0f0f0f0ff0f0f0f04c464d43", + "digest": [ + 239487879, + 4202600110, + 1114311674, + 2088372354 + ], + "digest_lanes_hex": "0e464b87fa7e96ae426b0bfa7c7a0882", + "digest_bytes_hex": "874b460eae967efafa0b6b4282087a7c", + "full_blake3_32B_hex": "874b460eae967efafa0b6b4282087a7c4c2fd5d18d01c16d29d433ea86e32be9", + "negative_controls": { + "swap_a_b": "326fa9b67a4e77fc5d680d57cff56be0", + "tag_changed": "6dcafcba246776cf73d5042f2036e7ba", + "tag_omitted": "09eae11ead2c6d2f99e612a127494a3d", + "truncate_high_half": "d1d52f4c6dc1018dea33d429e92be386", + "flags_parent": "67f483afa3936eedc0dcdb0254a63770", + "flags_no_root": "87cf577404850cb1989a6a530cca8efc", + "block_len_64": "0b0b9e9879ad36c5cb429a00a09cc832", + "block_len_32": "605518c738e366f439d7a0f5fc693f5b", + "counter_one": "c2a5d22b1454b91462c1ea6df361e430", + "cv_zero": "d77dec782519bcf963790949ad758f5f", + "lanes_big_endian": "a850f57c1cf70abcc5f7973ea5bac826", + "tag_slot_moved": "61941f28ab383680ccdfbea39fcde6b5", + "msg_perm_swapped": "c137c58dfb2d9b8ccdfc53b676047158" + }, + "controls_inapplicable": [ + "rounds_6_not_7" + ] + }, + { + "name": "boundary", + "a": [ + 0, + 1, + 4294967294, + 4294967295 + ], + "b": [ + 2147483648, + 2147483647, + 65536, + 65535 + ], + "message_bytes_hex": "0000000001000000feffffffffffffff00000080ffffff7f00000100ffff00004c464d43", + "digest": [ + 1019641822, + 4283204685, + 3695458577, + 3681139715 + ], + "digest_lanes_hex": "3cc67fdeff4c844ddc443911db69bc03", + "digest_bytes_hex": "de7fc63c4d844cff113944dc03bc69db", + "full_blake3_32B_hex": "de7fc63c4d844cff113944dc03bc69db29673a356520107663e5cbb376e5b1ed", + "negative_controls": { + "swap_a_b": "d1946d047814438bf55caa2d2eb37d97", + "tag_changed": "5eb0b5d0b41f1de169d4dccd86cdf899", + "tag_omitted": "a29ed5f9c7c302f536c9487e59f1cc64", + "truncate_high_half": "353a672976102065b3cbe563edb1e576", + "flags_parent": "e22222224323cfbf72a6fc843dc40b2b", + "flags_no_root": "795bc0422979d5333b4e1763591b1488", + "block_len_64": "0a972ab4f3aac0bee2abfdfbb2ca19a2", + "block_len_32": "585962c3bc2b45933481f3476a88a703", + "counter_one": "fdbe545eb73ea8452b4e68e66a1413d2", + "cv_zero": "e05be0a47b333eddebce39956107c0d9", + "lanes_big_endian": "6a42e84e3513e9be3753be5ab5f21bd6", + "tag_slot_moved": "f2746376852a7fc31fc4f16b567bf517", + "msg_perm_swapped": "80db89b19b8607082dafc7a5276bfa29" + }, + "controls_inapplicable": [ + "rounds_6_not_7" + ] + } + ], + "7": [ + { + "name": "zeros", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 2494038600, + 807496444, + 2349420159, + 3886468141 + ], + "digest_lanes_hex": "94a8024830216afc8c094e7fe7a6cc2d", + "digest_bytes_hex": "4802a894fc6a21307f4e098c2dcca6e7", + "full_blake3_32B_hex": "4802a894fc6a21307f4e098c2dcca6e7fc7d0fa72963ad16b7f2f5b3fe8ebf84", + "negative_controls": { + "tag_changed": "e2c470af3f8bac8bed4fa9b926a7b265", + "tag_omitted": "bf975024db4970bd00a8b92ff170b04c", + "truncate_high_half": "a70f7dfc16ad6329b3f5f2b784bf8efe", + "flags_parent": "a40745c3ba993df1f6ef2368bf2b403a", + "flags_no_root": "6396e07f44b6434d84102d89874103fd", + "block_len_64": "e8a536181c134e070ae800103913b680", + "block_len_32": "6988e18b556c32c9cd1e6db56cf02aaf", + "counter_one": "020a0686cae49f72e6b21d7b31a1ff6c", + "cv_zero": "ba7a5348e93f2d6aab6d8d6027fb8d81", + "tag_slot_moved": "37019100d59e5a03c70b121bca61a9c9", + "rounds_6_not_7": "a77af7138ece88c91918d4bbf67e206e" + }, + "controls_inapplicable": [ + "swap_a_b", + "lanes_big_endian", + "msg_perm_swapped" + ] + }, + { + "name": "unit_a", + "a": [ + 1, + 0, + 0, + 0 + ], + "b": [ + 0, + 0, + 0, + 0 + ], + "message_bytes_hex": "01000000000000000000000000000000000000000000000000000000000000004c464d43", + "digest": [ + 3104074695, + 1974443198, + 2882972316, + 1734279477 + ], + "digest_lanes_hex": "b9046bc775af9cbeabd6aa9c675f0135", + "digest_bytes_hex": "c76b04b9be9caf759caad6ab35015f67", + "full_blake3_32B_hex": "c76b04b9be9caf759caad6ab35015f67f58236136aa594f04f37d4fd228effdd", + "negative_controls": { + "swap_a_b": "86d366c5e620a872f8340f7bd08847c1", + "tag_changed": "2a8a3fb876902bc7b1fa23e1c9cd20f7", + "tag_omitted": "2bf60bb29b2edab71fa418c875f29f77", + "truncate_high_half": "133682f5f094a56afdd4374fddff8e22", + "flags_parent": "08b17808908f02afe03a1ea8f06c015c", + "flags_no_root": "b15b774249f199f98a2257b4abcc6f0e", + "block_len_64": "096e4192024ad5ee956e6fb5c8c76b29", + "block_len_32": "01917337c3db776d57e781bc01af174f", + "counter_one": "3498771c2478a631309609bc15a7e070", + "cv_zero": "52edf93124ce758dea7b9c6f0bc88302", + "lanes_big_endian": "a6332f74c17a0234cfe5008d91226d78", + "tag_slot_moved": "dc186367157214385cd1b500504c25b3", + "rounds_6_not_7": "d41793e643b503f8701c3d9a21761c9d" + }, + "controls_inapplicable": [ + "msg_perm_swapped" + ] + }, + { + "name": "unit_b", + "a": [ + 0, + 0, + 0, + 0 + ], + "b": [ + 1, + 0, + 0, + 0 + ], + "message_bytes_hex": "00000000000000000000000000000000010000000000000000000000000000004c464d43", + "digest": [ + 2262001349, + 3860899954, + 4164161403, + 3498592193 + ], + "digest_lanes_hex": "86d366c5e620a872f8340f7bd08847c1", + "digest_bytes_hex": "c566d38672a820e67b0f34f8c14788d0", + "full_blake3_32B_hex": "c566d38672a820e67b0f34f8c14788d04bdfd0fa1ab2d9631965cfb01294a0e1", + "negative_controls": { + "swap_a_b": "b9046bc775af9cbeabd6aa9c675f0135", + "tag_changed": "204fe79819fe95bb994beb42decc2d42", + "tag_omitted": "75f6d73b882e8f2a1b9ec0d42665b307", + "truncate_high_half": "fad0df4b63d9b21ab0cf6519e1a09412", + "flags_parent": "b432a51632028da6691d070542a2539e", + "flags_no_root": "1fa50c9255cde21972f31c790ed53d22", + "block_len_64": "2ac333251ad3b12a38cb70b9bf6331d2", + "block_len_32": "ba624e7ad5208f07caa67ba940d9b0a5", + "counter_one": "2afdb7d9d202628905bae28a6aa3aaa5", + "cv_zero": "09c940704a86900d024ce621e441ac1a", + "lanes_big_endian": "301fb0e21d8c754ad378664f1bfdaa80", + "tag_slot_moved": "853f0280598a6a393b4abd31c16dfe5e", + "msg_perm_swapped": "ac86706e425894de75541f2af213fec8", + "rounds_6_not_7": "9204d33ac3ce7c22023d183839247c70" + }, + "controls_inapplicable": [] + }, + { + "name": "all_ones", + "a": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "b": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "message_bytes_hex": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4c464d43", + "digest": [ + 512112064, + 2194416191, + 3337763018, + 1475985439 + ], + "digest_lanes_hex": "1e8635c082cc223fc6f238ca57f9c01f", + "digest_bytes_hex": "c035861e3f22cc82ca38f2c61fc0f957", + "full_blake3_32B_hex": "c035861e3f22cc82ca38f2c61fc0f95759f8100aa7ac004473813b9d103603a9", + "negative_controls": { + "tag_changed": "2cb5534b07659c90d771a72d864ab92d", + "tag_omitted": "8245b166f2a97682186cb1d1b1ec6fc9", + "truncate_high_half": "0a10f8594400aca79d3b8173a9033610", + "flags_parent": "82fbc95ca9163a5bc0f1dd672aaeb648", + "flags_no_root": "8aa9e67f0e7b0f860f186ba2cbf5c443", + "block_len_64": "ee164ab4e0c48eb037064d9cbde6b4b5", + "block_len_32": "ed805a7fd7d0b704b435bb43aa0ef5fb", + "counter_one": "bd84c3d0931a7112206eef7d8b071f40", + "cv_zero": "1c952e2f59d75d2a87e7dcad0d38f103", + "tag_slot_moved": "4241c2ea6473091c6cbc309c6a7fcd18", + "msg_perm_swapped": "59bffa2bd2c4957717d090b971aef6a4", + "rounds_6_not_7": "1cb39655466d7ca9a8e4463e4d33195f" + }, + "controls_inapplicable": [ + "swap_a_b", + "lanes_big_endian" + ] + }, + { + "name": "nibble_ramp", + "a": [ + 0, + 286331153, + 572662306, + 858993459 + ], + "b": [ + 1145324612, + 1431655765, + 1717986918, + 2004318071 + ], + "message_bytes_hex": "00000000111111112222222233333333444444445555555566666666777777774c464d43", + "digest": [ + 447364800, + 1725782825, + 3919861296, + 1641182463 + ], + "digest_lanes_hex": "1aaa3ec066dd5b29e9a4563061d274ff", + "digest_bytes_hex": "c03eaa1a295bdd663056a4e9ff74d261", + "full_blake3_32B_hex": "c03eaa1a295bdd663056a4e9ff74d261051f49096ec2345cde112bda36168bf4", + "negative_controls": { + "swap_a_b": "9b1bc27e411372d4c17e59a81456d5b7", + "tag_changed": "e1e9c09b28234be08daaa9085141a72d", + "tag_omitted": "2b5b974449a87c16c67a229ed64d19b8", + "truncate_high_half": "09491f055c34c26eda2b11def48b1636", + "flags_parent": "3746a1d838efe712ed936910a6706964", + "flags_no_root": "ff9bc313b53a7424dcea0fb104418ede", + "block_len_64": "ac3e02427b42c9772f48c5a64e349b53", + "block_len_32": "ad64fa41ec46f2a5241ddca634e0b62c", + "counter_one": "50a041c1211280795bc7896af7f24523", + "cv_zero": "9123202cac6abe23615ab6040e0791e1", + "tag_slot_moved": "7682355e6de49423ca2f03292e2b72df", + "msg_perm_swapped": "1cf4818630631acdf6738e38cd4ce185", + "rounds_6_not_7": "2ef9ed444b4ab3f56be64dc6dabef7b1" + }, + "controls_inapplicable": [ + "lanes_big_endian" + ] + }, + { + "name": "max_min", + "a": [ + 4294967295, + 0, + 4294967295, + 0 + ], + "b": [ + 0, + 4294967295, + 0, + 4294967295 + ], + "message_bytes_hex": "ffffffff00000000ffffffff0000000000000000ffffffff00000000ffffffff4c464d43", + "digest": [ + 3497144197, + 18127627, + 3188702941, + 1402725093 + ], + "digest_lanes_hex": "d0722f8501149b0bbe0fbedd539be2e5", + "digest_bytes_hex": "852f72d00b9b1401ddbe0fbee5e29b53", + "full_blake3_32B_hex": "852f72d00b9b1401ddbe0fbee5e29b53498355d8ff37cf71aba2f6d1e95e6ae6", + "negative_controls": { + "swap_a_b": "9b842c608deb5391b10534576e35db4f", + "tag_changed": "e1796104a4be7163f2b923085f5ac8c9", + "tag_omitted": "2b2d156743294a6def96740385f9f059", + "truncate_high_half": "d855834971cf37ffd1f6a2abe66a5ee9", + "flags_parent": "a9c1eec4b1094afab152e44f109c313e", + "flags_no_root": "dfe08e98f2bacdd2fbe157c23a51b49a", + "block_len_64": "8ae38d1449b51ab4254a168a4b8c7ac1", + "block_len_32": "8ea52d5571deef1a53ffe4b1189e0e31", + "counter_one": "0e4abd74283aa2c4ae44ac5c7ca59b33", + "cv_zero": "aedc1f103214e3dc88364f68ea7de936", + "tag_slot_moved": "471904a7d43bda1346beec7f522c8564", + "msg_perm_swapped": "ad83d77ac2bdd1644c6f669fe1fb26a9", + "rounds_6_not_7": "43594335d779c7e840424e199d340534" + }, + "controls_inapplicable": [ + "lanes_big_endian" + ] + }, + { + "name": "formula_1", + "a": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "b": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "message_bytes_hex": "04030201080706050c0b0a09100f0e0d14131211181716151c1b1a19201f1e1d4c464d43", + "digest": [ + 239178171, + 1294475087, + 1642291500, + 1239295299 + ], + "digest_lanes_hex": "0e4191bb4d281f4f61e3612c49de2543", + "digest_bytes_hex": "bb91410e4f1f284d2c61e3614325de49", + "full_blake3_32B_hex": "bb91410e4f1f284d2c61e3614325de4993839e4ee713a7f989f54bfa5a78a64e", + "negative_controls": { + "swap_a_b": "8e10203f32773a61cd54cd03f81f850f", + "tag_changed": "46d576163ecad7548e84b06befd53a3e", + "tag_omitted": "6ba8ecb688932bcb54c596d3078d8733", + "truncate_high_half": "4e9e8393f9a713e7fa4bf5894ea6785a", + "flags_parent": "ac1155b1a70d2c50d4bd507e484325b6", + "flags_no_root": "5af5e4d9103d18e3ab4240eaa7929e05", + "block_len_64": "2a5636f225c2848b7f3bbc85bde08c10", + "block_len_32": "385fb312a8b9695a763bfda04a35ab48", + "counter_one": "bbff0da051afcae951d6a1d4f43bf65a", + "cv_zero": "814230df7585d0eea79e87d3b824859c", + "lanes_big_endian": "63644844638abeec2f2c17f7dac51315", + "tag_slot_moved": "c16667b7f4284325d131f4d487efcfa1", + "msg_perm_swapped": "23d4f95a7355c40a268785786a93debb", + "rounds_6_not_7": "ba2a18977545c999b7269cda8a29378f" + }, + "controls_inapplicable": [] + }, + { + "name": "formula_2", + "a": [ + 3735928559, + 3405691582, + 2343432205, + 4277009102 + ], + "b": [ + 195936478, + 3512640997, + 3237998080, + 3131746989 + ], + "message_bytes_hex": "efbeaddebebafeca0df0ad8bcefaedfedec0ad0be5a55ed100eeffc0adaaaaba4c464d43", + "digest": [ + 3933277007, + 1645050021, + 3608577857, + 1159414982 + ], + "digest_lanes_hex": "ea710b4f620d78a5d7168741451b44c6", + "digest_bytes_hex": "4f0b71eaa5780d62418716d7c6441b45", + "full_blake3_32B_hex": "4f0b71eaa5780d62418716d7c6441b45c050bb850433986f958640d195cd66b5", + "negative_controls": { + "swap_a_b": "b56cbd5568f87e2ad3020a9f0758ba61", + "tag_changed": "5f0cc834562bfbd251b6851402ed5a6e", + "tag_omitted": "8cf8780e1f9c746529d2c957b524e452", + "truncate_high_half": "85bb50c06f983304d1408695b566cd95", + "flags_parent": "fab3b37497823189d29dc979b4542bbc", + "flags_no_root": "d25a2f0ab1bed69a1e1e6ba225288d66", + "block_len_64": "24a6a381f054eaee17d7bc6603b9dbc9", + "block_len_32": "4d498373fb0feaeed09500694ed05794", + "counter_one": "1c8ffb52a69de2cbdaa5ffa3ae8a354d", + "cv_zero": "e7acca0d8b034df298a4c512242c6562", + "lanes_big_endian": "ea9a81efbb2bd046005f1d1da5aa00f2", + "tag_slot_moved": "be709bdcb7d38429b9153e304fbd64fa", + "msg_perm_swapped": "9a08c38461f1827ec1334e883418f150", + "rounds_6_not_7": "2979a59877af5cde57855d1b30a0b8b7" + }, + "controls_inapplicable": [] + }, + { + "name": "formula_3", + "a": [ + 2139095041, + 2, + 2147483648, + 2147483647 + ], + "b": [ + 16711935, + 4278255360, + 252645135, + 4042322160 + ], + "message_bytes_hex": "0100807f0200000000000080ffffff7fff00ff0000ff00ff0f0f0f0ff0f0f0f04c464d43", + "digest": [ + 155998990, + 433989712, + 1904584668, + 618086215 + ], + "digest_lanes_hex": "094c5b0e19de28507185a7dc24d73f47", + "digest_bytes_hex": "0e5b4c095028de19dca78571473fd724", + "full_blake3_32B_hex": "0e5b4c095028de19dca78571473fd724947d5ef08101667ecc68ea3ee90bcb1f", + "negative_controls": { + "swap_a_b": "76d6be3be503a903d622c15fad5d0ffe", + "tag_changed": "88f6e6fea2020fbe39f7e2c0603697b3", + "tag_omitted": "30faba80996fb3827ac7fcfd45a87a47", + "truncate_high_half": "f05e7d947e6601813eea68cc1fcb0be9", + "flags_parent": "a02b070428e0d37ab4798693efc91850", + "flags_no_root": "4fc836faab17d7a6cdf4932f5af7e251", + "block_len_64": "4fdeccd53403a75c123acfce744f7cd5", + "block_len_32": "612d94a9ad301fb765a1fc752a5e3962", + "counter_one": "c296b8901573ec83231628ab5a065817", + "cv_zero": "4e79028da5bb21a258b9659c827aeb7c", + "lanes_big_endian": "909e43a12dcb7807e337544d72e89592", + "tag_slot_moved": "f5ad329b936354961838404f3b1f1796", + "msg_perm_swapped": "59f91f11a0ec0b4db375d9dcd1c0ac19", + "rounds_6_not_7": "0e464b87fa7e96ae426b0bfa7c7a0882" + }, + "controls_inapplicable": [] + }, + { + "name": "boundary", + "a": [ + 0, + 1, + 4294967294, + 4294967295 + ], + "b": [ + 2147483648, + 2147483647, + 65536, + 65535 + ], + "message_bytes_hex": "0000000001000000feffffffffffffff00000080ffffff7f00000100ffff00004c464d43", + "digest": [ + 3957399861, + 1041271354, + 3957028985, + 2265928208 + ], + "digest_lanes_hex": "ebe121353e108a3aebdb7879870f5210", + "digest_bytes_hex": "3521e1eb3a8a103e7978dbeb10520f87", + "full_blake3_32B_hex": "3521e1eb3a8a103e7978dbeb10520f875e8c8f5ddb149d62cf8f95dc32c22314", + "negative_controls": { + "swap_a_b": "5cdb06ad0ab4304564272756b6dd257a", + "tag_changed": "3cb0b4d96f250f53110c971713510a36", + "tag_omitted": "9b7d025fdd1e2a025c8d63cbf0601fb4", + "truncate_high_half": "5d8f8c5e629d14dbdc958fcf1423c232", + "flags_parent": "bf0bd898169e91b530479454a484d577", + "flags_no_root": "fdd6dd1d78ac2db4f0c3e1077033c577", + "block_len_64": "0142b9168da1635239e2301d9c3d9c5e", + "block_len_32": "0defac1dc58d502ef6c81da4f19911b4", + "counter_one": "4d2956ac665c0c876a25115b9b821498", + "cv_zero": "a766e3663dfe7ca11a25e64ef6b46b02", + "lanes_big_endian": "f0d852520f5ebc3ac7aa80ae64b98bd1", + "tag_slot_moved": "9602f886309b1d8ed6d38a7820fdb774", + "msg_perm_swapped": "e4361eb4a67824651a75d953427773cc", + "rounds_6_not_7": "3cc67fdeff4c844ddc443911db69bc03" + }, + "controls_inapplicable": [] + } + ] + }, + "control_discrimination": { + "swap_a_b": [ + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7" + ], + "tag_changed": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "tag_omitted": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "truncate_high_half": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "flags_parent": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "flags_no_root": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "block_len_64": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "block_len_32": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "counter_one": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "cv_zero": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "lanes_big_endian": [ + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7" + ], + "tag_slot_moved": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_a@6", + "unit_a@7", + "unit_b@6", + "unit_b@7", + "zeros@6", + "zeros@7" + ], + "msg_perm_swapped": [ + "all_ones@6", + "all_ones@7", + "boundary@6", + "boundary@7", + "formula_1@6", + "formula_1@7", + "formula_2@6", + "formula_2@7", + "formula_3@6", + "formula_3@7", + "max_min@6", + "max_min@7", + "nibble_ramp@6", + "nibble_ramp@7", + "unit_b@6", + "unit_b@7" + ], + "rounds_6_not_7": [ + "all_ones@7", + "boundary@7", + "formula_1@7", + "formula_2@7", + "formula_3@7", + "max_min@7", + "nibble_ramp@7", + "unit_a@7", + "unit_b@7", + "zeros@7" + ] + } +} \ No newline at end of file diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.py b/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.py new file mode 100644 index 000000000..bf092b313 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/socket_kats.py @@ -0,0 +1,210 @@ +""" +LAYER 2 VECTORS: emit and self-check the socket KATs, then cross-check against +the independently-produced table in `thoughts/blake3/socket-kats/socket_kats.json`. + +What this establishes: + * the two routes (byte level via the tree hasher, word level via one + compression) agree on every vector at BOTH round counts; + * every framing degree of freedom is DISCRIMINATED by at least one vector -- + i.e. the table can actually catch a chip that gets that choice wrong; + * at rounds = 7 the socket equals standard BLAKE3 of the 36-byte message, + truncated, which is the external cross-check a build phase can re-run as a + one-line `blake3::hash` assertion; + * my digests equal the parallel agent's, computed from two separately written + implementations of both the primitive and the framing. + +Run: python3 socket_kats.py [--write] +""" + +from __future__ import annotations + +import json +import os +import sys + +import blake3_oracle as ora +import socket_ref as sk + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT = os.path.join(HERE, "socket_kats.json") + +_PEER_CANDIDATES = [ + "/Users/maurofab/workspace/lambda_vm-blake3-impl/thoughts/blake3/socket-kats/socket_kats.json", + os.path.join(HERE, "..", "..", "..", "blake3", "socket-kats", "socket_kats.json"), +] + + +def vectors() -> list[tuple[str, list[int], list[int]]]: + """Fixed, written-out inputs -- nothing depends on an RNG. + + The structural vectors are deliberately degenerate (they are the inputs a + buggy chip is most likely to be tested on); the formula vectors exist + BECAUSE the degenerate ones cannot detect a byte-order or a swap error. + """ + v: list[tuple[str, list[int], list[int]]] = [ + ("zeros", [0, 0, 0, 0], [0, 0, 0, 0]), + ("unit_a", [1, 0, 0, 0], [0, 0, 0, 0]), + ("unit_b", [0, 0, 0, 0], [1, 0, 0, 0]), + ("all_ones", [0xFFFFFFFF] * 4, [0xFFFFFFFF] * 4), + ("nibble_ramp", [0x00000000, 0x11111111, 0x22222222, 0x33333333], + [0x44444444, 0x55555555, 0x66666666, 0x77777777]), + ("max_min", [0xFFFFFFFF, 0, 0xFFFFFFFF, 0], [0, 0xFFFFFFFF, 0, 0xFFFFFFFF]), + # Formula vectors: asymmetric, byte-distinct, a != b. + ("formula_1", [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20]), + ("formula_2", [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE], + [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD]), + ("formula_3", [0x7F800001, 0x00000002, 0x80000000, 0x7FFFFFFF], + [0x00FF00FF, 0xFF00FF00, 0x0F0F0F0F, 0xF0F0F0F0]), + ("boundary", [0, 1, 0xFFFFFFFE, 0xFFFFFFFF], + [0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF]), + ] + return v + + +def build() -> dict: + table: dict[str, list] = {"6": [], "7": []} + discriminated: dict[str, set[str]] = {name: set() for name in sk.CONTROLS} + problems: list[str] = [] + + for rounds in (7, 6): + fr = sk.honest(rounds) + for name, a, b in vectors(): + # Both routes; socket_digest() asserts they agree. + digest = sk.socket_digest(a, b, fr) + + msg = sk.message_bytes(a, b, fr) + full = ora.hash_bytes(msg, 32, rounds=rounds) + # The 7-round external identity, stated as an executable claim. + if rounds == 7: + want = [int.from_bytes(full[4 * i:4 * i + 4], "little") for i in range(4)] + if want != digest: + problems.append(f"{name}: 7-round != standard BLAKE3(msg)[..16]") + + controls_out = {} + inapplicable = [] + for cname, cfr in sk.CONTROLS.items(): + cfr = sk.Framing(**{**cfr.__dict__, "rounds": + (6 if cname == "rounds_6_not_7" else rounds)}) + if not sk.control_applicable(a, b, cfr, fr): + inapplicable.append(cname) + continue + cd = sk.socket_digest_wordlevel(a, b, cfr) + controls_out[cname] = "".join(f"{x:08x}" for x in cd) + if cd != digest: + discriminated[cname].add(f"{name}@{rounds}") + else: + problems.append( + f"CONTROL {cname} did NOT change the digest on {name}@{rounds}") + + table[str(rounds)].append({ + "name": name, + "a": a, + "b": b, + "message_bytes_hex": msg.hex(), + "digest": digest, + "digest_lanes_hex": "".join(f"{x:08x}" for x in digest), + "digest_bytes_hex": b"".join( + int(x).to_bytes(4, "little") for x in digest).hex(), + "full_blake3_32B_hex": full.hex(), + "negative_controls": controls_out, + "controls_inapplicable": inapplicable, + }) + + undiscriminated = [c for c, s in discriminated.items() if not s] + if undiscriminated: + problems.append(f"controls NEVER discriminated by any vector: {undiscriminated}") + + return { + "socket": "LFM_HASH 2-to-1 BLAKE3 compress (Option A + domain tag)", + "spec": { + "digest_lanes": sk.DIGEST_LANES, + "digest_bits": 128, + "domain_tag_ascii": sk.TAG_LFMC_ASCII.decode(), + "domain_tag_word": sk.TAG_LFMC, + "chaining_value_in": "BLAKE3 IV[0..8]", + "counter": 0, + "block_len": sk.BLOCK_LEN_LFMC, + "flags": sk.FLAGS_LFMC, + "flags_meaning": "CHUNK_START|CHUNK_END|ROOT", + "message_layout": "m[0..4]=a, m[4..8]=b, m[8]=tag, m[9..16]=0", + "truncation_window": "out[0..4] (the LOW four of 16 output words)", + "lane_serialisation": "one felt = one u32 = four little-endian bytes " + "(keccak_host convention, NOT word::pack_digest)", + }, + "rounds": table, + "control_discrimination": {c: sorted(s) for c, s in discriminated.items()}, + "_problems": problems, + } + + +def cross_check(built: dict) -> tuple[bool, str]: + """Recompute the PARALLEL AGENT's vectors with MY code. Two independently + written implementations of both the primitive and the framing must agree.""" + path = next((p for p in _PEER_CANDIDATES if os.path.exists(p)), None) + if path is None: + return False, "peer socket_kats.json NOT FOUND -- cross-check CANNOT RUN" + with open(path) as f: + peer = json.load(f) + + if peer.get("domain_tag_word") != sk.TAG_LFMC: + return False, (f"SPEC DISAGREEMENT: peer tag {peer.get('domain_tag_word')} " + f"vs mine {sk.TAG_LFMC}") + for k, mine in (("block_len", sk.BLOCK_LEN_LFMC), ("flags", sk.FLAGS_LFMC), + ("counter", 0), ("digest_lanes", 4)): + if peer.get(k) != mine: + return False, f"SPEC DISAGREEMENT on {k}: peer {peer.get(k)} vs mine {mine}" + + n = 0 + for rounds_key, entries in peer["rounds"].items(): + rounds = int(rounds_key) + fr = sk.honest(rounds) + for e in entries: + mine = sk.socket_digest(e["a"], e["b"], fr) + if [f"{x:08x}" for x in mine] != [f"{x:08x}" for x in e["digest"]]: + return False, (f"DIGEST MISMATCH on peer vector {e['name']}@{rounds}\n" + f" mine={[hex(x) for x in mine]}\n" + f" peer={[hex(x) for x in e['digest']]}") + if sk.message_bytes(e["a"], e["b"], fr).hex() != e["message_bytes_hex"]: + return False, f"MESSAGE MISMATCH on peer vector {e['name']}@{rounds}" + n += 1 + return True, (f"cross-check PASS: {n} peer vectors reproduced exactly " + f"(spec fields agree too)") + + +def main() -> int: + built = build() + problems = built.pop("_problems") + print("=" * 74) + print("LAYER 2 -- socket KATs") + print("=" * 74) + nvec = sum(len(v) for v in built["rounds"].values()) + ncontrol = sum(len(e["negative_controls"]) + for v in built["rounds"].values() for e in v) + print(f" vectors : {nvec} ({len(vectors())} inputs x 2 round counts)") + print(f" framing controls evaluated : {ncontrol}") + print(f" framing degrees of freedom : {len(sk.CONTROLS)}") + for c, s in built["control_discrimination"].items(): + print(f" {c:20s} discriminated by {len(s):2d} vector-instances") + + ok_cc, msg_cc = cross_check(built) + print(f" [{'PASS' if ok_cc else 'FAIL'}] {msg_cc}") + + if problems: + print("\n PROBLEMS:") + for p in problems: + print(f" - {p}") + + if "--write" in sys.argv: + with open(OUT, "w") as f: + json.dump(built, f, indent=1) + print(f"\n wrote {OUT}") + + ok = ok_cc and not problems + print("-" * 74) + print(f"LAYER 2: {'PASS' if ok else 'FAIL'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/shared/lfm-real-hash/gate-oracle/socket_ref.py b/thoughts/shared/lfm-real-hash/gate-oracle/socket_ref.py new file mode 100644 index 000000000..98e14d756 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/gate-oracle/socket_ref.py @@ -0,0 +1,247 @@ +""" +LAYER 2: the OPTION-A `LFM_HASH` 2-to-1 compress socket, as a reference function. + +This is the thing the chip must compute. `blake3_oracle.compress` is the +primitive; this file is the *framing* -- the six independent choices that sit +between "we have a correct f" and "we have a correct 2-to-1 compress", every one +of which is a way to be wrong while every primitive test stays green: + + 1. where a and b land in the 16 message words, + 2. what the chaining value h is, + 3. what the counter t is, + 4. what block_len is, + 5. what the flags byte is, + 6. which 4 of the 16 output words become the digest (the truncation window), + +plus, for the LFM socket specifically and *not* present in a syscall-shaped chip: + + 7. how a Goldilocks felt becomes four message bytes (the lane boundary). + +THE SPECIFICATION +----------------- +Byte-level (normative, and deliberately expressible as a library call): + + msg = LE32(a0)‖LE32(a1)‖LE32(a2)‖LE32(a3) (16 bytes) + ‖ LE32(b0)‖LE32(b1)‖LE32(b2)‖LE32(b3) (16 bytes) + ‖ "LFMC" ( 4 bytes) = 36 bytes + + digest_bytes = BLAKE3(msg)[0..16] + c_i = LE32^-1(digest_bytes[4i .. 4i+4]) for i in 0..4 + +Word-level (what the chip proves) -- 36 bytes is one block, so this is exactly +one compression: + + h = IV[0..8] (all eight words; the unkeyed default) + m[0..4] = a, m[4..8] = b + m[8] = 0x434D464C ("LFMC" read as one little-endian u32) + m[9..16] = 0 + t = 0 + block_len = 36 + flags = CHUNK_START|CHUNK_END|ROOT = 0x0B + digest = out[0..4] (the LOW four of the 16 output words) + +The two routes are computed here by separate code paths and asserted equal. +At rounds = 7 the byte-level route is literally `blake3::hash(a‖b‖"LFMC")[..16]`, +so the socket has an external anchor and needs no oracle in the chain. At +rounds = 6 no library computes it: that is assumption A6R, and it is the reason +the tag lives in the MESSAGE and not in `flags`/`t`/`h` -- any tag outside the +message would make even the 7-round socket a nonstandard invocation of f that no +library computes, throwing away the anchor for nothing. + +WHY THE DOMAIN TAG IS IN THE MESSAGE (independently re-derived, agrees with +`thoughts/blake3/socket-kats/SOCKET.md`). The message is fixed-length (36 bytes) +with the tag at a fixed offset, and `block_len` is itself an input to f, so the +encoding is unambiguous: distinct tags give distinct messages, and no length +ambiguity exists. Cost is 4 extra message bytes inside the same single block -- +zero extra compressions, and (see COLUMN_ROLE_MAP) zero extra columns, because +m[8] is a constant. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace + +import blake3_oracle as ora +from blake3_oracle import IV, MASK32 + +# Domain tags. A tag is never reused for a second purpose. +TAG_LFMC_ASCII = b"LFMC" # this socket: 2-to-1 compress / Merkle parent +TAG_LFMP_ASCII = b"LFMP" # reserved: the `permute` socket (NOT specified) +TAG_LFML_ASCII = b"LFML" # reserved: leaf domain (see obligation O5) + +TAG_LFMC = int.from_bytes(TAG_LFMC_ASCII, "little") # 0x434D464C +TAG_LFMP = int.from_bytes(TAG_LFMP_ASCII, "little") +TAG_LFML = int.from_bytes(TAG_LFML_ASCII, "little") + +FLAGS_LFMC = ora.CHUNK_START | ora.CHUNK_END | ora.ROOT # 0x0B +BLOCK_LEN_LFMC = 36 +DIGEST_LANES = 4 + + +@dataclass(frozen=True) +class Framing: + """Every framing degree of freedom, in one object. + + The honest socket is `HONEST`. A negative control is a `replace(HONEST, ...)` + applied to the CHIP side while the reference keeps `HONEST` -- which is what + makes the control suite systematic instead of ad hoc, and what lets the gate + and the reference share one definition of "the framing". + """ + rounds: int = 7 + cv: tuple[int, ...] = tuple(IV) # h[0..8] + tag_word: int = TAG_LFMC + counter: int = 0 + block_len: int = BLOCK_LEN_LFMC + flags: int = FLAGS_LFMC + a_slot: int = 0 # m[a_slot .. a_slot+4] = a + b_slot: int = 4 # m[b_slot .. b_slot+4] = b + tag_slot: int = 8 # m[tag_slot] = tag_word + out_window: int = 0 # digest = out[out_window .. +4] + lane_le: bool = True # lane -> 4 bytes, little-endian + msg_permutation: tuple[int, ...] = tuple(ora.MSG_PERMUTATION) + + +HONEST_7 = Framing(rounds=7) +HONEST_6 = Framing(rounds=6) + + +def honest(rounds: int) -> Framing: + return Framing(rounds=rounds) + + +# --------------------------------------------------------------------------- +# Lane boundary (choice 7). A digest cell is 4 lanes; a lane is a felt that +# MUST carry a u32. `keccak_host`'s convention: one felt = one u32 = four +# little-endian bytes. This is NOT `word::pack_digest` (8 bytes per lane). +# --------------------------------------------------------------------------- + +def lane_to_bytes(lane: int, le: bool = True) -> bytes: + if not 0 <= lane <= MASK32: + raise ValueError(f"lane {lane:#x} is not a u32 -- obligation O1 violated") + return int(lane).to_bytes(4, "little" if le else "big") + + +def bytes_to_lane(b: bytes, le: bool = True) -> int: + return int.from_bytes(b, "little" if le else "big") + + +def message_bytes(a: list[int], b: list[int], fr: Framing = HONEST_7) -> bytes: + """The normative 36-byte message.""" + assert len(a) == len(b) == DIGEST_LANES + out = b"".join(lane_to_bytes(x, fr.lane_le) for x in a) + out += b"".join(lane_to_bytes(x, fr.lane_le) for x in b) + out += int(fr.tag_word & MASK32).to_bytes(4, "little") + return out + + +# --------------------------------------------------------------------------- +# Route 1 -- byte level. At rounds = 7 this is a plain BLAKE3 hash. +# --------------------------------------------------------------------------- + +def socket_digest_bytelevel(a: list[int], b: list[int], + fr: Framing = HONEST_7) -> list[int]: + msg = message_bytes(a, b, fr) + full = ora.hash_bytes(msg, 32, rounds=fr.rounds) + window = full[4 * fr.out_window: 4 * fr.out_window + 16] + return [bytes_to_lane(window[4 * i:4 * i + 4]) for i in range(DIGEST_LANES)] + + +# --------------------------------------------------------------------------- +# Route 2 -- word level. This is what the chip proves. +# --------------------------------------------------------------------------- + +def socket_message_words(a: list[int], b: list[int], + fr: Framing = HONEST_7) -> list[int]: + m = [0] * 16 + for i in range(DIGEST_LANES): + m[fr.a_slot + i] = a[i] & MASK32 + m[fr.b_slot + i] = b[i] & MASK32 + m[fr.tag_slot] = fr.tag_word & MASK32 + if not fr.lane_le: + # A big-endian lane serialisation changes the message WORDS, because a + # word is read little-endian from the byte string. + for i in range(DIGEST_LANES): + m[fr.a_slot + i] = int.from_bytes(lane_to_bytes(a[i], False), "little") + m[fr.b_slot + i] = int.from_bytes(lane_to_bytes(b[i], False), "little") + return m + + +def socket_digest_wordlevel(a: list[int], b: list[int], + fr: Framing = HONEST_7) -> list[int]: + saved = list(ora.MSG_PERMUTATION) + ora.MSG_PERMUTATION[:] = list(fr.msg_permutation) + try: + out = ora.compress(list(fr.cv), socket_message_words(a, b, fr), + fr.counter, fr.block_len, fr.flags, rounds=fr.rounds) + finally: + ora.MSG_PERMUTATION[:] = saved + return out[fr.out_window: fr.out_window + DIGEST_LANES] + + +def socket_digest(a: list[int], b: list[int], fr: Framing = HONEST_7) -> list[int]: + """THE reference the gate checks the chip against. Both routes, asserted equal.""" + w = socket_digest_wordlevel(a, b, fr) + if fr.counter == 0 and fr.block_len == BLOCK_LEN_LFMC and \ + fr.flags == FLAGS_LFMC and tuple(fr.cv) == tuple(IV) and \ + (fr.a_slot, fr.b_slot, fr.tag_slot) == (0, 4, 8) and \ + tuple(fr.msg_permutation) == tuple(ora.MSG_PERMUTATION): + # The byte-level route only *exists* for the honest framing -- it is a + # call to the tree hasher, which fixes h/t/block_len/flags itself. + bl = socket_digest_bytelevel(a, b, fr) + assert w == bl, ( + "FRAMING CHECK FAILED: word-level and byte-level routes disagree\n" + f" word={[hex(x) for x in w]}\n byte={[hex(x) for x in bl]}") + return w + + +# --------------------------------------------------------------------------- +# The negative-control catalogue, as framing perturbations. +# --------------------------------------------------------------------------- + +CONTROLS: dict[str, Framing] = { + "swap_a_b": replace(HONEST_7, a_slot=4, b_slot=0), + "tag_changed": replace(HONEST_7, tag_word=TAG_LFMP), + "tag_omitted": replace(HONEST_7, tag_word=0), + "truncate_high_half": replace(HONEST_7, out_window=4), + "flags_parent": replace(HONEST_7, flags=ora.PARENT), + "flags_no_root": replace(HONEST_7, flags=ora.CHUNK_START | ora.CHUNK_END), + "block_len_64": replace(HONEST_7, block_len=64), + "block_len_32": replace(HONEST_7, block_len=32), + "counter_one": replace(HONEST_7, counter=1), + "cv_zero": replace(HONEST_7, cv=tuple([0] * 8)), + "lanes_big_endian": replace(HONEST_7, lane_le=False), + "tag_slot_moved": replace(HONEST_7, tag_slot=9), + "msg_perm_swapped": replace( + HONEST_7, + msg_permutation=tuple([ora.MSG_PERMUTATION[1], ora.MSG_PERMUTATION[0]] + + list(ora.MSG_PERMUTATION[2:]))), + "rounds_6_not_7": replace(HONEST_7, rounds=6), +} + + +def effective_trace(a: list[int], b: list[int], fr: Framing): + """Everything f actually sees: the initial state, the message schedule at + every round, and the output window. + + Two framings with identical traces compute identical digests, necessarily. + So a control whose trace equals the honest trace on some input is genuinely + INAPPLICABLE on that input -- not undetected. Deriving applicability this + way rather than hand-listing it is deliberate: a hand-list silently grows + stale as controls are added, and a stale entry is a control that looks + covered and is not. + """ + m = socket_message_words(a, b, fr) + sched = list(m) + scheds = [] + for r in range(fr.rounds): + scheds.append(tuple(sched)) + if r < fr.rounds - 1: + sched = [sched[fr.msg_permutation[i]] for i in range(16)] + init = (tuple(fr.cv), fr.counter & MASK32, (fr.counter >> 32) & MASK32, + fr.block_len & MASK32, fr.flags & MASK32) + return (fr.rounds, init, tuple(scheds), fr.out_window) + + +def control_applicable(a: list[int], b: list[int], + cfr: Framing, honest_fr: Framing) -> bool: + return effective_trace(a, b, cfr) != effective_trace(a, b, honest_fr) diff --git a/thoughts/shared/lfm-real-hash/leaf-convention-cost.py b/thoughts/shared/lfm-real-hash/leaf-convention-cost.py new file mode 100644 index 000000000..40090b319 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-convention-cost.py @@ -0,0 +1,73 @@ +""" +Pricing for the O1 leaf-convention options, from the gated census formulas. + +Validated the same way as `permute-socket-cost.py`: the compress formula must +reproduce the GATED census before it is allowed to price anything else. +""" + +import math + +# --- gated compress census (CHIP-GATE.md §3, reconciled to the built chip) --- +PREFIX, CELLS_PER_LANE, SENDS_PER_LANE = 28, 4, 2 +CELLS_PER_G, SENDS_PER_G = 60, 24 +CELLS_PER_OUTW, SENDS_PER_OUTW = 4, 4 +IO_SENDS = 6 + + +def compress_ce(rounds=7, extra_main=0): + g = 8 * rounds + main = PREFIX + 8 * CELLS_PER_LANE + g * CELLS_PER_G + 4 * CELLS_PER_OUTW + extra_main + sends = 8 * SENDS_PER_LANE + g * SENDS_PER_G + 4 * SENDS_PER_OUTW + IO_SENDS + return main + 3 * math.ceil(sends / 2) + + +assert compress_ce(7) == 5509 and compress_ce(6) == 4741, "formula must match the gate" +print(f"validated: compress = {compress_ce(7)} @7r / {compress_ce(6)} @6r\n") + +# --- LFM_BITDEC, ✓ VERIFIED from chips.rs/layout.rs ------------------------- +# main = NUM_COLUMNS - PREP_WIDTH = (130+64+2) - 130 = 66 +# sends = 1 receiver + 64 bit senders = 65 +BITDEC_CE = 66 + 3 * math.ceil(65 / 2) +# LFM_BALU: 4 main (A,B,C,OUT); ~4 LfmMem interactions +BALU_CE = 4 + 3 * math.ceil(4 / 2) +# felt_be_halves = 1 bit_dec + 64 mul/mul_add (32 per half x 2 halves) +FELT_BE_HALVES_CE = BITDEC_CE + 64 * BALU_CE +print(f"LFM_BITDEC per felt : {BITDEC_CE}") +print(f"LFM_BALU per op : {BALU_CE}") +print(f"felt_be_halves per felt : {FELT_BE_HALVES_CE} (option A's per-felt tax)\n") + +# --- FriToyV0 shape, ✓ VERIFIED from programs.rs / fixture.rs --------------- +NUM_QUERIES = 4 +# per query today: 3 leaves (1 compress each) + 4 + 4 + 3 path compresses +TODAY_PER_QUERY = 3 + 4 + 4 + 3 +TODAY_TRANSCRIPT = 11 +TODAY_TOTAL = NUM_QUERIES * TODAY_PER_QUERY + TODAY_TRANSCRIPT + +# A leaf covers 2 trace rows = 8 FIELD ELEMENTS. Four felts fill one compress +# input (2 cells x 4 lanes = 8 lanes = 4 felts x 2 halves), so a leaf becomes +# 2 felt-mode compresses + 1 combine = 3. +LEAF_AFTER = 3 +AFTER_PER_QUERY = 3 * LEAF_AFTER + 4 + 4 + 3 +AFTER_TOTAL = NUM_QUERIES * AFTER_PER_QUERY + TODAY_TRANSCRIPT + +# Felts needing a decomposition: leaf data only. Siblings and internal nodes are +# DIGESTS, already u32-laned by obligation O2. +LEAF_FELTS = NUM_QUERIES * 24 + 8 # 6 cells x 4 felts per query, + t0w/t1w + +print(f"FriToyV0 compresses today(counterfactual)={TODAY_TOTAL} after={AFTER_TOTAL}") +print(f"felts needing decomposition: {LEAF_FELTS}\n") + +base = TODAY_TOTAL * compress_ce(7) +opt_a = AFTER_TOTAL * compress_ce(7) + LEAF_FELTS * FELT_BE_HALVES_CE +# Option C: the canonicity gate lives IN the socket. Per felt: Z + GINV = 2 +# witness columns; 4 felts per row => +8 main columns, ZERO extra sends. +opt_c = AFTER_TOTAL * compress_ce(7, extra_main=8) + +print("FriToyV0 end-to-end, cell-equiv @7r") +print(f" counterfactual 'if felts fit' {base:>9,} (todays 369,103 shape)") +print(f" (A) felt_be_halves precedent {opt_a:>9,} {100*(opt_a/base-1):+.1f}%") +print(f" (C) in-socket felt mode {opt_c:>9,} {100*(opt_c/base-1):+.1f}%") +print(f" (B) stay off BLAKE3 {'n/a':>9} 0% (nothing is proved)") +print(f"\n C is {100*(1-opt_c/opt_a):.1f}% cheaper than A, and adds " + f"{compress_ce(7,8)-compress_ce(7)} cells/row rather than " + f"{FELT_BE_HALVES_CE} per felt.") diff --git a/thoughts/shared/lfm-real-hash/leaf-convention-options.md b/thoughts/shared/lfm-real-hash/leaf-convention-options.md new file mode 100644 index 000000000..3cc7b3bea --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-convention-options.md @@ -0,0 +1,278 @@ +# The O1 leaf convention — options note + +> # ✅ DECIDED — OPTION C WITH `LFML`, ratified by the user 2026-08-11 +> +> On this paper's presentation, with §6's unsettled items disclosed. The +> recommendation in §5 carried. +> +> **§6's open point is resolved BY DECISION, not by assumption:** `MODE_L` +> **implies felt-input semantics** — the cheaper form, which keeps the one-hot +> selector span contiguous. +> +> **Sequencing:** the `MODE_L` change builds on **committed** B1, not into B1's +> current uncommitted diff. Spec first: `leaf-spec/LEAF.md`. +> +> **Pricing correction (2026-08-11, post-build):** `FriToyV0` measures **93 +> compresses / 513,081 @7r**, not the 91 / 502,047 quoted in §4–§5. The `t0`/`t1` +> transcript absorbs are arbitrary field elements and need the felt path — two +> more `LFML` rows. §6 had flagged those absorbs as ✗ OPEN and the number was +> written as if they were settled. **The decision is unaffected:** option C is +> still ~12% under option A, and the ranking never depended on those two rows. +> +> This paper is now the record of *why*; `leaf-spec/` is the record of *what*. + +**Decision paper — the recommendation was adopted. NOT an implementation.** +§5 carries the recommendation; §4 carries the finding it rests on. + +**Date:** 2026-08-11. **Question:** how do arbitrary Goldilocks field elements +reach the BLAKE3 socket, given obligation O1 requires `u32` lanes? This is what +blocks `FriToyV0` and with it the second half of F3.4. + +Claims are ✓ VERIFIED (read the code, cited), ✓ EXECUTED (ran it), ? INFERRED. + +--- + +## 1. The blocker, and its exact shape + +✓ VERIFIED (`transcript-impl-report.md` §7.1, reproducing the builder's measured +run): `execute(fri_toy_program(), …, Blake3)` returns +`HasherRejected("BLAKE3 compress input lane is not a u32")`, with **124 of the +fixture's 128 committed column values ≥ 2^32**. + +The cause is structural, not a fixture accident: `FriToyV0` hashes **FRI data** — +Merkle leaves over LDE evaluations and folded ext values — and the evaluations of +a low-degree polynomial over a coset are arbitrary elements mod `p`. No choice of +polynomial changes that. + +**What is NOT blocked** — worth stating, because it bounds the problem: + +- **Digests are fine.** Obligation O2: the socket's output is four `u32`s by + construction, so every internal Merkle node and every sibling already + satisfies O1. Only *leaf data* is arbitrary. +- **The transcript is fine** except where it absorbs raw felts (`t0w`/`t1w`, the + terminal-polynomial coefficients). +- **`TrivialV0` is already retired** — F3.4 is closed for that entry. + +So the question is narrow: **a leaf-data encoding**, ~104 field elements per +`FriToyV0` proof. + +### 1.1 Why "just reduce mod 2^32" is not an option + +Stated because it is the tempting shortcut and it is the same bug O1 itself +names. If a felt `v` reached the hash by reduction, then `v` and `v + 2^32` +(where both are < p) would hash alike — the prover picks which. The encoding +must be a **checked decomposition, rejecting out-of-range inputs, not reducing +them**: the same reject-don't-reduce shape as O1. + +Concretely, any option must supply two things, and **neither alone suffices**: + +1. **Binding** — the halves are *the* halves of the committed felt: + `v = lo + 2^32·hi`, as a constraint, not a convention. +2. **Range + canonicity** — `lo, hi < 2^32` **and** `v < p`. Range alone is not + enough: `lo + 2^32·hi` ranges over `[0, 2^64)` while the field has `p ≈ 2^64 − + 2^32` elements, so without canonicity two distinct half-pairs collide onto one + felt and a prover opens one leaf two ways. + +--- + +## 2. Option A — the `felt_be_halves` precedent (bit-decompose per felt) + +The keccak path's existing shape. ✓ VERIFIED `transcript_replay.rs:743-761`: +`felt_be_halves` calls `bit_dec(v, 64)` and recomposes two 32-bit halves from the +bits with `mul`/`mul_add`. + +**Soundness.** Strong, and already reviewed: `LFM_BITDEC` supplies both +obligations. ✓ VERIFIED `chips.rs`: 64 booleanity constraints, plus the +canonicity witness pair — `G = (2^32−1) − top32`, `Z·G = 0` (so `G ≠ 0 ⇒ Z = 0`) +and `IS_REAL·(1 − Z − G·GINV) = 0` (so `G = 0 ⇒ Z = 1`). Binding comes from the +bus receiver, which reads the value as the **linear recomposition** `Σ 2^i·B_i` +rather than as a separate column — so there is no "is this the same value?" gap +at all. This is the machine's established canonicity idiom. + +**KAT-ability.** Unchanged: the socket still hashes `u32` lanes, so every +compress remains `blake3::hash(a‖b‖tag)[..16]`. + +**Cost — the problem.** ✓ EXECUTED (`leaf-convention-cost.py`, whose compress +formula reproduces the gated census first): + +| | per felt | +|---|---:| +| `LFM_BITDEC` row (66 main + 65 sends) | **165** | +| 64 × `LFM_BALU` recomposition ops | **640** | +| **total per felt** | **805** | + +104 felts ⇒ **+83,720 cell-equiv**, on top of the leaf restructuring every option +pays. `FriToyV0` end-to-end: **585,039 vs 369,103 = +58.5%**. + +**Blast radius.** Moderate: `FriToyV0`'s arena layout and program identity move; +no chip changes. **Gate impact: none** — the socket is untouched, so the pinned +board still describes it. + +--- + +## 3. Option B — keep `FriToyV0` off BLAKE3 (the honest do-nothing baseline) + +The registry entry stays on `Test`/`Poseidon`; nothing is built. + +**Cost:** zero. **Soundness:** nothing new to argue. **What it costs instead is +the claim.** The disclosure would have to read, permanently and precisely: + +> `TrivialV0` proves under BLAKE3 and its hashing is cryptographically +> meaningful. **`FriToyV0` does not.** Its Merkle authentication and its +> Fiat–Shamir transcript run under `TestPermutation`, which is not a hash; +> collisions are trivially constructible. The FRI verification that entry +> performs is cryptographically vacuous. + +That is the original F3.4 disclosure, surviving for the entry it was mostly +about. **The machine would ship a registry in which its only non-trivial program +cannot use its real hash** — and the reason would be an encoding gap, not a +cryptographic one, which is an uncomfortable thing to have to explain. + +Worth saying plainly: B is not absurd. The wrap is hash-neutral, so nothing in +production depends on this. But it leaves the interesting entry permanently on a +placeholder. + +--- + +## 4. ★ Option C — a felt-input mode in the socket (my finding; cheapest and simplest) + +The observation the other two miss: **the socket's existing O1 machinery already +does most of a half-decomposition.** Every input lane is byte-decomposed and +`AreBytes`-checked, and the lane identity pins `IN_lane = Σ bytes·2^{8k} < 2^32`. +So `lo` and `hi` are *already* forced to be `u32`s. The only missing piece is +**canonicity**. + +And canonicity over two halves is far cheaper than over 64 bits. ✓ EXECUTED +(200,007 cases including every boundary): + +``` +with lo, hi < 2^32 and v = lo + 2^32·hi: + v < p ⟺ NOT( hi = 2^32−1 AND lo ≥ 1 ) +``` + +because `p − 1 = 0xFFFFFFFF_00000000` — `hi = 2^32−1`, `lo = 0`. So the whole +canonicity check is *"if `hi` is maximal then `lo` is zero"*, which is the +**same `Z`/`GINV` trick `LFM_BITDEC` already uses**, applied to two halves +instead of 64 bits: + +``` +G = (2^32 − 1) − hi +idx a MU · Z · G = 0 (G ≠ 0 ⇒ Z = 0) deg 3 +idx b MU · (1 − Z − G·GINV) = 0 (G = 0 ⇒ Z = 1) deg 3 +idx c MU · Z · lo = 0 (hi maximal ⇒ lo = 0) deg 3 + MU · (v − lo − 2^32·hi) = 0 (binding) deg 2 +``` + +**Per felt: 2 witness columns (`Z`, `GINV`) and 4 constraints. Per row (4 felts): +8 columns, 16 constraints, ZERO extra sends.** Max degree stays 3. + +**Soundness.** Both obligations are met and neither is inherited on faith: +binding is the explicit identity; range comes from the *existing* lane machinery +(the same `AreBytes` sends WA1/WA2 already gate); canonicity is the `Z`/`GINV` +pair above. It is reject-don't-reduce: a non-canonical input has no satisfying +witness, so the row is unprovable rather than silently reduced. + +**KAT-ability.** Fully preserved. The message layout does not change — 4 felts +occupy the same 8 lanes 8 `u32`s did — so a felt-mode compress is still exactly +`blake3::hash(lo₀‖hi₀‖…‖tag)[..16]`. + +**Cost.** ✓ EXECUTED: **502,047 vs 369,103 = +36.0%**, and **14.2% cheaper than +option A**. The per-row price moves only `5,509 → 5,517`; the increase is almost +entirely the leaf restructuring that *every* option pays (a leaf covers 8 field +elements = 2 felt-mode compresses + 1 combine, so leaves go 1 → 3 compresses and +`FriToyV0` goes 67 → 91 compresses). + +**Blast radius.** Larger than A on the chip and smaller everywhere else: a new +preprocessed mode column, the canonicity block, an executor/trace arm — but no +`bit_dec` traffic, no 64-op recomposition per felt, and no memory round-trip. +`FriToyV0`'s program identity moves either way. + +**Gate impact.** Real but well-understood, and it is my work: the pinned board +must be re-transcribed (a new mode, 8 new columns, 16 new constraints), plus a +new width-audit pair in the field domain — *canonicity present → a +non-canonical felt is unprovable (UNSAT); canonicity dropped → the same felt +becomes provable (SAT)* — with the honest leg that canonical felts still prove. +That pair is the direct analogue of WA1/WA2 and is the thing that would make the +argument checked rather than asserted. + +--- + +## 4.1 The `LFML` question — live now, and my answer is yes + +The lead is right that this is the moment. ✓ VERIFIED and worth stating clearly: +**`FriToyV0` already performs leaf hashing today**, compressing raw trace rows +into leaves under the **`LFMC`** tag (`programs.rs`, the `leaf_a`/`leaf_b`/ +`l1_leaf` compresses). The claim "no leaf-hashing path exists" is false; O5's +safety today rests on **fixed depth alone** — every eDSL circuit is fixed-shape +at build time, so no variable-depth second-preimage confusion is reachable. + +That is a real argument but a fragile one: it is a property of every *current* +program, not of the construction, and nothing enforces it. + +**Recommendation: leaf compresses should use `LFML`.** O5's ratified wording +already requires it ("any future leaf-hashing path MUST use the reserved `LFML` +tag"), the leaf convention is being redesigned anyway, and program identity moves +regardless — so the cost of adopting it now is zero and the cost of adopting it +later is another re-bless. Doing so **retires the fixed-depth crutch**: with +leaves and parents in separate domains the confusion is closed structurally, and +a future variable-depth tree stops being a latent hazard. + +Mechanically this fits option C neatly: make the leaf mode `MODE_L`, carrying +both the `LFML` tag *and* felt-input semantics (leaves take felts; parents and +transcript steps take digests). One more preprocessed selector, one more +`TAG_SELECTOR` entry, and **M8's one-hot control extends to cover it unchanged**. + +--- + +## 5. Comparison and recommendation + +| | **A — `felt_be_halves`** | **B — stay off BLAKE3** | **★ C — in-socket felt mode** | +|---|---|---|---| +| binding | bus reads the recomposition | — | explicit identity | +| range | `LFM_BITDEC` booleanity | — | **existing** lane `AreBytes` | +| canonicity | 64-bit `Z`/`GINV` | — | 2-half `Z`/`GINV` (✓ EXECUTED) | +| per-felt tax | **805 cell-equiv** | 0 | **2 columns** | +| `FriToyV0` total | 585,039 (**+58.5%**) | n/a | **502,047 (+36.0%)** | +| KAT-able | yes | n/a | yes | +| chip change | none | none | new mode + canonicity block | +| gate impact | **none** | none | re-transcribe + 1 new audit pair | +| retires F3.4 for `FriToyV0` | yes | **no** | yes | + +**My recommendation: option C, with leaves under `LFML`.** + +1. **It is the cheapest and by a real margin** — 36% over the counterfactual + against A's 58.5%, because it adds 8 cells per row instead of 805 per felt. +2. **It reuses the machine's own canonicity idiom** rather than inventing one: + the `Z`/`GINV` pair is `LFM_BITDEC`'s, and the two-half criterion is executed, + not argued. +3. **It needs no new range machinery at all.** O1's existing lane check already + forces `u32` halves; only canonicity was missing. That is the whole finding. +4. **It closes O5's fixed-depth dependence** as a side effect, at zero marginal + cost, because program identity moves anyway. + +The honest case against C: it is the only option that touches the **chip**, so it +is the only one that invalidates the current pin and needs a re-gate. I am the +one who pays that, and it is a day's work of the kind just done twice — I do not +think it should drive the decision, but it should be visible. + +If cost is not a concern and minimising chip churn is, **A is a perfectly +defensible choice** — it is the reviewed, precedented path and costs the gate +nothing. **B should be chosen only deliberately**, with §3's disclosure written +down, not drifted into. + +--- + +## 6. What I could not settle + +- ? INFERRED: option A's 805/felt assumes `felt_be_halves`' 64 `mul`/`mul_add` + ops each cost one `LFM_BALU` row (4 main + ~4 interactions). The chip widths + are ✓ VERIFIED; the op count is read off the source loop; but no profile was + run and memory traffic for the 64 bit-senders is not priced. Treat A's figure + as a floor. +- ? INFERRED: the leaf restructuring (1 → 3 compresses) assumes a leaf keeps + covering 2 trace rows. A different leaf arity changes every option equally. +- ✗ OPEN: whether `t0w`/`t1w`'s absorbs want felt mode or a separate convention — + they are 8 of the 104 felts and do not change the ranking. +- ✗ OPEN: option C's exact constraint count depends on whether `MODE_L` implies + felt-input or the two are separate selectors. I assumed implied, which is + cheaper and keeps the one-hot span contiguous. diff --git a/thoughts/shared/lfm-real-hash/leaf-impl-report.md b/thoughts/shared/lfm-real-hash/leaf-impl-report.md new file mode 100644 index 000000000..d8719d718 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-impl-report.md @@ -0,0 +1,420 @@ +# The `LFML` leaf mode (option C) — implementation report + +**Status:** GREEN **after the post-review fixes of §10**. **★ `FriToyV0` proves +and verifies under BLAKE3 — F3.4 is retired.** One material deviation from the +spec's pricing, named in §6. + +⚠ **The first submission of this work carried a HIGH soundness defect (D1) that +this report did not disclose** — `MODE_L`'s unread input cells were pinned on the +BLAKE3 arm and on nothing else, which was a Fiat–Shamir break under `Test` and +`Poseidon`. It is fixed, single-sourced and regression-tested; §10 has the +record, and §7's board below is the post-fix state. **Date:** 2026-08-11. + +**Ground:** worktree `lambda_vm-blake3-impl`, branch `blake3-real-hash`, on +committed B1 (`9bcc9ee2`), uncommitted. **Spec:** `leaf-spec/LEAF.md`, binding. + +Claims are ✓ EXECUTED / ✓ VERIFIED / ✗ OPEN. + +--- + +## 0. Board + +| item | result | +|---|---| +| leaf KATs L1/L5/L6 — per-row vectors, 6 and 7 rounds | ✓ EXECUTED, PASS | +| L2/L3 — boundary felts and non-canonical rejects | ✓ EXECUTED, PASS | +| L4 — the predicate IS `v < p` | ✓ EXECUTED, PASS (boundaries + 24k sweep) | +| crate anchor: leaf == `blake3::hash(LE32(lanes)‖"LFML")[..16]` @7r | ✓ EXECUTED, PASS | +| **`FriToyV0` proves + verifies under BLAKE3** | ✓ **EXECUTED, PASS — the milestone** | +| `FriToyV0` + `TrivialV0` under Test / Poseidon / BLAKE3 | ✓ EXECUTED, 3/3 each | +| M9 — mode confusion, six ordered pairs | ✓ EXECUTED, all fire | +| M10 — a `MODE_L` row cannot skip canonicity | ✓ EXECUTED, fires, incl. the alias | +| negative canonicity leg in the assembled proof | ✓ EXECUTED, §5 | +| sweep for other `LFMC` leaf-hashing | ✓ EXECUTED, §4 — none found | +| D6 — hasher-parameterised fixture | ✓ done | +| full `lfm::` suite | **306 pass / 19 fail** — failure set `diff`-identical to the B1 baseline | +| `make fmt` + `make lint` (4 combos) + `blake3-6round` clippy | ✓ clean | +| **cost: `TrivialV0` 16,551 @7r** | ✓ EXECUTED, matches the spec exactly | +| **cost: `FriToyV0`** | ⚠ **513,081, not the spec's 502,047** — §6 | + +--- + +## 1. What was built + +A fourth preprocessed selector, `MODE_L`, carrying the `"LFML"` tag and +**felt-input semantics**. A leaf row reads ONE cell as four arbitrary Goldilocks +elements, splits each into a checked `lo`/`hi` `u32` pair, and hashes the eight +halves through the same socket every other mode uses. + +``` +v = lo + 2^32·hi , lo, hi < 2^32 +v < p ⟺ NOT( hi = 2^32−1 AND lo ≥ 1 ) (p − 1 = 0xFFFFFFFF_00000000) +``` + +Per felt: the halves binding plus three canonicity constraints in `LFM_BITDEC`'s +own `Z`/`GINV` idiom — **2 witness columns and 4 constraints per felt, zero new +sends, max degree still 3.** + +| | before | after | +|---|---|---| +| selectors | `MODE_C`, `MODE_P`, `MODE_T` | + **`MODE_L` at index 9** | +| `NUM_SELECTORS` / `PREP_WIDTH` | 3 / 12 | **4 / 13** (`MULT0..2` → 10..12) | +| socket value columns @7r | 3,436 | **3,444** (+8 canonicity witnesses) | +| framing constraints | 26 | **46** | +| live domains | `LFMC`, `LFMT` | + **`LFML`** | + +### File:line map + +| what | where | +|---|---| +| `MODE_L`, `NUM_SELECTORS` 4, `PREP_WIDTH` 13 | `prover/src/lfm/layout.rs:80-133` | +| `HashMode::Leaf` + `num_input_cells`/`num_output_cells` | `prover/src/lfm/instr.rs:47-115` | +| `LfmBuilder::leaf` | `prover/src/lfm/builder.rs:285-305` | +| `LfmHasher::leaf` / `leaf_out` (+ `HasherKind` dispatch) | `prover/src/lfm/hash.rs:92-118`, `:290-308` | +| `TAG_LFML`, `is_canonical`, `felt_halves`, `leaf_lanes`, `leaf_digest*` | `prover/src/lfm/blake3_socket.rs:195-380` | +| BLAKE3 `leaf`/`leaf_out` + the `admits` leaf arm | `prover/src/lfm/blake3_socket.rs:470-520` | +| `MU_COLUMNS` (3), `DIGEST_MODE_COLUMNS`, `CANON` block, `canon_z`/`canon_ginv` | `prover/src/lfm/blake3_socket.rs:455-560` | +| the leaf constraints (idx 26–45) | `prover/src/lfm/blake3_socket.rs:1130-1190` | +| `lanes_from_row` + `fill_canonicity_witness` | `prover/src/lfm/blake3_socket.rs:1045-1105` | +| the §2.2 warning, verbatim in substance | `prover/src/lfm/blake3_socket.rs:39-75` (module docs) | +| O5 retirement, rewritten | `prover/src/lfm/blake3_socket.rs:120-145` (module docs) | +| `edsl::leaf_hash_pair`, `SpongeVar::absorb_felts` | `prover/src/lfm/edsl.rs:98-175` | +| `fixture::host_leaf_hash_pair`, hasher-parameterised tree/prover | `prover/src/lfm/fixture.rs:136-300` | +| `FriToyV0`'s three leaf sites + the two data absorbs | `prover/src/lfm/programs.rs:629,637,676`, `:600-607` | +| leaf KAT vectors (generated from the spec JSON) | `prover/src/lfm/leaf_kats.rs` | +| leaf tests (14) | `prover/src/lfm/leaf_tests.rs` | + +--- + +## 2. KAT results — ✓ EXECUTED + +The Rust table is **rendered from `leaf_kats.json`**, not hand-copied. + +| | check | evidence | +|---|---|---| +| L1 | 5 leaf rows, lanes and digest, at 6 **and** 7 rounds | `every_leaf_vector_reproduces_at_both_round_counts` | +| L1′ | **the crate anchor** — `blake3::hash(LE32(lanes)‖"LFML")[..16]` @7r, message rebuilt byte-level | `seven_rounds_is_blake3_of_the_leaf_message` | +| L2 | six boundary felts round-trip, `p − 1` included (the tight case) | `every_boundary_felt_round_trips_through_its_halves` | +| L3 | three non-canonical values rejected, **not reduced** — and the test derives the alias each one collides with | `non_canonical_values_are_rejected_not_reduced` | +| L4 | predicate == `v < p` over every boundary, ±2,000 around the wrap, and a 20k stride | `the_canonicity_predicate_is_exactly_less_than_p` | +| L5 | `LFMC`/`LFMT`/`LFML` give three different digests from the SAME eight lanes, both round counts | `the_three_domains_differ_on_the_same_lanes` | +| L6 | an 8-felt leaf is 2 `LFML` + 1 `LFMC`, and the HOST path agrees | `an_eight_felt_leaf_is_two_leaf_rows_and_one_parent` | + +`leaf_tests`: **14 passed** at 7 rounds; **14 passed** under +`--features blake3-6round`. + +--- + +## 3. M9 / M10 — ✓ EXECUTED, both fire + +| | statement | result | +|---|---|---| +| **M9** | a row in domain X whose witness computes domain Y's digest | **all six ordered confusions rejected**; the three same-domain cases accepted (the honest control, in the same loop) | +| **M10** | a `MODE_L` row that skips canonicity | **rejected**, two ways | + +M10's second way is the one worth reading. It installs **the alias**: felt `0` +re-encoded as `(lo = 1, hi = 2^32−1)`. That is the *same field element* — the +binding constraint `v = lo + 2^32·hi` is satisfied — so nothing except canonicity +can catch it, and the test asserts the violated index is **`canon-c` for felt 0 +(idx 33)** specifically rather than "something fired". Without the block, one +felt would have two leaf digests, which is a collision in the felt→digest map. + +**What this makes checkable:** "`MODE_L` implies felt-input semantics" is now a +constraint, not a convention. + +--- + +## 4. The `LFMC` leaf-hashing sweep — ✓ EXECUTED, nothing else found + +Every `LFM_HASH` call site in non-test code, classified: + +| site | kind | verdict | +|---|---|---| +| `edsl::merkle_walk:187` | parent over two digests | correct as `LFMC` | +| `edsl::leaf_hash_pair:170` | parent over two leaf digests | correct as `LFMC` | +| `FriToyV0` ×3 (`programs.rs:629,637,676`) | **leaf over trace rows / folded ext values** | **moved to `MODE_L`** | +| `TrivialV0` ×3 (`programs.rs:60-62`) | raw arena data | **left as `LFMC`, deliberately** | + +`TrivialV0`'s three are the only remaining place raw arena data enters a +compress, and they form a **chain, not a tree** — there is no leaf and no parent, +so there is no confusion for `MODE_L` to separate. The consequence, recorded at +the call site: that program's arena words must be `u32`-laned under BLAKE3, which +its tests supply. Data that cannot be is exactly what `leaf` exists for. + +**O5 is retired and now enforced by the tag.** A leaf digest is +`BLAKE3(…‖"LFML")` and a parent is `BLAKE3(…‖"LFMC")`, so an internal node cannot +be replayed as a leaf whatever the tree's shape. Fixed depth remains true of +every current program but **is no longer load-bearing**. The module docs were +rewritten accordingly — the previous text said "nothing implements `LFML` yet" +and rested the argument on fixed depth. + +--- + +## 5. ★ The milestone, and its negative leg + +`leaf_tests::fri_toy_proves_and_verifies_under_blake3` — ✓ EXECUTED. A real +verification program, over real FRI data (LDE evaluations and folded extension +elements, **124 of the fixture's 128 committed values are ≥ 2^32**), proved under +the machine's real hash and accepted by the production verifier. The attested +public output is checked against the inner proof's own roots. + +The four replacement criteria the old tripwire's doc set: + +1. it is deleted only now that `FriToyV0` proves and verifies under BLAKE3 ✓ +2. the replacement is a **prove+verify**, not an execute ✓ +3. honest control: the same program proves under `Test` (and Poseidon) ✓ — + `fri_toy_proves_and_verifies_under_every_hasher`, 3/3 +4. ⚠ **RETIRED AS UNSATISFIABLE — not met, and it cannot be.** The criterion + asked for "a non-canonical arena value must make the proof fail, and fail FOR + canonicity". No such arena value exists: an arena word is `[FE; 4]` and every + `FE` is canonical by construction, so the input the criterion describes is + unconstructible. (By the same argument the `admits` leaf arm's canonicity + check is dead code on today's call paths — it is kept as the boundary's + statement, not as a reachable rejection.) + + **What I shipped in its place is NOT that test.** + `fri_toy_rejects_a_fixture_built_under_another_hasher` is a hasher-mismatch + test: it shows a fixture whose leaves were hashed differently fails to + authenticate. Useful, and it does exercise the leaf digests end-to-end — but + it is a root-mismatch rejection, not a canonicity one, and the first version + of this report presented it as satisfying criterion 4. It does not. + + **Canonicity's necessity is shown elsewhere, and adequately:** by M10's alias + leg (a non-canonical half-pair for a felt that the binding constraint accepts + and only `canon-c` rejects, `leaf_tests.rs`) and, in z3, by the oracle's WA8 + dropped-leg — canonicity removed ⇒ the same felt becomes provable. The + assembled-proof evidence criterion 4 wanted would need a trace tamper rather + than an arena value. + +Plus `the_fixture_data_is_still_not_u32_and_that_is_the_point`, which keeps the +premise visible: proving over `u32`-shaped data would have proved nothing about +the leaf mode. + +### D6 — the fixture, closed + +`HostTree::build` and `fixture_prove_columns` now take a `HasherKind`, and +`host_leaf_hash_pair` mirrors the machine's leaf exactly (2 `LFML` + 1 `LFMC`). +The review called this a completeness trap at exactly this milestone and it was: +with `TestPermutation` hard-coded, every BLAKE3 run would have failed inside a +query walk with an authentication error rather than at the mismatch. + +--- + +## 6. ⚠ DEVIATION — `FriToyV0` costs 513,081, not the spec's 502,047 + +**This is a spec premise that does not hold, not an implementation choice.** + +`LEAF.md` §5 says *"`FriToyV0` compresses 67 → **91** … transcript unchanged at +11"*. The transcript's step count IS unchanged at 11 — that half is right. But +two of the four cells `FriToyV0` absorbs are **`t0` and `t1`, the terminal +polynomial's coefficients** — arbitrary field elements, not digests. Absorbing +them raw hands the socket lanes that are not `u32`, so the row is unprovable +under BLAKE3. ✓ EXECUTED: the fixture panicked in +`Blake3Permutation::step` until this was fixed; the probe showed +`commitments[2]` and `[3]` each carry 3 non-`u32` lanes. + +**The fix, and why this shape:** data enters the transcript the same way it +enters a tree — through the leaf encoding. `SpongeVar::absorb_felts(c)` hashes +the cell to a digest under `"LFML"` and absorbs *that*, binding the data up to +the leaf hash's collision resistance. One uniform rule ("`absorb` for digests, +`absorb_felts` for data"), stated at both the machine and host sponge. + +**Cost:** two extra `LFML` rows. + +| | spec | built | note | +|---|---:|---:|---| +| `LFMC` rows | 56 | 56 | 4 queries × (3 leaf parents + 11 walk steps) | +| `LFMT` rows | 11 | 11 | the transcript, unchanged as §5 says | +| `LFML` rows | 24 | **26** | 4 × 3 data leaves × 2, **+2 terminal coefficients** | +| total rows | 91 | **93** | | +| cell-equiv @7r | 502,047 | **513,081** | +2.2% | + +`TrivialV0` is unaffected and reproduces the spec's **16,551** exactly (3 rows × +5,517), including §5's own correction that it is *not* cost-unchanged — the +canonicity witnesses exist on every row. + +**For the decision record:** the per-row price (5,517 @7r, 4,749 @6r) and +`TrivialV0` are exactly as ratified; only `FriToyV0`'s row count moved, and it +moved because a program that absorbs field data needs the leaf encoding there +too. If the leaf spec's 91 is quoted anywhere downstream it should be corrected +to 93. + +--- + +## 7. Verification + +| gate | result | +|---|---| +| full `lfm::` suite | **306 passed / 19 failed** — `diff`-identical failure set to the B1 baseline (the `fibonacci.elf` 19). From B1's 290: **+16 passes**, being 16 new `leaf_tests` less the deleted O1 tripwire, plus the transcript preamble's split assertions | +| `lfm::leaf_tests` | **16** pass @7r, 16 pass @6r | +| `lfm::blake3_socket_tests` | **34** pass @7r, 34 @6r — the same 34; the file has no `cfg`-gated test, and the earlier "35/34, one 7r-only" in this report was simply wrong | +| `lfm::transcript_tests` | 17 pass @7r, 17 @6r | +| `make fmt` + `make lint` (4 feature combos) | clean, exit 0 | +| `clippy --features blake3-6round` | clean, exit 0 | + +### Registry re-bless — once, riding `PREP_WIDTH` 12 → 13 + +All six `program_id`s moved: + +| entry | new (first 8 bytes) | +|---|---| +| `TrivialV0` | `7087e2838dae1171` | +| `FriToyV0` | `82b53911e83ceb53` | +| `KeccakChainV0` | `e830e1f5f9f1ebaf` | +| `KeccakSpongeV0` | `d4f94944580b18eb` | +| `TranscriptReplayV0` | `998273f096ab6b57` | +| `StatementReplayV0` | `788129775db248d2` | + +Diff scope: 19 modified files under `prover/src/lfm/` plus 2 new +(`leaf_kats.rs`, `leaf_tests.rs`). Nothing outside the LFM surface; the keccak +wrap path is untouched. + +--- + +## 8. What the oracle's re-gate needs — exposed, mirroring the `MODE_T` pass + +| the gate needs | where it is | +|---|---| +| the leaf selector | `cols::MODE_L` | +| the four-way mu | `cols::MU_COLUMNS` (3 entries: C, T, L) and `cols::DIGEST_MODE_COLUMNS` (C, T — the lane identity's gate) | +| the tag map, verbatim | `TAG_SELECTOR` — `(column, tag)` pairs, now three | +| `TAG_LFML` for the pin's checked set | `blake3_socket::TAG_LFML = 0x4C4D464C` | +| the canonicity witness columns | `cols::canon_z(i)` / `cols::canon_ginv(i)`, `cols::CANON` | +| the half lanes | `cols::leaf_lo_lane(i)` / `cols::leaf_hi_lane(i)` | +| constraint indices | 0–3 capacity, 4 mode-sum, 5 `MODE_P` pin, 6–13 lane identity (**digest modes only**), 14–21 unused `OUT`, 22–25 digest recomposition, **26–33 unread-`IN` pins** (`chips::hash::emit_unread_input_pins`, shared by all three arms), **34–49 leaf** (per felt: binding, canon-a, canon-b, canon-c — located by `blake3_socket::LEAF_IDX`), 50+ core | +| the predicate | `blake3_socket::is_canonical` | + +**WA8's honest leg is already covered on the Rust side** by +`m10_a_leaf_row_cannot_skip_canonicity`'s control and by +`fri_toy_proves_and_verifies_under_blake3`; the "canonicity dropped ⇒ SAT" leg +needs the gate, since it requires editing the constraint set. + +## 9. Open + +| item | status | +|---|---| +| WA8 / M8-four-way / M9 / M10 in z3, and the pin gaining `TAG_LFML` + pairwise-distinct | ✗ OPEN — the oracle's, `gate-oracle/` untouched by this build | +| tag tables marking `"LFML"` **live** rather than reserved | ✗ OPEN — flagged, not edited (the B1 pass established that these are the oracle's) | +| `LEAF.md` §5's 91 / 502,047 | ✗ OPEN — §6; needs correcting to 93 / 513,081 | +| single-domain hashers do not separate leaf from parent | ✗ OPEN by design — recorded at `LfmHasher::leaf_out`; under `Test`/`Poseidon` O5 still rests on fixed depth, as it did before. Neither is a production hash | + +--- + +## 10. Post-review fixes (leaf-verify.md D1–D6) + +The adversarial review confirmed the leaf mode's own machinery sound and found +one **HIGH soundness defect** in the other two arms, plus five claim/doc defects. +All are fixed. + +### ★ D1 — HIGH, soundness. `MODE_L`'s unread input cells were free under `Test` and `Poseidon`. + +**The defect, and it was mine.** `MODE_L` reads ONE cell. Three places were +taught that — the `LfmMem` receive, the validator's address-slot check, and the +BLAKE3 AIR's value-column pin — and two were not: `eval_test` and `eval_poseidon` +read `A_i = IN_i` for every `i < 8` in round 0. So on a leaf row under those +hashers, `IN4..8` received nothing from the bus, were pinned by nothing, and were +**read by the permutation the AIR proves** — four free Goldilocks felts, and +`leaf(c)` stopped being a function of `c`. The reviewer executed it end to end: +Poseidon proved AND verified with attacker junk in those columns. For any program +that absorbs data through `absorb_felts` — `FriToyV0` does — that is a complete +Fiat–Shamir break, since the prover re-randomises the junk and chooses `alpha`, +`zeta0`, `zeta1` and every query index with the public statement unchanged. + +**⚠ This report talked someone out of the fix before it was needed.** §1's earlier +text called the pin "hygiene rather than soundness". That was true of the BLAKE3 +arm in isolation and false as a general statement, and it is exactly the sentence +a reader would have cited to skip the other two arms. The comment is reworded at +the source and the claim is retracted here. + +**The fix** — `chips::hash::emit_unread_input_pins`, ONE derivation from +`HashMode::num_input_cells()`, called by all three arms: + +- for each input cell some mode does not read, `Σ(selectors of modes that do not + read it) · IN_col = 0`, four constraints per cell, degree 2; +- **both** unread cells are pinned, not only the one that broke: cell 2 was + previously safe because nothing read it, which is precisely the reasoning that + failed for cell 1. `MODE_SELECTORS` is the single mode↔column table it reads. +- constraint counts: `Test` 17 → 25, Poseidon 601 → 609, BLAKE3 framing 46 → 50 + (`NUM_CONSTRAINTS` 942 → 946 @7r). **No cell counts move** — constraints do not + enter the census — so §6's pricing is unchanged and ✓ EXECUTED: all six + `program_id`s are byte-identical to the pre-fix re-bless. + +**Regression tests**, `leaf_tests`: + +- `d1_the_unread_input_pins_are_load_bearing_under_every_hasher` — **shaped like + WA9**, on the oracle's suggestion, because "the junk row is rejected" would + pass for a set that rejected it incidentally and would say nothing about + whether the new constraints are needed. For each of the three arms: the honest + leaf row still satisfies every constraint (the mandatory honest control), and a + **consistent** forgery — junk in the unread cell with the whole rest of the row + rebuilt from it, which is what a prover controlling the trace would actually + submit — has a violated set that is **exactly** the four pins for that cell, + read from `chips::hash::unread_input_pin_base` rather than a literal. + + That equality carries both legs at once: WITH the pins the row is rejected, and + WITHOUT them — delete those four and every remaining constraint still evaluates + to zero on this row — it is **ACCEPTED**. That is the dropped-leg, and it is + what makes the pins necessary rather than merely present. On `Test` and + `Poseidon` that acceptance was the shipped behaviour and an executed + Fiat–Shamir break; on BLAKE3 the row is inert either way, which is precisely + the WA9 mirror-image the oracle named — the same constraint is hygiene on one + arm and soundness on the two whose round 0 reads `IN4..8`. +- `d1_the_pins_come_from_one_derivation` — the selector table is the layout's + one-hot span, and the pin count follows from `num_input_cells()`, so a mode + added later cannot acquire free columns by an arm forgetting a line. + +### D2 — MEDIUM. The transcript end-to-end vector modelled a transcript `FriToyV0` no longer runs. FIXED — **consumed from the oracle, not invented here.** + +`transcript_kats.rs` carried a `✓ VERIFIED against fri_toy_program_source` marker +on a vector built from `absorb2(t0w, t1w)`, but the program now does +`absorb_felts(t0w); absorb_felts(t1w)`. The transcript's own **step count is +unchanged at 11**, which is why nothing went red — and why the stale marker +mattered: the state vector is the one anchor from an independent reference. + +**The oracle found the identical staleness in its own `transcript_kats.json` and +regenerated it at both round counts.** `transcript_kats.rs` is re-rendered from +that file; the vector is theirs. + +**✓ EXECUTED cross-check, worth recording.** Before their regeneration was +visible I had composed a replacement myself from the same two oracle references +(`transcript_ref` chained, `leaf_ref` for the leaf step). The two agree +**bit-for-bit** across all 11 states at 7 rounds — two independent compositions +landing on the same vector. Mine is discarded; theirs is what ships. + +**Their convention, adopted:** `FRI_TOY_COMPRESSIONS` counts the preamble's total +socket cost — **13**, being 11 transcript steps plus the 2 `LFML` rows +`absorb_felts` adds — rather than transcript steps alone. That is the +decomposition that closes the total in the vectors instead of only in prose: +**4 queries × 20 + 13 = 93**, which is the number §6 measured. The tests assert +both halves and their sum, so neither can drift alone. + +The four replay sites now model the real preamble: the reference replay, the +`HostSponge` replay, the emitted preamble program, and the machine-vs-host +agreement test. The preamble's arena carries the terminal coefficients as FELTS, +which is what the leaf encoding consumes. + +### D3 — criterion 4 retired as unsatisfiable. §5, rewritten above. + +### D4 — the report's own arithmetic. Fixed in §0 and §7. + +`blake3_socket_tests` is **34 at both round counts** with no `cfg`-gated test — +the earlier "35 @7r / 34 @6r, one 7r-only" was wrong (the difference was the +tripwire I had already deleted). The suite total is now 306, and the delta from +B1's 290 is **+16 passes**, stated as such rather than as "+13". + +### D5 — the leaf/parent separation caveat was in the wrong place. FIXED. + +`instr.rs` and `layout.rs` stated "O5 retired by the tag" **hasher-independently** +in the ISA docs, while the correctly qualified version sat in `hash.rs` — exactly +backwards, since a single-domain hasher does not separate the domains at all. +Both ISA sites now say the mode is a machine-level SHAPE and that whether a leaf +and a parent are different functions is the hasher's business, pointing at +`LfmHasher::leaf_out`. + +### D6 — "hygiene rather than soundness". FIXED. + +The comment on the BLAKE3 pin now says the pin is **load-bearing on any arm whose +constraints read `IN`**, names the two that do, and records that it shipped +missing there — so the next reader is warned rather than reassured. + +--- diff --git a/thoughts/shared/lfm-real-hash/leaf-spec/LEAF.md b/thoughts/shared/lfm-real-hash/leaf-spec/LEAF.md new file mode 100644 index 000000000..27d7370f6 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-spec/LEAF.md @@ -0,0 +1,294 @@ +# The `LFML` leaf mode — specification + +**Status:** specification + reference + vectors + gate plan, written **before any +Rust exists**. **No chip code exists for this.** **Date:** 2026-08-11. + +**Decision this implements:** the user ratified **option C with `LFML`** +(`../leaf-convention-options.md`): arbitrary field elements reach the BLAKE3 +socket as **checked u32 halves inside the socket itself**, under a fourth +preprocessed selector `MODE_L` carrying the `"LFML"` tag. `MODE_L` **implies +felt-input semantics** — fixed by decision, not assumed. + +**Sequencing:** this builds on **committed B1**, not into B1's uncommitted diff. + +Claims are ✓ VERIFIED / ✓ EXECUTED / ? INFERRED / ✗ OPEN. + +--- + +## 0. Board + +| check | result | +|---|---| +| `leaf_kats.py` L1–L6 | **PASS**, both round counts | +| canonicity predicate == `v < p` | ✓ EXECUTED, **300,007 cases** incl. every boundary | +| non-canonical inputs rejected, not reduced | ✓ EXECUTED (L3) | +| `LFML` / `LFMC` / `LFMT` pairwise distinct on the same lanes | ✓ EXECUTED (L5) | +| `TAG_LFML` = `0x4C4D464C` round-trip | ✓ EXECUTED | +| `FriToyV0` **93** compresses / **513,081** @7r | ✓ EXECUTED — corrected from the spec's 91 / 502,047; the `t0`/`t1` felt absorbs need `absorb_felts` (+2 `LFML` rows). See the correction below (§ near the cost table). | + +--- + +## 1. Construction + +### 1.1 The felt↔halves boundary + +``` +v = lo + 2^32·hi , lo, hi ∈ [0, 2^32) +``` + +**Canonicity, and why it is cheap.** `p − 1 = 0xFFFFFFFF_00000000` — that is +`hi = 2^32−1`, `lo = 0`. So for halves already known to be `u32`: + +> **`v < p` ⟺ NOT( `hi = 2^32−1` AND `lo ≥ 1` )** + +✓ EXECUTED over 300,007 cases including every boundary. The socket's **existing** +O1 machinery (byte columns + `AreBytes` + the lane identity) already forces +`lo, hi < 2^32`; canonicity was the only missing piece. + +**Boundary table** — the cases the KATs pin: + +| felt | `hi` | `lo` | canonical? | +|---|---|---|---| +| `0` | `0x00000000` | `0x00000000` | ✓ | +| `1` | `0x00000000` | `0x00000001` | ✓ | +| `2^32 − 1` | `0x00000000` | `0xFFFFFFFF` | ✓ | +| `2^32` | `0x00000001` | `0x00000000` | ✓ | +| `p − 2^32` | `0xFFFFFFFE` | `0x00000001` | ✓ | +| `p − 1` | `0xFFFFFFFF` | `0x00000000` | ✓ **the tight case** | +| `p` | `0xFFFFFFFF` | `0x00000001` | ✗ **rejected** | +| `p + 1` | `0xFFFFFFFF` | `0x00000002` | ✗ rejected | +| `2^64 − 1` | `0xFFFFFFFF` | `0xFFFFFFFF` | ✗ rejected | + +**REJECT, DO NOT REDUCE.** A non-canonical input has no satisfying witness, so +the row is unprovable. Same shape as O1 itself; the host-side impl must refuse, +never wrap. + +### 1.2 Lane layout + +A leaf row hashes **four felts** = eight lanes = exactly one compress input: + +``` +lanes = [lo0, hi0, lo1, hi1, lo2, hi2, lo3, hi3] felt i at lanes 2i, 2i+1 +``` + +**Halves adjacent** is load-bearing: it lets the canonicity gate read one pair of +neighbouring lanes rather than reaching across the row. + +### 1.3 Byte serialization (normative — the crate-KAT anchor) + +Each lane is four **little-endian** bytes, in lane order, then the tag: + +``` +msg = LE32(lo0)‖LE32(hi0)‖…‖LE32(lo3)‖LE32(hi3)‖"LFML" (36 bytes) +digest = BLAKE3(msg)[0..16] read back as four LE u32 lanes +``` + +✓ EXECUTED (L1): identical to the word-level route at both round counts. At 7 +rounds this is a plain `blake3::hash` call — **the crate-KAT property survives +because the message layout is byte-identical to a digest-mode compress.** + +### 1.4 Leaf structure: 1 → 3 compresses + +A `FriToyV0` leaf covers **two trace rows = eight field elements** +(`NUM_COLS = 4`). Four felts per row ⇒ + +``` +d0 = LFML(f0..f3) leaf row 1 +d1 = LFML(f4..f7) leaf row 2 +leaf = LFMC(d0, d1) ordinary parent +``` + +Two `LFML` rows + one `LFMC` parent. ✓ EXECUTED (L6). + +--- + +## 2. `MODE_L` — layout and constraints + +### 2.1 Layout, applying the builder's §7.2 lesson + +`MODE_L` **must sit inside the contiguous selector run**: the admission +validator's one-hot check reads `NUM_SELECTORS` from `MODE_C`, so a selector +parked past the multiplicities would be outside that check and silently +unchecked. That is the mistake §7.2 already caught once; it must not be repeated. + +| index | column | change | +|---:|---|---| +| 6 | `MODE_C` | — | +| 7 | `MODE_P` | — | +| 8 | `MODE_T` | — | +| **9** | **`MODE_L`** | **NEW** | +| 10 | `MULT0` | shifted 9 → 10 | +| 11 | `MULT1` | shifted 10 → 11 | +| 12 | `MULT2` | shifted 11 → 12 | + +**`NUM_SELECTORS` 3 → 4. `PREP_WIDTH` 12 → 13.** + +`TAG_SELECTOR` gains `(cols::MODE_L, TAG_LFML)`; `MU_COLUMNS` becomes +`MODE_C + MODE_T + MODE_L`. + +### 2.2 The constraints + +Per felt `i ∈ 0..4`, with `lo = lane(2i)`, `hi = lane(2i+1)`, `v = IN_i`: + +``` +binding MU_L · ( v − lo − 2^32·hi ) = 0 degree 2 +canon-a MU_L · Z_i · G_i = 0 degree 3 +canon-b MU_L · ( 1 − Z_i − G_i·GINV_i ) = 0 degree 3 +canon-c MU_L · Z_i · lo = 0 degree 3 + +where G_i = (2^32 − 1) − hi MU_L = MODE_L +``` + +`canon-a` gives `G ≠ 0 ⇒ Z = 0`; `canon-b` gives `G = 0 ⇒ Z = 1`; `canon-c` then +says *hi maximal ⇒ lo zero*. This is **`LFM_BITDEC`'s own `Z`/`GINV` idiom** +(✓ VERIFIED `chips.rs`), applied to two halves instead of 64 bits — the machine's +established canonicity shape, not a new invention. + +**Cost: 2 witness columns (`Z_i`, `GINV_i`) and 4 constraints per felt** — 8 +columns and 16 constraints per row, **zero extra sends**. Max degree stays **3**. + +**Range comes free.** `lo` and `hi` are ordinary input lanes, so the existing +lane identity plus `AreBytes` already force them `< 2^32` — the same machinery +WA1/WA2 gate. **This is the whole reason option C is cheap** and it must be +stated in the chip's own docs, because a future reader who does not see it may +"helpfully" add a redundant range check or, worse, remove the lane identity +believing the canonicity gate subsumes it. It does not: canonicity assumes the +`u32` bound, it does not establish it. + +--- + +## 3. Gate extension plan + +### 3.1 New width-audit pair (field domain) + +The direct analogue of WA1/WA2, and the item that makes §2.2 checked rather than +asserted: + +| | check | expected | +|---|---|---| +| **WA8** | canonicity present → a non-canonical felt is **unprovable** | UNSAT | +| **WA8** | canonicity **dropped** → the same felt becomes provable | **SAT** | +| **WA8** | honest leg → canonical felts still prove | **SAT** | + +The honest leg is not optional: a "fix" that rejected every felt would pass the +first two on its own. + +### 3.2 M-controls, extended + +- **M8 over a four-way one-hot.** The existing M8 (idx 4 pins the mode *sum*, not + the selectors) extends unchanged — with four selectors a fractional split still + forges any tag as a blend, so the registrar's one-hot check remains the + load-bearing mechanism. Verify M8 fires with `MODE_L` in the span. +- **M9 (new) — mode confusion.** An `LFML` row computing an `LFMC` or `LFMT` + digest → **SAT** (detected), and the mirror images. Three tags now means six + ordered confusions; L5 already shows the three are pairwise distinct at the + reference level. +- **M10 (new) — felt-input semantics is implied.** A row with `MODE_L = 1` that + skips the canonicity block → **SAT**. This is what pins "`MODE_L` implies + felt-input" as a constraint rather than a convention. + +### 3.3 Pin + +`artifact_pin.py` must resolve **`TAG_LFML`** into the checked set and assert the +three tags are **pairwise distinct** (today it checks only `LFMC ≠ LFMT`). +`framing_consts` and `cols` regions will both drift; `eval` will too. + +--- + +## 4. O5 — what closes, and what the fixed-depth argument becomes + +✓ VERIFIED, and the framing needs stating plainly: **`FriToyV0` already performs +leaf hashing today**, compressing raw trace rows into leaves under the **`LFMC`** +tag. The claim "no leaf-hashing path exists" is false. O5's safety today rests on +**fixed depth alone** — every eDSL circuit is fixed-shape at build time, so no +variable-depth second-preimage confusion is reachable. + +**With `LFML` live, that changes structurally.** Leaves and parents occupy +different domains by construction: a leaf digest is `BLAKE3(…‖"LFML")` and a +parent is `BLAKE3(…‖"LFMC")`, so an internal node can no longer be replayed as a +leaf regardless of tree shape. + +**What fixed depth still buys: nothing that O5 needs.** It remains true of every +current program and is worth keeping as a property, but it stops being +load-bearing for second-preimage resistance. + +> **O5's obligation becomes: RETIRED, and enforced by the tag rather than by +> review.** The rule "any leaf-hashing path MUST use `LFML`" stops being a +> review checklist item and becomes a mechanical fact — a leaf row is one with +> `MODE_L` set, and `MODE_L` selects `LFML`. The reviewer's job shrinks to +> *"is this row's mode right?"*, which the one-hot check and M9/M10 answer. + +? INFERRED and worth a build-time check: whether any *existing* program besides +`FriToyV0` compresses non-digest data under `LFMC`. If one does, it is a leaf +path that must move to `MODE_L` in the same pass. + +--- + +## 5. Program impact + +| | today (post-B1) | with `MODE_L` | note | +|---|---:|---:|---| +| per-row price @7r | 5,509 | **5,517** | +8 cells: the canonicity witnesses exist on every row | +| `FriToyV0` compresses | 67 (blocked) | **93** | leaves 1→3, **plus 2 LFML rows for the `t0`/`t1` felt absorbs** — see the correction below | +| `FriToyV0` cell-equiv @7r | — | **513,081** | ✓ EXECUTED against the built chip | +| `TrivialV0` compresses | 3 | 3 | unchanged in count | +| `TrivialV0` cell-equiv @7r | 16,527 | **16,551** | ⚠ **not unchanged** — see below | + +> ⚠ **CORRECTION TO THIS SPEC — mine, and worth reading as a lesson.** §5 +> originally said **91 compresses / 502,047**. The truth is **93 / 513,081**: the +> transcript's `absorb2(t0w, t1w)` absorbs the terminal-polynomial coefficients, +> which are **arbitrary field elements**, so they must go through the leaf/felt +> path — two more `LFML` rows, +11,034 cell-equiv. +> +> **The failure was not arithmetic, it was leaving an open item open.** The +> options note flagged exactly this at §6: *"whether `t0w`/`t1w`'s absorbs want +> felt mode or a separate convention"* — recorded as ✗ OPEN. Then this spec +> asserted "transcript unchanged at 11" and put a definite number in a table. +> **An open question carried into a concrete figure stops looking open.** The +> ranking is unaffected (C stays ~12% under A), but the number was wrong for two +> documents until the build measured it. +> +> ⚠ **Correction to the brief:** `TrivialV0` is **not** cost-unchanged. It gains +> **+24 cell-equiv** (3 rows × 8 columns), because the canonicity witness columns +> are part of the AIR and therefore exist on *every* compress row, leaf or not. +> Small, but the brief said "unchanged" and the census must not carry a claim the +> formula contradicts. + +**Registry re-bless:** rides the `PREP_WIDTH` 12 → 13 change — the preprocessed +roots move, so all entries are re-blessed **once**, in the same pass. ✓ Consistent +with how B1's re-bless was sequenced. + +**Tripwire replacement.** `blake3_socket_tests::fri_toy_is_still_blocked_by_o1_and_no_longer_by_the_sponge` +asserts (a) no permute remains, (b) fixture values are not `u32`-laned, (c) the +refusal is specifically O1, (d) the honest control under `Test`. Its own doc says +it must be replaced when O1 closes. **Replacement criteria:** + +1. delete it only when `FriToyV0` **proves and verifies** under `Blake3`; +2. the replacement is a **prove+verify**, not an execute — an execute-only test + proves nothing about the chip; +3. keep an honest control that the same program still proves under `Test`; +4. add a **negative** leg: a deliberately non-canonical arena value must make the + proof fail, and fail *for canonicity*, not for some other reason. + +Criterion 4 is the one most likely to be skipped, and it is the one that shows +the canonicity gate is doing work in the assembled program rather than only in +the unit test. + +--- + +## 6. Open + +| item | status | +|---|---| +| the same identity against the Rust `blake3` **crate** | ✗ DEFERRED — needs cargo | +| WA8 / M9 / M10 against a real chip | ✗ OPEN — needs the build | +| any other program leaf-hashing under `LFMC` (§4) | ✗ OPEN — build-time sweep | +| tag tables gain `"LFML"` as **live** rather than reserved | ✗ OPEN — one pass, with the build | + +## 7. Files + +| file | what | +|---|---| +| `leaf_ref.py` | the reference: halves boundary, canonicity predicate, leaf compress | +| `leaf_kats.py`, `leaf_kats.json` | L1–L6 incl. boundary felts and non-canonical rejects | +| `../leaf-convention-options.md` | why option C was chosen (the decision record) | diff --git a/thoughts/shared/lfm-real-hash/leaf-spec/leaf_kats.json b/thoughts/shared/lfm-real-hash/leaf-spec/leaf_kats.json new file mode 100644 index 000000000..d4857a4a3 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-spec/leaf_kats.json @@ -0,0 +1,389 @@ +{ + "mode": "LFML leaf (felt-input, option C)", + "tag_ascii": "LFML", + "tag_word": 1280132684, + "felts_per_row": 4, + "lane_order": "[lo0, hi0, lo1, hi1, lo2, hi2, lo3, hi3] \u2014 halves adjacent", + "byte_serialization": "each lane as 4 little-endian bytes, in lane order, then the 4 tag bytes", + "canonicity": "v < p <=> NOT(hi == 2^32-1 AND lo >= 1)", + "rounds": { + "7": { + "leaf_rows": [ + { + "name": "zeros", + "felts": [ + "0", + "0", + "0", + "0" + ], + "lanes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "digest": [ + 1017299827, + 2031355749, + 1047555248, + 3835764026 + ], + "digest_hex": "3ca2c373791407653e706cb0e4a11d3a" + }, + { + "name": "boundary_mix", + "felts": [ + "0", + "1", + "18446744069414584320", + "4294967296" + ], + "lanes": [ + 0, + 0, + 1, + 0, + 0, + 4294967295, + 0, + 1 + ], + "digest": [ + 1140485700, + 3943323121, + 2877313884, + 2954089848 + ], + "digest_hex": "43fa6e44eb0a55f1ab80535cb013d578" + }, + { + "name": "all_p_minus_1", + "felts": [ + "18446744069414584320", + "18446744069414584320", + "18446744069414584320", + "18446744069414584320" + ], + "lanes": [ + 0, + 4294967295, + 0, + 4294967295, + 0, + 4294967295, + 0, + 4294967295 + ], + "digest": [ + 382399115, + 2486454156, + 3385271256, + 1934521085 + ], + "digest_hex": "16caf28b9434478cc9c723d8734e72fd" + }, + { + "name": "ramp", + "felts": [ + "72623859790382856", + "1230066625199609624", + "2387509390608836392", + "3544952156018063160" + ], + "lanes": [ + 84281096, + 16909060, + 353769240, + 286397204, + 623257384, + 555885348, + 892745528, + 825373492 + ], + "digest": [ + 1971853178, + 2006291185, + 2531936965, + 932064554 + ], + "digest_hex": "7588177a779592f196ea4ac5378e2d2a" + }, + { + "name": "u32_edges", + "felts": [ + "4294967295", + "4294967296", + "18446744065119617025", + "1" + ], + "lanes": [ + 4294967295, + 0, + 0, + 1, + 1, + 4294967294, + 1, + 0 + ], + "digest": [ + 358117891, + 1115319308, + 2578175543, + 3516456933 + ], + "digest_hex": "15587203427a6c0c99abd637d198dfe5" + } + ], + "fri_leaf": { + "felts": [ + "18446744069414584320", + "0", + "1", + "4294967296", + "12345678901234567", + "4294967295", + "18446744065119617025", + "999" + ], + "digest": [ + 1649555383, + 2154463104, + 3083905982, + 851712224 + ], + "digest_hex": "625237b7806a7f80b7d0abbe32c418e0", + "compresses": 3 + } + }, + "6": { + "leaf_rows": [ + { + "name": "zeros", + "felts": [ + "0", + "0", + "0", + "0" + ], + "lanes": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "digest": [ + 2557777634, + 1732849878, + 3606705929, + 3187414574 + ], + "digest_hex": "987496e2674930d6d6f9f709bdfc162e" + }, + { + "name": "boundary_mix", + "felts": [ + "0", + "1", + "18446744069414584320", + "4294967296" + ], + "lanes": [ + 0, + 0, + 1, + 0, + 0, + 4294967295, + 0, + 1 + ], + "digest": [ + 27291842, + 2002304836, + 3470155112, + 1426027047 + ], + "digest_hex": "01a070c27758bf44ced65d6854ff7227" + }, + { + "name": "all_p_minus_1", + "felts": [ + "18446744069414584320", + "18446744069414584320", + "18446744069414584320", + "18446744069414584320" + ], + "lanes": [ + 0, + 4294967295, + 0, + 4294967295, + 0, + 4294967295, + 0, + 4294967295 + ], + "digest": [ + 2528259573, + 2408577808, + 2675574121, + 2324101380 + ], + "digest_hex": "96b22df58f8ffb109f7a05698a86f904" + }, + { + "name": "ramp", + "felts": [ + "72623859790382856", + "1230066625199609624", + "2387509390608836392", + "3544952156018063160" + ], + "lanes": [ + 84281096, + 16909060, + 353769240, + 286397204, + 623257384, + 555885348, + 892745528, + 825373492 + ], + "digest": [ + 1928037103, + 3328938581, + 654530270, + 2779182067 + ], + "digest_hex": "72eb82efc66b9255270356dea5a6f3f3" + }, + { + "name": "u32_edges", + "felts": [ + "4294967295", + "4294967296", + "18446744065119617025", + "1" + ], + "lanes": [ + 4294967295, + 0, + 0, + 1, + 1, + 4294967294, + 1, + 0 + ], + "digest": [ + 2029179454, + 1580812704, + 1019564234, + 4087569903 + ], + "digest_hex": "78f2d23e5e3949a03cc550caf3a35def" + } + ], + "fri_leaf": { + "felts": [ + "18446744069414584320", + "0", + "1", + "4294967296", + "12345678901234567", + "4294967295", + "18446744065119617025", + "999" + ], + "digest": [ + 3209263337, + 1853253886, + 3413660228, + 1483997368 + ], + "digest_hex": "bf4978e96e7668fecb785244587400b8", + "compresses": 3 + } + } + }, + "l2_boundary_roundtrip": [ + { + "name": "zero", + "felt": "0", + "lo": 0, + "hi": 0, + "canonical": true + }, + { + "name": "one", + "felt": "1", + "lo": 1, + "hi": 0, + "canonical": true + }, + { + "name": "u32_max", + "felt": "4294967295", + "lo": 4294967295, + "hi": 0, + "canonical": true + }, + { + "name": "two_pow_32", + "felt": "4294967296", + "lo": 0, + "hi": 1, + "canonical": true + }, + { + "name": "p_minus_2_32", + "felt": "18446744065119617025", + "lo": 1, + "hi": 4294967294, + "canonical": true + }, + { + "name": "p_minus_1", + "felt": "18446744069414584320", + "lo": 0, + "hi": 4294967295, + "canonical": true + } + ], + "l3_non_canonical_rejected": [ + { + "name": "p", + "value": "18446744069414584321", + "lo": 1, + "hi": 4294967295, + "canonical": false, + "rejected": true + }, + { + "name": "p_plus_1", + "value": "18446744069414584322", + "lo": 2, + "hi": 4294967295, + "canonical": false, + "rejected": true + }, + { + "name": "two_pow_64_minus_1", + "value": "18446744073709551615", + "lo": 4294967295, + "hi": 4294967295, + "canonical": false, + "rejected": true + } + ] +} \ No newline at end of file diff --git a/thoughts/shared/lfm-real-hash/leaf-spec/leaf_kats.py b/thoughts/shared/lfm-real-hash/leaf-spec/leaf_kats.py new file mode 100644 index 000000000..aa4928e87 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-spec/leaf_kats.py @@ -0,0 +1,185 @@ +""" +LFML leaf-mode KATs. + + L1 crate-KAT identity: every leaf row equals BLAKE3(halves ‖ "LFML")[..16] at + 7 rounds, computed by two separate routes and asserted equal. + L2 BOUNDARY felts round-trip through the halves boundary: 0, 1, 2^32-1, + 2^32, p-2^32, p-1. + L3 NON-CANONICAL inputs are REJECTED, not reduced: p, p+1, 2^64-1 have no + valid half-pair, and the chip predicate refuses the pairs that encode them. + L4 the canonicity predicate agrees with `v < p` exhaustively on the boundary + and over a large random sample. + L5 DOMAIN SEPARATION: an LFML leaf row over the same eight lanes differs from + an LFMC parent and from an LFMT transcript step. + L6 a FriToyV0-shaped leaf (8 field elements) costs exactly 3 compresses and is + reproducible end to end. + +Run: python3 leaf_kats.py [--write] +""" + +from __future__ import annotations + +import json +import os +import random +import sys + +import leaf_ref as lf + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "gate-oracle")) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "transcript-spec")) +import socket_ref as sk # noqa: E402 +import transcript_ref as tr # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT = os.path.join(HERE, "leaf_kats.json") +P = lf.P + + +def hexlanes(c): + return "".join(f"{x:08x}" for x in c) + + +def l1_crate_identity(rounds: int): + cases = [ + ("zeros", [0, 0, 0, 0]), + ("boundary_mix", [0, 1, P - 1, 2**32]), + ("all_p_minus_1", [P - 1] * 4), + ("ramp", [0x0102030405060708, 0x1112131415161718, + 0x2122232425262728, 0x3132333435363738]), + ("u32_edges", [2**32 - 1, 2**32, P - 2**32, 1]), + ] + out = [] + for name, felts in cases: + w = lf.leaf_compress(felts, rounds) + b = lf.leaf_compress_bytelevel(felts, rounds) + if w != b: + return False, f"L1 route mismatch on {name}@{rounds}", [] + out.append({ + "name": name, "felts": [str(f) for f in felts], + "lanes": lf.leaf_lanes(felts), + "digest": w, "digest_hex": hexlanes(w), + }) + return True, f"L1 PASS: {len(cases)} leaf rows, word route == byte route", out + + +def l2_boundary_roundtrip(): + rows = [] + for name, v in lf.BOUNDARY_FELTS: + lo, hi = lf.felt_halves(v) + if lf.halves_felt(lo, hi) != v: + return False, f"L2 FAIL: {name} does not round-trip", [] + rows.append({"name": name, "felt": str(v), "lo": lo, "hi": hi, + "canonical": True}) + return True, f"L2 PASS: {len(rows)} boundary felts round-trip", rows + + +def l3_non_canonical_rejected(): + rows = [] + for name, v in lf.NON_CANONICAL: + try: + lf.felt_halves(v) + except ValueError: + pass + else: + return False, f"L3 FAIL: {name} ({v:#x}) was ACCEPTED", [] + # and the raw pair that would encode it must fail the chip predicate + lo, hi = v & lf.MASK32, (v >> 32) & lf.MASK32 + if lf.is_canonical(lo, hi): + return False, (f"L3 FAIL: the pair encoding {name} passes the chip " + f"predicate — canonicity is not being enforced") + rows.append({"name": name, "value": str(v), "lo": lo, "hi": hi, + "canonical": False, "rejected": True}) + return True, f"L3 PASS: {len(rows)} non-canonical inputs rejected, not reduced", rows + + +def l4_predicate_exhaustive_on_boundary(): + MAXH = lf.MAX_HI + cases = [(0, MAXH), (1, MAXH), (lf.MASK32, MAXH), (0, MAXH - 1), + (lf.MASK32, MAXH - 1), (0, 0), (1, 0)] + rng = random.Random(11) + cases += [(rng.randrange(2**32), rng.randrange(2**32)) for _ in range(300000)] + for lo, hi in cases: + if ((lo + (hi << 32)) < P) != lf.is_canonical(lo, hi): + return False, f"L4 FAIL at lo={lo:#x} hi={hi:#x}" + return True, (f"L4 PASS: predicate == (v < p) on {len(cases)} cases " + f"including every boundary") + + +def l5_domain_separation(rounds: int): + felts = [0x0102030405060708, 0x1112131415161718, + 0x2122232425262728, 0x3132333435363738] + lanes = lf.leaf_lanes(felts) + a, b = lanes[0:4], lanes[4:8] + leaf = lf.leaf_compress(felts, rounds) + parent = sk.socket_digest_wordlevel(a, b, sk.Framing(rounds=rounds)) + step = tr.compress_t(a, b, rounds) + if leaf == parent: + return False, "L5 FAIL: LFML leaf == LFMC parent" + if leaf == step: + return False, "L5 FAIL: LFML leaf == LFMT transcript step" + if parent == step: + return False, "L5 FAIL: LFMC parent == LFMT transcript step" + return True, ("L5 PASS: LFML / LFMC / LFMT are pairwise distinct on the same " + "eight lanes") + + +def l6_fri_leaf(rounds: int): + felts = [P - 1, 0, 1, 2**32, 12345678901234567, 2**32 - 1, P - 2**32, 999] + d = lf.leaf_over_8_felts(felts, rounds) + return True, "L6 PASS: 8-felt leaf = 3 compresses (2 LFML + 1 LFMC)", { + "felts": [str(f) for f in felts], + "digest": d, "digest_hex": hexlanes(d), "compresses": 3, + } + + +def main() -> int: + print("=" * 74) + print("LFML LEAF-MODE KATs (option C, ratified)") + print("=" * 74) + ok = True + doc = { + "mode": "LFML leaf (felt-input, option C)", + "tag_ascii": lf.TAG_LFML_ASCII.decode(), + "tag_word": lf.TAG_LFML, + "felts_per_row": lf.FELTS_PER_LEAF_ROW, + "lane_order": "[lo0, hi0, lo1, hi1, lo2, hi2, lo3, hi3] — halves adjacent", + "byte_serialization": "each lane as 4 little-endian bytes, in lane order, then the 4 tag bytes", + "canonicity": "v < p <=> NOT(hi == 2^32-1 AND lo >= 1)", + "rounds": {}, + } + for rounds in (7, 6): + good, msg, rows = l1_crate_identity(rounds) + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] {msg}") + g6, m6, leaf = l6_fri_leaf(rounds) + ok &= g6 + print(f" [{'PASS' if g6 else 'FAIL'}] {m6} @{rounds}r") + doc["rounds"][str(rounds)] = {"leaf_rows": rows, "fri_leaf": leaf} + + for fn in (l2_boundary_roundtrip, l3_non_canonical_rejected): + good, msg, rows = fn() + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] {msg}") + doc[fn.__name__] = rows + for fn in (l4_predicate_exhaustive_on_boundary,): + good, msg = fn() + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] {msg}") + good, msg = l5_domain_separation(7) + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] {msg}") + + if "--write" in sys.argv: + with open(OUT, "w") as f: + json.dump(doc, f, indent=1) + print(f"\n wrote {OUT}") + print("-" * 74) + print(f"LFML LEAF KATs: {'PASS' if ok else 'FAIL'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/shared/lfm-real-hash/leaf-spec/leaf_ref.py b/thoughts/shared/lfm-real-hash/leaf-spec/leaf_ref.py new file mode 100644 index 000000000..ff797de55 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-spec/leaf_ref.py @@ -0,0 +1,151 @@ +""" +THE LFML LEAF MODE — reference implementation (option C, ratified 2026-08-11). + +A leaf row hashes FOUR arbitrary Goldilocks field elements. Each felt occupies +TWO lanes as checked u32 halves, so four felts fill exactly the socket's eight +input lanes — the message layout is byte-identical to a digest-mode compress and +the crate-KAT property survives untouched. + + v = lo + 2^32 * hi, lo, hi in [0, 2^32) + +CANONICITY, and why it is cheap. `p - 1 = 0xFFFFFFFF_00000000`, i.e. hi = 2^32-1 +and lo = 0. So for lo, hi already known to be u32: + + v < p <==> NOT( hi == 2^32-1 AND lo >= 1 ) + +which is just "if hi is maximal then lo is zero". The socket's EXISTING O1 +machinery (byte columns + AreBytes + the lane identity) already forces lo and hi +to be u32; canonicity was the only missing piece, and it costs two witness +columns per felt rather than a 64-bit decomposition. + +THE DECOMPOSITION IS CHECKED, NOT REDUCING. A non-canonical input has no +satisfying witness, so the row is unprovable — the same reject-don't-reduce shape +as O1 itself. `felt_halves` raises rather than wrapping, mirroring that. +""" + +from __future__ import annotations + +import os +import sys + +_GATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "gate-oracle") +sys.path.insert(0, _GATE) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "transcript-spec")) + +import blake3_oracle as ora # noqa: E402 +import socket_ref as sk # noqa: E402 + +P = 2**64 - 2**32 + 1 # Goldilocks +MASK32 = 0xFFFFFFFF +MAX_HI = 0xFFFFFFFF + +TAG_LFML_ASCII = b"LFML" +TAG_LFML = int.from_bytes(TAG_LFML_ASCII, "little") # 0x4C4D464C + +FELTS_PER_LEAF_ROW = 4 # 4 felts = 8 lanes = one compress input + + +# --------------------------------------------------------------------------- +# The felt <-> halves boundary +# --------------------------------------------------------------------------- + +def is_canonical(lo: int, hi: int) -> bool: + """The chip's canonicity predicate, stated exactly as the constraints do.""" + if not (0 <= lo <= MASK32 and 0 <= hi <= MASK32): + return False + return not (hi == MAX_HI and lo >= 1) + + +def felt_halves(v: int) -> tuple[int, int]: + """v -> (lo, hi). REJECTS rather than reduces, matching the AIR.""" + if not 0 <= v < P: + raise ValueError( + f"{v:#x} is not a canonical Goldilocks element; the leaf mode " + f"REJECTS it (reject-don't-reduce, obligation O1)") + lo, hi = v & MASK32, (v >> 32) & MASK32 + assert is_canonical(lo, hi), "canonical felt must pass the chip predicate" + return lo, hi + + +def halves_felt(lo: int, hi: int) -> int: + if not is_canonical(lo, hi): + raise ValueError(f"({lo:#x}, {hi:#x}) is not a canonical half-pair") + return lo + (hi << 32) + + +def leaf_lanes(felts: list[int]) -> list[int]: + """Four felts -> eight lanes, LOW half first within each felt. + + Lane order is `[lo0, hi0, lo1, hi1, lo2, hi2, lo3, hi3]`, so felt `i` + occupies lanes `2i` and `2i+1`. Keeping a felt's two halves ADJACENT is what + lets the canonicity gate read one pair of neighbouring lanes. + """ + assert len(felts) == FELTS_PER_LEAF_ROW + lanes: list[int] = [] + for v in felts: + lo, hi = felt_halves(v) + lanes += [lo, hi] + return lanes + + +# --------------------------------------------------------------------------- +# The leaf compress +# --------------------------------------------------------------------------- + +def leaf_compress(felts: list[int], rounds: int = 7) -> list[int]: + """One LFML row: four felts -> one digest cell. + + Framing is the socket's, with `m[8] = TAG_LFML`; the eight lanes are the + felts' halves rather than eight u32s. + """ + lanes = leaf_lanes(felts) + fr = sk.Framing(rounds=rounds, tag_word=TAG_LFML) + return sk.socket_digest_wordlevel(lanes[0:4], lanes[4:8], fr) + + +def leaf_compress_bytelevel(felts: list[int], rounds: int = 7) -> list[int]: + """The library-shaped route — the external anchor. + + BYTE SERIALIZATION, stated exactly: each of the eight lanes is written as + four LITTLE-ENDIAN bytes in lane order, then the four tag bytes `"LFML"`. + So a felt contributes its low half's 4 bytes then its high half's 4 bytes: + + msg = LE32(lo0)‖LE32(hi0)‖…‖LE32(lo3)‖LE32(hi3)‖"LFML" (36 bytes) + digest = BLAKE3(msg)[0..16], read back as four LE u32 lanes + + At 7 rounds this is a plain `blake3::hash` call. + """ + lanes = leaf_lanes(felts) + msg = b"".join(int(x).to_bytes(4, "little") for x in lanes) + TAG_LFML_ASCII + assert len(msg) == 36 + full = ora.hash_bytes(msg, 32, rounds=rounds) + return [int.from_bytes(full[4 * i:4 * i + 4], "little") for i in range(4)] + + +def leaf_over_8_felts(felts: list[int], rounds: int = 7) -> list[int]: + """A FriToyV0 leaf covers TWO trace rows = EIGHT field elements. + + Three compresses, per the ratified pricing: two LFML rows (4 felts each) + and one ordinary LFMC parent combining them. + """ + assert len(felts) == 8 + d0 = leaf_compress(felts[0:4], rounds) + d1 = leaf_compress(felts[4:8], rounds) + return sk.socket_digest_wordlevel(d0, d1, sk.Framing(rounds=rounds)) + + +# Boundary felts the KATs must pin, including the non-canonical rejects. +BOUNDARY_FELTS = [ + ("zero", 0), + ("one", 1), + ("u32_max", 2**32 - 1), + ("two_pow_32", 2**32), + ("p_minus_2_32", P - 2**32), + ("p_minus_1", P - 1), +] +NON_CANONICAL = [ + ("p", P), + ("p_plus_1", P + 1), + ("two_pow_64_minus_1", 2**64 - 1), +] diff --git a/thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py b/thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py new file mode 100644 index 000000000..5d0c21fd4 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py @@ -0,0 +1,382 @@ +""" +RATE-4 KAT GENERATOR — renders the Rust vector tables for the widened socket. + +The socket's message grew from 8 lanes to TWELVE and its `block_len` from 36 to +52 when the leaf gained a chaining accumulator in the message (COMMIT.md §1.2, +the `RATE = 4` construction). That moves EVERY digest in all THREE domains — +`LFML`, `LFMC` and `LFMT` — because `block_len` is `v[14]` and cannot be made +mode-dependent (COMMIT.md §1.4.4 H9). So every pinned vector re-blesses, and +this is the script that re-pins them. + + msg = LE32(lane0..lane11) ‖ tag (52 bytes) + + LFML lanes = acc[0..4] ‖ (lo_i ‖ hi_i for each of four felts) + LFMC lanes = a[0..4] ‖ b[0..4] ‖ 0^4 (the third input cell, pinned) + LFMT lanes = state ‖ operand ‖ 0^4 + +★ THE VECTORS COME FROM THE ORACLE, NOT FROM THE RUST. Every digest below is +computed by `blake3_oracle.hash_bytes` — a from-scratch Python BLAKE3 written +before any of this Rust existed — over a message this script serialises itself. +Nothing here reads the implementation under test, which is the only thing that +makes the tables a specification rather than a recording. 52 < 64 keeps a row a +single block, so at 7 rounds each vector is also a plain `blake3::hash` call and +the crate anchor survives the widening. + +The INPUTS are carried over unchanged: the socket and transcript tables are +rewritten digest-line by digest-line out of the existing Rust, so their diff is +exactly the digests and a reviewer can see that nothing structural moved. The +leaf table is re-rendered whole, because the leaf row genuinely gained an input. + +Run: python3 rate4_kat_gen.py [--check] +""" + +from __future__ import annotations + +import os +import re +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(_HERE, "..", "gate-oracle")) + +import blake3_oracle as ora # noqa: E402 + +P = 2**64 - 2**32 + 1 # Goldilocks +MASK32 = 0xFFFFFFFF + +NUM_LANES = 12 # 4 accumulator lanes + 4 felts' halves +ACC_LANES = 4 +FELTS_PER_LEAF = 4 +BLOCK_LEN = 4 * (NUM_LANES + 1) # 52 + +TAG_LFMC = int.from_bytes(b"LFMC", "little") +TAG_LFML = int.from_bytes(b"LFML", "little") +TAG_LFMT = int.from_bytes(b"LFMT", "little") + +RUST = os.path.join(_HERE, "..", "..", "..", "..", "prover", "src", "lfm") + + +# --------------------------------------------------------------------------- +# The construction +# --------------------------------------------------------------------------- + +def _u32le(x: int) -> bytes: + if not 0 <= x <= MASK32: + raise ValueError(f"{x:#x} is not a u32 — obligation O1 (reject, never reduce)") + return int(x).to_bytes(4, "little") + + +def socket_digest(lanes: list[int], tag: int, rounds: int) -> list[int]: + """One row: twelve lanes and a tag -> four digest lanes. + + Serialised as a plain byte string and hashed by the oracle's `hash_bytes`, + which for any input under 64 bytes is exactly one compression with h = IV, + t = 0, block_len = len, flags = CHUNK_START|CHUNK_END|ROOT. That IS the + socket's framing, which is why the anchor holds. + """ + assert len(lanes) == NUM_LANES + msg = b"".join(_u32le(x) for x in lanes) + int(tag).to_bytes(4, "little") + assert len(msg) == BLOCK_LEN + full = ora.hash_bytes(msg, 32, rounds=rounds) + return [int.from_bytes(full[4 * i:4 * i + 4], "little") for i in range(4)] + + +def digest_lanes(a: list[int], b: list[int]) -> list[int]: + """A digest row's lanes: the two cells it reads, then the pinned zeros.""" + return list(a) + list(b) + [0] * (NUM_LANES - 8) + + +def felt_halves(v: int) -> tuple[int, int]: + """v -> (lo, hi). REJECTS rather than reduces, matching the AIR.""" + if not 0 <= v < P: + raise ValueError(f"{v:#x} is not a canonical Goldilocks element") + lo, hi = v & MASK32, (v >> 32) & MASK32 + assert not (hi == MASK32 and lo >= 1), "canonical felt must pass the predicate" + return lo, hi + + +def leaf_lanes(acc: list[int], felts: list[int]) -> list[int]: + """A leaf row's twelve lanes: the accumulator, then the felts' halves. + + Halves stay ADJACENT within a felt and start above the accumulator, so felt + `i` occupies lanes `4 + 2i` and `4 + 2i + 1`. + """ + assert len(acc) == ACC_LANES and len(felts) == FELTS_PER_LEAF + lanes = list(acc) + for v in felts: + lo, hi = felt_halves(v) + lanes += [lo, hi] + return lanes + + +def leaf_digest(acc: list[int], felts: list[int], rounds: int) -> list[int]: + """ONE compression that absorbs four felts AND chains the accumulator.""" + return socket_digest(leaf_lanes(acc, felts), TAG_LFML, rounds) + + +def leaf_chain(felts: list[int], rounds: int) -> list[int]: + """A wide leaf: the felts absorbed four at a time into one chain. + + The chain starts at the zero cell here, NOT at COMMIT.md §1.3's shape + header — these fixture leaves are fixed-shape by the program that builds + them and have no width to bind. A commitment layer over arbitrary-width + openings must open the chain at the header instead. + """ + assert len(felts) % FELTS_PER_LEAF == 0 + acc = [0] * ACC_LANES + for j in range(0, len(felts), FELTS_PER_LEAF): + acc = leaf_digest(acc, felts[j:j + FELTS_PER_LEAF], rounds) + return acc + + +def leaf_chain_compressions(num_felts: int) -> int: + """One compression per RATE felts — no fold, so no `2 *`.""" + return -(-num_felts // FELTS_PER_LEAF) + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + +def lanes_rs(v: list[int]) -> str: + return "[" + ", ".join(f"0x{x:08X}" for x in v) + "]" + + +def rewrite_digests(path: str, inputs: list[str], digests: dict[str, int], + tag: int, check: bool) -> tuple[int, int]: + """Recompute a table's digest fields IN PLACE from its own input fields. + + The inputs are read back out of the Rust rather than restated here, so this + cannot quietly re-pin a vector to a different input than the one the table + claims — and the resulting diff is exactly the digest lines. + """ + src = open(path).read() + field = lambda name: rf"{name}: \[((?:0x[0-9A-Fa-f]{{8}}(?:, )?)+)\]," + cells = [[int(x, 16) for x in m.group(1).split(", ")] + for m in re.finditer(field(inputs[0]), src)] + other = [[int(x, 16) for x in m.group(1).split(", ")] + for m in re.finditer(field(inputs[1]), src)] + assert len(cells) == len(other), f"{path}: {inputs} counts disagree" + + moved = 0 + for name, rounds in digests.items(): + wanted = [socket_digest(digest_lanes(a, b), tag, rounds) + for a, b in zip(cells, other)] + it = iter(wanted) + def sub(m, it=it): + nonlocal moved + new = lanes_rs(next(it)) + if m.group(0) != f"{name}: {new},": + moved += 1 + return f"{name}: {new}," + src = re.sub(field(name), sub, src) + assert next(it, None) is None, f"{path}: {name} count != input count" + + if check: + if src != open(path).read(): + raise SystemExit(f"STALE: {path} does not match the oracle") + else: + open(path, "w").write(src) + return len(cells), moved + + +LEAF_HEADER = '''//! LEAF-mode KATs for the LFM `"LFML"` domain, at 6 and 7 rounds. +//! +//! GENERATED — do not hand-edit. Rendered by +//! `thoughts/shared/lfm-real-hash/leaf-spec/rate4_kat_gen.py` from +//! `gate-oracle/blake3_oracle.py`, a Python BLAKE3 written **before any Rust +//! existed**. These vectors are a specification the implementation is checked +//! against, not a recording of what the implementation happened to do. +//! +//! A leaf row hashes FOUR arbitrary Goldilocks elements AND chains an +//! accumulator, in ONE compression (COMMIT.md §1.2). The accumulator is a digest +//! cell and fills lanes 0–3; each felt occupies two lanes above it as checked +//! `u32` halves, `[lo0, hi0, …, lo3, hi3]`. So the message is +//! `LE32(acc ‖ halves) ‖ "LFML"` — 52 bytes, still one BLAKE3 block, so the +//! crate-KAT anchor survives the widening. + +/// One leaf row: the chaining accumulator, four felts, the twelve lanes they +/// become, and the digest at each round count. +pub struct LeafVector { + pub name: &'static str, + pub acc: [u32; 4], + pub felts: [u64; 4], + pub lanes: [u32; 12], + /// Digest at 6 rounds (the A6R variant; no library computes it). + pub digest_6: [u32; 4], + /// Digest at 7 rounds — `blake3::hash(LE32(lanes) ‖ "LFML")[..16]`. + pub digest_7: [u32; 4], +} + +''' + +# The five ratified felt inputs, each now carried by a DIFFERENT accumulator, so +# a row that dropped the accumulator from its preimage could not reproduce the +# table. `acc_ignored_control` is that discrimination made explicit: same felts +# as `zeros`, nonzero accumulator, and the suite asserts the digests differ. +LEAF_CASES = [ + ("zeros", [0, 0, 0, 0], [0, 0, 0, 0]), + ("boundary_mix", [0x00000000, 0x00000001, 0xFFFFFFFE, 0xFFFFFFFF], + [0, 1, P - 1, 2**32]), + ("all_p_minus_1", [0xFFFFFFFF] * 4, [P - 1] * 4), + ("ramp", [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10], + [0x0102030405060708, 0x1112131415161718, + 0x2122232425262728, 0x3132333435363738]), + ("u32_edges", [0x80000000, 0x7FFFFFFF, 0x00010000, 0x0000FFFF], + [2**32 - 1, 2**32, P - 2**32, 1]), + ("acc_ignored_control", [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20], + [0, 0, 0, 0]), +] + +FRI_FELTS = [P - 1, 0, 1, 2**32, 12345678901234567, 2**32 - 1, P - 2**32, 999] + + +def render_leaf(path: str, check: bool) -> int: + out = [LEAF_HEADER] + out.append(f"pub const LEAF_VECTORS: [LeafVector; {len(LEAF_CASES)}] = [\n") + for name, acc, felts in LEAF_CASES: + out.append(" LeafVector {\n") + out.append(f' name: "{name}",\n') + out.append(f" acc: {lanes_rs(acc)},\n") + out.append(" felts: [" + ", ".join(f"{v}u64" for v in felts) + "],\n") + out.append(f" lanes: {lanes_rs(leaf_lanes(acc, felts))},\n") + out.append(f" digest_6: {lanes_rs(leaf_digest(acc, felts, 6))},\n") + out.append(f" digest_7: {lanes_rs(leaf_digest(acc, felts, 7))},\n") + out.append(" },\n") + out.append("];\n") + + tail = open(os.path.join(RUST, "leaf_kats.rs")).read() + keep = tail[tail.index("/// A boundary felt and the halves"):] + keep = keep[:keep.index("/// The eight-felt `FriToyV0` leaf")] + out.append("\n" + keep) + + out.append('''/// The eight-felt `FriToyV0` leaf: ONE `LFML` chain, two rows, no fold. +pub struct FriLeafVector { + pub felts: [u64; 8], + pub digest_6: [u32; 4], + pub digest_7: [u32; 4], + /// Compressions the whole leaf costs — 3 before the accumulator moved into + /// the message, 2 after (COMMIT.md §1.4.1: the RATE, measured). + pub compresses: usize, +} + +pub const FRI_LEAF: FriLeafVector = FriLeafVector { +''') + out.append(" felts: [\n") + for v in FRI_FELTS: + out.append(f" {v}u64,\n") + out.append(" ],\n") + out.append(f" digest_6: {lanes_rs(leaf_chain(FRI_FELTS, 6))},\n") + out.append(f" digest_7: {lanes_rs(leaf_chain(FRI_FELTS, 7))},\n") + out.append(f" compresses: {leaf_chain_compressions(len(FRI_FELTS))},\n") + out.append("};\n") + + src = "".join(out) + if check: + if src != open(path).read(): + raise SystemExit(f"STALE: {path} does not match the oracle") + else: + open(path, "w").write(src) + return len(LEAF_CASES) + + +SQUEEZE_MARK = int.from_bytes(b"SQ00"[:4], "little") if False else 811225427 + +MAIN_ROOT = [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10] +L1_ROOT = [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20] +T0W = [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE] +T1W = [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD] +NUM_QUERIES = 4 +QUERY_BITS = 4 + + +def fri_toy_transcript(rounds: int) -> tuple[list[list[int]], list[list[int]]]: + """The `FriToyV0` preamble, op by op — the K2 end-to-end vector. + + `absorb(main_root), squeeze, squeeze, absorb(l1_root), squeeze, + absorb_felts(t0w), absorb_felts(t1w), 4x squeeze_bits`. The two data absorbs + go through the LEAF encoding and absorb the digest, which is what makes this + vector move with the leaf construction and not only with `block_len`. + """ + state = [0, 0, 0, 0] + idx = 0 + states, outputs = [], [] + + def absorb(operand): + nonlocal state + state = socket_digest(digest_lanes(state, operand), TAG_LFMT, rounds) + states.append(list(state)) + + def squeeze(): + nonlocal state, idx + outputs.append(list(state)) + sq = [SQUEEZE_MARK, idx, 0, 0] + state = socket_digest(digest_lanes(state, sq), TAG_LFMT, rounds) + idx += 1 + states.append(list(state)) + + absorb(MAIN_ROOT) + squeeze() + squeeze() + absorb(L1_ROOT) + squeeze() + for cell in (T0W, T1W): + # DATA: leaf-hashed from the chain start, then the digest absorbed. + absorb(leaf_digest([0] * ACC_LANES, cell, rounds)) + for _ in range(NUM_QUERIES): + squeeze() + return states, outputs + + +def render_end_to_end(path: str, check: bool) -> int: + src = open(path).read() + moved = 0 + for rounds, const in ((7, "FRI_TOY_7"), (6, "FRI_TOY_6")): + states, outputs = fri_toy_transcript(rounds) + body = [" states: [\n"] + for s in states: + body.append(f" {lanes_rs(s)},\n") + body.append(" ],\n") + for name, o in (("alpha", outputs[0]), ("zeta0", outputs[1]), + ("zeta1", outputs[2])): + body.append(f" {name}: {lanes_rs(o[:3])},\n") + bits = [] + for q in range(NUM_QUERIES): + lane0 = outputs[3 + q][0] + bits.append("[" + ", ".join(str((lane0 >> k) & 1) + for k in range(QUERY_BITS)) + "]") + body.append(" query_bits: [" + ", ".join(bits) + "],\n") + + pat = re.compile(rf"(pub const {const}: EndToEndVector = EndToEndVector \{{\n).*?(\}};\n)", + re.S) + new = pat.sub(lambda m: m.group(1) + "".join(body) + m.group(2), src) + if new != src: + moved += 1 + src = new + + if check: + if src != open(path).read(): + raise SystemExit(f"STALE: {path} end-to-end vectors") + else: + open(path, "w").write(src) + return moved + + +def main() -> None: + check = "--check" in sys.argv + n, moved = rewrite_digests(os.path.join(RUST, "blake3_socket_kats.rs"), + ["a", "b"], {"digest_6": 6, "digest_7": 7}, + TAG_LFMC, check) + print(f"socket : {n} vectors, {moved} digests moved") + n, moved = rewrite_digests(os.path.join(RUST, "transcript_kats.rs"), + ["state", "operand"], + {"result_6": 6, "result_7": 7}, TAG_LFMT, check) + print(f"transcript: {n} vectors, {moved} digests moved") + moved = render_end_to_end(os.path.join(RUST, "transcript_kats.rs"), check) + print(f"end-to-end: {moved} FriToyV0 vectors re-pinned") + n = render_leaf(os.path.join(RUST, "leaf_kats.rs"), check) + print(f"leaf : {n} vectors re-rendered (the row gained an input)") + + +if __name__ == "__main__": + main() diff --git a/thoughts/shared/lfm-real-hash/leaf-spec/run-kats.log b/thoughts/shared/lfm-real-hash/leaf-spec/run-kats.log new file mode 100644 index 000000000..4c0e754f1 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-spec/run-kats.log @@ -0,0 +1,13 @@ +========================================================================== +LFML LEAF-MODE KATs (option C, ratified) +========================================================================== + [PASS] L1 PASS: 5 leaf rows, word route == byte route + [PASS] L6 PASS: 8-felt leaf = 3 compresses (2 LFML + 1 LFMC) @7r + [PASS] L1 PASS: 5 leaf rows, word route == byte route + [PASS] L6 PASS: 8-felt leaf = 3 compresses (2 LFML + 1 LFMC) @6r + [PASS] L2 PASS: 6 boundary felts round-trip + [PASS] L3 PASS: 3 non-canonical inputs rejected, not reduced + [PASS] L4 PASS: predicate == (v < p) on 300007 cases including every boundary + [PASS] L5 PASS: LFML / LFMC / LFMT are pairwise distinct on the same eight lanes +-------------------------------------------------------------------------- +LFML LEAF KATs: PASS diff --git a/thoughts/shared/lfm-real-hash/leaf-verify.md b/thoughts/shared/lfm-real-hash/leaf-verify.md new file mode 100644 index 000000000..bab9c5696 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/leaf-verify.md @@ -0,0 +1,422 @@ +# `MODE_L` / `LFML` leaf mode — adversarial verification + +**Verdict: ONE SOUNDNESS DEFECT (D1), high severity, in the field-native arms — +the BLAKE3 arm itself is sound.** Plus two claim/coverage defects and three +documentation defects. The leaf mode's *own* machinery — the canonicity block, +the lane-identity gating change, the tag separation, the transcript layering — +is **CONFIRMED SOUND** under adversarial reading and execution. + +**Ground:** worktree `lambda_vm-blake3-impl`, branch `blake3-real-hash`, on +committed B1 (`9bcc9ee2`), uncommitted. Nothing fixed, nothing committed. All +probe edits reverted and md5-verified back to their pre-probe bytes. + +Claims are ✓ EXECUTED / ✓ VERIFIED (read + traced) / ✗ UNVERIFIABLE. + +--- + +## 0. Verdict table + +| # | target | verdict | +|---|---|---| +| 1a | leaf half-lanes bound by byte decomposition + `AreBytes` | **CONFIRMED SOUND** ✓ VERIFIED | +| 1b | same-linear-form: lane == message word in leaf mode | **CONFIRMED SOUND** ✓ VERIFIED | +| 1c | a mode where neither lane identity nor leaf block applies but the core runs | **CONFIRMED SOUND** (impossible) ✓ VERIFIED | +| 2 | canonicity algebra (idx 30–45), `Z`/`GINV` abuse, the alias | **CONFIRMED SOUND** ✓ VERIFIED + ✓ EXECUTED | +| 3 | `absorb_felts` + transcript layering, call sites, sweep | **SOUND under BLAKE3**; **broken under Test/Poseidon by D1** | +| 4 | selector shift, by-name discipline, one-hot span, #915 mults | **CONFIRMED SOUND** ✓ VERIFIED | +| 5 | `TrivialV0` left on `LFMC` | **CONFIRMED SOUND** ✓ VERIFIED | +| 6 | `HostTree`/`HostSponge` hasher parameterisation (D6) | **CONFIRMED SOUND** ✓ VERIFIED | +| 7 | claim verification by execution | **1 DEVIATION** (D4) — everything else reproduces | +| 8 | hygiene, registry re-bless, O5 doc accuracy | **3 DOC DEFECTS** (D5, D6, D7) | +| — | **the leaf mode's own AIR under the OTHER two hashers** | ★ **DEFECT D1** | + +--- + +## ★ D1 — `MODE_L` rows leave four value columns free under `Test` and `Poseidon` + +**Severity: HIGH.** Soundness. Not disclosed anywhere in the report or the spec. +✓ EXECUTED, two independent ways, with a BLAKE3 control that fires. + +### The defect + +`MODE_L` reads ONE input cell. Three places were updated to say so, and one was +not: + +| what | where | leaf-aware? | +|---|---|---| +| the bus receive for the 2nd cell | `chips.rs:635` `reads_two() = Sum3(MODE_C, MODE_T, MODE_P)` | ✓ excludes `MODE_L` | +| the validator's address-slot pin | `validator.rs:232` `ins[mode.num_input_cells()..] == Addr(0)` | ✓ | +| the BLAKE3 AIR's value-column pin | `blake3_socket.rs:1324-1327` (idx 26–29) `mode_l · IN_{4+j} = 0` | ✓ | +| **the `Test` AIR** | `chips.rs:755-762` — round 0 reads `A_i = IN_i` for **i < 8** | ✗ **no pin** | +| **the `Poseidon` AIR** | `chips.rs:838-845` — round 0 reads `A_i = IN_i` for **i < 8** | ✗ **no pin** | + +So on a `MODE_L` row under `Test` or `Poseidon`, columns `IN4..IN8` + +- receive nothing from `LfmMem` (multiplicity is literally zero there), and +- are pinned by no constraint, and +- **are read by the permutation the AIR proves.** + +Four Goldilocks felts of free prover choice per leaf row. `leaf(c)` stops being +a function of `c`. + +**Why it was missed** is worth recording: the codebase already tolerates +unconstrained `IN` lanes — `executor.rs:112-114` says "lanes 8–11 are +unconstrained on those rows" — and that was safe *because nothing reads them*. +`MODE_L` is the first mode whose unread cell is nevertheless read by an AIR. + +### Executed leg 1 — the AIR admits it (probe, since reverted) + +A `MODE_L` row over fixed data `IN0..4`, built twice: `IN4..8 = 0` and +`IN4..8 = [1,2,3,4]`, each with the honest permutation output. Every constraint +of the set evaluated with the production `ProverEvalFolder`: + +``` +TEST arm : violations [] for both rows; digests differ + honest [8100894340827603473, 14773174770469813971, …] + forged [1250454659320479132, 7922735088962689630, …] +POSEIDON : violations [] for both rows; digests differ + honest [18148906729156086505, 16418236894463812223, …] + forged [507135091515794632, 11308586702453336741, …] +BLAKE3 control: violations [26, 27, 28, 29] ← the pin fires +``` + +### Executed leg 2 — end-to-end, bus included + +Executor patched to write attacker junk into `state[4..8]` **and** `in_cols[4..8]` +on every `Leaf` row, with the host `LfmHasher::leaf_out` default given the same +junk (an attacker controls both the trace and the arena hints, so this is +exactly their position). Then +`leaf_tests::fri_toy_proves_and_verifies_under_every_hasher`, `LFM_LEAF_JUNK` set: + +``` +Test → proved and VERIFIED +Poseidon → proved and VERIFIED +Blake3 → proved, then verification FAILED ← the control +``` + +The LogUp bus balances, the proof verifies against the same `program_id`. This +is not an AIR-only artefact. + +### Consequence (reasoning, not executed) + +- **Merkle-root binding survives.** `FriToyV0`'s walks end at a root pinned to a + public input, and hitting a fixed root still needs a preimage; the four free + felts do not help (Poseidon's capacity is separately pinned to zero on a leaf + row by idx 0–3). +- **Fiat–Shamir does NOT survive.** `programs.rs:618-619` derives every challenge + after `absorb_felts(t0w); absorb_felts(t1w)`, and `absorb_felts` is + `leaf` → `absorb` (`edsl.rs:106-109`). The leaf digest there is compared with + nothing, so a prover re-randomises the junk, recomputes forward — no inversion, + no search — and **chooses `alpha`, `zeta0`, `zeta1` and all four query + indices**, with the public statement (both roots) unchanged. That is a + complete FS break for any program that absorbs data. + +### Why this matters today, and where it does not + +All six `LFM_REGISTRY` entries are `hasher: HasherKind::Test` +(`registry.rs:271,355,439,523,607,691`) and `HasherKind::default() == Test` +(`hash.rs:198`). `TestPermutation` is already non-cryptographic, so D1 adds no +*new* exploit to the blessed configuration. **Poseidon is where it bites**: it is +a named production candidate, it is exercised by +`fri_toy_proves_and_verifies_under_every_hasher`, and MODE_L breaks its FS +soundness where B1's `absorb2` did not. BLAKE3 — the intended production hasher — +is unaffected. + +### Fix shape (not applied) + +Mirror idx 26–29 in `eval_test` and `eval_poseidon`: `mode_l · IN_{4+j} = 0` for +`j ∈ 0..4`, degree 2, four constraints each. The honest-path control matters here +— `leaf_out`'s default already writes zeros there, so honest rows keep proving. + +--- + +## 1. The lane-identity gating change — CONFIRMED SOUND + +The highest-risk item in the brief, and it holds. + +**(a) All eight half-lanes ARE byte-bound on a leaf row.** ✓ VERIFIED. +`bitwise_interactions` (`blake3_socket.rs:946-957`) sends `AreBytes` over +`(lane_byte(l,0), lane_byte(l,1))` and `(lane_byte(l,2), lane_byte(l,3))` for +all 8 lanes, with multiplicity `mu() = Sum3(MODE_C, MODE_T, MODE_L)` — +`MODE_L` **is** in the sum (`blake3_socket.rs:913-919`), so all 32 byte columns +are range-checked on a leaf row. The canonicity block's `u32` premise is +therefore established, not assumed. The trap the spec's §2.2 warning names is +not present. + +**(b) The same-linear-form property is preserved.** ✓ VERIFIED. `lo` and `hi` in +the leaf binding are `word_expr` over `cols::lane_byte(2i, ·)` / `lane_byte(2i+1, ·)` +(`blake3_socket.rs:1345-1352`), and `message_word_ref(i)` for `i < 8` is +`WordRef::Cols(word_cols(cols::lane_byte(i, 0)))` (`:726`) — **the same columns**. +So `IN_i = m[2i] + 2^32·m[2i+1]` is an identity over the very words the mixing +core consumes. A leaf row cannot bind one value and hash another. + +**(c) No mode admits the core with neither gate.** ✓ VERIFIED. +`mu = digest_mu + mode_l` by construction (`MU_COLUMNS` = C,T,L; +`DIGEST_MODE_COLUMNS` = C,T), so `mu ≠ 0 ⇒ digest_mu ≠ 0 or mode_l ≠ 0`, and each +of those makes its block bite (a nonzero field scalar does not weaken +`s·(x) = 0`). Fractional-selector rows such as `MODE_C = x, MODE_T = −x, +MODE_L = 1` do exist in the AIR's solution set — they satisfy idx 4 and 5 and +blend `m[8]` to an arbitrary field element — but that is the **pre-existing** +M5/M6 class, answered by the registrar's one-hot check, which now covers +`MODE_L` (§4). No new hole. + +--- + +## 2. The canonicity block (idx 30–45) — CONFIRMED SOUND + +With `lo, hi < 2^32` established by (1a) and `mode_l = 1`: + +| constraint | with `hi = 2^32−1` (`G = 0`) | with `hi ≠ 2^32−1` (`G ≠ 0`) | +|---|---|---| +| canon-b `1 − Z − G·GINV` | forces `Z = 1` **whatever `GINV` is** | with canon-a's `Z = 0`, forces `GINV = G^{-1}` | +| canon-a `Z·G` | vacuous | forces `Z = 0` | +| canon-c `Z·lo` | forces `lo = 0` | vacuous | + +`Z` is **fully determined** by `hi` in both branches, so **no prover-chosen +`GINV` can make the check vacuous** — the one abuse the brief asked about. +No wraparound is possible: `hi ≤ 2^32−1 ≪ p`, so `G = (2^32−1) − hi` is zero +exactly when `hi` is maximal. + +The accepted set is exactly `{(lo,hi) : ¬(hi = 2^32−1 ∧ lo ≥ 1)}`, whose +complement is exactly the `2^32 − 1` pairs encoding `v ∈ [p, 2^64)` — i.e. +**exactly `v < p`**, tight at `p − 1 = (lo 0, hi 2^32−1)`. + +**Indices, recomputed independently:** `LEAF_IDX = 26`, `base = 30 + 4i`, so +felt `i`'s (binding, canon-a, canon-b, canon-c) sit at `30+4i … 33+4i`: +felt 0 → 30,31,32,**33**; felt 1 → 34–**37**; felt 2 → 38–**41**; +felt 3 → 42–**45**. The report's "canon-c for felt 0 is idx 33" ✓, and the other +seven of the eight canon-c/others land where claimed. Framing total +`4+1+1+8+8+4+4+16 = 46 = CORE_IDX` ✓. + +**The alias** ✓ EXECUTED (`m10_a_leaf_row_cannot_skip_canonicity`, and +independently re-derived here): `1 + 2^32·(2^32−1) = p ≡ 0`, so felt `0` has a +second half-pair; the binding constraint is satisfied by it and **only canon-c +catches it**. The test asserts the index rather than "something fired", which is +the right shape. + +**Degree** stays 3: canon-a/b/c are `mode_l · (deg-2)`, binding is `mode_l · +(deg-1)`. Pinned by `the_arm_emits_its_constraints_at_degree_3` ✓ EXECUTED. + +--- + +## 3. `absorb_felts` and the transcript layering + +**(a) The two-step is mirrored exactly.** ✓ VERIFIED. +Machine `edsl.rs:106-109`: `let d = b.leaf(c); self.absorb(b, d.as_cell())`. +Host `fixture.rs:113-116`: `let d = self.hasher.leaf(c); self.absorb(&d)`. +Same order, same count (1 `LFML` + 1 `LFMT` per call). Under BLAKE3 the binding +is to the data up to leaf-hash collision resistance, i.e. the socket's already +declared 64-bit birthday bound — **not a new weakening**, since a direct absorb +would have had the same bound through the compress. Under Test/Poseidon this +argument is void — see D1. + +**(b) Every call site is the right one.** ✓ VERIFIED, one by one: + +| site | absorbs | call | correct? | +|---|---|---|---| +| `programs.rs:608` | `main_root` | `absorb` | ✓ digest | +| `programs.rs:612` | `l1_root` | `absorb` | ✓ digest | +| `programs.rs:618-619` | `t0w`, `t1w` | `absorb_felts` | ✓ data | +| `fixture.rs:262` | `main_tree.root()` | `absorb` | ✓ | +| `fixture.rs:295` | `l1_tree.root()` | `absorb` | ✓ | +| `fixture.rs:322-323` | `t0`, `t1` | `absorb_felts` | ✓ | + +**(c) Sweep — no other raw-felt absorb, and no other `LFMC` leaf.** ✓ VERIFIED, +performed independently of the report's §4 and reaching the same four sites. +`SpongeVar` appears in exactly one program (`programs.rs:606`); the only +`compress` call sites in non-test code are `edsl.rs:170` (parent over two leaf +digests), `edsl.rs:187` (`merkle_walk` parent), `fixture.rs:146,170` (the host +mirrors) and `programs.rs:69-71` (`TrivialV0`, §5). Nothing else. + +--- + +## 4. The selector shift — CONFIRMED SOUND + +- **By-name discipline held.** ✓ VERIFIED. No literal `9`/`10`/`11` reaches a + hash path: `compiler.rs:343-350` uses `layout::hash::MODE_*`/`MULT*`, + `validator.rs:316,415` likewise, `blake3_socket.rs:593` re-exports by name, and + `airs.rs:187-189,428-432` reads `hash::num_columns(hasher)` and + `layout::hash::PREP_WIDTH` — which is why `airs.rs` needed no edit at all. +- **The one-hot span covers all four.** ✓ VERIFIED. `MODE_C=6, MODE_P=7, + MODE_T=8, MODE_L=9` and `one_hot(&g.hash, "LFM_HASH", MODE_C, NUM_SELECTORS=4)` + (`validator.rs:313-318`) walks columns 6..10 — `MODE_L` inside, `MULT0..2` + (10,11,12) outside. The §7.2 mistake is not repeated. +- **#915 multiplicity bounding follows the move.** ✓ VERIFIED. + `validator.rs:415` lists `vec![hash::MULT0, hash::MULT1, hash::MULT2]` by name, + so `check_mult_ranges` reads 10/11/12. +- **`is_real` widened correctly**: `selector_sum(MODE_C, 4)` (`chips.rs:624`). +- Note (pre-existing, restated because `MODE_L` widens it): `validate` is called + only from tests — it is a *registration-time* gate, and the runtime binding is + `program_id`. Correct as designed; the one-hot claim rests on every registered + program having a `validate` test (`machine_tests.rs:32,152,315,607,1324,…`). + +--- + +## 5. `TrivialV0` left on `LFMC` — CONFIRMED SOUND + +✓ VERIFIED. `programs.rs:69-71` is `compress(h0,h1) → compress(d0,l2) → +compress(d1,h3)` — a **chain**: every compress after the first consumes the +previous one's digest as its left operand and there is no level structure, so +there is no leaf/parent pair for `MODE_L` to separate. The recorded argument is +at `programs.rs:60-68` and says exactly this, including the consequence. +Its BLAKE3 arenas are `word_of(&[u32; 4])` (`blake3_socket_tests.rs:1344-1350`), +so obligation O1 is satisfied by construction. ✓ + +--- + +## 6. Fixture hasher parameterisation (D6) — CONFIRMED SOUND + +✓ VERIFIED. `TestPermutation` no longer appears anywhere in `fixture.rs` (import +removed; grep confirms zero occurrences). Every hash goes through the +`HasherKind`: `HostTree::build(hasher, …)`, `HostSponge::with_hasher`, +`host_leaf_hash_pair(hasher, …)`, both tree constructions and both transcript +absorbs. `host_leaf_hash_pair` is +`hasher.compress(&hasher.leaf(c0), &hasher.leaf(c1))` (`fixture.rs:146`) — +2 `LFML` + 1 `LFMC`, the same association and order as `edsl::leaf_hash_pair` +(`edsl.rs:168-170`). ✓ + +--- + +## 7. Claim verification by execution + +| claim | executed | verdict | +|---|---|---| +| full `lfm::` @7r = 304 pass / 19 fail | **304 / 19 / 7 ignored** (255.8s) | ✓ exact | +| full `lfm::` @6r | **304 / 19 / 7 ignored** (266.5s) | ✓ exact | +| the 19 are the pre-existing set | **name-for-name identical at both round counts**; module split 7 `epoch_tests` / 6 `epoch_verify_tests` / 1 `logup_tests` / 5 `machine_tests` — identical to the B1 record (`b1-verify.md` §8) | ✓ | +| `leaf_tests` 14/14 both counts | **14 @7r**; 6r totals identical ⇒ 14 | ✓ | +| `transcript_tests` 17/17 | **17 @7r** | ✓ | +| **`blake3_socket_tests` 35 @7r / 34 @6r, "one 7r-only"** | **34 @7r** | ✗ **D4** | +| the milestone + negative leg + three-hasher control | all in the 304 | ✓ | +| `TrivialV0` 16,551 and `FriToyV0` 93 rows / 513,081 | `the_programs_cost_what_the_leaf_spec_priced_them_at` asserts `(56,11,26)`, `93`, `16_551`, `513_081` — passes | ✓ | +| `make lint` (fmt + 4 combos) | see §10 | — | +| leaf KATs really rendered from the spec JSON | all **12** digests (5 rows × 2 round counts + `fri_leaf` × 2) present verbatim in `leaf_kats.rs` | ✓ | + +### D4 — `blake3_socket_tests` is 34, not "35 @7r / 34 @6r" + +**Severity: LOW (claims accuracy).** ✓ EXECUTED: 34 tests pass at 7 rounds. The +file contains exactly 34 `#[test]` and **no** `cfg`-gated ones (grep for +`cfg(…blake3-6round…)` / `cfg(not` returns nothing), so there is no +"7-round-only by construction" test. HEAD had 35; this diff deletes the O1 +tripwire and adds none. Both halves of the report's §7 cell are wrong. + +Corollary the report also gets wrong: it says "+13 passes are the new leaf +tests", but B1's record is 290 passed and this is 304 — **+14**, while the net +test-count change is +13 (14 new leaf tests − 1 deleted tripwire; 318 → 331 +`#[test]` under `prover/src/lfm/`). The +1 does not close from either side's +records. Not a defect in the change, but the "290 → 304" arithmetic in the +report is not the one the files support; the per-module numbers executed above +are the authoritative ones. + +--- + +## 8. D2 — the transcript KAT no longer models `FriToyV0`, and still claims to + +**Severity: MEDIUM (stale verified-claim + real coverage loss).** Not disclosed. + +`transcript_kats.rs:76-79` says the end-to-end vector's op sequence is + +> `absorb, squeeze, squeeze, absorb, squeeze, **absorb2**, 4× squeeze_bits`, +> ✓ VERIFIED against `programs::fri_toy_program_source` + +and `transcript_tests.rs:203-205` repeats it. But `fri_toy_program_source` no +longer contains `absorb2(t0w, t1w)` — `programs.rs:618-619` is now +`absorb_felts(t0w); absorb_felts(t1w)`, i.e. leaf-then-absorb. The model that +replays the preamble was not updated: `transcript_tests.rs:416` still calls +`sponge.absorb2(&mut b, h[2], h[3])`, and the host replay at `:304` and `:574` +likewise. + +- The **step count** claim survives (2 absorbs either way, so `FRI_TOY_COMPRESSIONS + = 11` still holds and the cost test still passes) — which is precisely why + nothing went red. +- The **state** claim does not. The end-to-end vector, which is the one anchor + rendered from an independent Python reference, now pins a transcript + `FriToyV0` does not run. `FriToyV0`'s actual challenge derivation is left + checked only by machine-vs-host agreement — and host and machine were changed + together, so a shared error in the leaf convention would not be caught. + +The `✓ VERIFIED` marker on a claim that the same change set falsified is the +part worth flagging: it is exactly the marker a future reader will trust. + +--- + +## 9. Documentation defects + +**D3 — spec criterion 4 was replaced, not met, and the board says otherwise.** +Severity: LOW-MEDIUM (claims accuracy). `LEAF.md` §5 criterion 4 requires "a +deliberately non-canonical arena value must make the proof fail, and fail *for +canonicity*". The delivered negative leg is +`fri_toy_rejects_a_fixture_built_under_another_hasher` — a **hasher-mismatch** +test whose failure mode is a Merkle-walk root mismatch, not canonicity; the +mismatched fixture's arena values are ordinary canonical `FE`s. The report's §5 +lists it under criterion 4 without saying it is a substitution, and its board row +reads "negative canonicity leg **in the assembled proof** — ✓ EXECUTED". There is +no such leg: canonicity is exercised only at the chip level (`M10`). +In fairness the criterion as written is **unsatisfiable** — every `FE` is +canonical by construction, so a non-canonical arena value cannot be built +(`admits`' leaf arm at `blake3_socket.rs:528-536` is dead code by the same +argument, as its own comment concedes). The right disposition is to record the +criterion as retired-because-impossible, not to mark it met. + +**D5 — the O5 retirement is asserted hasher-independently where it is +BLAKE3-only.** Severity: LOW. `instr.rs:75-79` (`HashMode::Leaf`, a +hasher-independent ISA doc) states flatly: "Leaves and parents now occupy +different hash domains by construction (`"LFML"` vs `"LFMC"`), so an internal +node cannot be replayed as a leaf whatever the tree's shape." Under `Test` and +`Poseidon` that is **false** — `leaf`, `transcript` and `compress` are the same +function. `layout.rs:82-89`'s selector/domain table has the same problem. The +caveat *is* recorded correctly at `hash.rs:104-108` (`leaf_out`) and the report's +§9 last row names it, so this is a placement defect rather than an omission: the +weakened statement lives in the hasher-specific file and the unqualified one in +the hasher-independent files, which is backwards. + +**D6 — `blake3_socket.rs:1319-1323` calls the `IN4..8` pin "hygiene rather than +soundness".** Severity: LOW, but it is the comment that would talk a reader out +of the fix D1 needs. On the BLAKE3 arm the claim is true (its message lanes come +from the byte columns, not from `IN`), but the comment reads as a general +statement about the mode, and the identical pin is load-bearing on the two arms +that do read `IN4..8`. + +**D7 — diff scope has grown by one file since the report.** Severity: NONE, +noted for the record. `thoughts/blake3/socket-kats/SOCKET.md` is now modified +(the `"LFML"` row flipped from "reserved" to "LIVE"), which closes the report's +§9 open item. It appeared at 12:54, after this review started, and is another +workstream's edit — not part of the 19+2. Code scope is otherwise exactly as +claimed: 19 modified `.rs` all under `prover/src/lfm/`, 2 new +(`leaf_kats.rs`, `leaf_tests.rs`), nothing in `crypto/`, `executor/` or +`prover/src/tables/`. No `println!`/`dbg!`/`TODO`/`FIXME` anywhere in the diff. + +--- + +## 10. Registry re-bless and lint + +Registry: all six entries re-blessed in one pass, and each is pinned by a drift +test that also covers roots/log_heights/keccak_rnd_chunks/hasher — all six pass +inside the 304. The report's `program_id` prefixes reproduce +(`TrivialV0` → `7087e2838dae1171` at `registry.rs:273-277` ✓, spot-checked). + +`make lint` — ✓ EXECUTED, **exit 0, clean**, all combos including the +`lambda-vm-prover/cuda` pass. Reproduces the report. + +**Probe hygiene:** the three files touched by the executed probes +(`hash.rs`, `executor.rs`, `mod.rs`) were snapshotted before editing and +restored after; md5s match byte-for-byte and `prover/src/lfm/zz_probe.rs` is +deleted. The working tree is exactly as handed over. + +--- + +## 11. What a follow-up should do + +1. **Fix D1** — four constraints in each of `eval_test` and `eval_poseidon`, + with an honest-path control (`leaf_out`'s zeros must still prove). Consider + deriving the pin from `HashMode::num_input_cells()` in one place so the next + mode cannot repeat it. +2. **Re-point the transcript end-to-end KAT at the real `FriToyV0` preamble** + (D2), or delete the `✓ VERIFIED against fri_toy_program_source` claim and say + plainly that the vector models an `absorb2` transcript. +3. Correct §7's `blake3_socket_tests` cell (D4) and the criterion-4 disposition + (D3). +4. Move the leaf/parent-separation caveat into `instr.rs` and `layout.rs` (D5). +5. The z3 oracle's WA8 "canonicity dropped ⇒ SAT" leg remains the only thing + that can show the block is *necessary* rather than merely satisfied; nothing + here substitutes for it. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/FIX-PLAN.md b/thoughts/shared/lfm-real-hash/merge-plan/FIX-PLAN.md new file mode 100644 index 000000000..032752ad3 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/FIX-PLAN.md @@ -0,0 +1,109 @@ +# Fix plan: LFM recursion-machine bus-balance divergence after the main merge + +**Context.** Merging `origin/main` into `blake3-real-hash` (worktree `lambda_vm-blake3-merge`, +branch `blake3-real-hash-mainmerge`). The artifact-feature reconciliation is done and green +(round-trip 11/11). The remaining breakage: **20 LFM machine proof/verify tests fail**, all on +one root cause, plus **2 trivial HINT bookkeeping tests**. This plan fixes both. Nothing is +committed; the pristine campaign tip is tagged `blake3-campaign-preMerge`. + +## Diagnosis (evidence, not hypothesis) + +Instrumented `Verifier::multi_verify_views` and `verify_against` (throwaway `eprintln`s, to be +removed). Findings for `machine_proves_the_sample_replay`: + +1. The proof **proves** fine; only **verify** fails. +2. Every failure is the **cross-table LogUp bus-balance** check (`total != expected_bus_balance` + at `verifier.rs:1442`). No other check fires — composition-parts, `ood_blocks_well_formed`, + preprocessed-commitment match, per-table `verify_rounds_2_to_4` (incl. #909's width check) all + **pass**. So per-table STARK verification is correct; only the cross-table binding is off. +3. Per-table contributions (14 tables, all `has_interaction`): sum = `5597…836`, expected = + `16884…021` — different. Not a sign flip, not a missing table, not zero. +4. Ruled out as the cause (branch vs `origin/main`, byte-identical or unchanged): + `LOGUP_NUM_CHALLENGES` (2), `compute_alpha_powers`, `build_accumulated_column_from_terms` + (the L computation), the Phase-A transcript absorption order (the LFM machine's hand-rolled + `replay_transcript_phase_a_view` matches `multi_verify_views` Phase A exactly), and the + fiat-shamir/transcript module (untouched by the merge). + +**Conclusion.** The LFM machine hand-rolls its cross-table binding — `expected_public_balance` +(`prover/src/lfm/proof.rs`) and `replay_transcript_phase_a_view` (`prover/src/lib.rs`) — to mirror +crypto/stark's LogUp convention. The merge's large crypto/stark batch (prover rewrite #877/#875/#863 +et al.) shifted that convention in a way the obvious diffs don't reveal, so the branch's hand-rolled +mirror disagrees. **main's crypto/stark stays authoritative; the hand-rolled LFM binding adapts** — +exactly as with the artifact feature. + +This is **soundness-critical**: `expected_public_balance` is the recursion verifier's cross-table +check. A wrong fix could make the machine accept invalid proofs. The fix must be validated by BOTH +positive (valid proofs verify) AND negative (tampered proofs still rejected) controls. + +## Step 1 — PIN the exact convention (decisive, before any fix) + +Compare the SAME `sample()` proof's internals on the pristine branch vs the merge: +- In `lambda_vm-blake3-impl` @ `ed1b7785` (branch, test passes) and in `lambda_vm-blake3-merge` + (merge, test fails), print: `z`, `alpha`, each table's `bus_table_contribution`, and `expected`. +- **Outcome A:** `z`/`alpha` differ ⇒ challenge derivation changed (unlikely — transcript module + untouched). Fix targets the replay. +- **Outcome B:** `z`/`alpha` identical but per-table contributions differ ⇒ main's aux/LogUp + column construction changed the L values ⇒ the fix is either in how the LFM machine reads/sums + contributions or in `expected_public_balance`'s target formula. +- **Outcome C:** contributions identical, only `expected` differs ⇒ the target formula in + `expected_public_balance` is stale ⇒ fix it directly. +- Also inspect the LfmPublic **send token layout** (how the LFM chips send `(index, v0..v3)` to the + LfmPublic bus) vs `expected_public_balance`'s hard-coded fingerprint `busid + index·α + Σ v_l·α^{2+l}` + vs main's actual bus-interaction fingerprint alpha-power assignment. A shifted alpha-power offset + is the leading suspect. + +Deliverable: the exact convention that shifted, named with file:line on both sides. + +## Step 2 — FIX the hand-rolled binding + +Scope is confined to the **branch's** hand-rolled binding — NOT crypto/stark: +- `prover/src/lfm/proof.rs::expected_public_balance` (the fingerprint/target formula), and/or +- `prover/src/lib.rs::replay_transcript_phase_a_view` (the challenge replay), +- and any sibling that mirrors the same convention (`compute_expected_commit_bus_balance_view`, + `absorb_lfm_statement`). +Update them to main's pinned convention. No edits under `crypto/stark/` (main's IR/verifier remain). + +## Step 3 — HINT bookkeeping (independent, trivial) + +Add the `HINT` design-table entry to the LFM design census and update the one epoch-budget constant +(`lfm::constraint_tests::constraint_leg_instruction_census`, `continuation_epoch_constraint_leg_cost`). +These are unrelated to the bus-balance fix; done in the same pass because they're the last 2 of the 22. + +## Step 4 — VALIDATE + +- Remove ALL throwaway diagnostics (verifier.rs, proof.rs, lib.rs). Confirm `git diff` under + `crypto/stark/` is only the intended merge content (no diagnostics, no logic changes). +- **Positive:** all 20 machine proof/verify tests pass; full `lfm::` returns to a clean baseline + (the 19 pre-existing `fibonacci.elf` failures only, modulo the HINT tables now passing). +- **Negative controls (mandatory, soundness):** the existing tamper/rejection tests + (`tampered_l2g_binding_rejects`, the output-swap-hazard tests, any "must NOT verify" tests) still + REJECT. A fix that makes the balance always pass is as wrong as the bug. +- Artifact round-trip suite (`constraint_artifact`) stays 11/11. +- Chip gate `artifact_pin.py --check` still green (BLAKE3 chip untouched by any of this). +- Cross-version / whole-suite sanity: full lib suite failure set vs the `blake3-campaign-preMerge` + baseline shows only pre-existing fixture/env failures — zero new. + +## Step 5 — REVIEW + FINALIZE + +- Adversarial review of the binding change (it is the recursion verifier's cross-table soundness + check): confirm the new formula matches main's convention AND that negative controls hold. +- Commit the merge; fast-forward `blake3-real-hash` to the merged branch; push → PR #930 up to date. +- Keep `blake3-campaign-preMerge` as the recoverable pristine point. + +## Rollback + +Merge is uncommitted in a dedicated worktree; the pristine tip is tagged and pushed. Any failure ⇒ +`git reset --hard blake3-campaign-preMerge` (or discard the worktree). Zero risk to PR #930 until the +final fast-forward. + +## Risk register + +- **R1 (high impact):** wrong binding formula → machine accepts invalid proofs. Mitigation: negative + controls in Step 4 are mandatory and gate the commit. +- **R2:** the convention shift is in main's TRACE/aux construction (Outcome B), not the formula — + fix might need to touch how contributions are read, not just `expected`. Mitigation: Step 1 pins + which, before any edit. +- **R3:** more than one convention shifted at once. Mitigation: Step 1 compares ALL of z/alpha/ + per-table-contrib/expected, catching multiple divergences together. +- **R4:** the fix passes the sample test but not other programs (join, splice, keccak variants). + Mitigation: Step 4 runs the full 20, not one. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/JUDGE-VERDICT.md b/thoughts/shared/lfm-real-hash/merge-plan/JUDGE-VERDICT.md new file mode 100644 index 000000000..faba674a1 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/JUDGE-VERDICT.md @@ -0,0 +1,405 @@ +# JUDGE VERDICT — FIX-PLAN.md adversarial review + +**Ruling: REVISE, THEN IMPLEMENT.** The plan's process skeleton survives; its diagnosis and its +Step-2 scope do not. The attacker is right on the root cause, and I verified it independently. +The corrected fix is mechanical and fail-closed and needs no further human review pass — but two +carve-outs do, and they are named in §7. + +All code citations are from the merge worktree `/Users/maurofab/workspace/lambda_vm-blake3-merge` +(branch `blake3-real-hash-mainmerge`, `HEAD = ed1b7785`, `MERGE_HEAD = 58160b6f`, merge +uncommitted). `origin/main` = `528a8411`. Everything marked ✓ VERIFIED I read out of the tree or +the diff myself; I ran no cargo and made no edits. + +--- + +## 1. Ruling on the central dispute + +**The attacker's diagnosis is correct. The plan's is falsified.** + +The root cause is a **sender/receiver multiplicity mismatch on `BusId::Hwsl` (id 9)** inside the +LFM machine, created by the merge: main deleted the HWSL sends from the production `KECCAK_RND` +chip *and* from the production receiver-side collector, but the LFM machine's **forked** receiver +-side collector — branch-only code the merge never touched — still emits them. + +### Fix location + +``` +prover/src/lfm/keccak_adapter.rs:361-366 // theta HWSL push (20 per round) +prover/src/lfm/keccak_adapter.rs:441-446 // rho HWSL push (100 per round) +prover/src/lfm/keccak_adapter.rs:306,319 // the stale pinned count `24 * 1148` +prover/src/lfm/keccak_probe.rs:201-205 // the same count, asserted +``` + +### The verification chain, link by link + +1. ✓ **Main removed the sends.** `git diff ed1b7785 origin/main -- prover/src/tables/keccak_rnd.rs` + filtered to `BusId::` / section comments yields exactly two removals and nothing else: + `--- Theta: HWSL for rotated C (20) ---` (hunk `@@ -587,48 +596,8 @@`) and + `--- Rho: HWSL (100) ---` (hunk `@@ -717,53 +686,8 @@`), each replaced by a comment saying the + shift is now enforced by an inline μ-gated linear identity in `KeccakRndConstraints`. Capacity + `1371 → 1031` (`keccak_rnd.rs:446`), and the new module comment at `keccak_rnd.rs:439` states + *"The θ/ρ halfword shifts no longer emit HWSL lookups (120 sends/row removed) … The matching + HWSL multiplicities are likewise dropped on the BITWISE side (`collect_bitwise_from_keccak`)."* + The surrounding `AreBytes` blocks are **unchanged** on both sides — the delta is exactly the + 120 HWSL sends per row, nothing more. + +2. ✓ **Main removed the matching receives, in the production collector.** + `git show origin/main:prover/src/tables/trace_builder.rs` has **zero** occurrences of + `BitwiseOperationType::Hwsl`. `git show ed1b7785:…` has two, at `:2427` (theta) and `:2510` + (rho), inside `collect_bitwise_from_keccak` (`:2343` branch / `:2447` main). Main changed both + sides in lockstep. The production path is self-consistent. + +3. ✓ **The LFM machine takes main's sender side automatically.** + `prover/src/lfm/airs.rs:21` imports `crate::tables::{bitwise, keccak_rc, keccak_rnd}`; + `airs.rs:239` reads `keccak_rnd::bus_interactions().len()`; `airs.rs:488-494` builds the LFM + `KECCAK_RND` AIRs from `keccak_rnd::bus_interactions()` and `keccak_rnd::KeccakRndConstraints`. + The trace comes from main's own generator (`prover/src/lfm/trace.rs:166` + `.map(keccak_rnd::generate_keccak_rnd_trace)`), which is why per-table STARK verification still + passes — the trace does satisfy main's new inline identities. + +4. ✓ **The LFM machine does NOT take main's receiver side.** + `prover/src/lfm/trace.rs:176` feeds the BITWISE multiplicity histogram from + `keccak_adapter::bitwise_ops_for(&keccak_ops)` — the branch's fork, documented as such at + `keccak_adapter.rs:306-315` (*"the per-round half of `trace_builder::collect_bitwise_from_keccak`, + forked rather than called"*). ✓ `git diff ed1b7785 -- prover/src/lfm/keccak_adapter.rs` is + **empty**: the merge did not touch it. It still pushes `BitwiseOperationType::Hwsl` at `:362` + (5×4 = 20/round) and `:442` (5×5×4 = 100/round). + +5. ✓ **Those two sites are the only HWSL in the whole LFM module.** + `grep -rn "Hwsl" prover/src/lfm/` returns exactly `keccak_adapter.rs:362` and `:442`. So in the + LFM AIR set the Hwsl bus now has **receivers with no senders at all** — a pure one-sided + imbalance, not a subtle re-weighting. + +6. ✓ **The count arithmetic closes.** Hand-counting the pushes per round in `bitwise_ops_for`: + theta XOR chain 160 + theta (20 HWSL + 20 AreBytes) + theta Dxz 40 + theta final 200 + + rho (100 HWSL + 200 AreBytes) + chi 400 + iota 8 = **1148**, matching the pin at `:319`. + Removing the 120 HWSL gives **1028**. + +7. ✓ **No other embedded table drifted.** `prover/src/tables/bitwise.rs` and + `prover/src/tables/keccak_rc.rs` are byte-identical branch↔main; `keccak.rs` differs by one + `#[derive(Clone, Copy)]`; `types.rs` differs only by the branch's own `LfmMem = 32` / + `LfmRange = 33` / `LfmPublic = 34` additions, which survive the merge with no collision + (`prover/src/tables/types.rs:363-373`; `Hwsl = 9` at `:283`). + +### The passing/failing split matches this theory and nothing else + +✓ `keccak_ops` is derived **only** from `records.keccak` (`prover/src/lfm/trace.rs:145-155`), i.e. +from explicit `Instr::KeccakF` rows — never from the hash chip. ✓ `HasherKind::default() = Test` +(`prover/src/lfm/hash.rs:196-199`), and `build_artifacts` uses the default +(`prover/src/lfm/registry.rs:117-119`). ✓ Every `KECCAK_RND` bus interaction is gated +`Multiplicity::Column(cols::MU)` (`keccak_rnd.rs:446ff`), so padding rows send nothing. + +Therefore a program with **zero keccak permutations** feeds `bitwise_ops_for(&[])` → no HWSL +receives → balanced; a program with **any** keccak permutation is unbalanced. That is exactly the +observed split: + +- `trivial_program_source` (`prover/src/lfm/programs.rs:31-79`) uses `b.compress` (the hash chip + under `TestPermutation`) and **no** `keccak_f`/absorb → passes. So do the BLAKE3 suites. +- Every one of the 20 failures is keccak-touching: the keccak_* / sponge / chain / merkle-walk + tests obviously; `splice`/`append_ext`/`transcript_replay`/`statement_replay` are keccak + absorbs; `fri_tests` goes through `edsl::keccak_merkle_walk` (`prover/src/lfm/fri.rs:564`); + `join_tests` through `prover/src/lfm/sub_proof.rs:256,268,289` + (`edsl::keccak_leaf_hash` / `keccak256` / `keccak_merkle_walk`); `program_id_*` are keccak folds. + +--- + +## 2. Why the plan's diagnosis is dead + +✓ The plan's premise — *"the merge's large crypto/stark batch shifted [the LogUp] convention"* — +is refuted by the diff. `git diff ed1b7785 origin/main -- crypto/stark/src/lookup.rs` is 67+/79− +and its **first hunk starts at line 834**. `compute_alpha_powers` (`:73`), every +`accumulate_fingerprint*` impl (`:274`, `:377`, `:626`, `:742`), `add_combined_terms` and the whole +alpha-offset assignment are all **above** the first hunk and therefore untouched. The later hunks +(`@@ -1134`, `-1177`, `-1210`, `-1223`, `-1242`, `-1270`, `-1278`, `-1299`, `-1372`) are: `Arc`-wrap +of `constraint_program`, removal of the branch's `precaptured_program`, a `OnceCell` for lazily +materializing host main columns on the GPU-resident aux path, and a `#[derive(Clone)]`. **No value +semantics.** Defenders A and B reached the same conclusion by region hashing; I confirmed it by +hunk boundaries. Step 1's "leading suspect" (a shifted alpha-power offset) is dead. + +--- + +## 3. Ruling on the soundness-regression claim (attacker Finding 2) — UPHELD, and stronger + +The attacker says patching `expected_public_balance` to match would fold an unmatched-bus residual +into the verifier target and permanently blind the cross-table check, with every named control +staying green. **I uphold that, and I find the situation is worse than stated.** + +`expected_public_balance` (`prover/src/lfm/proof.rs:247-276`) is a pure function of +`(claimed_public, z, alpha)`. The Hwsl residual is `Σ over the trace's HWSL lookup multiset of +mult/(z − fingerprint)` — a function of the *keccak trace contents*, which the verifier does not +have and which differs per program and per input. So **no formula change to +`expected_public_balance` can compensate for it.** The only edits that would turn the 20 tests +green are the degenerate ones Defender B enumerates: return a constant, drop the dependence on +`claimed_public`, or derive the target from the proof's own `bus_table_contribution()` values. The +last of those is the one that "works," and it is a total soundness break — `expected_public_balance` +is the recursion verifier's only cross-table binding, since LfmPublic has no in-trace receiver +(`proof.rs:215-222`). + +So Step 2 is not merely aimed at the wrong file. **Executed as written with "make the 20 pass" as +the acceptance criterion, it has exactly one reachable answer, and that answer is catastrophic.** +R1 names this risk and Step 4's controls cannot see it: the residual is independent of +`claimed_public`, so `tampered_claimed_public_word_rejects` still rejects. The plan's headline +mitigation does not mitigate its headline risk. + +The same reasoning kills `replay_transcript_phase_a_view` as a target: changing it moves `z`/`α`, +which would break the per-table OOD composition checks — and those **pass**. Nothing in the +verifier binding can be the cause. + +--- + +## 4. Ruling on "20 failures = one root cause" — FALSE, as both the attacker and Defender B argued + +✓ **`keccak_probe::adapter_probe_proves_real_permutations` and +`keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` cannot be touched by the +plan's fix at all.** `keccak_probe.rs:126-143` verifies with a hardcoded `&FEE::zero()` expected +balance through `Verifier::multi_verify_views` on the **production** AIRs plus a local adapter. It +calls neither `verify_against` nor `lfm_verify` nor `expected_public_balance` nor +`replay_transcript_phase_a_view`. Both tests **are** explained by the HWSL mismatch +(`:211 round_trip(|_|{}) == Ok(true)`; `:285 assert!(verify_proof(...))`). + +⚠️ **`keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` is not a negative +control.** `keccak_probe.rs:284-289` asserts the proof **verifies**, documenting an open +tag-uniqueness hazard. Step 4's instruction that "the output-swap-hazard tests" must "still REJECT" +would invert its meaning. Both defenders flagged this; it is correct and it must be struck. + +✓ **`machine_tests::preprocessed_tags_close_the_output_swap_hazard` is not explained by either +theory.** `machine_tests.rs:441-444` is its only verify assertion and it is *negative* +(`assert!(!lfm_verify(...))`) — a universally-rejecting verifier **satisfies** it. Its remaining +failure modes are all prove/compile-side: `assert_ne!(tag(0), tag(1))` at `:429`, +`prove_keccak_chain_with_tamper(...).expect("locally consistent")` at `:440` (proving must +*succeed*), and `.expect_err(...)` + `matches!(err, ProvingError::PrecomputedCommitmentMismatch)` +at `:455-459`. Main rewrote the prover's preprocessed / split-tree commit path, which is the +plausible independent cause. Nobody has read this test's actual failure message. + +✓ **The two census failures are a third bucket, prover-side, and one of them must not be +re-blessed.** `constraint_leg_instruction_census` dies at `constraint_tests.rs:438-442` +(`panic!("no design entry for {label}")`) on main's new HINT table — that half is genuinely +bookkeeping. But `continuation_epoch_constraint_leg_cost` is not: ✓ HINT appears in **neither** +`SPLIT_FAMILIES` (`constraint_tests.rs:1491-1494`) nor `FIXED` (`:1497-1508`), so the −1018 delta +(62 375 vs the pinned `63_393` at `:1566-1569`) **cannot** come from HINT. Defender A is right that +the plan misattributes it. I add the likely true attribution: **`KECCAK_RND` is in `FIXED` +(`:1502`), and main's HWSL→inline-identity swap is precisely a change to `KeccakRndConstraints` +(the `@@ -900,26 +824,99 @@` hunk).** That is the same main change as the root cause, and it should +be checked first. The test's own doc comment (`:1563-1565`) says a mismatch "is a finding about the +epoch, not about this pass" — so the number must be attributed, not pasted. + +⚠️ **`program_id_matches_production_on_the_real_fixture` and +`program_id_folds_pages_in_the_production_layout` each carry a digest `assert_eq!` before their +`verify_against`** (`machine_tests.rs:3722-3726` then `:3728-3738`). Both are keccak folds so HWSL +explains them, but which assertion fires is unknown. The first also depends on +`proof_fixture::load_or_generate(&fixture_cache())` (`machine_tests.rs:3690`) and hard-asserts +`pages.is_empty()` at `:3709-3713`, so it is fixture-sensitive — the attacker's baseline- +comparability caution (Finding 11) is legitimate. + +**Conclusion: at least four buckets, not one.** Bucketing by actual panic message is mandatory and +costs one test run. + +--- + +## 5. Is the corrected fix mechanical enough to implement directly? YES + +I rule that the `keccak_adapter` reconciliation may be implemented **directly, without a further +human review pass**, for four reasons I verified: + +1. **It is a deletion of two 6-line push blocks plus three number updates.** No new logic. +2. **It is fail-closed.** The edit changes what multiplicities the *prover* claims BITWISE was + looked up for. Get it wrong in either direction and the bus fails to balance and the proof is + **rejected**. Unlike the plan's Step 2, there is no way for this edit to make the verifier + accept more than it should — it cannot weaken a check, because it is not on a check. +3. **Its blast radius does not reach any pinned identity.** ✓ BITWISE is not among the 11 committed + groups in `LfmArtifacts` (`prover/src/lfm/registry.rs:133-152`: const_, balu, xalu, select, + bitdec, hash, keccak, lanes, hint, public, range; slot 11 is the KECCAK_RND sentinel), and + multiplicities live in the main trace, not in `bitwise::NUM_PRECOMPUTED_COLS = 11` + (`prover/src/tables/bitwise.rs:101`). So `artifacts.roots`, `program_id`, the `registry_drift_*` + tests and the in-circuit `statement_replay.rs:164-190` mirror are all **untouched**. The + attacker's Finding 6 (registry drift / in-circuit mirror) is a real risk *for the plan's fix* + and a non-risk for the correct one. +4. **It restores an invariant that has an external oracle** — main's own + `collect_bitwise_from_keccak`, which the fork's doc comment already names as its source. The + correct post-state is not a judgement call; it is "the fork agrees with its documented origin + again." + +Contrast with the plan's proposed fix, which would edit the recursion verifier's only cross-table +soundness check on a false diagnosis. That is the difference between the two verdicts. + +--- + +## 6. Which debate strengthenings are adopted + +**Adopted (mandatory).** Per-bus residual measurement before any edit (attacker §1; Defender A +§6.3) — ✓ the machinery exists: `crypto/stark/src/bus_debug.rs` behind the `debug-checks` feature +(`crypto/stark/Cargo.toml`, `prover/Cargo.toml`), runtime filter `DEBUG_BUS_ID` at +`bus_debug.rs:66`, and `BusId::Hwsl = 9`. Note: the `DEBUG_BUS_TRACKER=1` form recorded in project +memory does not appear in `bus_debug.rs`; use the feature + `DEBUG_BUS_ID`. Bucket all 22 by panic +message (attacker §3; Defender B S3; Defender A §6.4). Correct negative controls, run positive and +negative in the same run (Defender B S2; attacker §5). Index-permutation control (Defender A §6.7). +`git add` the five unmerged paths (attacker §9) — ✓ confirmed still unmerged: +`crypto/stark/src/lookup.rs`, `prover/src/continuation.rs`, +`prover/src/tests/constraint_program_{device_,}tests.rs`, `prover/src/tests/ood_window_ir_tests.rs`; +until they are added, `git diff -- crypto/stark/` prints `lookup.rs | Unmerged` and inspects +nothing. Empty-diff cleanup gate on `verifier.rs` (Defender B S7; Defender A §6.8) — ✓ confirmed +all ~50 changed lines are diagnostics, and ✓ the `DBG909` insert at `verifier.rs:1447-1450` **stole +the `#[cfg(not(feature = "test_fiat_shamir"))]` attribute** from `error!`, plus de-indent damage at +`:1311`, `:1315`, `:1363`. Push the tag before the branch push (Defender B S1; Defender A §6.9). +Separate commit for the HINT/census items (Defender B S4). Attribute the −1018 before re-pinning +(Defender A §6.5, Defender B S4). + +**Adopted (should).** `cargo test -p stark` including main's new +`opening_width_tests.rs` / `aux_opening_width_tests.rs` (attacker §8). Checkpoint-commit the +resolved merge rather than leaving 1 700+ conflict-resolved lines in one index (Defender A §6.9). +Baseline-comparability check on fixture state (attacker §11). + +**Rejected.** Defender A §6.1's conclusion that "what remains is Outcome C (the target's inputs)" +— Outcome C is dead too; the answer is the trace side, which Step 2 could not express. Defender A +§6.2/Defender B S5's same-tree A/B as the *opening* move — it is a good experiment but the per-bus +dump is strictly more decisive and equally cheap, so it goes first. Step 4's framing of the +output-swap-hazard tests as negative controls — struck outright (see §4). + +**Both defenders deserve credit for conceding the load-bearing points** (Outcomes A and B dead by +diff; wrong primary control; single-root-cause unverified; `replay_transcript_phase_a_view` reaches +the production VM verifier at `prover/src/lib.rs:1442` and `prover/src/continuation.rs:896`, while +`expected_public_balance` has exactly one caller at `proof.rs:220` — ✓ both verified). Their +defense of the plan's *shape* stands. Their defense of its *content* does not survive the +`keccak_adapter` finding, which neither of them located. + +--- + +## 7. The corrected plan + +### Step 0 — CONFIRM the diagnosis (no edits) + +1. Per-bus residual on one failing test: + `DEBUG_BUS_ID=9 cargo test --release -p lambda-vm-prover --features debug-checks --lib + lfm::machine_tests::machine_proves_the_sample_replay -- --nocapture`. + **Expect the residual on `Hwsl` (9), receiver-side, zero senders.** If it lands on `LfmPublic` + (34), `LfmMem` (32) or `LfmRange` (33) instead, **stop** and re-open this verdict. +2. Bucket all 22 by actual panic message: + `cargo test --release -p lambda-vm-prover --lib lfm:: -- --nocapture 2>&1 | grep -B2 -A5 panicked`. + Record the bucket for each. Expect ≥4 buckets (§4). + +### Step 1 — FIX the fork + +Delete the two HWSL pushes: `prover/src/lfm/keccak_adapter.rs:361-366` (theta) and `:441-446` +(rho). Update the pinned per-round count `1148 → 1028` at `keccak_adapter.rs:306` (doc), `:319` +(capacity), and `keccak_probe.rs:201-205` (assertion). Refresh the doc comment at +`keccak_adapter.rs:306-315` to record that main dropped the θ/ρ HWSL lookups in favour of inline +μ-gated identities, so the fork is again the per-round half of main's collector. + +**Prohibited without new evidence and an explicit escalation:** any edit to +`expected_public_balance`, `replay_transcript_phase_a_view`, `compute_expected_commit_bus_balance_view`, +`absorb_lfm_statement`, or anything under `crypto/stark/`. The scope rule is Defender B's, and it +is better than a directory boundary: **no edit whose blast radius reaches `prover/src/lib.rs:1442`.** + +### Step 2 — CONTROLS (positive and negative in the same run) + +Honest-path, must be GREEN: `machine_tests.rs:36 trivial_program_proves_and_verifies`; +`machine_tests.rs:66 different_arena_values_change_the_public_output_not_the_program` (carries +positive `:81-83`, cross-claim negative `:85-87`, and distinctness `:80` in one body — the single +best gate); `blake3_probe.rs:521 falsification_control_the_untampered_proof_verifies`. + +Rejection, must stay RED for the prover: `machine_tests.rs:52 tampered_claimed_public_word_rejects` +(primary); `machine_tests.rs:2248 tampered_statement_or_root_rejects`; +`framework_probe.rs:156,169,188`; `keccak_probe.rs:219,228,237`; +`blake3_probe.rs:529,542,556,564,575`; +`logup_tests.rs:478 the_closure_cannot_sum_a_contribution_the_constraints_rejected`. + +Must ACCEPT (not a negative control): +`keccak_probe.rs:261 duplicate_tag_output_swap_accepts_demonstrating_hazard`. + +Executor-level only, keep but do not treat as the binding's gate: +`machine_tests.rs:3593 tampered_l2g_binding_rejects` (its first vectors reject inside +`super::executor::execute`; only the coherent-swap leg at `:3647` reaches `verify_against`). + +New control to add: an **index-permutation** vector — build `claimed` by swapping two entries' +`index` fields with all lane values untouched, assert `lfm_verify` returns `false`. Nothing in the +suite currently covers the `index·α` term (`proof.rs:261`). + +### Step 3 — HINT / census, in a SEPARATE commit + +Add the HINT row to `DESIGN_INSTR`; expect the census to then print a per-table design-vs-emitter +mismatch list (`constraint_tests.rs:447-452`) — that list is the attribution the budget delta +needs. **Do not re-pin `63_393` until the −1018 is attributed.** First hypothesis to test: +`KECCAK_RND`, whose constraint set main rewrote in the same change as the root cause. + +### Step 4 — VALIDATE + +- `lfm::` failure set equals the `blake3-campaign-preMerge` baseline set (306/19, the + `recursion/fibonacci.elf` fixture failures) — set equality, not counts. Confirm the fixture state + matches the baseline worktree before comparing. +- `cargo test -p stark`, explicitly including `opening_width_tests` and `aux_opening_width_tests`. +- The main VM and continuation verify paths are untouched by this fix; if that ever stops being + true, `prover/src/tests/prove_elfs_tests.rs` and the continuation suite join the gate. +- `constraint_artifact` 11/11 and `artifact_pin.py --check` remain as **regression-only** guards; + they do not exercise `keccak_adapter.rs`, `proof.rs` or `lib.rs` and must not be cited as gates + for this change. + +### Step 5 — CLEANUP (mechanical, not eyeball) + +1. `git add` the five unmerged paths first, or the crypto/stark diff check inspects nothing. +2. `git diff origin/main -- crypto/stark/src/verifier.rs` must be **exactly empty** — including + reverting `let ok = (0..num_queries).all(…); …; ok` back to the direct return, and restoring + `#[cfg(not(feature = "test_fiat_shamir"))]` to its `error!`. +3. `git diff --name-only origin/main -- crypto/stark/` must reduce to exactly the seven artifact- + feature files (`constraint_ir/artifact.rs`, `constraint_ir/artifact_tests.rs`, + `constraint_ir/mod.rs`, `constraint_ir/device.rs`, `constraints/builder.rs`, `lookup.rs`, + `traits.rs`). Note the plan's "No edits under `crypto/stark/`" is already false: the merge + deliberately re-adds `with_precaptured` / `precaptured_constraint_program`, which main deleted. +4. Remove the `LFM_BUS_DEBUG` block at `prover/src/lfm/proof.rs:224-240`. Campaign diagnostics are + identifiable by the markers `W909_DEBUG`, `DBG909`, `LFM_BUS_DEBUG`. **Do not** strip main's own + `LAMBDA_VM_TIMELINE_JSON` / `LAMBDA_VM_TRACE_BUILDERS` instrumentation — that is merge content. +5. `make fmt` and `make lint` from the repo root (the de-indented `error!` sites are also a + formatting failure). + +### Step 6 — FINALIZE + +`git push origin blake3-campaign-preMerge` **before** the branch push — ✓ the tag is currently +local-only, and after the fast-forward `refs/heads/blake3-real-hash` stops being the remote copy of +`ed1b7785`. Then commit (binding fix and HINT/census as separate commits), fast-forward, push. + +### Escalation gates (the only things that need a further review pass) + +- **G1.** If Step 0's residual is not on `Hwsl`, or if bucketing shows failures the HWSL theory + cannot explain *and* they point at the verifier binding — stop, do not edit, re-open. +- **G2.** Any change to `expected_public_balance` or `replay_transcript_phase_a_view` — human + review, mandatory. Both defenders and the attacker agree these are the recursion verifier's and + the production VM verifier's soundness surface. +- **G3.** Re-pinning `continuation_epoch_constraint_leg_cost` — the attribution must be written + down and read by a human before the constant moves. "Investigate, never re-bless" + (`machine_tests.rs:113-117`) is the house rule and it applies here. + +Everything else: implement directly. + +--- + +## 8. Verification log (what I read myself) + +| Claim | Status | Evidence | +|---|---|---| +| Main dropped 120 HWSL sends/round from `KECCAK_RND` | ✓ VERIFIED | `keccak_rnd.rs:439,446`; diff hunks `@@ -587,48 +596,8` / `@@ -717,53 +686,8` | +| Only the HWSL blocks changed in `bus_interactions()` | ✓ VERIFIED | filtered diff shows two `- BusId::Hwsl` and no other `BusId::` line | +| Main dropped the matching receives in production | ✓ VERIFIED | `origin/main:trace_builder.rs` has 0 `BitwiseOperationType::Hwsl`; `ed1b7785` has 2 (`:2427`, `:2510`) | +| LFM AIR built from main's `keccak_rnd::bus_interactions()` | ✓ VERIFIED | `airs.rs:21,239,488-494` | +| LFM receiver side is the branch fork, merge-untouched | ✓ VERIFIED | `trace.rs:176`; `git diff ed1b7785 -- keccak_adapter.rs` empty | +| Fork still emits 120 HWSL/round | ✓ VERIFIED | `keccak_adapter.rs:361-366` (20), `:441-446` (100) | +| Those are the only HWSL in `prover/src/lfm/` | ✓ VERIFIED | `grep -rn Hwsl prover/src/lfm/` → 2 hits | +| Per-round total is 1148; 1148 − 120 = 1028 | ✓ VERIFIED | hand count of `bitwise_ops_for` `:324-509` | +| Interactions μ-gated ⇒ padding sends nothing | ✓ VERIFIED | `Multiplicity::Column(cols::MU)` throughout `keccak_rnd.rs:446ff` | +| `keccak_ops` comes only from `records.keccak` | ✓ VERIFIED | `trace.rs:145-155`; `executor.rs:440,522` | +| `HasherKind::default() = Test`; trivial has no keccak | ✓ VERIFIED | `hash.rs:196-199`; `registry.rs:117-119`; `programs.rs:31-79` | +| crypto/stark LogUp fingerprint math unchanged | ✓ VERIFIED | first `lookup.rs` hunk at line 834; all fingerprint fns above it | +| `keccak_probe` verifies against hardcoded `FEE::zero()` | ✓ VERIFIED | `keccak_probe.rs:126-143` | +| `duplicate_tag_…_hazard` asserts ACCEPT | ✓ VERIFIED | `keccak_probe.rs:284-289` | +| `preprocessed_tags_…` only verify assert is negative | ✓ VERIFIED | `machine_tests.rs:441-444`; prove-side legs `:429`, `:440`, `:455-459` | +| HINT absent from both epoch-budget lists | ✓ VERIFIED | `constraint_tests.rs:1491-1494`, `:1497-1508`; pin at `:1566-1569` | +| `KECCAK_RND` is in `FIXED` (budget delta suspect) | ✓ VERIFIED | `constraint_tests.rs:1502` | +| Census panics before its mismatch assert | ✓ VERIFIED | `constraint_tests.rs:438-442` vs `:447-452` | +| `expected_public_balance` has exactly one caller | ✓ VERIFIED | `proof.rs:220` / `:247` | +| `replay_transcript_phase_a_view` reaches the VM verifier | ✓ VERIFIED | `lib.rs:989,1014,1442`; `continuation.rs:896` | +| In-circuit mirror exists | ✓ VERIFIED | `statement_replay.rs:164-190`; third mirror `machine_tests.rs:2101` | +| BITWISE not in `LfmArtifacts.roots` ⇒ program_id safe | ✓ VERIFIED | `registry.rs:133-152`; `bitwise.rs:101` | +| Five paths still unmerged in the index | ✓ VERIFIED | `git diff --name-only --diff-filter=U` | +| `verifier.rs` diagnostics stole a `#[cfg]` | ✓ VERIFIED | diff at `verifier.rs:1447-1450`; de-indents at `:1311,:1315,:1363` | +| `let ok = …; ok` refactor is semantics-preserving | ✓ VERIFIED | diff at `verifier.rs:240-263` | +| `debug-checks` + `DEBUG_BUS_ID` exist; `Hwsl = 9` | ✓ VERIFIED | `bus_debug.rs:8-14,66`; both Cargo.toml; `types.rs:283` | +| Tag `blake3-campaign-preMerge` not pushed | ? INFERRED | both defenders ran `git ls-remote --tags origin`; I did not re-run | +| `trivial_program_proves_and_verifies` passes in the merge tree | ? INFERRED | absent from `reconcile-report.md` §5's list; not re-run | diff --git a/thoughts/shared/lfm-real-hash/merge-plan/PLAN.md b/thoughts/shared/lfm-real-hash/merge-plan/PLAN.md new file mode 100644 index 000000000..834a4ed7b --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/PLAN.md @@ -0,0 +1,68 @@ +# Merge plan: bring `blake3-real-hash` up to date with `main` + +**Goal:** merge current `origin/main` into the campaign branch so it is testable/explorable +against latest, keeping main's constraint-IR redesign authoritative and adapting the branch's +build-time constraint-artifact feature to it. PR #930 ends up up to date. + +Grounded in two read-only investigations (this dir): `main-ir-spec.md`, `artifact-feature-map.md`. + +## The core problem, settled + +Main redesigned the device-IR: `DeviceProgram::lower()` now runs a liveness slot allocator and +encodes operands as `kind<<29 | payload` (OPK-tagged slots/uniforms), drops dead nodes + uniform +leaves, replaces per-node `dim` with a `res` slot word, and adds `num_base_slots`/`num_ext_slots`. +The OLD `lower()` was a 1:1 image of `ConstraintProgram` with **node-index** operands and per-node +`dim`. The branch's artifact serialized that OLD (node-index) form and its consumers +(`validate_self`, `program()`, the census, and the whole `prover/src/lfm/` recursion machine) +assume node-index operands. + +New `lower()` is **lossy and one-way** → `DeviceProgram → ConstraintProgram` is impossible, and +`eval_program`/`eval_program_verifier` need the node-index `ConstraintProgram` form. So the artifact +**must** keep a node-index form. `ir.rs` (`ConstraintProgram`/`Op`/`Dim`) is byte-identical on both +branches and carries no serde/rkyv derives. + +## The chosen approach — A (decoupled) + +The artifact owns a POD node type `ArtifactNode { op, a, b, dim }` (rkyv, `#[repr(C)]`, node-index +operands) — i.e. exactly the OLD `DeviceNode` — decoupled from main's now-slot-based `DeviceNode`. +It serializes that; `program()` lifts it to a `ConstraintProgram` (unchanged); `device_program()` +re-derives the flat blob through main's production `DeviceProgram::lower(&self.program())`. +Soundness preserved: the device blob goes through the same `lower()` the prover/GPU use, and +`program()` still lifts to the `ConstraintProgram` the compiled folders are pinned against. + +## Steps + +**Setup (keep the clean branch pristine until validated):** +1. Tag the clean tip: `git tag blake3-campaign-preMerge ed1b7785`. +2. Dedicated worktree on a new branch: `git worktree add ../lambda_vm-blake3-merge -b blake3-real-hash-mainmerge blake3-real-hash`. + +**Merge + mechanical conflicts (known from the trial merge):** +3. `git merge --no-commit --no-ff origin/main`. +4. Resolve 5 conflicts: `lookup.rs` (main's `Arc` + our `precaptured_program`, both), `continuation.rs` + (`#[derive(Clone,Copy)] pub(crate)`), 3 test files (keep our generic `production_airs()` iteration). +5. HINT coverage gap: add `HINT` to `production_airs()`, `NUM_PRODUCTION_AIRS` 28→29. +6. Shared IR files (`device.rs`, `ir.rs`, `interp.rs`, `builder.rs`, `gpu_interp.rs`): main's versions win. + +**Artifact reconciliation (approach A — the real work, artifact.rs):** +7. Define `ArtifactNode { op:u32, a:u32, b:u32, dim:u32 }` (rkyv derives, `#[repr(C)]`) + local + `DIM_BASE`/`DIM_EXT` consts in `artifact.rs`. +8. `ConstraintArtifact.nodes: Vec` (drop the DeviceNode dependence + the broken + slot-size fields from the trial merge). +9. `capture()`: build `ArtifactNode`s via the OLD 1:1 map from `ConstraintProgram` (op tag, node-index + a/b, dim), take `roots`/`num_base` from `prog` — NOT from `DeviceProgram::lower`. +10. `device_program()`: `DeviceProgram::lower(&self.program())`. +11. `program()` / `validate_self()`: unchanged (they already assume node-index — now correct). +12. Tests: `constraint_artifact_tests.rs` (census `DIM_BASE` import → artifact's) and + `lfm/constraint_tests.rs:658` (`DeviceNode{...,dim}` literal → `ArtifactNode`). +13. `prover/src/lfm/*` — NO change (they read `artifact.program()`). + +**Validation (round-trip FIRST — the direct signal that broke):** +14. `cargo check` → `constraint_artifact` suite (MUST pass) → `lfm::` (expect 306/19) → + full lib suite categorized (confirm only pre-existing fixture/env failures, nothing in the edited + modules) → chip gate `artifact_pin.py --check` (BLAKE3 chip unchanged by the merge). +15. Baseline: run the full lib suite on `blake3-campaign-preMerge` too, to diff pre-existing vs new. + +**Finalize:** +16. Adversarial review of the artifact reconciliation (soundness-adjacent). +17. Once green + reviewed: fast-forward `blake3-real-hash` to the merged branch, push → PR #930 up to date. + Keep `blake3-campaign-preMerge` tag as the recoverable pristine point. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/artifact-feature-map.md b/thoughts/shared/lfm-real-hash/merge-plan/artifact-feature-map.md new file mode 100644 index 000000000..95957c106 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/artifact-feature-map.md @@ -0,0 +1,385 @@ +# ConstraintArtifact feature map + OLD→NEW device-IR diff + +READ-ONLY investigation. Branch `blake3-real-hash` @ `ed1b7785` vs `origin/main` @ `58160b6f`. +Working tree: `/Users/maurofab/workspace/lambda_vm-blake3-impl`. + +## TL;DR + +- **Recommendation: Approach A** — decouple the artifact's wire format from + `device.rs`. The artifact should own a POD node type equal to the *OLD* + `DeviceNode { op, a, b, dim }` (rkyv-derived, **node-index operands**, per-node + `dim`), keep serializing that, and re-derive the device blob at read time via + main's `DeviceProgram::lower(&self.program())`. This preserves the node-index + operand model that `program()`, `validate_self()`, the census, **and the entire + `prover/src/lfm/` recursion-machine lowering** are built on. Only two functions + change materially (`capture`, `device_program`) plus import-path fixes. +- **Approach B (store main's slot-form `DeviceProgram`) is infeasible**, not just + risky: main's `DeviceNode` has no rkyv derives, main's `lower()` is lossy + (drops uniform leaves + dead nodes, slot-encodes operands), so `program()` + cannot invert it — the round-trip's `prog.nodes == captured.nodes` assertion can + never hold — and the LFM machine's per-node model has no meaning on the reduced + slot graph. +- **THE key device-IR diff:** OLD `lower()` did **NOT** slot-encode operands; it + produced a 1:1 image of `ConstraintProgram` with `a`/`b` as **raw node indices** + and a per-node `dim`. NEW `lower()` runs a liveness slot allocator, encodes each + operand as `kind << 29 | payload` (slot or uniform-table index), drops uniform + leaves and dead nodes, replaces `dim` with a `res` slot word, and adds + `num_base_slots`/`num_ext_slots`. `DeviceNode` also lost its rkyv derives. +- **Scope is bigger than the 3 named files.** `prover/src/lfm/constraints.rs` + (`analyze`/`differential_program`/`ood_frame_words`), `constraint_tests.rs`, + `join_tests.rs`, `epoch_verify*.rs`, and the `compute_constraint_artifacts` + binary all consume `ConstraintArtifact`. All but one read it through + `artifact.program()` (node-index `Op`), so Approach A leaves them untouched. + +--- + +## 0. Where the feature lives, and why main breaks it + +The `artifact` module is **branch-only**. `origin/main`'s +`crypto/stark/src/constraint_ir/mod.rs` does **not** declare `pub mod artifact;` +and does **not** re-export `AirShape/ArtifactError/ArtifactMeta/ConstraintArtifact` +(HEAD's mod.rs line 45 does; main's does not). So this is an additive feature that +was written against HEAD's `device.rs`; main independently rewrote `device.rs`. + +The break is entirely at the `device.rs` seam: + +| symbol the artifact imports from `device.rs` | HEAD | origin/main | +|---|---|---| +| `DeviceNode` fields | `{ op, a, b, dim }` | `{ op, a, b, res }` (✓ VERIFIED, new_device.rs:118) | +| `DeviceNode` rkyv derives | present (old_device.rs:76) | **absent** (`derive(Clone, Copy, Debug, PartialEq, Eq)`, new_device.rs:117) | +| `DIM_BASE` / `DIM_EXT` consts | present (old_device.rs:67,69) | **gone** (grep: none in new_device.rs) | +| operand model in `nodes` | raw node indices | slot-encoded `OPK_* << 29 \| payload` | +| `DeviceProgram` extra fields | — | `num_base_slots`, `num_ext_slots` (new_device.rs:169-171) | +| `roots` entries | node ids | `slot \| RES_EXT_BIT` (new_device.rs:162-164, 333-344) | + +`ir.rs` is **byte-identical** between HEAD and main (`diff` = identical). +`ConstraintProgram`, `Op`, `Dim` are unchanged — and, importantly, **none of them +is rkyv-serializable** (plain `derive(Clone, Debug)` / `derive(..., Hash, Debug)`; +`ConstraintProgram` holds `FieldElement`). That is *why* the artifact carries +its own POD projection rather than serializing `ConstraintProgram` directly. + +--- + +## 1. What `ConstraintArtifact` serializes + +Struct (`artifact.rs:226-243`), all fields rkyv: + +```rust +#[derive(Clone, Debug, PartialEq, Eq, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ConstraintArtifact { + pub nodes: Vec, // <-- imported from device.rs + pub base_consts: Vec, + pub ext_consts: Vec<[u64; 3]>, + pub roots: Vec, + pub num_base: u32, + pub meta: Vec, // {constraint_idx:u32, kind:u8, end_exemptions:u32} + pub shape: AirShape, // width/step/offsets/next_row_cols/... scalars +} +``` + +- **rkyv derives:** `ConstraintArtifact`, `ArtifactMeta` (artifact.rs:112), + `AirShape` (artifact.rs:166) all derive `rkyv::{Archive, Serialize, Deserialize}`. + The `nodes: Vec` field requires **`DeviceNode: rkyv::*`**, satisfied + only by HEAD's `device.rs` (old_device.rs:76). On main this field would not + compile — the first, hardest breakage. +- **`to_bytes`/`from_bytes`** (artifact.rs:656-668): `rkyv::to_bytes` / + `rkyv::from_bytes` with `rancor::Error`; `from_bytes` runs `validate_self()`. +- **It stores the FLAT `DeviceProgram` form, not `ConstraintProgram`.** The five + program fields (`nodes/base_consts/ext_consts/roots/num_base`) are copied + straight out of `DeviceProgram::lower(prog)` in `capture()` (artifact.rs:302, + 333-338). BUT — and this is the load-bearing subtlety — HEAD's `DeviceProgram` + is a **faithful 1:1 image** of `ConstraintProgram`: same node count, same order, + per-node `dim`, and `a`/`b` = node indices. So "the flat DeviceProgram form" and + "a serializable ConstraintProgram" are the *same bytes* on HEAD. That equivalence + is exactly what main's slot-encoding `lower()` destroys. + +--- + +## 2. Every dependency on the OLD (node-index) operand model + +Each site below reads `n.a`/`n.b`/`roots` as **node indices** and/or reads `n.dim`. +Under main's slot encoding these words are `kind<<29|payload` and `res` slot words, +and many nodes are eliminated — so each site is a breakage point. + +### 2a. `artifact.rs` + +- **`validate_self()` operand check** (artifact.rs:476-514). The closure + ```rust + let check_id = |x: u32| if (x as usize) < i { Ok(()) } else { Err(... "references node {x}, which is not strictly earlier") }; + ... + OP_ADD | OP_SUB | OP_MUL => { check_id(n.a)?; check_id(n.b)?; } + OP_NEG | OP_EMBED => check_id(n.a)?, + ``` + interprets `n.a`/`n.b` as **node ids** and enforces topological order + (`id i references only < i`). Under slot encoding this is meaningless (operand is + `OPK_* << 29 | slot`). Also reads `n.dim` against `DIM_BASE/DIM_EXT` + (artifact.rs:471-472) — tags that no longer exist on main. +- **`program()` reconstruction** (artifact.rs:388-451). Rebuilds a + `ConstraintProgram` by a linear walk that reads `n.a`/`n.b` as node ids + (`OP_ADD => Op::Add(n.a, n.b)`, artifact.rs:413) and `n.dim` → `Dim` + (artifact.rs:420-424). This is the **inverse of the OLD 1:1 lower** and is the + method the LFM machine and the round-trip oracle both depend on. +- **`device_program()`** (artifact.rs:366-374). Cheap field copy that reconstructs + a `DeviceProgram` from the stored fields — valid only because the stored form IS + the device form on HEAD. On main a `DeviceProgram` also needs + `num_base_slots/num_ext_slots`, which the artifact does not store. +- **`capture()`** (artifact.rs:302, 333-338). `let dev = DeviceProgram::lower(prog)` + then copies `dev.nodes/roots/...`. On main this yields **slot-form** nodes and + slot-encoded roots of a *different length* — the artifact would silently store + the wrong thing even if it compiled. + +### 2b. `constraint_artifact_tests.rs` (the round-trip + census) + +- **`constraint_op_census`** (test lines 424-467). The `v_base` propagation reads + operands as node ids: + ```rust + OP_NEG => (v_base[n.a as usize], true), + _ => (v_base[n.a as usize], v_base[n.b as usize]), // line 447 + v_base[i] = ba && bb && n.dim == DIM_BASE; // line 451 + ``` + Both `n.a/n.b`-as-index and `n.dim` break on main. `DIM_BASE` is imported from + `stark::constraint_ir::device` (test line 397) — gone on main. +- **`leg_instructions`** helper (test lines 925-955) — same pattern + (`v_base[n.a as usize]`, `n.dim == DIM_BASE`), imports `DIM_BASE` from + `device` (line 927). +- **`check_air_artifact`** (test lines 99-133) asserts `prog.nodes == captured.nodes`, + `prog.dims == captured.dims`, `prog.roots == captured.roots` — i.e. `program()` + must reproduce the captured `ConstraintProgram` exactly (see §4). +- The fusability/DCE pass in `constraint_op_census` (test lines 527-567) reads + `uses[n.a as usize]`/`nodes[n.a as usize].op` as node ids. + +### 2c. `crypto/stark/src/constraint_ir/artifact_tests.rs` + +- `validate_self_rejects_a_forward_reference` (lines 196-207) sets + `artifact.nodes[last].a = last` and expects rejection — depends on the node-index + topological invariant. +- `validate_self_rejects_an_out_of_range_constant` (lines 220-235) reads + `n.op == OP_CONST_BASE` and mutates `node.a` as a `base_consts` index. +- `lift_is_the_inverse_of_lower` (lines 138-166) asserts + `artifact.program().nodes == air.constraint_program().nodes` (+ dims/roots/consts). + +### 2d. `prover/src/lfm/` (the recursion machine — NOT in the task's file list, but the largest consumer) + +- **`lfm/constraints.rs::analyze`** (constraints.rs:286-290) calls + `artifact.program()` and works over `prog.nodes[i]` as `Op` with **node-index + operands** (`Op::Add(a,b)` → `konst[a]`/`konst[b]`, fanout counting, DCE, + MulAdd-fusion, `differential_program`). This whole subsystem consumes the + *node-index `Op` form via `program()`* — it never touches the device slot blob. + **Approach A leaves it untouched; Approach B would require rewriting all of it.** +- **`lfm/constraint_tests.rs:658`** is the one place that constructs a raw + `DeviceNode { op, a, b, dim: DIM_EXT }` and pushes onto `injected.nodes` — a + direct dependency on the OLD `DeviceNode` shape (has `dim`, node-index `a`/`b`, + rkyv). This is a mechanical rename under Approach A. +- `join_tests.rs`, `epoch_verify.rs`/`epoch_verify_tests.rs`, + `bin/compute_constraint_artifacts.rs` use `ConstraintArtifact::capture` / + `from_bytes` / `program()` — all node-index / `program()`-mediated. + +--- + +## 3. The OLD → NEW device-IR diff (precise) + +### 3a. Did OLD `lower()` slot-encode operands? **NO — raw node indices.** + +OLD `lower()` (old_device.rs:125-170) is a pure 1:1 map over `prog.nodes.zip(dims)`: + +```rust +let (op, a, b) = match *op { + Op::Add(a, b) => (OP_ADD, a, b), // a,b are NODE IDS, passed through verbatim + Op::Sub(a, b) => (OP_SUB, a, b), + Op::Mul(a, b) => (OP_MUL, a, b), + Op::Neg(a) => (OP_NEG, a, 0), + Op::Embed(a) => (OP_EMBED, a, 0), + ... +}; +DeviceNode { op, a, b, dim } // per-node dim carried +... +roots: prog.roots.clone(), // roots = node ids, verbatim +num_base: prog.num_base as u32, +``` + +No slots, no liveness, no elimination. `nodes.len() == prog.nodes.len()`, order +preserved. This is what makes it a serializable mirror of `ConstraintProgram`. + +### 3b. NEW `lower()` slot-encodes and eliminates (new_device.rs:195-358) + +- **Liveness slot allocator** with per-class free lists (`free_base`/`free_ext`), + `num_base_slots`/`num_ext_slots` counters, operand slots freed at last use, roots + pinned (new_device.rs:246-344). +- **Operand encoding** `kind << OPK_SHIFT(29) | payload` (new_device.rs:86-105, + 263-274): `OPK_BASE_SLOT/EXT_SLOT/BASE_CONST/EXT_CONST/RAP/ALPHA/OFFSET`. An + arithmetic operand is a **slot index or a uniform-table index**, never a node id. +- **Uniform-leaf propagation** (new_device.rs:174-227): `Op::ConstBase/ConstExt/ + RapChallenge/AlphaPow/TableOffset` are *not materialized as nodes* unless they + are themselves roots; operands reference the uniform tables directly. +- **Dead-node elimination**: a node materializes only if `used[i]` (new_device.rs:225-227). +- So `nodes.len() < prog.nodes.len()` in general, order/indices no longer match + `ConstraintProgram`, and lowering is **lossy** (uniforms/dead nodes gone). + +### 3c. `DeviceNode` field diff + +| | OLD | NEW | +|---|---|---| +| fields | `op, a, b, **dim**` | `op, a, b, **res**` | +| `dim` | `DIM_BASE`/`DIM_EXT` per node | removed | +| `res` | — | result slot; bit31 (`RES_EXT_BIT`) = ext class, low bits = slot | +| derives | `+ rkyv::{Archive,Serialize,Deserialize}` | **no rkyv** | + +### 3d. `DeviceProgram` field diff + +| | OLD | NEW | +|---|---|---| +| `nodes/base_consts/ext_consts/num_base` | yes | yes | +| `roots` | node ids | `slot \| RES_EXT_BIT` | +| `num_base_slots` | — | **added** (base `u64` slot-class size) | +| `num_ext_slots` | — | **added** (ext `[u64;3]` slot-class size) | +| derives | `Clone, Debug` | `Clone, Debug` (unchanged; neither is rkyv) | + +### 3e. `eval_device_program` diff + +OLD (old_device.rs:247-324): forward pass into a flat `Vec` indexed by node +id; `binop` reads `values[a]`/`values[b]`, dim-driven base/ext. NEW +(new_device.rs:390-497): two slot files (`base_slots`/`ext_slots`), decodes each +operand via `load_base`/`load_ext` on its `OPK_*` kind, writes `res` slot; roots +read back by slot. Semantically bit-identical, structurally different — and the +round-trip test calls `eval_device_program` on whatever `device_program()` returns, +so under Approach A it must return a **main**-lowered `DeviceProgram`. + +### 3f. `ConstraintProgram`/`Op`/`Dim` (ir.rs): **identical** on both. +Not rkyv on either side (relevant to Approach A feasibility — see §5). + +--- + +## 4. The round-trip contract (what the failing tests assert) + +`all_table_artifacts_roundtrip_and_match_folders` → `check_air_artifact` +(constraint_artifact_tests.rs:72-264), for each of `NUM_PRODUCTION_AIRS` AIRs: + +1. `capture` → `validate_against(air)` accepts. +2. `to_bytes` → `from_bytes` → `validate_against(air)` accepts (wire hop). +3. **Structural identity**: `prog = artifact.program()` must equal the AIR's own + `air.constraint_program()` in `nodes`, `dims`, `roots`, `num_base`, + `base_consts`, `ext_consts` (lines 101-124). ⇒ **`program()` must reconstruct + the captured `ConstraintProgram` bit-for-bit.** +4. **Three evaluation oracles agree with the compiled folders** over 100 random + trials: + - `eval_program(&prog, ...)` (prover shape) vs `compute_transition_prover`. + - `eval_device_program(&dev, ...)` with `dev = artifact.device_program()` (flat + blob) vs the prover folder. + - `eval_program_verifier(&prog, ...)` (OOD shape) vs `compute_transition`. + +`production_airs_accept_a_precaptured_program` (lines 1237-1287): install +`artifact.program()` into a fresh AIR via `with_precaptured`, assert pointer +identity (no re-capture) and folder agreement. + +`constraint_op_census` / `epoch_chunk_multiplier` / +`continuation_epoch_constraint_leg` / `continuation_epoch_chunk_counts_measured`: +walk `artifact.nodes` (node-index + `dim`) to count constraint-leg instructions; +assert a loose ceiling (`instr < 200_000`) and a fixed epoch sub-proof composition +(24 intermediate / 25 final). + +**What must hold for all of these to pass:** (a) `nodes: Vec` must be +rkyv-serializable; (b) `program()` must be the exact inverse of the capture-time +lowering (`prog.nodes == captured.nodes`); (c) `device_program()` must produce a +`DeviceProgram` that `eval_device_program` evaluates to the folder result; (d) the +census must be able to read per-node `dim` and node-index operands. (b) and (d) are +**impossible from main's slot form**; they are trivially preserved by keeping the +OLD node-index form (Approach A). + +--- + +## 5. Reconciliation — two approaches + +### Approach A — artifact owns the node-index wire form; re-lower at read time ✅ RECOMMENDED + +Keep the artifact storing a POD node array identical to the **OLD** `DeviceNode` +(`{ op, a, b, dim }`, rkyv, node-index operands), owned by the artifact module +instead of imported from `device.rs`. Derive the device blob on demand. + +Is `ConstraintProgram`/`Op` rkyv on main? **No** (ir.rs derives are plain; it holds +`FieldElement`). So we cannot serialize `ConstraintProgram` directly — which is +fine, because the artifact already carries its own POD projection. Approach A = +*retain that projection* and stop piggy-backing it on `device.rs`'s type. + +**Sites to change (concrete):** + +1. **New owned node type in `artifact.rs`** — e.g. `ArtifactNode { op:u32, a:u32, + b:u32, dim:u32 }` with `#[repr(C)]` + `rkyv::{Archive,Serialize,Deserialize}` + + `Clone,Copy,Debug,PartialEq,Eq`. Verbatim copy of the OLD `DeviceNode`. Define + `DIM_BASE`/`DIM_EXT` (u32) here too (gone from `device.rs`). Reuse main's still- + exported `OP_*` tags and `pack_var`/`unpack_var` (unchanged on main), or re-home + them alongside the node type for full decoupling. +2. **`ConstraintArtifact.nodes`** field: `Vec` (was `Vec`). +3. **`capture()`**: replace `let dev = DeviceProgram::lower(prog)` + field copies + with a **1:1 map** of `prog.nodes.zip(prog.dims)` into `ArtifactNode` (i.e. the + OLD `lower` body, old_device.rs:126-158), and `roots = prog.roots.clone()`, + `num_base = prog.num_base`. (The linearity/shape logic is unchanged.) +4. **`device_program()`**: return `DeviceProgram::lower(&self.program())` — re-lower + through main's production lowering so the blob has correct + slots/`res`/`num_base_slots`. (Now non-trivial instead of a field copy; still + guest-safe: `program()` is a POD walk, `lower()` is a slot scan, no capture.) +5. **`program()`**: unchanged except imports (`DIM_BASE/DIM_EXT`, `OP_*` from the + new home). Reads `n.a/n.b` as node ids, `n.dim` → `Dim`. +6. **`validate_self()`**: unchanged except imports. Node-index/topo check stays + valid because the artifact's own form is node-index. +7. **Tests**: `constraint_artifact_tests.rs` and `lfm/constraint_tests.rs:658` + swap `stark::constraint_ir::device::{DeviceNode, DIM_BASE, DIM_EXT}` for the + artifact's node type / DIM tags. Census logic unchanged. `artifact_tests.rs` + unchanged except the same import move. +8. **`lfm/constraints.rs` and the rest of `lfm/`**: **no change** — they consume + `artifact.program()` (node-index `Op`), which is byte-identical to before. + +**Soundness:** the device blob is produced by the *same* `DeviceProgram::lower` +the prover/GPU use, so `eval_device_program` agreement with the folder is inherited +from main's own device tests. The artifact's own form is validated by +`validate_self` (topo order, in-range consts/roots, dense meta) exactly as today. +No new trust surface: `program()` still lifts to the `ConstraintProgram` the folders +are pinned against, and `validate_against` still gates shape/metadata. + +**Risk:** low. Two functions change behavior (`capture`, `device_program`); the rest +is renames. The node-index operand model — the thing the LFM machine, the census, +and `program()` all assume — is preserved verbatim. + +### Approach B — store main's slot-form `DeviceProgram`; decode slots everywhere ❌ INFEASIBLE + +1. **Serialization**: main's `DeviceNode` has no rkyv derives and `DeviceProgram` + isn't rkyv either — would have to add rkyv to `device.rs` (and its `res` word, + `num_base_slots/num_ext_slots`). Touches main's file. +2. **`program()` cannot be written**: main's `lower()` drops uniform leaves and dead + nodes and slot-encodes operands. There is no function from the slot graph back to + the original `ConstraintProgram.nodes`. ⇒ the round-trip's `prog.nodes == + captured.nodes` (and `lift_is_the_inverse_of_lower`) can never pass. +3. **The census / DCE / fanout / MulAdd-fusion** all count *per `ConstraintProgram` + node* with node-index operands. On the reduced slot graph these quantities are + different numbers (uniforms and dead nodes already removed) and the operand words + are slot/uniform indices, not node ids — every one of §2b/§2d would need a + semantic rewrite, and several have no slot-graph analogue. +4. **`lfm/constraints.rs`** is a node-index `Op` lowering fed by `program()`; without + a working `program()` the entire recursion-machine constraint leg has no input. +5. **`validate_self`'s** topological/node-index invariant would be replaced by a + slot-range check — losing the "references strictly earlier node" guarantee the + falsification tests pin. + +Approach B fails at step 2 alone. + +--- + +## Appendix — file/line index + +- Feature: `crypto/stark/src/constraint_ir/artifact.rs` (struct 226-243; capture + 297-362; device_program 366-374; program 388-451; validate_self 463-563; + validate_against 578-652; to/from_bytes 656-668). +- Unit tests: `crypto/stark/src/constraint_ir/artifact_tests.rs`. +- Round-trip + census: `prover/src/tests/constraint_artifact_tests.rs` + (check_air_artifact 72-264; all_table_...match_folders 279-295; census 394-606; + leg_instructions 925-955). +- OLD device.rs (HEAD): `git show HEAD:crypto/stark/src/constraint_ir/device.rs` + (lower 125-170 = node-index 1:1; DeviceNode 75-82 w/ `dim`+rkyv; DIM_* 67-69). +- NEW device.rs (main): `git show origin/main:...` (lower 195-358 = slot alloc; + DeviceNode 116-123 w/ `res`, no rkyv; OPK_* 86-105; RES_EXT_BIT 109; + DeviceProgram 149-172 w/ num_base_slots/num_ext_slots). +- ir.rs: identical HEAD vs main; `Op`/`Dim`/`ConstraintProgram` not rkyv. +- mod.rs: main omits `pub mod artifact;` and the artifact re-exports (branch-only). +- Extra consumers (all node-index / `program()`-mediated): `prover/src/lfm/ + constraints.rs` (analyze 286-290 calls `artifact.program()`), `lfm/ + constraint_tests.rs` (incl. raw `DeviceNode{...,dim}` at :658), `lfm/join_tests.rs`, + `lfm/epoch_verify*.rs`, `prover/src/bin/compute_constraint_artifacts.rs`. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/debate-attacker.md b/thoughts/shared/lfm-real-hash/merge-plan/debate-attacker.md new file mode 100644 index 000000000..0d80e4b77 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/debate-attacker.md @@ -0,0 +1,435 @@ +# Adversarial attack on FIX-PLAN.md — attacker's brief + +**Verdict: the plan should be sent back for revision.** + +Its central conclusion — *"main's crypto/stark batch shifted the LogUp convention, so the +branch's hand-rolled binding must adapt"* — is **falsified by the diff**. I also found what +is almost certainly the real root cause, in a file the plan places out of scope. Executing +Step 2 as written would most likely bake an unbalanced-bus residual into a verifier +soundness check, and **none of the plan's named negative controls would catch it**. + +All paths are in the merge worktree `/Users/maurofab/workspace/lambda_vm-blake3-merge` +unless stated otherwise. I was read-only (no cargo), so execution-dependent claims are +marked; everything marked ✓ VERIFIED was established by reading the code or the diff. + +--- + +## 1. CRITICAL — The likely real root cause is in `prover/src/lfm/keccak_adapter.rs`, which the plan never mentions + +✓ VERIFIED by reading both sides of the diff. + +Main deleted **120 `BusId::Hwsl` sender interactions per keccak round** from the +**production** KECCAK_RND chip, replacing them with inline μ-gated linear identities. + +`git diff ed1b7785 origin/main -- prover/src/tables/keccak_rnd.rs`: + +- `Vec::with_capacity(1371)` → `Vec::with_capacity(1031)` +- the `--- Theta: HWSL for rotated C (20) ---` block is removed, replaced by + `--- Theta: rotate-C-by-1 shift is enforced by an inline μ-gated linear identity + (see KeccakRndConstraints), not an HWSL lookup. ---` +- the `--- Rho: HWSL (100) ---` block is removed +- the new module comment states: *"The matching HWSL multiplicities are likewise dropped + on the BITWISE side (`collect_bitwise_from_keccak`)."* + +Main updated the production receiver side accordingly: +`prover/src/tables/trace_builder.rs`, +418/−41. + +**The LFM machine embeds those production chips.** `prover/src/lfm/airs.rs:21`: + +```rust +use crate::tables::{bitwise, keccak_rc, keccak_rnd}; +``` + +and `airs.rs:239` (`let rnd_interactions = keccak_rnd::bus_interactions().len();`) and +`airs.rs:488-494` build the LFM KECCAK_RND AIRs from `keccak_rnd::bus_interactions()` and +`keccak_rnd::KeccakRndConstraints`. So the LFM AIR set picked up main's deletion +automatically, at merge time, silently. + +**But the LFM machine has its own forked copy of the receiver-side multiplicity +collection, and it is branch-only code the merge never touched.** +`prover/src/lfm/keccak_adapter.rs:306-318`: + +```rust +/// BITWISE lookups the `KECCAK_RND` rows of `ops` send: exactly `24 * 1148` per +/// permutation. +/// +/// This is the per-round half of `trace_builder::collect_bitwise_from_keccak`, +/// forked rather than called: ... +pub fn bitwise_ops_for(ops: &[KeccakAdapterOperation]) -> Vec { + let mut out = Vec::with_capacity(ops.len() * 24 * 1148); +``` + +It still pushes `BitwiseOperationType::Hwsl` in the Theta loop +(`keccak_adapter.rs:361-366`, 20 per round) and in the Rho loop +(`keccak_adapter.rs:441-446`, 100 per round) — **exactly the 120 sends main deleted**. +The per-round count `1148` is pinned in the capacity and is now stale. + +**Consequence.** The LFM BITWISE chip receives 120 HWSL lookups per round that KECCAK_RND +no longer sends. With circular LogUp constraints there are no boundary constraints on the +accumulator, so **proving still succeeds** and the imbalance surfaces only as a nonzero +residual in `total` at `crypto/stark/src/verifier.rs:1448`. That is precisely the reported +symptom, and it predicts the observed failing/passing split exactly: every keccak-touching +program fails; `trivial_program_*` (no keccak) passes. + +**Why this destroys the plan.** This is the plan's own **Outcome B**, and the fix lives in +`prover/src/lfm/keccak_adapter.rs` — not in `expected_public_balance`, not in +`replay_transcript_phase_a_view`. Step 2's enumerated scope cannot reach it. + +**To close:** before any edit, dump the **per-bus** residual, not just the per-table total. +`crypto/stark/src/lookup.rs` already has `compute_debug_bus_sums_batched`, and +`DEBUG_BUS_TRACKER=1` enables per-bus balance reporting in release. Run it on +`machine_proves_the_sample_replay` and confirm whether the residual lands on +`BusId::Hwsl`/`Bitwise` or on `LfmPublic`. That single measurement decides the whole plan +and should have preceded it. + +--- + +## 2. CRITICAL — Step 2 as written is a genuine soundness regression, and R1's mitigation is blind to it + +If the residual is an unmatched-HWSL constant (Finding 1), then "adjust +`expected_public_balance` to main's convention" means folding that residual into the +verifier's expected target at `prover/src/lfm/proof.rs:247-276`. The LfmPublic target would +then absorb an arbitrary unbalanced-bus remainder, and the cross-table check at +`crypto/stark/src/verifier.rs:1448` would **permanently stop detecting unmatched HWSL +lookups in the recursion machine**. That is the recursion verifier's only cross-table +binding. + +Critically, **every negative control the plan names would stay green.** The residual is a +constant independent of the claimed public words, so tampering a value lane still moves +`expected` and still rejects. R1 says "negative controls in Step 4 are mandatory and gate +the commit" — but the controls are structurally blind to this exact failure mode. The +plan's headline mitigation does not mitigate its headline risk. + +**To close:** add a control sensitive to a *constant offset* in `expected`, not just to word +tampering — cheapest is asserting the per-bus residual is zero on every bus except +LfmPublic, which is the same measurement Finding 1 requires anyway. + +--- + +## 3. CRITICAL — "20 failures, one root cause" is falsified; at least three cannot be the hand-rolled binding + +✓ VERIFIED by reading the tests. The diagnosis is **n = 1** +(`machine_proves_the_sample_replay`) generalized to 20 without checking. + +### (a) Both `keccak_probe` failures never touch the hand-rolled binding at all + +`prover/src/lfm/keccak_probe.rs:126-143`: + +```rust +fn verify_proof(opts: &ProofOptions, adapter: &AdapterAir, + proof: &stark::proof::stark::MultiProof) -> bool { + let rnd_air = create_keccak_rnd_air(opts); + let rc_air = create_keccak_rc_air(opts).with_preprocessed(...); + let bw_air = create_bitwise_air(opts).with_preprocessed(...); + let refs: Vec = vec![adapter, &rnd_air, &rc_air, &bw_air]; + let mut vt = transcript(); + Verifier::multi_verify_views(&refs, MultiProofView::Owned(proof), &mut vt, &FEE::zero()) +} +``` + +Expected bus balance is a **hard-coded `FEE::zero()`**. This path calls neither +`verify_against`, nor `lfm_verify`, nor `expected_public_balance`, nor +`replay_transcript_phase_a_view`. Its AIR set is the **production** VM AIRs plus a local +adapter. Yet `keccak_probe::adapter_probe_proves_real_permutations` and +`keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` are both in the +new-failure list (`reconcile-report.md:281-282`). **No change to `proof.rs` or `lib.rs` can +fix them.** They *are* explained by Finding 1. + +(The reconcile-report itself noticed this — §5.3, *"keccak_probe.rs contains zero +occurrences of `artifact`, yet two of its tests are in the new-failure list"* — and used it +to exonerate the artifact reconciliation. The FIX-PLAN then folded them into the LFM +binding, which is equally impossible.) + +### (b) `preprocessed_tags_close_the_output_swap_hazard` cannot fail from a bus-balance mismatch + +`prover/src/lfm/machine_tests.rs:415-460`. Its **only** verify assertion is negative: + +```rust + assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "with distinct tags the swapped outputs must no longer balance" + ); // :441-444 +``` + +A universal verify failure **satisfies** that. Its remaining failure modes are all +prove-side or compile-side: + +- `assert_ne!(tag(0), tag(1), "keccak tags must be distinct")` — `:429` +- `.expect("locally consistent")` on `prove_keccak_chain_with_tamper` — `:440`, proving + must **succeed** +- `expect_err(...)` + `matches!(err, ProvingError::PrecomputedCommitmentMismatch)` — + `:455-459` + +Main rewrote the prover's preprocessed / split-tree commit path (`crypto/stark/src/prover.rs`, ++1725 lines; the diff comment reads *"Preprocessed tables also carry a handle with +`trace_dev` (the split-tree path)"*), a plausible independent cause. **Neither the plan's +theory nor mine explains this test** — which is the point: nobody has read its actual +failure message. + +**To close:** capture the actual assertion/panic message for all 20 and bucket them. This is +one `cargo test ... 2>&1 | grep -B2 -A5 panicked` away and should have preceded the plan. + +--- + +## 4. HIGH — "The convention shifted" is contradicted by the diff *and* by the tests that still pass + +✓ VERIFIED. + +**From the crypto/stark side.** `git diff ed1b7785 origin/main -- crypto/stark/src/lookup.rs` +is 246 lines and touches only: + +- `Arc`-wrapping `constraint_program` (perf) +- a new `Clone` impl for `AirWithBuses` and `#[derive(Clone)]` on `AuxiliaryTraceBuildData` +- removal of `with_precaptured` / `precaptured_program` (the branch's own feature, also + removed from `crypto/stark/src/traits.rs`) +- lazy host `main_cols_cell` for the GPU-resident aux path + +**`compute_alpha_powers` (`lookup.rs:73`), `add_combined_terms` (`:265-370`), and the entire +alpha-offset assignment (`:624-800`) are untouched.** There is no shifted fingerprint +convention in crypto/stark to adapt to. + +Phase A is likewise preserved on the prover side: the `prover.rs` diff shows the same +absorption sequence (precomputed root if preprocessed, then main root, in index order), +followed by the same two-challenge sample. `crypto/stark/src/verifier.rs:1305-1350` matches +`prover/src/lib.rs:994-1002` exactly. Outcome A is dead. + +**From the test side.** `prover/src/lfm/machine_tests.rs:36 +trivial_program_proves_and_verifies` asserts a **positive** verify through +`lfm_verify` → `verify_against` → `expected_public_balance`, with non-empty public words +(proved by `:52 tampered_claimed_public_word_rejects` indexing `claimed[0].1[0]`, and by +`:81-87` which asserts both a positive verify and a cross-claim reject). **None of those +three is in the new-failure list.** So `expected_public_balance` computes the correct value +today for at least one program — and a formula edit would break them. + +**This also retires Step 1's stated "leading suspect"** (a shifted alpha-power offset). The +LFM_PUBLIC sender is `direct(cols::INDEX)` plus `word(cols::V0)` +(`prover/src/lfm/chips.rs:1340-1349`) — five single-alpha elements — and +`proof.rs:253-265` maps them to α¹…α⁵ over `bus = BusId::LfmPublic = 34` +(`prover/src/tables/types.rs:373`), consistent with its own doc comment at `proof.rs:245` +and with the byte-identical `add_combined_terms`. Step 1 is pointed at the wrong thing. + +--- + +## 5. HIGH — The named negative control is the wrong test, and one "control" is inverted + +✓ VERIFIED. + +**`tampered_l2g_binding_rejects` — the plan's headline soundness gate — contains no +negative verify assertion at all.** `prover/src/lfm/machine_tests.rs:3593-3660`: its tamper +vectors go through + +```rust + let err = super::executor::execute(&program, &arenas, &super::hash::TestPermutation) + .err() + .unwrap_or_else(|| panic!("{what}: must not execute")); +``` + +— an **executor-level** reject, no proof and no verifier. Its coherent branch then asserts +the proof **proves** and compares `published_root(&proved.public_words, 0)` in host Rust. +It cannot detect an `expected_public_balance` that accepts everything. + +**Worse, Step 4 lists "the output-swap-hazard tests" among controls that must "still +REJECT".** `prover/src/lfm/keccak_probe.rs:261-290 +duplicate_tag_output_swap_accepts_demonstrating_hazard` asserts the **opposite**: + +```rust + let proof = prove_traces(&opts, &adapter, &mut traces).expect("locally consistent"); + assert!( + verify_proof(&opts, &adapter, &proof), + "documents the tag-uniqueness obligation: with duplicate tags the swapped \ + outputs still balance the bus, so the verifier cannot catch the forgery" + ); +``` + +It must **ACCEPT** — it documents an open hazard. Under the plan's framing, that test +staying red would be misread as "control holding". + +**The controls that actually guard this code go unnamed:** +`machine_tests.rs:52` (`tampered_claimed_public_word_rejects`), `machine_tests.rs:84-87` +(cross-claim), `wrap_tests.rs:490` and `:510`, `constraint_tests.rs:1365`, +`blake3_socket_tests.rs:1417`. + +Two caveats the plan must state and does not: + +1. They currently pass **vacuously** while everything rejects. The gate must be "positive + AND negative green in the same run", not "the reject tests still reject". +2. Every one of them perturbs a **value** lane. Nothing covers the `index·α` term or a + permutation of words, so a formula edit that dropped or mis-weighted `index·α` would + keep every control green. + +**To close:** name the right tests, require positive+negative in one run, add an +index-permutation vector. + +--- + +## 6. HIGH — Scope is understated: the convention is also hand-rolled *in-circuit*, and other consumers share the code + +✓ VERIFIED. + +**In-circuit mirror.** `prover/src/lfm/statement_replay.rs:164-190` (`replay_phase_a`) is an +**LfmBuilder / in-circuit** mirror of `replay_transcript_phase_a_view`, doc-linked as such +at `:164-167` (*"Mirrors `crate::replay_transcript_phase_a_view` — for each air, the +preprocessed commitment when it has one, then the main trace root, and finally `z` and `α` +…"*). `machine_tests.rs:2101` mirrors it a third time. + +If Step 1 landed on Outcome A, the fix would have to change the **compiled LFM program**, +which moves `artifacts.roots` and `artifacts.program_id`, which breaks every +`registry_drift_*` test (`machine_tests.rs:118`, `:230`, `:511`, `:754`) — whose own doc +says *"A failure here means the trivial program, a chip layout, the commit pipeline or the +digest changed: investigate, never re-bless"* (`machine_tests.rs:113-117`). Step 2's file +list omits `statement_replay.rs` entirely, and the plan never mentions registry +re-blessing. + +**Non-LFM consumers.** `replay_transcript_phase_a_view` is not LFM-only. Reached via +`compute_expected_commit_bus_balance_view` (`prover/src/lib.rs:1007-1016`) from: + +- `prover/src/lib.rs:1442` — the **VM** verifier +- `prover/src/continuation.rs:896` — the **continuation** verifier +- `prover/src/lfm/logup_tests.rs:1201`, `prover/src/lfm/epoch_tests.rs:743` + +Editing it changes the VM and continuation verifiers. The plan's validation runs neither +suite. + +--- + +## 7. HIGH — Step 2's scope contradicts the plan's own R2 + +R2 concedes the shift may be in main's trace/aux construction (Outcome B). But Step 2 offers +only two edit targets, both verifier-side. Under Outcome B the correct fix is on the LFM +trace/chip side (`prover/src/lfm/keccak_adapter.rs`, `trace.rs`, `chips.rs`) — none of which +is in scope. "Patch `expected` until it matches the contributions" is precisely the move R1 +warns against, and Step 2 provides no exit from it. + +R2's mitigation ("Step 1 pins which") is not a mitigation: pinning *which* outcome holds +does nothing if Step 2 only has verifier-side edits available. **Outcome B needs an explicit +third branch: if the contributions changed, do NOT touch `expected` — find why the trace +changed.** + +--- + +## 8. MEDIUM — Validation omits the one suite that gates the merge's actual risk + +Step 4 gates on `lfm::`, `constraint_artifact` (11/11), `artifact_pin.py --check`, and a +full-lib baseline diff. **It never runs `cargo test -p stark`.** + +Main added `crypto/stark/src/tests/opening_width_tests.rs` (+532) and +`crypto/stark/src/tests/aux_opening_width_tests.rs` (+715) — the executable soundness tests +for #909, the very check the diagnosis claims to have cleared — and the merged +`verifier.rs` has been hand-edited (Finding 10). Those tests are the gate and they are not +in the plan. + +Separately, `constraint_artifact` 11/11 and `artifact_pin.py` do not exercise `proof.rs` or +`lib.rs` at all. Listing them as gates for **this** change creates false assurance. + +--- + +## 9. MEDIUM — The tree is not in the state the plan describes, and Step 4's verification command will not do what it says + +✓ VERIFIED. `git diff --name-only --diff-filter=U` shows five paths still in **unmerged +index state**: + +``` +crypto/stark/src/lookup.rs +prover/src/continuation.rs +prover/src/tests/constraint_program_device_tests.rs +prover/src/tests/constraint_program_tests.rs +prover/src/tests/ood_window_ir_tests.rs +``` + +The working files are resolved (`grep -c '^<<<<<<<' crypto/stark/src/lookup.rs` → 0) but +were never `git add`ed. Consequently `git diff --stat -- crypto/stark/` literally prints + +``` + crypto/stark/src/lookup.rs | Unmerged +``` + +so Step 4's *"Confirm `git diff` under `crypto/stark/` is only the intended merge content"* +would **silently skip `lookup.rs`** — the very file whose convention the entire plan is +about. + +**To close:** `git add` the resolved paths first, and state the check as +`git diff origin/main -- crypto/stark/` returning **empty**. + +--- + +## 10. MEDIUM — The in-tree diagnostics already changed crypto/stark semantics + +✓ VERIFIED. `crypto/stark/src/verifier.rs:1448-1454` in the merged tree: + +```rust + if total != *expected_bus_balance { + #[cfg(not(feature = "test_fiat_shamir"))] + eprintln!("DBG909 FAIL: BUS BALANCE total={total:?} expected={expected_bus_balance:?}"); +error!( + "LogUp bus does not balance: ...", + total, expected_bus_balance + ); +``` + +On clean main (`git show origin/main:crypto/stark/src/verifier.rs`, the +`LogUp bus does not balance` block, ~:1413-1418) that `#[cfg]` guards `error!`. The +inserted `eprintln!` **stole the attribute**, leaving `error!` unguarded under +`test_fiat_shamir`. The same de-indent-to-column-0 damage is at `:1315`, `:1323`, `:1363`. + +Also present: a `W909_DEBUG` block at `verifier.rs:263-292` that restructured +`trace_opening_widths_well_formed`'s `(0..num_queries).all(...)` into +`let ok = ...; ...; ok`, and a `std::env::var("LFM_BUS_DEBUG")` lookup on **every verify** +at `prover/src/lfm/proof.rs:224-240`. + +Step 4's "remove ALL" is right; the acceptance criterion should be an **empty** diff, not +"only the intended merge content". + +--- + +## 11. LOW-MEDIUM — Baseline comparability is not established + +`reconcile-report.md` §3 says `executor/program_artifacts/asm/` was copied into the merge +worktree, while the `ed1b7785` baseline worktree had *"neither `asm/` nor `recursion/` +present"* (§5). It claims both-ways measurement, but the FIX-PLAN Step 4 gate — full lib +suite vs `blake3-campaign-preMerge` — **has not been run at all** (PLAN.md step 15 is still +open). + +Several failing tests are fixture-driven: `machine_tests.rs:3706` +(`program_id_matches_production_on_the_real_fixture`, via +`proof_fixture::load_or_generate(&fixture_cache())` at `:3690`) and `machine_tests.rs:4282` +(`the_register_derivation_proves_and_verifies`, via `proof_fixture::fixture_options()`), so +fixture asymmetry can move tests between the "pre-existing 19" and "new 22" buckets and +invalidate the "22 new, 0 fixed" diff. + +Additionally `machine_tests.rs:3709-3713` hard-asserts `pages.is_empty()` about that +fixture, and main's private-page OFFSET change (reconcile-report §4, `continuation.rs:240` +now calls `with_preprocessed(page::private_page_preprocessed_commitment(opts), +page::NUM_PREPROCESSED_COLS_PRIVATE)`) touches page preprocessing. That deserves an explicit +check rather than an assumption. + +--- + +## 12. LOW — Note for the judge on what the plan gets right + +To avoid a strawman reading: the plan's *process* instincts are sound. Step 1 (pin before +fixing), R1 (this is soundness-critical), the rollback story, and the insistence on both +positive and negative controls are all correct and should survive revision. The defects are +that Step 1's three outcomes are already decidable from the diff and were not decided; that +Step 2's scope was fixed *before* Step 1 ran, so it cannot express the outcome the evidence +actually supports; and that the controls named in Step 4 are the wrong tests. + +--- + +## What must change before this plan is implementable + +1. **Measure the per-bus residual** on one failing test (`DEBUG_BUS_TRACKER=1` / + `compute_debug_bus_sums_batched`), to confirm or refute that the imbalance is on + HWSL/Bitwise rather than LfmPublic. Decisive, cheap, and it settles Findings 1, 2, 4, 7 + at once. +2. **Get the actual failure message for all 20** and bucket them — not one test generalized + to twenty. +3. **Rewrite Step 2** with `prover/src/lfm/keccak_adapter.rs` (and the trace/chip side + generally) in scope, plus an explicit prohibition on editing `expected_public_balance` + unless Step 1 proves the formula itself is stale. +4. **Name the correct negative controls**, state the positive-and-negative-in-one-run rule, + and add an index-permutation vector. +5. **Add `cargo test -p stark`** (incl. `opening_width_tests` / `aux_opening_width_tests`) + to the gate; drop `constraint_artifact` / `artifact_pin.py` as gates for this change or + label them as regression-only. +6. **`git add` the five unmerged paths** so the crypto/stark diff check actually inspects + `lookup.rs`; restate the criterion as an empty diff vs `origin/main`. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/debate-defender-A.md b/thoughts/shared/lfm-real-hash/merge-plan/debate-defender-A.md new file mode 100644 index 000000000..2fdfee68c --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/debate-defender-A.md @@ -0,0 +1,426 @@ +# Defender A — the FIX-PLAN is sound and ready + +Adversarial review of `thoughts/shared/lfm-real-hash/merge-plan/FIX-PLAN.md`. +Position: **the plan should be implemented**, with the strengthenings in §6. + +All code citations are from the merge worktree +`/Users/maurofab/workspace/lambda_vm-blake3-merge` (branch +`blake3-real-hash-mainmerge`, `git merge origin/main` uncommitted, `MERGE_HEAD` +present). Revisions referenced: `ed1b7785` = pre-merge campaign tip = tag +`blake3-campaign-preMerge` = `blake3-real-hash`; `origin/main` = `528a8411`. + +Confidence markers per the house rule: ✓ VERIFIED = I read the file or ran the +comparison; ? INFERRED = rests on a measurement in `reconcile-report.md` that I +did not re-run. + +--- + +## Verdict + +The plan's **structure** — pin before fix, confine the fix to the branch's +hand-rolled mirror, gate on controls in *both* directions, keep rollback bounded +— is correct and ready to implement. Its **diagnosis narrative** is the weak +part, and the most useful thing I can do for it is show that two of its three +Step-1 outcomes are already dead by diff, and that a discriminating control it +never names is sitting in the same test file. Neither finding changes the plan's +direction; both make Step 1 cheaper and sharper. + +--- + +## 1. The diagnosis is correct *in kind* + +**Claim.** The failure is the cross-table LogUp bus balance, per-table STARK +verification is fine, and the cause lives in the LFM's hand-rolled binding +rather than in main's `crypto/stark`. + +**Evidence that the observation is real, not inferred.** The instrumentation +that produced the plan's §2 findings is still in the worktree, and it prints a +*distinct* message for every rejection path upstream of the balance check: + +| site | print | +|---|---| +| `crypto/stark/src/verifier.rs:1290` | `DBG909 FAIL: composition parts, table {idx}` | +| `crypto/stark/src/verifier.rs:1302` | `DBG909 FAIL: ood_blocks_well_formed, table {idx}` | +| `crypto/stark/src/verifier.rs:1314` | `DBG909 FAIL: preprocessed commitment MISMATCH table {idx}` | +| `crypto/stark/src/verifier.rs:1322` | `DBG909 FAIL: preprocessed commitment MISSING table {idx}` | +| `crypto/stark/src/verifier.rs:1362` | `DBG909 FAIL: missing bus_public_inputs table {idx}` | +| `crypto/stark/src/verifier.rs:1417` | `DBG909 FAIL: verify_rounds_2_to_4 table {idx}` | +| `crypto/stark/src/verifier.rs:1450` | `DBG909 FAIL: BUS BALANCE total=… expected=…` | + +plus a `W909_DEBUG` block inside `trace_opening_widths_well_formed` (#909's +width pin) that dumps the expected/actual precomputed/main/aux split. ✓ VERIFIED +by reading the diff of `crypto/stark/src/verifier.rs` against `origin/main`. So +"only the balance fired, everything else passed" is an *observed* fact with +per-check granularity, not a deduction. + +**Evidence the plan's eliminations hold.** + +- ✓ VERIFIED `LOGUP_NUM_CHALLENGES = 2` and `LOGUP_CHALLENGE_ALPHA = 1` + (`crypto/stark/src/lookup.rs:102,105`), so the replay's `(z, alpha)` ordering + at `prover/src/lib.rs:1000-1002` matches the consumer's + `challenges[0]` / `challenges[LOGUP_CHALLENGE_ALPHA]` at `lookup.rs:1736-1737`. +- ✓ VERIFIED the fingerprint the doc comment at `prover/src/lfm/proof.rs:245-246` + advertises is what the code computes: `powers[i] = α^{i+1}` + (`proof.rs:253-257`), so `acc = BusId::LfmPublic + index·α + Σ_l v_l·α^{2+l}` + (`proof.rs:261-265`). +- ✓ VERIFIED `BusId::LfmPublic = 34` survives the merge with no collision from + main's new tables (`prover/src/tables/types.rs:368-373`; the branch→main diff + shows 32/33/34 are branch-only additions and main added no id in that range). + +**A structural argument the plan does not make, stronger than the ones it does.** +`verify_against` forks the replay transcript *before* handing the same object to +the verifier: + +``` +prover/src/lfm/proof.rs:218 let mut replay = transcript.clone(); +prover/src/lfm/proof.rs:219 let (z, alpha) = crate::replay_transcript_phase_a_view(&refs, view, &mut replay); +prover/src/lfm/proof.rs:242 Verifier::multi_verify_views(&refs, view, &mut transcript, &expected) +``` + +`multi_verify_views` re-runs the identical Phase A absorption +(`crypto/stark/src/verifier.rs:1279-1337`) on `transcript` and samples its own +`lookup_challenges` (`:1344-1350`), which flow into `verify_rounds_2_to_4` +(`:1410-1416`). Since those per-table checks **pass**, the challenges the +verifier used must equal the ones the prover used — a wrong `alpha` would break +the OOD composition check. Therefore the surviving degrees of freedom are the +target formula and its inputs, which is exactly where the plan aims Step 2. This +argument does not require trusting any diff. + +--- + +## 2. The scope is right: main authoritative, the LFM binding adapts + +**Claim.** Fixing only `prover/src/lfm/proof.rs` + `prover/src/lib.rs` and leaving +`crypto/stark` as main's is the correct direction. + +**Evidence — a counting argument, not an inspection.** + +1. ✓ VERIFIED the merge touched exactly **two** files under `prover/src/lfm/` + (`git diff --stat ed1b7785 -- prover/src/lfm/`): + `constraint_tests.rs` (6 lines — the `ArtifactNode` import move from the + artifact reconciliation) and `proof.rs` (+18 — the `LFM_BUS_DEBUG` diagnostic + block at `proof.rs:224-240`). Every other LFM file is byte-identical to + `ed1b7785`, where all 20 of these tests passed. +2. ✓ VERIFIED main's LogUp math is byte-identical across all three revisions: + `crypto/stark/src/lookup.rs` lines 1-833 — which contain + `compute_alpha_powers` (`:73`) and all four `accumulate_fingerprint` / + `accumulate_fingerprint_with` / `accumulate_fingerprint_from_step` impls + (`:274, :377, :626, :742` in branch numbering) plus the packing shifts — + hash to `29d4849da6bda633b2fe37235319e14a` on `ed1b7785`, on `origin/main`, + and in the merged worktree. The slice from `fn compute_logup_term_column` to + EOF (the fingerprint loop, multiplicities, `build_accumulated_column_from_terms`, + the debug bus sums) hashes to `00bd7a930244cb5817270e5a6bf9f4f8` on all three. +3. ✓ VERIFIED `replay_transcript_phase_a_view` (`prover/src/lib.rs:989-1003`) + hashes to `422b0f7436f50c57717f860a99d94930` on `ed1b7785`, `origin/main` and + the worktree; `compute_commit_bus_offset` (`prover/src/lib.rs:947-984`) to + `05fadc35e83bbdf0576d85af3881728c` on all three. Both are **main's own code on + the VM's live verify path**, and main is green. +4. ✓ VERIFIED the only `crypto/stark` semantic change main brought to the + verifier is #909's opening-width pin: the `ed1b7785 → origin/main` diff of + `verifier.rs` is five hunks, all introducing `trace_opening_widths_well_formed` + (`verifier.rs:199-263` post-merge) and its call site (`:1628-1640`) plus + comments. Phase A, Phase B and the balance check carry **no hunk**. + +So main's generic machinery is internally consistent and externally validated by +main's own CI; the only code that *mirrors* its convention from outside is the +LFM binding. Fixing the mirror is the only direction that does not fork +`crypto/stark` from main. + +**Precedent.** This is the same call the artifact reconciliation already made +and validated: main's `DeviceProgram::lower` stayed authoritative and the +branch's serialization decoupled into `ArtifactNode`, with +`device_program()` re-deriving through main's `lower` +(`reconcile-report.md` §2). That reconciliation is green — round-trip 11/11, +`stark --lib constraint_ir` 39/0. + +--- + +## 3. Step 1 (pin before fix) + Step 4 (negative gate) make this safe + +`expected_public_balance` is the recursion verifier's only cross-table check: +the LfmPublic bus has no in-trace receiver, so the target *is* the binding +(`proof.rs:215-222`, and the balance check itself at `verifier.rs:1438-1459` is +the last gate before `return true`). A formula edit that "makes the 20 pass" by +weakening the target is a silent soundness break, and it would be invisible to +the rejection tests: `tampered_claimed_public_word_rejects` +(`prover/src/lfm/machine_tests.rs:52-63`) passes whether the binding is correct +*or* uniformly broken. Demanding both directions is therefore not ceremony, it +is the only gate design that discriminates. + +The codebase already uses that idiom by name — +`prover/src/lfm/blake3_probe.rs:521 falsification_control_the_untampered_proof_verifies` +sits directly above five tamper tests. The plan is consistent with the house +rule recorded in memory ("every soundness fix needs a test asserting honest +proofs STILL verify"). + +✓ VERIFIED the plan's named control exists and is in the right module: +`tampered_l2g_binding_rejects` at `prover/src/lfm/machine_tests.rs:3593`. + +Step 1's ordering is also right for a second reason the plan states in R2: if +the shift turns out to be in how contributions are *produced* rather than in the +target, the fix site changes entirely. Editing first and measuring second would +mean editing `expected_public_balance` — the soundness-critical function — on a +guess. + +--- + +## 4. Rollback bounds the downside + +✓ VERIFIED: + +- `blake3-campaign-preMerge` → `ed1b7785964568d237567dd0ee83162e9db87d58`. +- `blake3-real-hash` still at `ed1b7785` in its own worktree + (`/Users/maurofab/workspace/lambda_vm-blake3-impl`); the merge lives only in + `/Users/maurofab/workspace/lambda_vm-blake3-merge` on the throwaway branch + `blake3-real-hash-mainmerge`. +- `MERGE_HEAD` present — nothing is committed. PR #930 cannot move until the + deliberate fast-forward in Step 5. + +Two corrections to that section are in §6.9. + +--- + +## 5. Summary of the defense + +| plan claim | status | +|---|---| +| Failure is the cross-table balance; per-table STARK is correct | ✓ VERIFIED (per-check instrumentation) | +| Challenge derivation unchanged | ✓ VERIFIED (byte-identical replay + no verifier hunk) | +| LogUp L-value math unchanged | ✓ VERIFIED (two region hashes across three revisions) | +| Fix belongs on the LFM side, not in `crypto/stark` | ✓ sound (counting argument, §2) | +| Negative controls are mandatory | ✓ sound, and the named one exists (`machine_tests.rs:3593`) | +| Rollback is bounded | ✓ VERIFIED, minus the "pushed" claim (§6.9) | +| "The convention shifted"; alpha-power offset is the leading suspect | ✗ **ruled out by diff** (§6.1) — redirect Step 1 | +| All 20 share one root cause | ? unverified assumption (§6.4) | +| Step 3 is trivial bookkeeping caused by HINT | ✗ **wrong cause** for the budget half (§6.5) | + +--- + +## 6. Concrete strengthenings + +### 6.1 Retire Outcomes A and B — they are dead by diff + +Per §2 items 2-4: neither the challenge derivation nor the LogUp column +construction changed between `ed1b7785` and `origin/main`. That includes the +plan's stated **leading suspect** — the alpha-power offset of the LfmPublic +sender token. `accumulate_fingerprint` is byte-identical, so the slot layout +(`bus_id` at α⁰, values from α¹ upward, `lookup.rs:1759-1774`) cannot have +shifted. Step 1 should not spend a build on Outcome A or on the alpha-offset +hypothesis. What remains is Outcome C (the target's *inputs*, not its shape) and +R2 (main's prover changed what goes into L). + +### 6.2 The discriminating control the plan is missing — re-aim Step 1 at it + +? INFERRED, from `reconcile-report.md` §5's failure list (a measurement I did +not re-run): **`trivial_program_proves_and_verifies` +(`prover/src/lfm/machine_tests.rs:36-49`) passes in the merged tree.** It is not +among the 22 new failures, it needs no ELF fixture (so it cannot be one of the +19 pre-existing `recursion/fibonacci.elf` failures), and ✓ VERIFIED it is not +`#[ignore]`d. + +It exercises the *identical* binding end to end — `lfm_prove` → `lfm_verify` +→ `resolve` → `verify_against` → `replay_transcript_phase_a_view` + +`expected_public_balance` + `multi_verify_views` — and its public word vector is +non-empty (✓ VERIFIED: `machine_tests.rs:59` indexes `claimed[0].1[0]`, and +`machine_tests.rs:52-63` asserts a tamper on it rejects). + +If that holds, **"the merge shifted the convention and the hand-rolled mirror is +stale" is false as a blanket statement**, and the divergence is +*program-dependent*. Both sides use the same 14 chips — ✓ VERIFIED +`NUM_LFM_CHIPS = 14` (`prover/src/lfm/airs.rs:50`), all registry entries carry +`keccak_rnd_chunks: 1` (`prover/src/lfm/registry.rs:270,354,438,522`), and the +plan itself reports 14 tables for the failing `machine_proves_the_sample_replay` +— so chunk count and table count are *not* the difference. Trace content and +height are, which moves the leading suspect to main's prover rewrite +(#877/#875/#863: padding, aux-build path selection, `resident_aux_ok`, +row counts) or to a program-shape assumption baked into the LFM programs. + +**Action.** Replace Step 1's cross-worktree comparison with a *within-tree* +differential: instrument once, run `trivial_program_proves_and_verifies` (passes) +and `machine_proves_the_sample_replay` (`machine_tests.rs:916-949`, fails) +side by side, and diff `z`, `alpha`, the 14 per-table contributions and +`expected`. One build instead of two, no cross-worktree fixture skew, and it +isolates *what about the bigger program* matters — which the cross-tree +comparison cannot tell you. + +Confirm the premise first, it is one command: + +``` +cargo test --release -p lambda-vm-prover --lib \ + lfm::machine_tests::trivial_program_proves_and_verifies -- --nocapture +``` + +If that test in fact fails, §6.2 collapses and the plan's original Step 1 stands +unchanged — so this costs nothing to check and settles the whole framing. + +### 6.3 Falsify the scope claim for free, before editing anything + +The tree already ships per-bus attribution: `crypto/stark/src/bus_debug.rs` +(`log_interaction` `:223`, `analyze_mismatches` `:105`, `print_summary` `:264`, +env selector `DEBUG_BUS_ID=`), fed by +`per_bus_sums` / `per_bus_sender_sums` / `per_bus_receiver_sums` populated at +`crypto/stark/src/lookup.rs:1346-1371` and exposed on `BusPublicInputs` +(`lookup.rs:1642-1655`). All of it is behind `--features debug-checks` +(`bus_debug.rs:8-14`). + +One run tells you **which bus** is unbalanced: `LfmPublic` (34), `LfmMem` (32) or +`LfmRange` (33). If it is either of the latter two — internal buses that must net +to zero in trace — then `expected_public_balance` is *innocent* and Step 2's +scope is wrong. This is the single cheapest falsification of the plan's core +hypothesis, it requires no new code, and it should run before Step 1's bespoke +`eprintln`s. + +### 6.4 Verify "all 20 share one root cause" instead of assuming it + +The plan's §2 evidence comes from instrumenting one test +(`machine_proves_the_sample_replay`) but its opening asserts all 20 fail "on one +root cause". The `DBG909` prints are unconditional, so a single run groups all +20 by which check fired: + +``` +cargo test --release -p lambda-vm-prover --lib lfm:: -- --nocapture 2>&1 | grep -E "DBG909|W909" +``` + +Two of the 20 look like a *different* cause: +`machine_tests::program_id_matches_production_on_the_real_fixture` and +`machine_tests::program_id_folds_pages_in_the_production_layout` are about the +**production** table layout, which main moved (HINT added, +`NUM_PRODUCTION_AIRS` 28→29 per `PLAN.md` step 5). If those two are a separate +item, R4's mitigation ("run the full 20") would report a partial fix as a +regression and cost a debugging cycle. + +### 6.5 Step 3 is not trivial, and it should run FIRST as a diagnostic + +The plan calls the two census failures "unrelated bookkeeping" with a single +stale constant. ✓ VERIFIED that is wrong for the budget half: + +`continuation_epoch_constraint_leg_cost` computes +`design_intermediate = families_unfused + fixed_unfused + l2g_unfused` +(`prover/src/lfm/constraint_tests.rs:1547`), summing over +`SPLIT_FAMILIES` — 14 labels, `constraint_tests.rs:1491-1494` — plus +`FIXED[..9]` — `constraint_tests.rs:1497-1508` — plus `L2G_MEMORY`. +**HINT appears in neither list.** So the observed −1018 delta (computed 62,375 +vs the pinned `63_393` at `constraint_tests.rs:1566-1569`) cannot come from HINT +being added; it comes from one or more of those 24 existing tables' constraint +counts moving on main. Re-pinning the constant to 62,375 without attributing the +delta destroys exactly the signal the test claims for itself: + +``` +prover/src/lfm/constraint_tests.rs:1563-1565 +// The design's §8.2.2 arithmetic, reproduced from the emitter's own unfused +// counts. A mismatch means the epoch composition changed, which is a finding +// about the epoch, not about this pass. +``` + +Related: `constraint_leg_instruction_census` **panics** at +`constraint_tests.rs:438-442` (`no design entry for {label}`) *before* reaching +its post-loop `mismatches` assertion (accumulated at `:447-452`). So adding the +HINT row to `DESIGN_INSTR` will very likely surface a per-table design-vs-emitter +mismatch list — and that list is precisely the attribution the budget delta +needs, and may name the production tables whose shape changed. Given #4's +suspicion about the `program_id_*` failures, that is a plausible common thread. + +**Action.** Move Step 3 ahead of Step 2 (it is a read-only census; it does not +touch the binding), record the per-table deltas in the merge notes, and pin the +new budget with the attribution written down rather than as a bare number. + +### 6.6 Name the Step-4 controls, in both directions + +Step 4 says "all 20 pass" and "the existing tamper tests still REJECT". Make it +an explicit list so the gate is checkable by someone who did not write it. + +Honest-path (must stay green — these catch an over-broad fix): + +- `prover/src/lfm/machine_tests.rs:36` `trivial_program_proves_and_verifies` +- `prover/src/lfm/machine_tests.rs:66` `different_arena_values_change_the_public_output_not_the_program` — its `:81-83` assertion is honest-path +- `prover/src/lfm/blake3_probe.rs:521` `falsification_control_the_untampered_proof_verifies` + +Rejection (must stay red for the prover): + +- `prover/src/lfm/machine_tests.rs:52` `tampered_claimed_public_word_rejects` +- `prover/src/lfm/machine_tests.rs:85-87` the cross-claim assertion inside `different_arena_values_…` +- `prover/src/lfm/machine_tests.rs:3593` `tampered_l2g_binding_rejects` +- `prover/src/lfm/framework_probe.rs:188` `b0_tampered_witness_value_breaks_balance` +- `prover/src/lfm/framework_probe.rs:169` `b0_verifier_rejects_wrong_preprocessed_root` +- `prover/src/lfm/framework_probe.rs:156` `b0_prover_rejects_mismatched_preprocessed_root` +- `prover/src/lfm/keccak_probe.rs:219,228,237` (tampered output byte / input byte / padding-row multiplicity) +- `prover/src/lfm/blake3_probe.rs:529,542,556,564,575` +- `prover/src/lfm/logup_tests.rs:478` `the_closure_cannot_sum_a_contribution_the_constraints_rejected` + +### 6.7 Add the one control that does not exist yet + +Every current tamper mutates a public **value** (`machine_tests.rs:59`: +`claimed[0].1[0] += 1`). A binding that dropped or shifted the `index·α` term at +`prover/src/lfm/proof.rs:261` would still reject all of those — but would accept +a **permutation of the public words** (same lanes, swapped indices). That is +precisely the error class Step 2 is most likely to introduce, and nothing in the +suite catches it. + +Add to `tampered_claimed_public_word_rejects`, or as a sibling: build `claimed` +by swapping the `index` fields of two entries (or reversing the vector) with all +lane values untouched, and assert `lfm_verify` returns `false`. Cheap, and it is +the control that makes an alpha-offset regression impossible to land. + +### 6.8 Make the Step-4 cleanup gate mechanical, not a judgement call + +Step 4 says "confirm `git diff` under `crypto/stark/` is only the intended merge +content". That is unfalsifiable as written. It can be made exact: + +✓ VERIFIED the pre-merge branch made **zero** changes to +`crypto/stark/src/verifier.rs` (the `ed1b7785 → origin/main` diff of that file is +five hunks, all main-side additions). ✓ VERIFIED the worktree's current 48-line +delta vs `origin/main` in that file is **entirely diagnostics** — the `W909_DEBUG` +block plus the `DBG909` `eprintln`s — and it includes a **structural rewrite of a +soundness check**: `trace_opening_widths_well_formed`'s body was changed from +`(0..num_queries).all(…)` into `let ok = (0..num_queries).all(…); …; ok`. + +So the gate is: **`git diff origin/main -- crypto/stark/src/verifier.rs` must be +empty.** And `git diff --stat origin/main -- crypto/stark/` must reduce to +exactly this allow-list (current state, minus the verifier line): + +``` +crypto/stark/src/constraint_ir/artifact.rs 771 ++ (new — artifact feature) +crypto/stark/src/constraint_ir/artifact_tests.rs 394 ++ (new) +crypto/stark/src/constraint_ir/mod.rs 8 + (re-exports incl. ArtifactNode) +crypto/stark/src/constraint_ir/device.rs 2 +- (one derive: PartialEq, Eq) +crypto/stark/src/constraints/builder.rs 2 +- (one derive: rkyv Archive/Serialize/Deserialize) +crypto/stark/src/lookup.rs 67 +- (precaptured_program re-addition) +crypto/stark/src/traits.rs 43 +- (precaptured_constraint_program re-addition) +crypto/stark/src/verifier.rs 0 (MUST be empty after cleanup) +``` + +Note this also corrects the plan's wording: "No edits under `crypto/stark/`" is +already literally false — the merge deliberately re-adds `with_precaptured` / +`precaptured_constraint_program`, which main deleted (`lookup.rs:968-1007` and +`traits.rs:270-317` on `ed1b7785`; removed on main). State the Step-2 boundary as +*this allow-list*, not as a directory. + +The `LFM_BUS_DEBUG` block at `prover/src/lfm/proof.rs:224-240` must go too. + +### 6.9 Two fixes to the rollback section + +- ✗ The tag is **not pushed**. `git ls-remote --tags origin` returns nothing for + `blake3-campaign-preMerge`; the plan asserts "tagged and pushed". The pristine + tip is separately recoverable via the `blake3-real-hash` remote branch (PR + #930), so the exposure is small, but the claim is untrue and the fix is one + command: `git push origin blake3-campaign-preMerge`. +- "Uncommitted" is presented purely as a safety property, but it is also the + risk: a large conflict resolution (`prover.rs` alone is 1,725 changed lines + between the two sides) exists only in one worktree's index, with no recovery + point. `blake3-real-hash-mainmerge` is a disposable branch — commit the + resolved merge **now** as a checkpoint and do the fix as follow-up commits; + Step 5's "commit the merge" becomes a squash before the fast-forward. That + keeps the bounded downside *and* adds a checkpoint, instead of trading one for + the other. + +--- + +## 7. What would change my position + +If §6.3 shows the unbalanced bus is `LfmMem` or `LfmRange` rather than +`LfmPublic`, Step 2's scope (`expected_public_balance` / the replay) is wrong and +the plan needs a new Step 2 — the internal buses balance in-trace, so a mismatch +there is a trace/prover-side finding, not a target-formula one. Everything else +in the plan (ordering, gates, rollback, the "main stays authoritative" call) +survives that outcome unchanged, which is itself an argument that the plan's +skeleton is the right one. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/debate-defender-B.md b/thoughts/shared/lfm-real-hash/merge-plan/debate-defender-B.md new file mode 100644 index 000000000..ea17841f7 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/debate-defender-B.md @@ -0,0 +1,443 @@ +# Defender B — execution safety, reversibility, and process risk management + +**Position:** the plan in `FIX-PLAN.md` is SOUND and READY to execute. + +**Angle:** not the correctness of the diagnosis (that is Defender A's). This is an +argument about *process*: whether pin→fix→validate→review is the right risk-managed +shape for a soundness-critical merge reconciliation, whether the operation is safely +reversible, and whether the validation actually discriminates a real fix from a +weakened check. + +**Method note.** Everything below marked ✓ VERIFIED was read out of the tree or the +git refs in `/Users/maurofab/workspace/lambda_vm-blake3-merge` during this review. +Claims marked ? INFERRED are derived from `reconcile-report.md`'s recorded +measurements rather than re-measured by me, and I say so at each site. I made no +edits and ran no cargo. + +--- + +## Verdict + +Execute it. The central process choice — pin the convention with a decisive +measurement *before* touching a line — is the correct defense against this bug's +specific temptation, and the isolation makes the operation genuinely reversible. + +I found four concrete strengthenings. One of them (§S1) is a real defect in the +rollback story that must be fixed before Step 5 runs. + +--- + +## Claim 1 — Pin-before-edit is the right order, and this tree shows why + +The "made it pass by weakening a check" failure mode is not hypothetical here. +`expected_public_balance` (`prover/src/lfm/proof.rs:247`) is the only thing standing +between the LFM machine and accepting a proof against a public output it did not +produce. Every degenerate repair to it turns all 20 tests green in one step: + +- return a constant; +- drop the dependence on `claimed_public`; +- derive the target from the proof's own `bus_table_contribution()` values. + +The bug *presents* as "a number doesn't match" (`total = 5597…836` vs +`expected = 16884…021`), which is precisely the presentation that invites a tuned +constant. Nothing about the symptom distinguishes "the formula is stale" from "the +formula is right and the inputs moved." + +Step 1 defuses this by making its **deliverable a named divergence with file:line on +both sides**, before any edit is permitted. That converts the task from "make the +numbers agree" (which has infinitely many answers, almost all wrong) into "restore a +stated convention" (which has one). Only the second framing has a *detectable* wrong +answer. + +**Evidence that this discipline is already load-bearing in this tree.** The diagnosis +pass left live instrumentation inside a soundness-critical file. ✓ VERIFIED: + +- `crypto/stark/src/verifier.rs` currently differs from `origin/main` by 50 changed + lines, and **every one of them is diagnostic** — a `W909_DEBUG` env-gated block, a + set of `DBG909 FAIL:` eprintlns, and a `let ok = …` binding that exists only to hold + the debug block. +- I checked whether any of it altered control flow. It does not: every insertion is + print-then-fall-through, and the `if !ok && std::env::var("W909_DEBUG").is_ok()` + block still returns `ok`. No check was removed or weakened during diagnosis. + +So the plan's Step 4 requirement that `crypto/stark/` contain only intended merge +content is a necessary check on real residue, not ceremony. §S6 below makes it +mechanical. + +--- + +## Claim 2 — Isolation and reversibility check out (with one defect) + +I verified the refs directly rather than taking the plan's word for them. + +✓ VERIFIED: + +| fact | value | +|---|---| +| `blake3-campaign-preMerge` | `ed1b7785964568d237567dd0ee83162e9db87d58` | +| `blake3-real-hash` (local) | same commit | +| `blake3-real-hash-mainmerge` HEAD | same commit | +| `refs/heads/blake3-real-hash` on origin | same commit | +| `MERGE_HEAD` in the merge worktree | present (`58160b6f…`) — merge genuinely uncommitted | +| merge worktree `.git` | an 82-byte link file, i.e. a linked worktree | + +Consequences, each of which is what "safe and reversible" has to mean concretely: + +1. **Nothing is committed.** `git merge --abort` or `git reset --hard + blake3-campaign-preMerge` restores the pristine tree with no history to rewrite. +2. **Discarding the worktree is free.** Because `.git` is a link file, `git worktree + remove` drops the working copy without touching the shared object store. There are + 16 worktrees on this machine (`git worktree list`); none of them can be corrupted + by this operation. +3. **PR #930 is untouched.** `origin/blake3-real-hash` is still the pristine campaign + tip, so the PR shows pre-merge content and there is zero external exposure until + the deliberate final fast-forward in Step 5. + +### ⚠️ S1 — The rollback claim is currently false, and Step 5 is what makes it matter + +`FIX-PLAN.md:96` states the pristine tip is "tagged **and pushed**." + +✓ VERIFIED: it is tagged but **not pushed**. `git ls-remote --tags origin` returns no +match for `blake3-campaign-preMerge` (exit 1, empty). + +Today this is harmless only by coincidence: the sole remote copy of `ed1b7785` is +`refs/heads/blake3-real-hash`, which happens to point at it. **Step 5's +fast-forward-and-push is the exact moment that stops being true.** After that push, +the only remote record of the pristine campaign tip is gone and the tag meant to +replace it exists on one laptop. + +**Fix: push the tag before the final push, not after.** This is a one-command change +to the sequencing and it is the difference between "recoverable from anywhere" and +"recoverable until this disk fails." + +--- + +## Claim 3 — The validation is sufficient + +### 3a. Set-equality is the strongest element, and it is already baselined + +Running all 20 rather than the one instrumented test is the plan's R4 mitigation and +it matters. But the more powerful criterion is Step 4's last bullet: the full `lfm::` +failure set must equal the pristine baseline's *set*. + +? INFERRED (from `reconcile-report.md` §5, a recorded measurement I did not re-run): +the baseline at `ed1b7785` is 306 passed / 19 failed, the 19 being the +`recursion/fibonacci.elf` fixture set, measured in the branch's own worktree with +identical fixture state. + +Set-equality is strictly stronger than a pass count, because it flags a test that +newly *passes* for the wrong reason as loudly as one that newly fails. Given R1 +(a fix that makes the balance always pass), that direction is the one that matters. + +### 3b. Orthogonal guards on what must not move + +- Artifact round-trip `constraint_artifact` at 11/11 — pins that `program()` still + reproduces `air.constraint_program()` bit-for-bit, i.e. that the LFM machine's input + is unchanged. +- Chip pin `artifact_pin.py --check` — ✓ VERIFIED the script exists at + `thoughts/shared/lfm-real-hash/gate-oracle/artifact_pin.py`. + +Both are already green and neither is downstream of the binding fix, so they are +genuine independent guards rather than restatements of the same signal. + +### 3c. ⭐ S2 — The single most important negative control (the plan names the wrong one) + +The plan lists `tampered_l2g_binding_rejects` first. That is the **wrong primary +tripwire**: it exercises epoch-root binding, and ✓ VERIFIED at +`machine_tests.rs:3617-3629` its first three tamper vectors reject inside +`super::executor::execute` — guest-side asserts that never reach the balance check at +all. Only its final coherent-swap leg (`:3647`) touches `verify_against`. + +**The control that must hold is `tampered_claimed_public_word_rejects` — +`prover/src/lfm/machine_tests.rs:52`, asserting at `:62`.** + +✓ VERIFIED, its body: + +```rust +let mut claimed = proved.public_words.clone(); +claimed[0].1[0] = &claimed[0].1[0] + FE::from(1u64); // :59 +let ok = lfm_verify(LfmProgramKind::TrivialV0, &proved.proof, &claimed, &opts) + .expect("registry entry exists"); +assert!(!ok, "a tampered claimed public word must reject"); // :62 +``` + +It holds the proof **fixed** and perturbs exactly one lane of `claimed_public`. It +therefore fails if and only if `expected_public_balance` stops depending injectively +on the claimed words — which is R1, stated exactly. No other test in the suite +isolates that variable. + +**Why it does not fall into the "attack rejected" trap.** A negative control is +worthless if the fix rejects everything, because then it passes for free. This one is +paired with a live positive control on the same program and the same code path: +`trivial_program_proves_and_verifies` at `:36`, asserting `ok` at `:48`. **Run the +pair, and treat either half failing as a stop.** (This is the user's own +`feedback-honest-control-catches-overbroad-fix` rule applied to the specific test.) + +**Best single test, if only one gates the commit:** +`different_arena_values_change_the_public_output_not_the_program` at `:66`. ✓ VERIFIED +it carries both directions in one body over two genuinely different proofs: + +- `:80` — `assert_ne!(a.public_words, b.public_words)` +- `:81-83` — proof `b` against `b`'s words must **verify** +- `:85-87` — proof `b` against `a`'s words must **reject** + +That is positive control, negative control, and a proof that the two statements are +actually distinct, in one test. + +### 3d. The tripwire set, with file:line + +All ✓ VERIFIED present and ? INFERRED currently-passing (none appears in +`reconcile-report.md` §5's enumerated 22 new failures): + +| test | file:line | what it pins | assertion site | +|---|---|---|---| +| `tampered_claimed_public_word_rejects` | `machine_tests.rs:52` | claimed-word injectivity — **primary** | `:62` | +| `different_arena_values_change_the_public_output_not_the_program` | `machine_tests.rs:66` | positive + cross-claim negative in one body | `:81`, `:85` | +| `trivial_program_proves_and_verifies` | `machine_tests.rs:36` | the honest-path pair for the primary | `:48` | +| `tampered_statement_or_root_rejects` | `machine_tests.rs:2248` | statement byte and Phase-A root each move z/alpha; claiming honest words rejects | `:2261`, `:2265-2276` | +| `tampered_l2g_binding_rejects` | `machine_tests.rs:3593` | coherent epoch-root reorder rejects | `:3647` | + +**Category error to avoid.** Step 4 says "the output-swap-hazard tests" as if they +were negative controls. They are not one thing: + +- `preprocessed_tags_close_the_output_swap_hazard` (`machine_tests.rs:415`) asserts + **rejection** at `:441-444` — and see §S3, it is currently *failing*, so it is not + available as a tripwire until the fix lands. +- `keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard` + (`prover/src/lfm/keccak_probe.rs:261`) asserts **acceptance** — it documents a known + hazard. ✓ VERIFIED from its body (`ops[1].tag = ops[0].tag; // the whole point: + duplicate tag`) and its single `assert!` at `:285`. Treating it as a negative + control would invert its meaning. + +--- + +## Claim 4 — HINT (Step 3) in the same pass is fine, with one condition + +The two HINT items are **data, not logic**, and they touch no verification path, so +they cannot mask or be masked by the binding fix: + +- `lfm::constraint_tests::constraint_leg_instruction_census` — a missing design-table + row. ? INFERRED from `reconcile-report.md` §5: the census machinery "ran fine and + printed a full, sane per-table node/leaf/fused/emitted table — the node-index walk + over `artifact.nodes` works; it is the *design table* that lacks the new row." That + is direct evidence the mechanism is healthy and only the pinned data is stale. +- `lfm::constraint_tests::continuation_epoch_constraint_leg_cost` — one pinned + constant, `left: 62375 right: 63393`. + +**Condition (S4).** That second item re-blesses a pinned constant, which is precisely +the anti-pattern the drift tests exist to catch ("investigate, never re-bless" — +`machine_tests.rs:113-116`, the registry drift doc comment, states the house rule). +The 1018-cell delta should be **attributed** to HINT's constraint legs before it is +pasted in. If it cannot be attributed, it is a second finding, not bookkeeping. + +**Sequencing improvement.** Put the two HINT changes in a **separate commit** from the +binding fix. You keep the plan's efficiency of doing both in one pass, while leaving +the soundness-critical diff reviewable on its own in Step 5. This removes the only +credible objection to combining them at zero cost. + +--- + +## Concrete strengthenings + +### S1 — Push the tag before the final push +See Claim 2. `FIX-PLAN.md:96`'s "tagged and pushed" is ✓ VERIFIED false; the tag is +local-only. Step 5's fast-forward destroys the only remote copy of `ed1b7785`. +**Move `git push origin blake3-campaign-preMerge` to before the branch push.** + +### S2 — Primary negative control is `tampered_claimed_public_word_rejects` +See §3c. `machine_tests.rs:52`/`:62`, run paired with `trivial_program_proves_and_verifies` +(`:36`/`:48`). If one test gates the commit, make it +`different_arena_values_change_the_public_output_not_the_program` (`:66`), which +carries both directions. + +### ⭐ S3 — "All 20 share one root cause" is not established, and one test contradicts it + +Only `machine_proves_the_sample_replay` was instrumented. The plan generalizes from +n=1 to 20. + +✓ VERIFIED counterexample: `preprocessed_tags_close_the_output_swap_hazard` +(`machine_tests.rs:415`) is in the failing set (? INFERRED from `reconcile-report.md` +§5's list), but its **only** verify-side assertion is a *negative* one: + +```rust +assert!( + !lfm_verify(LfmProgramKind::KeccakChainV0, &proof, &public, &opts).expect("registered"), + "with distinct tags the swapped outputs must no longer balance" +); // :441-444 +``` + +A globally-broken verify **satisfies** that assertion. So this test's failure cannot +be explained by the bus-balance theory. It must come from one of: + +- leg 1 — `assert_ne!(tag(0), tag(1), "keccak tags must be distinct")` at `:429` + (compiler-side); +- leg 2's prove — `prove_keccak_chain_with_tamper(…).expect("locally consistent")` at + `:440` (prover-side); +- leg 3 — `.expect_err("preprocessed tags cannot be rewritten")` at `:455` plus + `matches!(err, ProvingError::PrecomputedCommitmentMismatch)` at `:457` + (prover/commitment-side). + +All three are prover- or compiler-side, contradicting the diagnosis's "the proof +proves fine; only verify fails." + +This is the plan's own **R3** ("more than one convention shifted at once") showing up +with a name attached. **Recommendation: classify all 22 failures by their actual panic +message before Step 2 concludes single-root-cause.** It costs one test run and it +either confirms the theory across the cluster or saves a wasted fix. This *supports* +the plan's structure — R3 is already in the risk register — it just supplies the +evidence that R3 has materialized. + +Related note: `program_id_matches_production_on_the_real_fixture` and +`program_id_folds_pages_in_the_production_layout` each contain a digest `assert_eq!` +*before* their positive `verify_against` (✓ VERIFIED at `:3723` then `:3728-3738`, and +`:3780` then `:3785-3795`). Either assertion could be the failing one. Same +classification argument applies. + +### ⭐ S5 — A sharper and cheaper Step 1 than the cross-worktree diff + +Step 1 as written compares the same program across two trees +(`lambda_vm-blake3-impl` @ `ed1b7785` vs `lambda_vm-blake3-merge`). There is a better +controlled experiment available **inside the merge tree alone**. + +? INFERRED from `reconcile-report.md` §5 (no `trivial_*` test appears among the +enumerated 22 new failures, and the 19 pre-existing are the `fibonacci.elf` set): +**TrivialV0 proves and verifies today, while `machine_proves_the_sample_replay` +fails.** Confirm this first — it costs two test names. + +✓ VERIFIED that both run the identical path: `verify_against` (`proof.rs:185`) → +`crate::replay_transcript_phase_a_view` (`proof.rs:219`) → `expected_public_balance` +(`proof.rs:220`). ✓ VERIFIED TrivialV0 has non-empty public words, because +`machine_tests.rs:59` indexes `claimed[0].1[0]`. + +Therefore a stale fingerprint layout — the plan's **Outcome C**, and its stated +leading suspect ("a shifted alpha-power offset") — would break **both** programs. It +does not. So: + +- Running the same-tree A/B **falsifies Outcome C in a single run**, without needing + the second worktree at all. +- It localizes the divergence to whatever *differs between the programs* — chip/table + set, `keccak_rnd_chunks`, preprocessed tag rows — which the cross-tree diff does not + isolate, because it varies the tree instead of the program. + +**Recommendation: run the same-tree passing-vs-failing A/B first; keep the +cross-worktree diff as confirmation, not as the opening move.** Same deliverable, +fewer moving parts, and it discriminates the plan's own Outcome A/B/C trichotomy +faster. + +Supporting ✓ VERIFIED facts that narrow this further, all of which back the plan's +"ruled out" list: + +- `crypto/stark/src/traits.rs` — main changed **nothing** in the interaction/ + preprocessed surface. `git diff HEAD origin/main -- crypto/stark/src/traits.rs` + filtered to `fn |interaction|preprocess|num_aux|trace_layout|bus` yields exactly one + line, `- fn precaptured_constraint_program(`, which is the branch's own feature. + So `has_trace_interaction` / `is_preprocessed` / `num_auxiliary_rap_columns` + semantics are unchanged by the merge. +- `crypto/stark/src/lookup.rs` — main's change is the `Arc` wrap of + `constraint_program` plus a hand-written `Clone` impl. `max_bus_elements` exists on + both sides (7 occurrences at branch HEAD, 8 on main — the extra is the new `Clone` + body), so it is not a new bus-layout field. +- All LFM chips are preprocessed through a single site, + `prover/src/lfm/airs.rs:348` (`.with_preprocessed(root, num_prep)`), so Phase-A + absorption of `precomputed_commitment()` is uniform across LFM programs and does + **not** discriminate TrivialV0 from the failing set. + +### ⭐ S6 — Blast radius: `replay_transcript_phase_a_view` is NOT LFM-local + +Step 2 says "Scope is confined to the **branch's** hand-rolled binding — NOT +crypto/stark," then lists `prover/src/lib.rs::replay_transcript_phase_a_view` as a fix +target. Those two statements are in tension, and the second is the dangerous one. + +✓ VERIFIED call graph: + +- `replay_transcript_phase_a_view` is defined at `prover/src/lib.rs:989`. +- It is called at `prover/src/lib.rs:1014`, inside + `compute_expected_commit_bus_balance_view`. +- That function is called at **`prover/src/lib.rs:1442` — the production VM verify + path** — and at `prover/src/continuation.rs:896`, plus `lfm/epoch_tests.rs:743`, + `lfm/logup_tests.rs:1201`, and roughly a dozen sites in + `prover/src/tests/prove_elfs_tests.rs`. +- `replay_transcript_phase_a_view` is *also* called directly at `lfm/proof.rs:219` and + `lfm/logup_tests.rs:1364`. + +**Editing it changes the main VM's verifier**, not just the LFM machine's. The plan's +scope sentence would not catch that, because the file is under `prover/` rather than +`crypto/stark/`. + +By contrast, ✓ VERIFIED `expected_public_balance` (`prover/src/lfm/proof.rs:247`) is a +private `fn` with **exactly one caller**, `proof.rs:220`. It is genuinely LFM-local. + +**Recommendation:** +1. Strongly prefer landing the fix in `expected_public_balance`. +2. If it must land in the shared replay, the acceptance gate has to include the main + VM's own verify tests (`prover/src/tests/prove_elfs_tests.rs`) and the continuation + path, not just `lfm::`. +3. State the rule in Step 2 as "no edit whose blast radius reaches + `lib.rs:1442`," which is the property that actually matters, rather than a + directory boundary. + +### S7 — Make "crypto/stark is clean" mechanical instead of eyeball + +Step 4 asks to "Confirm `git diff` under `crypto/stark/` is only the intended merge +content." That diff is 4435 insertions across 23 files. Reviewing it by eye for stray +diagnostics is not a check. Three binary tests replace it: + +**Test 1 (decisive).** `git diff origin/main -- crypto/stark/src/verifier.rs` must be +**exactly empty**. ✓ VERIFIED that all 50 of its currently-changed lines are +diagnostic — including the `let ok = (0..num_queries).all(…)` refactor, which exists +only to hold the `W909_DEBUG` block, and which must revert to the direct +`(0..num_queries).all(…)` return. + +**Test 2.** `git diff origin/main -- crypto/stark/` must reduce to **exactly seven +files**, all of them the branch's artifact feature. ✓ VERIFIED the current residual +set is: + +``` +crypto/stark/src/constraint_ir/artifact.rs (+771) +crypto/stark/src/constraint_ir/artifact_tests.rs (+394) +crypto/stark/src/constraint_ir/device.rs (rkyv derives) +crypto/stark/src/constraint_ir/mod.rs (ArtifactNode re-export) +crypto/stark/src/constraints/builder.rs (PartialEq/Eq derive) +crypto/stark/src/lookup.rs (with_precaptured + precaptured_program) +crypto/stark/src/traits.rs (precaptured_constraint_program) +crypto/stark/src/verifier.rs ← MUST DISAPPEAR from this list +``` + +**Test 3 (free).** `cargo fmt --check`. ✓ VERIFIED that inserting the prints +de-indented four `error!(` call sites to column zero (the diff shows +`- error!(` / `+error!(`). Incomplete diagnostic removal is +therefore also a formatting failure. Per the user's global convention, `make fmt` and +`make lint` from the repo root are the right invocations, not per-package clippy. + +**Two corrections to Step 4's diagnostic inventory** (✓ VERIFIED by grepping the +working-tree diff for `eprintln|env::var|dbg!|println!`): + +1. `prover/src/lib.rs` contains **zero** campaign diagnostics. Step 4 over-names it. + The actual removal set is `crypto/stark/src/verifier.rs` plus the `LFM_BUS_DEBUG` + block at `prover/src/lfm/proof.rs:224-240`. +2. **Do not strip main's own instrumentation.** `LAMBDA_VM_TIMELINE_JSON` and + `LAMBDA_VM_TRACE_BUILDERS` arrived with the merge alongside + `crypto/stark/src/instruments.rs` (+79 lines) and are legitimate merge content. + Deleting them while "removing diagnostics" would be its own regression. The + campaign diagnostics are identifiable by their markers: `W909_DEBUG`, `DBG909`, + `LFM_BUS_DEBUG`. + +--- + +## Summary of recommended plan amendments + +| # | Amendment | Where | Cost | +|---|---|---|---| +| S1 | Push `blake3-campaign-preMerge` **before** the Step 5 branch push | Step 5 / Rollback | one command | +| S2 | Name `tampered_claimed_public_word_rejects` (`machine_tests.rs:52`) the primary negative control, run paired with `trivial_program_proves_and_verifies` (`:36`) | Step 4 | none | +| S3 | Classify all 22 failures by actual panic message before concluding one root cause; `preprocessed_tags_close_the_output_swap_hazard` (`:415`) already contradicts it | Step 1 | one test run | +| S4 | Attribute the 62375→63393 delta to HINT before re-blessing; separate commit from the binding fix | Step 3 / Step 5 | small | +| S5 | Run the same-tree TrivialV0-vs-sample A/B first; cross-worktree diff becomes confirmation | Step 1 | negative (cheaper) | +| S6 | Scope rule = "no edit whose blast radius reaches `lib.rs:1442`"; prefer `expected_public_balance` (one caller) over `replay_transcript_phase_a_view` (production VM verify) | Step 2 | none | +| S7 | Replace the eyeball diff check with: `git diff origin/main -- crypto/stark/src/verifier.rs` empty, residual = the 7 artifact files, `make fmt` clean | Step 4 | none | + +None of these changes the plan's shape. S1 is a correctness fix to the rollback +story; S3 and S5 sharpen Step 1 within its own stated Outcome A/B/C frame; S6 and S7 +replace prose scope boundaries with mechanical ones. The plan's process — +pin, then fix, then validate positively *and* negatively, then review — is the right +one, and I recommend executing it with these seven amendments. diff --git a/thoughts/shared/lfm-real-hash/merge-plan/main-ir-spec.md b/thoughts/shared/lfm-real-hash/merge-plan/main-ir-spec.md new file mode 100644 index 000000000..6c32b0ce6 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/main-ir-spec.md @@ -0,0 +1,472 @@ +# main's constraint-IR device representation & operand model + +Source of truth: `origin/main`, module `crypto/stark/src/constraint_ir/`. +Files read via `git show origin/main:`: `device.rs`, `ir.rs`, `interp.rs`, +`mod.rs`, `builder.rs`, `gpu_interp.rs`. All line numbers below refer to those +files on `origin/main`. + +This spec exists so a follow-up can adapt a build-time serialization feature to +main's current IR. The headline for that: **main has NO serde/rkyv derives on +any of these types** — see §5. + +--- + +## 0. TL;DR + +- **Two IR forms.** `ConstraintProgram` (`ir.rs`) is the high-level, field-generic + node form: a topologically ordered `Vec` where each `Op` references its + operands by **node-id** (`u32` index into `nodes`, id `i` only references `< i`). + `DeviceProgram` (`device.rs`) is the flat, concrete-Goldilocks POD form: a + `Vec` (16-byte `#[repr(C)]` structs) where operands are **slot-encoded + words**, not node-ids — and where uniform leaves and dead nodes have been + dropped entirely. `DeviceProgram::lower(&ConstraintProgram)` is the one-way map. + +- **Serializability.** Neither `ConstraintProgram`, `Op`, `Dim`, `DeviceProgram`, + nor `DeviceNode` derives `serde` or `rkyv` on main. `Op`/`Dim`/`DeviceNode` are + `Copy + Eq + Hash`-friendly PODs (trivially serializable if a feature adds the + derives); `ConstraintProgram`/`DeviceProgram` carry `FieldElement`/`[u64;3]` + const tables, so serializing the high-level form directly requires deriving on + `ConstraintProgram` + `Op` + `Dim` (and a field-element strategy), whereas the + flat `DeviceProgram` is already all-`u64`/POD and is the cheaper serialization + target. (§5) + +- **Interpreter input forms.** `eval_program` / `eval_program_verifier` / + `eval_program_base` (in `interp.rs`) all consume a **`ConstraintProgram`** + (node-index walk). `eval_device_program` (in `device.rs`) and the whole GPU + path (`gpu_interp.rs`) consume a **`DeviceProgram`** (slot walk). (§4) + +--- + +## 1. THE OPERAND ENCODING (device.rs) + +`DeviceNode` is the flat instruction (`device.rs:116-123`): + +```rust +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeviceNode { + pub op: u32, // OP_* tag + pub a: u32, // operand word 0 (encoding depends on op) + pub b: u32, // operand word 1 (encoding depends on op) + pub res: u32, // result slot, RES_EXT_BIT selects class +} +``` + +16 bytes, `#[repr(C)]`, 1:1 device upload. + +### 1a. OP_* tags (`device.rs:63-84`) + +| Const | Value | Meaning | `a`/`b` meaning | +|---|---|---|---| +| `OP_CONST_BASE` | 0 | base literal (root-pinned uniform only) | `a` = raw `base_consts` index, `b`=0 | +| `OP_CONST_EXT` | 1 | ext literal (root-pinned uniform only) | `a` = raw `ext_consts` index, `b`=0 | +| `OP_VAR` | 2 | trace-cell read | `a`/`b` = packed `Op::Var` fields (§1d) | +| `OP_RAP_CHALLENGE` | 3 | RAP challenge (root-only) | `a` = raw `rap_challenges` index | +| `OP_ALPHA_POW` | 4 | LogUp alpha power (root-only) | `a` = raw `logup_alpha_powers` index | +| `OP_TABLE_OFFSET` | 5 | LogUp table offset (root-only) | no operands | +| `OP_ADD` | 6 | `a + b` | `a`,`b` = **OPK-encoded** operands (§1b) | +| `OP_SUB` | 7 | `a - b` | `a`,`b` = OPK-encoded operands | +| `OP_MUL` | 8 | `a * b` | `a`,`b` = OPK-encoded operands | +| `OP_NEG` | 9 | `-a` | `a` = OPK-encoded operand | +| `OP_EMBED` | 10 | base→ext embed | `a` = OPK-encoded operand | + +Note the split-personality of `a`/`b`: for the arithmetic ops (6–10) they are +**OPK-encoded operand words**; for `OP_VAR` they are **packed var fields**; for the +root-pinned uniform leaves (0,1,3,4) `a` is a **raw table index** (not kind-tagged, +because the tag already tells the walker which table). + +### 1b. OPK operand encoding — for arithmetic-op operand words `a`/`b` + +Encoding scheme (`device.rs:86-105`): `enc = (kind << OPK_SHIFT) | payload`. + +- `OPK_SHIFT = 29` (`device.rs:89`) — 3-bit kind occupies bits **29–31**. +- `OPK_PAYLOAD_MASK = (1 << OPK_SHIFT) - 1 = 0x1FFF_FFFF` (`device.rs:91`) — 29-bit + payload occupies bits **0–28**. + +Operand KINDs (`device.rs:92-105`): + +| Const | Value | Kind | Payload = | +|---|---|---|---| +| `OPK_BASE_SLOT` | 0 | base (`u64`) scratch slot | slot index | +| `OPK_EXT_SLOT` | 1 | ext (`[u64;3]`) scratch slot | slot index | +| `OPK_BASE_CONST` | 2 | base constant | `base_consts` index | +| `OPK_EXT_CONST` | 3 | ext constant | `ext_consts` index | +| `OPK_RAP` | 4 | RAP challenge (uniform) | `rap_challenges` index | +| `OPK_ALPHA` | 5 | alpha power (uniform) | `logup_alpha_powers` index | +| `OPK_OFFSET` | 6 | LogUp table offset (uniform) | (payload unused) | + +Decode (as done in `eval_device_program`, `device.rs:403-425`): +`kind = enc >> OPK_SHIFT` (29); `payload = enc & OPK_PAYLOAD_MASK`. + +`load_base` accepts only `OPK_BASE_SLOT` / `OPK_BASE_CONST` (panics otherwise, +`device.rs:403-410`). `load_ext` accepts all seven kinds, embedding base slots/ +consts into the extension (`device.rs:411-425`). + +**Worked example — `0x4000_0001` as an OPERAND word:** +- `kind = 0x4000_0001 >> 29 = 0b010 = 2 = OPK_BASE_CONST`. +- `payload = 0x4000_0001 & 0x1FFF_FFFF = 0x1 = 1`. +- ⇒ this operand is `base_consts[1]`. + +(Caution: the same 32-bit value means something different in a `res`/`roots` +word — see §1c. In a `res` word `0x4000_0001` has `RES_EXT_BIT` (bit 31) clear, so +it would be base slot index `0x4000_0001` — a distinct, non-OPK interpretation.) + +### 1c. `res` word and `roots` entries — a DIFFERENT encoding + +`RES_EXT_BIT = 1 << 31` (`device.rs:109`). Used in a node's `res` word **and** in +every `roots` entry: + +- bit **31** set ⇒ ext (`[u64;3]`) slot class; clear ⇒ base (`u64`) slot class. +- low **31** bits (bits 0–30) = the slot index. + +Built at `device.rs:326-329` (per-node `res`) and `device.rs:339-342` / +`device.rs:161-163` (roots): `res = slot` for `Dim::Base`, `res = slot | RES_EXT_BIT` +for `Dim::Ext`. Decoded at `device.rs:428-429` and `486-487`: +`res_slot = res & !RES_EXT_BIT`, `res_ext = (res & RES_EXT_BIT) != 0`. + +**This is a 1-bit class tag at bit 31, NOT the 3-bit OPK kind at bits 29–31.** An +operand word and a `res`/`roots` word are decoded by two different schemes; do not +conflate them. + +### 1d. `OP_VAR` field packing (`device.rs:125-143`) + +`pack_var(main, offset, row, col) -> (a, b)`: +- `a = col as u32` (only low 16 bits are meaningful). +- `b = ((main as u32) << 16) | ((offset as u32) << 8) | (row as u32)`. + +So in `b`: bit **16** = `main`; bits **8–15** = `offset` (u8); bits **0–7** = `row` +(u8). `a` bits **0–15** = `col` (u16). + +`unpack_var(a, b) -> (main, offset, row, col)` (`device.rs:136-143`): +`col = (a & 0xFFFF)`, `main = (b >> 16) & 1`, `offset = (b >> 8) & 0xFF`, +`row = b & 0xFF`. + +--- + +## 2. THE TWO IR FORMS + +### 2a. `ConstraintProgram` — high-level node form (`ir.rs:85-107`) + +```rust +#[derive(Clone, Debug)] +pub struct ConstraintProgram { + pub nodes: Vec, // topologically ordered; id i refs only < i + pub dims: Vec, // per-node result dim, parallel to nodes + pub base_consts: Vec>, // base literals (indexed by Op::ConstBase) + pub ext_consts: Vec>, // ext literals (indexed by Op::ConstExt) + pub roots: Vec, // per-constraint root node-id + pub num_base: usize, // # leading base-rooted constraints +} +``` + +Fields, all `pub`: +- `nodes: Vec` — the instruction arena. +- `dims: Vec` — parallel to `nodes`, result dim of each node. +- `base_consts: Vec>` — base-field literal table. +- `ext_consts: Vec>` — extension-field literal table. +- `roots: Vec` — node-id of each constraint's value, indexed by `constraint_idx`. +- `num_base: usize` — count of leading base-`Dim`-rooted constraints (prover writes + these to `base_evals`; the rest, always ext/LogUp, to `ext_evals`). + +Methods (`ir.rs:109-...`): `len()`, `is_empty()`, `next_row_trace_reads(main_width)` +(tests/tooling only — derives the next-row read set from the captured IR). + +The `Op` enum (`ir.rs:40-83`), `#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]`: + +```rust +pub enum Op { + ConstBase(u32), // base_consts[idx] + ConstExt(u32), // ext_consts[idx] + Var { main: bool, offset: u8, row: u8, col: u16 }, // trace cell read + RapChallenge { idx: u16 }, // rap_challenges[idx] (ext, uniform) + AlphaPow { idx: u16 }, // logup_alpha_powers[idx] (ext, uniform) + TableOffset, // LogUp L/N (ext, uniform) + Add(u32, u32), // nodes[a] + nodes[b] + Sub(u32, u32), // nodes[a] - nodes[b] + Mul(u32, u32), // nodes[a] * nodes[b] + Neg(u32), // -nodes[a] + Embed(u32), // base -> ext embed +} +``` + +**Operands are node-ids.** `Add/Sub/Mul(a,b)`, `Neg(a)`, `Embed(a)` carry `u32` +indices into `nodes` (id `i` references only `< i`). `ConstBase/ConstExt(idx)` +carry `u32` indices into the const side-tables (so `Op` stays field-free +`Copy + Eq + Hash`, per the module docs `ir.rs:11-17`). + +The `Dim` enum (`ir.rs:26-34`), `#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)]`: +`Base` (default) | `Ext`. + +### 2b. `DeviceProgram` — flat POD form (`device.rs:149-172`) + +```rust +#[derive(Clone, Debug)] +pub struct DeviceProgram { + pub nodes: Vec, // flat 16-byte ops; uniform leaves & dead nodes dropped + pub base_consts: Vec, // canonical raw base limbs + pub ext_consts: Vec<[u64; 3]>, // canonical raw ext limbs + pub roots: Vec, // per-constraint root slot (slot | RES_EXT_BIT) + pub num_base: u32, // # base-rooted constraints -> base_evals + pub num_base_slots: u32, // size of base (u64) slot class, per thread + pub num_ext_slots: u32, // size of ext ([u64;3]) slot class, per thread +} +``` + +Fields, all `pub`: +- `nodes: Vec` — flat instruction list; operands reference **slots** + (or uniform tables), not node-ids. Uniform leaves and dead nodes are absent. +- `base_consts: Vec` — raw base limbs (`FieldElement::value()` copies, `device.rs:346`). +- `ext_consts: Vec<[u64; 3]>` — raw ext limbs (`encode_ext`, `device.rs:347,373-376`). +- `roots: Vec` — per-constraint root **slot** word, `slot | RES_EXT_BIT`. +- `num_base: u32` — same meaning as `ConstraintProgram::num_base`, narrowed to `u32`. +- `num_base_slots: u32` — count of `u64` scratch slots per thread. +- `num_ext_slots: u32` — count of `[u64;3]` scratch slots per thread. + +### 2c. The difference, stated explicitly + +`ConstraintProgram.nodes` is a node arena **indexed by node-id**, and every +arithmetic `Op` names its operands by those node-ids; the array is dense (every +captured node present, including uniform leaves) and field-generic +(`FieldElement`/`` const tables). `DeviceProgram.nodes` is a **compacted, +slot-addressed** array: `lower` drops uniform leaves (propagated into operand +words) and dead nodes, assigns each surviving node a reusable scratch **slot** +via liveness scan, and rewrites operands as slot-encoded words (`OPK_* << 29 | +payload`) pointing at slots or uniform tables — never at node positions. Constants +are demoted from `FieldElement` to raw `u64`/`[u64;3]` limbs. In short: +**node-id operands + generic field ⟶ slot-encoded operands + raw limbs, with +uniform/dead nodes removed.** + +--- + +## 3. `DeviceProgram::lower(&ConstraintProgram) -> DeviceProgram` (device.rs:195-358) + +Signature (`device.rs:200`): +`pub fn lower(prog: &ConstraintProgram) -> Self` +— concrete Goldilocks only. **`dims` IS an input**: `ConstraintProgram` carries the +parallel `dims: Vec`, and `lower` reads `prog.dims[i]` / `prog.dims[j]` for +slot-class decisions (`device.rs:260, 270, 304, 339`). Dims are not recomputed. + +Algorithm: + +1. **Bound check** (`device.rs:201-205`): `n = prog.nodes.len()` must be + `<= OPK_PAYLOAD_MASK` (2^29−1), else panic — the 29-bit slot/payload space. + +2. **Liveness pass** (`device.rs:208-220`): compute `used[j]` and `last_use[j]` + (max consumer node-id) for every node by scanning `operands(op)` (the up-to-two + operand node-ids, `device.rs:187-193`). Then mark `is_root[r]` and force + `used[r]=true` for each root. + +3. **Emit set** (`device.rs:225-227`): node `i` materializes iff + `used[i] && (!is_uniform_leaf(nodes[i]) || is_root[i])`. `is_uniform_leaf` + (`device.rs:175-184`) = `ConstBase|ConstExt|RapChallenge|AlphaPow|TableOffset`. + Uniform leaves are propagated into operands and only kept as nodes when they are + themselves constraint roots. + +4. **Slot allocator — linear scan with per-class free lists** (`device.rs:246-331`). + State: `slot_of[i]` (init `UNASSIGNED=u32::MAX`), `free_base: Vec`, + `free_ext: Vec`, counters `num_base_slots`, `num_ext_slots`. For each + emitted node `i` in order: + - **Encode operands** while operand slots are still live (`enc_operand`, + `device.rs:263-274`): if operand `j` is not emitted (a propagated uniform + leaf) → `enc_uniform` (`device.rs:229-244`) emits `OPK_BASE_CONST/EXT_CONST/ + RAP/ALPHA/OFFSET << 29 | idx`. Else → `OPK_BASE_SLOT`/`OPK_EXT_SLOT << 29 | + slot_of[j]`, class chosen by `prog.dims[j]`. + - **Build `(tag, a, b)`** per op (`device.rs:276-296`): uniform-leaf & `OP_VAR` + nodes stash raw indices / packed var fields; arithmetic ops store the encoded + operand words. + - **Free dead operand slots** (`device.rs:300-310`): for each operand `j`, if + `emitted[j] && !is_root[j] && last_use[j]==i && slot_of[j]!=UNASSIGNED`, push + `slot_of[j]` onto the matching free list and reset `slot_of[j]=UNASSIGNED` + (the reset guards the `a==b` double-free). Roots are pinned (never freed). + - **Allocate result slot** (`device.rs:314-324`): pop from the matching free + list, else bump the class counter (`num_base_slots`/`num_ext_slots`). A slot + freed this same node may be reused (kernel reads operands before writing res). + Record `slot_of[i]`. + - **Emit `DeviceNode`** (`device.rs:326-330`) with `res = slot` (base) or + `slot | RES_EXT_BIT` (ext). + + ⇒ `num_base_slots` / `num_ext_slots` end as the **max-live-set per class**, not + the node count (root pins excepted). + +5. **Roots** (`device.rs:333-344`): map each `prog.roots[c]` node-id through + `slot_of[..]` to `slot | (RES_EXT_BIT if Dim::Ext)`. + +6. **Const tables** (`device.rs:346-347`): `base_consts` = raw `u64` via + `c.value()`; `ext_consts` = `[u64;3]` via `encode_ext`. + +7. **Assemble** (`device.rs:349-357`): `num_base = prog.num_base as u32`. + +--- + +## 4. THE INTERPRETERS + +### 4a. `interp.rs` — node-index walkers over `ConstraintProgram` + +All three take a `&ConstraintProgram` and walk `prog.nodes` by node-index +(shared `run`, `interp.rs:60-113`, which builds a parallel `Vec` indexed +1:1 with `nodes`). + +- `eval_program_base` (`interp.rs:150-171`) — minimal single-root, main-only, + base result, for the per-constraint diff test: + ```rust + pub fn eval_program_base( + prog: &ConstraintProgram, + constraint_idx: usize, + main_row: &[FieldElement], + ) -> FieldElement + ``` + +- `eval_program` (`interp.rs:178-217`) — full **prover** entry; requires + `TransitionEvaluationContext::Prover`; writes base-rooted → `base_evals`, + ext-rooted → `ext_evals`: + ```rust + pub fn eval_program( + prog: &ConstraintProgram, + ctx: &TransitionEvaluationContext, + base_evals: &mut [FieldElement], + ext_evals: &mut [FieldElement], + ) + ``` + +- `eval_program_verifier` (`interp.rs:225-262`) — full **verifier** entry; + requires `TransitionEvaluationContext::Verifier`; writes every constraint into + `ext_evals` (base roots embedded): + ```rust + pub fn eval_program_verifier( + prog: &ConstraintProgram, + ctx: &TransitionEvaluationContext, + ext_evals: &mut [FieldElement], + ) + ``` + +### 4b. `device.rs` — flat slot walker over `DeviceProgram` + +- `eval_device_program` (`device.rs:389-497`) — CPU model of the GPU kernel; + consumes a `&DeviceProgram` and walks `dev.nodes` decoding slot-encoded + operands (dim-split slot files `base_slots`/`ext_slots`), in raw limbs: + ```rust + pub fn eval_device_program( + dev: &DeviceProgram, + main: &[Vec], + aux: &[Vec<[u64; 3]>], + rap_challenges: &[[u64; 3]], + alpha_powers: &[[u64; 3]], + table_offset: [u64; 3], + base_evals: &mut [u64], + ext_evals: &mut [[u64; 3]], + ) + ``` + +### 4c. GPU path (gpu_interp.rs) — consumes `DeviceProgram` + +`#[cfg(feature = "cuda")]`. Both entry points — `try_eval_composition_gpu` +(`gpu_interp.rs`) and `try_eval_program_gpu` — take a **generic +`&ConstraintProgram`**, but immediately funnel through `lower_and_pack` +(TypeId-gates the Goldilocks tower, `unsafe`-reinterprets to the concrete program, +then calls `DeviceProgram::lower`). Everything handed to the CUDA FFI is the +**lowered `DeviceProgram`** (via `pack_nodes` → 2×`u64` per node: +`op | a<<32`, `b | res<<32`; plus `dev.base_consts`, `flatten_ext3(dev.ext_consts)`, +`dev.roots`, `num_base_slots`, `num_ext_slots`). The lowering is cached +process-wide by content fingerprint (`lowering_cache`, `program_fingerprint`, +`program_eq`). **So the GPU consumes the `DeviceProgram` (slot) form**, produced +on demand from the `ConstraintProgram` at dispatch time. + +--- + +## 5. SERIALIZABILITY — main derives NEITHER serde NOR rkyv + +Verified by grepping the module for `rkyv|Archive|Serialize|Deserialize|serde`: +zero hits in `ir.rs`, `device.rs`, `builder.rs`, `interp.rs`, `mod.rs`, +`gpu_interp.rs`. The only derives present are the standard traits. + +Exact derive lines: + +- `Dim` (`ir.rs:26`): `#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)]` +- `Op` (`ir.rs:40`): `#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]` +- `ConstraintProgram` (`ir.rs:85`): `#[derive(Clone, Debug)]` +- `DeviceNode` (`device.rs:117`): `#[derive(Clone, Copy, Debug, PartialEq, Eq)]` +- `DeviceProgram` (`device.rs:149`): `#[derive(Clone, Debug)]` +- `Expr` (`builder.rs:29`, builder handle, not part of the program): `#[derive(Clone, Copy, Debug)]` + +Implications for a build-time serialization feature: + +- **`DeviceProgram` is the cheap target.** `DeviceNode` is a 16-byte `#[repr(C)]` + POD of four `u32`s; `DeviceProgram`'s other fields are `Vec` / `Vec<[u64;3]>` + / `Vec` / `u32`. Adding `rkyv(Archive, Serialize, Deserialize)` (or serde) + is mechanical — no field-element or generic-tower obstacle. This matches how the + gpu path already treats it as flat `u64` blobs. + +- **Serializing the high-level `ConstraintProgram` directly is more involved.** It + is generic `` and holds `Vec>` / `Vec>` + const tables, so a derive must either (a) bound the field types with the + serialization traits, or (b) fix the concrete Goldilocks tower and serialize the + const tables as raw limbs (the same `to_raw`/`value()` trick `lower` and + `program_fingerprint` use). `Op` and `Dim` themselves are trivially derivable + (plain `u32`/enum payloads, already `Copy + Eq + Hash`). + +- Consequence for the merge: if the incoming feature stores the **high-level** + form, it needs derives on `ConstraintProgram + Op + Dim` plus a field-element + serialization strategy; if it stores the **flat** form, it only needs derives on + `DeviceProgram + DeviceNode`. Main provides neither today; both are additive. + +--- + +## 6. mod.rs — public exports (mod.rs:29-45) + +```rust +pub mod builder; +pub mod device; +#[cfg(feature = "cuda")] +pub mod gpu_interp; +pub mod interp; +pub mod ir; + +#[cfg(test)] +mod tests; + +pub use builder::{Expr, IrBuilder}; +pub use device::{DeviceNode, DeviceProgram, eval_device_program}; +pub use interp::{eval_program, eval_program_base, eval_program_verifier}; +pub use ir::{ConstraintProgram, Dim, Op}; +``` + +Re-exported from `constraint_ir`: +- from `builder`: `Expr`, `IrBuilder` +- from `device`: `DeviceNode`, `DeviceProgram`, `eval_device_program` +- from `interp`: `eval_program`, `eval_program_base`, `eval_program_verifier` +- from `ir`: `ConstraintProgram`, `Dim`, `Op` + +**NOT re-exported (must be reached via `device::`):** the `OP_*` tag constants, +the `OPK_*` operand-kind constants, `OPK_SHIFT`, `OPK_PAYLOAD_MASK`, `RES_EXT_BIT`, +`pack_var` / `unpack_var`, and `DeviceProgram::lower`. The `cuda`-gated +`gpu_interp` (`try_eval_composition_gpu`, `try_eval_program_gpu`, the +`u64↔FieldElement` reinterpret helpers) is a `pub mod` but nothing is re-exported +at the `constraint_ir` root. + +--- + +## Appendix: full constant table (device.rs) + +| Name | Value | Role | +|---|---|---| +| `OP_CONST_BASE` | 0 | tag | +| `OP_CONST_EXT` | 1 | tag | +| `OP_VAR` | 2 | tag | +| `OP_RAP_CHALLENGE` | 3 | tag | +| `OP_ALPHA_POW` | 4 | tag | +| `OP_TABLE_OFFSET` | 5 | tag | +| `OP_ADD` | 6 | tag | +| `OP_SUB` | 7 | tag | +| `OP_MUL` | 8 | tag | +| `OP_NEG` | 9 | tag | +| `OP_EMBED` | 10 | tag | +| `OPK_SHIFT` | 29 | operand kind bit position | +| `OPK_PAYLOAD_MASK` | `0x1FFF_FFFF` | operand payload mask (bits 0–28) | +| `OPK_BASE_SLOT` | 0 | operand kind | +| `OPK_EXT_SLOT` | 1 | operand kind | +| `OPK_BASE_CONST` | 2 | operand kind | +| `OPK_EXT_CONST` | 3 | operand kind | +| `OPK_RAP` | 4 | operand kind | +| `OPK_ALPHA` | 5 | operand kind | +| `OPK_OFFSET` | 6 | operand kind | +| `RES_EXT_BIT` | `1 << 31` | `res`/`roots` ext-slot class bit | diff --git a/thoughts/shared/lfm-real-hash/merge-plan/reconcile-report.md b/thoughts/shared/lfm-real-hash/merge-plan/reconcile-report.md new file mode 100644 index 000000000..a8ae5a724 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/merge-plan/reconcile-report.md @@ -0,0 +1,340 @@ +# Artifact-feature reconciliation to main's constraint IR — report + +Approach A, per `artifact-feature-map.md` §5. Worktree +`/Users/maurofab/workspace/lambda_vm-blake3-merge` (branch +`blake3-real-hash-mainmerge`, `git merge origin/main` still in progress, +everything left uncommitted). + +**Status: GREEN for the artifact feature.** Round-trip suite 11/11. The 22 extra +`lfm::` failures are pre-existing main-drift, measured against the pre-merge +branch — see §5, they are the lead's, not this task's. + +--- + +## 1. Diff summary + +``` + crypto/stark/src/constraint_ir/artifact.rs | 150 +++++++++++++++++++++----- + crypto/stark/src/constraint_ir/mod.rs | 2 +- + prover/src/lfm/constraint_tests.rs | 6 +- + prover/src/tests/constraint_artifact_tests.rs | 35 ++++-- + 4 files changed, 154 insertions(+), 39 deletions(-) +``` + +Line numbers are post-edit. + +### `crypto/stark/src/constraint_ir/artifact.rs` + +| site | change | +|---|---| +| :16-20 (module doc) | item 1 of the bundle no longer claims to be `DeviceProgram`'s form; points at `ArtifactNode` | +| :91 | `use super::device::{DeviceNode, DeviceProgram}` → `use super::device::DeviceProgram` | +| :105-146 | **NEW** `DIM_BASE: u32 = 0` (:110) / `DIM_EXT: u32 = 1` (:112) and `pub struct ArtifactNode { op, a, b, dim }` (:139), `#[repr(C)]` + `Clone, Copy, Debug, PartialEq, Eq, rkyv::{Archive, Serialize, Deserialize}` — verbatim shape and const values of the OLD `device::DeviceNode` / `DIM_*` (checked against `git show HEAD:crypto/stark/src/constraint_ir/device.rs`, old lines 67-69 and 75-82) | +| :278 | `ConstraintArtifact.nodes: Vec` → `Vec` | +| :347-410 | `capture()` — `DeviceProgram::lower(prog)` removed; 1:1 node-index map transplanted at :360-392, const tables at :394-410 (body in §2) | +| :433-439 | `capture()` return — `nodes`/`base_consts`/`ext_consts` are the locals above, `roots`/`num_base` now come from `prog`, not `dev` | +| :474-476 | `device_program()` — field copy → `DeviceProgram::lower(&self.program())` | +| :493 / :568 | `program()` and `validate_self()` — dropped `DIM_BASE, DIM_EXT` from the `super::device::{…}` import lists; they now resolve to the module's own consts. **No logic change**: both still read `n.a`/`n.b` as node ids and `n.dim` as a `DIM_*` tag | + +`OP_*` tags and `pack_var`/`unpack_var` are still imported from `device::` — +unchanged on main, and they mean the same thing in both forms; only the operand +encoding differs. + +### `crypto/stark/src/constraint_ir/mod.rs` + +`:45` — added `ArtifactNode` to the `pub use artifact::{…}` re-export list +(parallel to `device::DeviceNode` being re-exported at `:47`). `DIM_BASE` / +`DIM_EXT` are deliberately NOT re-exported, mirroring main's treatment of +`OP_*` / `RES_EXT_BIT` (reachable via `artifact::`). + +### Tests — import moves only, node-index logic untouched + +- `prover/src/tests/constraint_artifact_tests.rs:396` (`constraint_op_census`) + and `:927` (`leg_instructions`): `DIM_BASE` now from + `stark::constraint_ir::artifact`, the `OP_*` list still from + `…::device`. The `v_base[n.a as usize]` / `n.dim == DIM_BASE` propagation is + byte-identical. +- `prover/src/lfm/constraint_tests.rs:638,658,662` + (`dead_nodes_are_eliminated`): `DeviceNode` → `ArtifactNode`, and + `device::DIM_EXT` → `artifact::DIM_EXT`. +- `crypto/stark/src/constraint_ir/artifact_tests.rs`: **no change needed** — it + never imported `DeviceNode` or `DIM_*`, only mutates `artifact.nodes[i].a` / + reads `.op`, and those field names are identical on `ArtifactNode`. (The map + predicted an import move here; there was none to make.) + +--- + +## 2. The transplanted `capture()` body + +```rust +use super::device::{ + OP_ADD, OP_ALPHA_POW, OP_CONST_BASE, OP_CONST_EXT, OP_EMBED, OP_MUL, OP_NEG, + OP_RAP_CHALLENGE, OP_SUB, OP_TABLE_OFFSET, OP_VAR, pack_var, +}; + +let prog = air.constraint_program(); + +// A 1:1 projection of the captured program — same node count, same +// order, operands left as node ids. Deliberately NOT +// `DeviceProgram::lower`: that is the slot-allocating lowering, and its +// output cannot be lifted back (see `ArtifactNode`). +let nodes: Vec = prog + .nodes + .iter() + .zip(prog.dims.iter()) + .map(|(op, dim)| { + let dim = match dim { + Dim::Base => DIM_BASE, + Dim::Ext => DIM_EXT, + }; + let (op, a, b) = match *op { + Op::ConstBase(idx) => (OP_CONST_BASE, idx, 0), + Op::ConstExt(idx) => (OP_CONST_EXT, idx, 0), + Op::Var { main, offset, row, col } => { + let (a, b) = pack_var(main, offset, row, col); + (OP_VAR, a, b) + } + Op::RapChallenge { idx } => (OP_RAP_CHALLENGE, idx as u32, 0), + Op::AlphaPow { idx } => (OP_ALPHA_POW, idx as u32, 0), + Op::TableOffset => (OP_TABLE_OFFSET, 0, 0), + Op::Add(a, b) => (OP_ADD, a, b), + Op::Sub(a, b) => (OP_SUB, a, b), + Op::Mul(a, b) => (OP_MUL, a, b), + Op::Neg(a) => (OP_NEG, a, 0), + Op::Embed(a) => (OP_EMBED, a, 0), + }; + ArtifactNode { op, a, b, dim } + }) + .collect(); + +let base_consts: Vec = prog.base_consts.iter().map(|c| *c.value()).collect(); +let ext_consts: Vec<[u64; 3]> = prog + .ext_consts + .iter() + .map(|x| { + let limbs = x.value(); + [*limbs[0].value(), *limbs[1].value(), *limbs[2].value()] + }) + .collect(); +``` + +and the return now reads + +```rust +Self { + nodes, + base_consts, + ext_consts, + roots: prog.roots.clone(), + num_base: prog.num_base as u32, + meta: /* unchanged */, + shape: /* unchanged */, +} +``` + +`device.rs`'s `encode_ext` is private to that module (`fn encode_ext`, not +`pub`), so the ext-limb encoding is inlined above rather than imported. It is the +same three-limb `value()` copy, and `program()`'s `FieldElement::from_raw` walk +is its exact inverse — pinned by the round-trip's `base_consts` / `ext_consts` +equality assertions. + +`device_program()` is now: + +```rust +pub fn device_program(&self) -> DeviceProgram { + DeviceProgram::lower(&self.program()) +} +``` + +so the slot encoding exists in exactly one place (main's `lower`) and cannot +drift from what the prover and the GPU path build. + +--- + +## 3. Round-trip suite — the hard oracle + +`cargo test --release -p lambda-vm-prover --lib constraint_artifact` + +``` +test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 913 filtered out +``` + +Every test the task named as previously failing now passes: + +| test | result | +|---|---| +| `all_table_artifacts_roundtrip_and_match_folders` | ok | +| `production_airs_accept_a_precaptured_program` | ok | +| `constraint_op_census` | ok | +| `epoch_chunk_multiplier` | ok | +| `continuation_epoch_constraint_leg` | ok | +| `continuation_epoch_chunk_counts_measured` | ok | +| `global_memory_private_input_is_a_second_shape_not_a_second_program` | ok (after the §4 fix) | +| `artifacts_are_invariant_across_trace_length` | ok | +| `artifacts_are_invariant_across_proof_options` | ok | +| `parameterized_airs_vary_per_parameter_value` | ok | +| `an_artifact_does_not_validate_against_a_different_table` | ok | + +This is the contract of `artifact-feature-map.md` §4 discharged in full: +`prog.nodes == captured.nodes` bit-for-bit (plus dims/roots/num_base/consts) for +every production AIR, and all three evaluation oracles — `eval_program`, +`eval_device_program` on the re-lowered blob, `eval_program_verifier` — agreeing +with the compiled folders over 100 random trials each. + +Also green, and directly in scope: + +`cargo test --release -p stark --lib constraint_ir` → **39 passed, 0 failed** +(includes `artifact_tests`: `lift_is_the_inverse_of_lower`, +`validate_self_rejects_a_forward_reference`, +`validate_self_rejects_an_out_of_range_constant`, `ExemptConstraints`). + +`cargo check --release -p lambda-vm-prover` → clean. +`rustfmt --check` on all four touched files → clean. +`cargo clippy --release -p stark -p lambda-vm-prover --all-targets` → **no +errors**; the warnings are all the pre-existing `op_ref` class and none land in +a line this task touched. + +### Fixture note (not a code change) + +Three of these tests read ELFs from `executor/program_artifacts/asm/`, which is +gitignored build output and did not exist in this fresh worktree. I copied the +directory in from `/Users/maurofab/workspace/lambda_vm` so the tests would +actually run rather than fail fast on a missing file. `make compile-*` would +produce the same thing. It changed nothing about the `lfm::` numbers below +(measured both ways, §5). + +--- + +## 4. Deviation from the map: one stale test assertion, fixed + +`global_memory_private_input_is_a_second_shape_not_a_second_program` +(`prover/src/tests/constraint_artifact_tests.rs:1151`) failed after the +reconciliation, but **not because of it** — it got past the program-equality +assertions and died on a shape assertion about the AIR: + +``` +a private-input page is not preprocessed — the verifier never recomputes its +genesis column from the ELF +``` + +That is main's private-page OFFSET soundness fix landing on a branch-era +expectation. Verified directly: + +- branch `HEAD:prover/src/continuation.rs:234` — `if config.is_private_input { return air; }` (no preprocessing at all) +- `origin/main:prover/src/continuation.rs:240` — returns + `air.with_preprocessed(page::private_page_preprocessed_commitment(opts), page::NUM_PREPROCESSED_COLS_PRIVATE)` + +So on main a private-input page **is** preprocessed; it commits OFFSET alone +(`NUM_PREPROCESSED_COLS_PRIVATE = 1`) while an ELF page commits OFFSET and INIT +(`global_memory::NUM_PREPROCESSED_COLS = 2`). INIT stays a main-trace column +because it is the private input; OFFSET must be committed because it is the +row's address and leaving it prover-chosen lets a genesis token name an +arbitrary address. + +The test's thesis — *a second shape, not a second program* — is still exactly +right and still worth pinning, so I updated it to main's semantics rather than +deleting it: both variants assert `is_preprocessed`, the two +`num_precomputed_columns` are asserted against the two named constants, and the +"differ ONLY in the preprocessed fields" normalization now normalizes +`num_precomputed_columns` alone. Doc comment updated to match ("preprocess +OFFSET only" rather than "built non-preprocessed"). + +**This is a judgement call the lead should sanity-check** — it is a test +expectation changed to follow main, in a file otherwise touched only by import +moves. + +No other deviations. No main-drift compile errors outside the artifact feature +turned up; the `lookup.rs` `precaptured_program` Clone the lead already added was +the only one. + +--- + +## 5. `lfm::` — 41 failures, and why 22 of them are not this task's + +`cargo test --release -p lambda-vm-prover --lib lfm::` in the merge worktree: + +``` +test result: FAILED. 284 passed; 41 failed; 7 ignored +``` + +That is **not** the expected 306/19. I measured the pre-merge baseline rather +than assume, running the same command in the branch's own worktree +`/Users/maurofab/workspace/lambda_vm-blake3-impl` @ `ed1b7785` (clean, and with +the identical fixture state — neither `asm/` nor `recursion/` present): + +``` +test result: FAILED. 306 passed; 19 failed; 7 ignored +``` + +Same 332 tests either side, so nothing was added or removed. Diffing the two +failure lists: **22 new, 0 fixed.** The 19 pre-existing are the +`recursion/fibonacci.elf` set (`run make compile-recursion-elfs`), exactly as +expected. + +The 22 new ones: + +``` +lfm::constraint_tests::constraint_leg_instruction_census +lfm::constraint_tests::continuation_epoch_constraint_leg_cost +lfm::fri_tests::the_fri_leg_proves_and_verifies +lfm::join_tests::the_join_proves_and_verifies +lfm::keccak_probe::adapter_probe_proves_real_permutations +lfm::keccak_probe::duplicate_tag_output_swap_accepts_demonstrating_hazard +lfm::machine_tests::append_ext_proves_and_verifies +lfm::machine_tests::chunked_sponge_proves_and_verifies +lfm::machine_tests::chunking_does_not_change_what_is_proved +lfm::machine_tests::keccak_chain_proves_and_verifies +lfm::machine_tests::keccak_merkle_walk_authenticates_a_real_opening +lfm::machine_tests::keccak_sponge_proves_and_verifies +lfm::machine_tests::keccak_sponge_reference_lengths_prove_and_verify +lfm::machine_tests::machine_proves_the_sample_replay +lfm::machine_tests::permutations_may_be_reassigned_across_chunk_boundaries +lfm::machine_tests::preprocessed_tags_close_the_output_swap_hazard +lfm::machine_tests::program_id_folds_pages_in_the_production_layout +lfm::machine_tests::program_id_matches_production_on_the_real_fixture +lfm::machine_tests::splice_proves_and_verifies +lfm::machine_tests::statement_replay_proves_and_verifies +lfm::machine_tests::the_register_derivation_proves_and_verifies +lfm::machine_tests::transcript_replay_proves_and_verifies +``` + +### Why none of these is the reconciliation + +The argument rests on a passing oracle, not on inspection: + +1. **`lfm/` consumes exactly one thing from the artifact — `program()`** — and + the round-trip suite asserts `program()` reproduces `air.constraint_program()` + bit-for-bit for every production AIR. So the LFM machine's input is provably + identical to what it was pre-merge. +2. **`device_program()` — the only other function whose output changed — has one + caller in the entire tree**: `constraint_artifact_tests.rs:97`, which passes. + `grep` over `prover/src` and `crypto/stark/src` finds no other call site, and + none in `lfm/`. +3. **`keccak_probe.rs` contains zero occurrences of `artifact`**, yet two of its + tests are in the new-failure list. + +### What they actually are + +Two are pinned design tables invalidated by main adding a table: + +- `constraint_leg_instruction_census` dies on **`no design entry for HINT`**. + HINT is new on main (`grep -c hint prover/src/tables/mod.rs`: 0 at branch + `HEAD`, 1 in the merged tree). Note the census itself ran fine and printed a + full, sane per-table node/leaf/fused/emitted table — the node-index walk over + `artifact.nodes` works; it is the *design table* that lacks the new row. +- `continuation_epoch_constraint_leg_cost`: `the design's intermediate-epoch + budget no longer reproduces, left: 62375 right: 63393` — same cause, the + production AIR set and its constraint counts moved. + +The other 20 are LFM machine proof/verify failures ("the machine proof of +sample() must verify", "the joined run must verify", "the registered keccak256 +program must verify", …). `git log HEAD..origin/main -- prover/src/lfm/ +crypto/stark/src/` shows main brought in **#909 "pin each trace-opening column +width to the AIR, not just their sum"** among others; the recursion machine +hand-builds the verifier it proves, so a verifier-side wire change plus a new +production table is the shape of drift that breaks this whole cluster at once. + +**Recommendation:** treat the 22 as a separate reconciliation item for whoever +owns the LFM machine in this merge. The HINT design-table entry looks like the +cheapest first thread to pull — it is a known-missing row, and the budget number +downstream of it is a single pinned constant. diff --git a/thoughts/shared/lfm-real-hash/permute-socket-cost.py b/thoughts/shared/lfm-real-hash/permute-socket-cost.py new file mode 100644 index 000000000..3772af14d --- /dev/null +++ b/thoughts/shared/lfm-real-hash/permute-socket-cost.py @@ -0,0 +1,79 @@ +""" +Cost model for the candidate permute sockets, VALIDATED against the gated census. + +The gate's own model (`gate-oracle/chip_model.py`) is pinned to the committed +chip, so it must not be edited for a costing exercise. Instead the per-item costs +are re-expressed as a closed formula here and the formula is CHECKED against the +gated numbers first: if it cannot reproduce compress at both round counts, it is +not allowed to price anything else. + +Per-item costs, all from the gated model: + frozen socket prefix 28 cells (12 IN + 4 S + 12 OUT; MU is preprocessed) + input lane 4 cells, 2 AreBytes sends (a lane's 4 byte columns) + G-instance 60 cells, 24 sends (16 ByteAlu[XOR] + 8 AreBytes) + output word 4 cells, 4 ByteAlu[XOR] sends + host LfmMem tuples 6 sends + aux 3 * ceil(sends / 2) +""" + +import math + +PREFIX = 28 +CELLS_PER_LANE, SENDS_PER_LANE = 4, 2 +CELLS_PER_G, SENDS_PER_G = 60, 24 +CELLS_PER_OUTW, SENDS_PER_OUTW = 4, 4 +IO_SENDS = 6 + + +def census(rounds: int, lanes: int, out_words: int) -> dict: + num_g = 8 * rounds + main = (PREFIX + lanes * CELLS_PER_LANE + num_g * CELLS_PER_G + + out_words * CELLS_PER_OUTW) + sends = (lanes * SENDS_PER_LANE + num_g * SENDS_PER_G + + out_words * SENDS_PER_OUTW + IO_SENDS) + aux = 3 * math.ceil(sends / 2) + return {"main": main, "sends": sends, "aux": aux, "cell_equiv": main + aux} + + +# --- VALIDATION: the formula must reproduce the GATED compress census --------- +GATED = {7: {"main": 3436, "sends": 1382, "aux": 2073, "cell_equiv": 5509}, + 6: {"main": 2956, "sends": 1190, "aux": 1785, "cell_equiv": 4741}} + +print("VALIDATION -- formula vs the gated compress census (lanes=8, out=4)") +ok = True +for r, want in GATED.items(): + got = census(r, lanes=8, out_words=4) + match = got == want + ok &= match + print(f" {r}r: {got} {'MATCH' if match else 'MISMATCH vs ' + str(want)}") +if not ok: + raise SystemExit("formula does not reproduce the gated census -- refusing to price") + +print("\nCOMPRESS socket (as built): lanes=8 (two cells), out=4 (one cell)") +for r in (7, 6): + print(f" {r}r: {census(r, 8, 4)}") + +print("\nOPTION A permute socket: lanes=12 (three cells), out=12 (three cells)") +for r in (7, 6): + c = census(r, 12, 12) + ratio = c["cell_equiv"] / census(r, 8, 4)["cell_equiv"] + print(f" {r}r: {c} = {ratio:.3f} x one compress") + +print("\nPER-PROGRAM (FriToyV0: 10 permutes + 56 compresses; counted from " + "programs.rs)") +for r in (7, 6): + comp = census(r, 8, 4)["cell_equiv"] + perm = census(r, 12, 12)["cell_equiv"] + a = 10 * perm + 56 * comp + # Option B: the sponge becomes compress-based; 11 compresses replace the + # 10 permutes (see the options paper for the op-by-op derivation). + b = (11 + 56) * comp + print(f" {r}r option A: {a:,} cell-equiv option B: {b:,} " + f"B/A = {b/a:.3f}") + +print("\nTrivialV0: 2 compresses + 1 permute") +for r in (7, 6): + comp = census(r, 8, 4)["cell_equiv"] + perm = census(r, 12, 12)["cell_equiv"] + print(f" {r}r option A: {2*comp + perm:,} option B (3 compresses): " + f"{3*comp:,}") diff --git a/thoughts/shared/lfm-real-hash/permute-socket-options.md b/thoughts/shared/lfm-real-hash/permute-socket-options.md new file mode 100644 index 000000000..8c85f1819 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/permute-socket-options.md @@ -0,0 +1,539 @@ +# The `LFM_HASH` permute socket — options paper + +**This is a decision paper for the user. It is NOT an implementation and NOT a +unilateral pick.** §7 carries a recommendation, clearly marked as mine. + +> **DECIDED — 2026-08-11: the user ratified OPTION B in its B1 form** (the +> Fiat–Shamir sponge becomes a compress-based chain for ALL hashers; no permute +> socket is ever built; `MODE_P` stays pinned to 0 permanently). Decision made +> on this paper's presentation with §6's unsettled items disclosed. Next steps +> per §7: TAG_LFMT allocation, `SpongeVar`+`HostSponge` rewritten together, +> `TrivialV0`'s raw permute resolved, registry re-bless sequenced once, gate +> re-run with the two-tag framing. +> +> **Post-decision update (§8, same day):** the A-TSP research completed AFTER +> ratification and was reported to the user: A-TSP is citable (not novel), so +> §7's reason 1 was overstated at decision time; and the T-sponge entropy-loss +> caveat applies to BOTH options' squeeze runs (§8.3), so that axis separates +> nothing. The decision was reaffirmed on reasons 2–5, and B's transcript spec +> must now carry its own squeeze-run iteration bound (assigned). + +**Date:** 2026-08-11. **Scope:** what to do about `LFM_HASH`'s `permute` mode +under BLAKE3 — the thing that keeps the F3.4 disclosure half-retired. +**Ground:** worktree `lambda_vm-blake3-impl`, branch `blake3-real-hash`, head +`2957c3f9`. No cargo run; costs are priced from the gated census, not estimated. + +Claims are ✓ VERIFIED (read the code, cited), ✓ EXECUTED (ran it), ? INFERRED +(derived, reasoning shown), or ✗ OPEN. + +--- + +## 1. The situation, verified rather than assumed + +**The sponge.** ✓ VERIFIED `edsl.rs:16-60`. `SpongeVar` is an overwrite-rate +duplex: state is **3 cells** (rate = cells 0–1, capacity = cell 2). + +```rust +absorb2(c0, c1): state = permute([c0, c1, state[2]]) // OVERWRITE the rate +squeeze_cell(): out = state[0]; state = permute(state) +``` + +Absorb **overwrites** the rate rather than XOR-ing into it, and the capacity +cell is carried unchanged. `squeeze_ext` takes lanes 0–2 of a squeezed cell; +`squeeze_bits` takes the bit decomposition of lane 0. + +**⚠ The single most important fact in this paper, and it changes the shape of +the decision.** ✓ VERIFIED `edsl.rs:6-10`, quoted in full: + +> *"The duplex sponge here is the machine side of the test transcript and is +> mirrored bit-exactly by `fixture::HostSponge`. Like `TestPermutation` itself it +> is **NOT a production construction** — the real transcript lands with the +> ecosystem hash decision; this one exists so the protocol loop can be built and +> measured now."* + +The sponge is **already scheduled for replacement**, by the same decision this +paper serves. Redesigning it is therefore *the planned work*, not a detour — and +that removes most of the usual objection to option B. + +**Who actually needs `permute`.** ✓ VERIFIED by grepping the whole module: + +| user | how | ops | +|---|---|---| +| `FriToyV0` (`programs.rs:524-640`) | via `SpongeVar` | **10 permutes**, 56 compresses | +| `TrivialV0` (`programs.rs:17-40`) | a **raw `b.permute(...)` call**, not the sponge | 1 permute, 2 compresses | + +Nothing else. The 10 permutes are 6 in the preamble (`absorb`, `squeeze_ext`×2, +`absorb`, `squeeze_ext`, `absorb2`) plus 1 per query × `NUM_QUERIES = 4` +(`squeeze_bits`). The 56 compresses are 4 queries × 14 (leaf + 4-level walk, +twice, plus an L1 leaf + 3-level walk). ✓ VERIFIED against `fixture.rs:26-40` +for the shape constants. + +Note `TrivialV0` calls `permute` **directly**, so it is blocked by this decision +independently of whatever happens to the sponge. Any option that removes the +permute socket must say what happens to that call. + +**What the wrap needs: nothing.** ✓ VERIFIED (F3.4, `F3-hash-chips.md:184-205`): +the epoch verifier hashes with keccak throughout and emits no `Instr::Hash` at +all, so its `LFM_HASH` group is empty. **The wrap is not blocked by this +decision.** Only the two registered `LFM_HASH` programs are. + +**Today's behaviour under BLAKE3 is loud, not silent.** ✓ VERIFIED: the AIR pins +`MODE_P = 0` (constraint idx 5), `admits()` rejects a `Permute` row naming why, +and `Blake3Permutation::permute` panics rather than returning a value the chip +does not prove. So the status quo is *safe*; it is merely incomplete. There is +no soundness fire here, which means this decision can be made on design merit +rather than under pressure. + +**A6R already covers the transcript.** ✓ VERIFIED `A6R-signoff.md:56-66` — the +signed statement reads *"...suitable as a 2-to-1 compression for Merkle hashing +**and as a PRF for Fiat–Shamir**"*, and the sheet says outright that it "covers +the transcript sponge as well as the Merkle compress". So using BLAKE3 for the +transcript invokes **no assumption that is not already signed**. This matters: +it means options A and B differ in *construction* risk, not in *primitive* risk. + +**What Fiat–Shamir actually needs here.** ? INFERRED, and it is the crux. The +protocol is public-coin: every absorbed value (`main_root`, `l1_root`, `t0`, +`t1`) is a public commitment, and every squeezed value (`alpha`, `zeta0`, +`zeta1`, query bits) is a public challenge. The requirement is that a challenge +be a random-oracle function of everything committed before it, so the prover +cannot grind or predict it before committing. **Secrecy of the capacity is not +required** — there is no secret in the transcript. That observation is what makes +option B's much simpler construction legitimate; a sponge's capacity buys +security against an adversary who sees only the rate, which is not the threat +model here. + +--- + +## 2. Option A — a compress-derived transform under the reserved `"LFMP"` tag + +The `SOCKET.md` §7 direction (which that document is careful to label *"a sketch, +not a decision — unreviewed"*). + +**Framing.** `h = IV`; `m[0..12] = state`; `m[12] = "LFMP"`; `m[13..16] = 0`; +`t = 0`; `block_len = 52`; `flags = 0x0B`; new state = `out[0..12]`. + +**Security property required.** The state-update map must behave as a random +transformation on 12 words. Note precisely what it is *not*: `out[0..12]` is 12 +of the 16 output words of a compression function, so it is **not a permutation** +— it is non-invertible. The standard sponge proof is for a random *permutation*; +this needs the random-*transformation* variant (the "T-sponge" of +Bertoni–Daemen–Peeters–Van Assche), which gives essentially the same bound. That +is a defensible but **different theorem**, and it is a construction assumption +that does not exist today. + +> **Named assumption this option would add — it must be signable, like A6R:** +> **A-TSP.** *The overwrite-rate duplex with rate 2 cells and capacity 1 cell, +> instantiated with `T(state) = BLAKE3-compress(IV, state‖"LFMP"‖0, t=0, +> block_len=52, flags=0x0B)[0..12]`, is indifferentiable from a random oracle up +> to ~2^64 queries.* At 7 rounds this rests on BLAKE3 plus the T-sponge theorem; +> at 6 rounds it additionally invokes A6R. + +**KAT-ability: ✓ EXECUTED, and it is good news.** `out[0..16]` is exactly the +first 64 bytes of BLAKE3's XOF stream over the 52-byte message, so +`out[0..12]` = XOF bytes 0..48. I ran this against my anchored oracle: + +``` +msg = LE32(state[0..12]) ‖ "LFMP" (52 bytes) +XOF 64B == out[0..16] : True +first 12 (the new state) : True +32B hash == out[0..8] : True +``` + +So at 7 rounds a permute is a direct `blake3::Hasher::finalize_xof()` assertion +against the crate — **the exact property the 7-round decision was bought for is +preserved.** This is option A's strongest point. + +**Cost.** ✓ EXECUTED via `permute-socket-cost.py`, whose formula is validated by +reproducing the gated compress census to the unit before it prices anything: + +| | main | sends | aux | cell-equiv | vs one compress | +|---|---:|---:|---:|---:|---:| +| compress (as built), 7r | 3,436 | 1,382 | 2,073 | **5,509** | 1.000 | +| **A permute, 7r** | 3,484 | 1,422 | 2,133 | **5,617** | **1.020** | +| compress, 6r | 2,956 | 1,190 | 1,785 | **4,741** | 1.000 | +| **A permute, 6r** | 3,004 | 1,230 | 1,845 | **4,849** | **1.023** | + +**One permute ≈ 1.02 compressions** — the mixing core is identical and only the +I/O differs (12 input lanes instead of 8, 12 output words instead of 4). +`FriToyV0` total: **364,674** cell-equiv at 7r. + +**Blast radius.** Large — it is a **second socket**. A second mode in +`blake3_socket.rs` with its own column layout (12 lanes, 12 output words), its +own constraint indices, `NUM_CONSTRAINTS` change, deleting the `MODE_P = 0` pin +(itself a currently-gated constraint), an executor arm, a trace filler, the host +`permute` impl replacing its panic, a KAT file, and a registry re-bless. +Roughly the size of the compress arm again. + +**Gate extension: HIGH feasibility.** The G-core theorem T1 is untouched — same +mixing core, same contracts. Only the framing theorems (T2/T3) and the KATs +change, and my `Framing` dataclass already parameterises `tag_word`, `tag_slot` +and `out_window`; it needs a lane count and a variable window *width*. Every +negative control transfers. The `MODE_P = 0` audit (B0a) would have to be +re-derived, since idx 5 is exactly what this option deletes. + +**F3.4:** fully retired for BLAKE3 programs. + +--- + +## 3. Option B — make the sponge compress-based, so the socket never exists + +Keep `LFM_HASH` **compress-only by design**. `MODE_P` stays pinned to 0 +permanently. No permute socket is ever built. + +**Construction.** State is **1 cell** (128 bits) — the chaining value, which is +BLAKE3's own native shape. + +``` +absorb(c) : state = compress_T(state, c) 1 compress +absorb2(c0, c1) : state = compress_T(compress_T(state, c0), c1) 2 compresses +squeeze_cell() : state = compress_T(state, DOMAIN); out = state 1 compress +``` + +**Domain separation, and the neat part: no new socket is needed for it.** A +transcript step must not be replayable as a Merkle parent, so it needs its own +tag — but the *shape* is unchanged (2 cells in, 1 cell out). Only the constant +`m[8]` differs. Make it a linear form over the **preprocessed** mode columns, +`m[8] = MODE_C·TAG_LFMC + MODE_T·TAG_LFMT`: prover-unchosen, essentially free in +cells, no new layout. ? INFERRED but well-supported — the existing arm already +computes `S_k = MODE_P·IN + MODE_C·IV` in exactly this shape (idx 0–3). + +**Security property required — and this is option B's real advantage.** This is +the textbook Fiat–Shamir transcript: a hash chain. What it needs is that the +chain is collision-resistant and the challenge derivation is a random oracle, +which is **precisely and only what A6R already asserts**. There is no T-sponge +theorem, no capacity argument, no overwrite-mode analysis, and (per §1) no need +for a secret capacity in a public-coin protocol. + +> **New named assumption required: NONE beyond A6R.** That is the difference +> between B and A, and it is worth more than the 1.2% cost gap between them. + +**Security bound.** State is 1 cell = 128 bits → ~64-bit collision resistance, +by the birthday bound. Identical to option A's (whose capacity is also one +128-bit cell) and identical to the digest's. All three options land on the same +number, because it is dictated by `HASH_DIGEST_FELTS = 4`, not by the +construction. Nothing here makes it worse. + +**KAT-ability.** Every transcript step is an ordinary socket compress, so it is +KAT-able exactly as the compress socket already is — `blake3::hash(a‖b‖tag)` +truncated. **No new KAT machinery at all.** + +**Cost.** ✓ EXECUTED. `FriToyV0`'s 10 permutes become 11 compresses (1 + 1 + 1 + +1 + 1 + 2 + 4, from the op-by-op derivation above): + +| program | option A | option B | B/A | +|---|---:|---:|---:| +| `FriToyV0`, 7r | 364,674 | 369,103 | **+1.2%** | +| `FriToyV0`, 6r | 313,986 | 317,647 | +1.2% | +| `TrivialV0`, 7r | 16,635 | **16,527** | **−0.6%** | + +**Cost is a tie.** It does not decide this. + +**Blast radius.** Moderate, and mostly in code that is already marked +provisional: `edsl.rs`'s `SpongeVar` (~45 lines), `fixture.rs`'s `HostSponge` +mirror (~45 lines), and `TrivialV0`'s raw `b.permute` call. **`programs.rs` need +not change at all** if `SpongeVar` keeps its public method signatures — +`fri_toy_program_source` calls only `absorb`/`absorb2`/`squeeze_ext`/ +`squeeze_bits`. Registry re-bless: all `program_id`s move, but Phase 3 moves +them anyway when the hasher tag enters the preimage, so this is close to free at +the protocol level if sequenced with Phase 3. + +**Does the eDSL fork per hasher?** The lead asked explicitly, and the answer +matters: + +- **B1 — the sponge becomes compress-based for ALL hashers. ★ the right + version.** `Test` and `Poseidon` both implement `compress`, so nothing breaks. + One transcript construction, one security argument, one host mirror. It also + opens the door to dropping `permute` from `LfmHasher` entirely later. +- **B2 — fork per hasher (permute-based for Test/Poseidon, compress-based for + BLAKE3). ✗ reject.** Two transcript constructions means two security + arguments, two host mirrors, and a program whose *meaning* depends on which + hasher verified it. That is a trap, not a compromise. + +Costs of B, stated honestly: it changes a shared construction that the +Test/Poseidon paths currently exercise green; it removes `TrivialV0`'s +deliberate permute coverage (that program would need to either drop the call or +be retained as a Test/Poseidon-only fixture); and if the ecosystem later wants a +genuine 12-felt sponge for some other protocol, B does not provide one — though +**A can always be added later on top of B**, which is not true in reverse. + +**Gate extension: TRIVIAL.** The gated surface does not change at all. The +75/75 board already covers the compress socket, and the two-tag variant is a +`Framing.tag_word` parameter my model already has. The `MODE_P = 0` audit (B0a) +stays valid *permanently* instead of being deleted. + +**F3.4:** fully retired for BLAKE3 programs. + +--- + +## 4. Option C — mixed hasher: BLAKE3 compress, Poseidon permute + +**Cost — and this is C's case.** ✓ EXECUTED, derived from +`chips.rs:515-578` + `poseidon.rs:48-53` (30 rounds, 8 full + 22 partial, S-box +on all 12 lanes in full rounds and lane 0 in partial): + +Poseidon's arm is **584 appended witness cells + 28 shared prefix = 612 main**, +601 constraints, and — because it is pure field arithmetic with **no BITWISE +traffic at all** — only the 6 `LfmMem` sends, so aux ≈ 9. **≈ 621 cell-equiv per +permute, ~9× cheaper than a BLAKE3 permute (5,617).** + +| program | option A | **option C** | C vs A | +|---|---:|---:|---:| +| `FriToyV0`, 7r | 364,674 | **314,714** | **−13.7%** | + +That is a real saving and it should not be dismissed. + +**What it costs instead.** + +- **Two primitives in the trusted base**, two KAT stories, two parameter + sign-offs. Poseidon-Goldilocks's round counts and MDS would need their own + review; ✓ VERIFIED the file cites Plonky3 and pins a known-answer vector, but + "matches Plonky3's vector" establishes *correctness of transcription*, not + *security of the parameters for this use*. +- **F3.3 blocks it today.** ✓ VERIFIED (recorded in the same findings file): the + registry verify path cannot reach Poseidon; it is measurement-only. That must + be fixed first. +- **The registry binds one hasher per entry.** Phase 3 added `hasher: + HasherKind` as a single field folded into `lfm_program_id`. A mixed machine + needs either two fields or a composite variant + (`Blake3CompressPoseidonPermute`) — a wire-format and digest-preimage change, + and a new way for the binding to be got wrong. +- **The disclosure gets stranger, not simpler.** F3.4 *would* be retired — both + are real hashes — but it is replaced by a standing note that *this machine's + Merkle tree and its Fiat–Shamir transcript rest on different primitives*. That + is an unusual sentence to have to write, and it doubles the cryptanalytic + surface a reviewer must cover. + +**Gate extension.** My gate says nothing about Poseidon and would not; a second +gate for the Poseidon arm is a separate project of comparable size to the BLAKE3 +one. Feasible in QF-BV but far less natural — Poseidon is field arithmetic, so +a bit-vector model is the wrong tool and it would want a field-domain gate +throughout. + +--- + +## 5. Comparison + +| | **A — `"LFMP"` transform** | **B — compress-based sponge** | **C — mixed hasher** | +|---|---|---|---| +| new named assumption | **A-TSP** (T-sponge instantiation) | **none beyond A6R** | Poseidon parameter sign-off | +| security argument | random-transformation duplex; new theorem | textbook FS hash chain | two independent arguments | +| security bound | ~64-bit (128-bit capacity) | ~64-bit (128-bit state) | ~64-bit / Poseidon-dependent | +| KAT-able vs `blake3` crate | ✓ EXECUTED (XOF 64B = `out[0..16]`) | ✓ same as compress, no new machinery | partially — Poseidon has no published crate KAT | +| cost / permute-equivalent | 5,617 (1.02× compress) | 5,509 (1 compress) | **621 (0.11×)** | +| `FriToyV0` total, 7r | 364,674 | 369,103 (+1.2%) | **314,714 (−13.7%)** | +| blast radius | **large** — a second socket, layout, executor, filler, KATs | moderate — `edsl.rs` + `fixture.rs` + one call site | large — F3.3 fix, registry shape, second gate | +| `MODE_P` | un-pinned; idx 5 deleted | **stays pinned to 0 permanently** | un-pinned (Poseidon uses it) | +| gate extension | high feasibility, real work | **trivial — surface unchanged** | separate project, wrong tool | +| primitives in TCB | 1 | **1** | 2 | +| retires F3.4 | yes | yes | yes | +| reversible? | adds a permanent socket | **yes — A can be added later on top** | registry shape change is sticky | + +--- + +## 6. What I could not settle + +- ~~✗ **The T-sponge bound for this exact construction**~~ → **RESOLVED in §8** + (2026-08-11). It is citable: Eurocrypt 2008 covers random transformations, and + duplex + overwrite mode compose onto it. **But** transformation-based sponges + carry a cryptanalytic caveat (entropy loss under iteration) that applies to + option B as well — see §8.2/§8.3. Read §8 before using §7. +- ✗ **Poseidon's cost is a column count, not a measurement.** 621 cell-equiv is + derived from the AIR's own `const fn`s (✓ VERIFIED arithmetic) but nothing was + proved or benched. +- ✗ **Whether the ecosystem's real transcript will want a sponge shape.** If the + eventual production transcript is specified as a sponge by an external + standard, B's chain would have to be revisited. Nobody has told me what that + transcript is; `edsl.rs:6-10` says it "lands with the ecosystem hash decision", + which is this one. + +--- + +## 7. ★ MY RECOMMENDATION (the decision is the user's) + +**Take option B — redesign the sponge as a compress-based chain, in its B1 form +(compress-based for all hashers), and never build a permute socket.** + +Five reasons, in the order I weight them: + +1. **It needs no new assumption.** A6R already covers PRF-for-Fiat–Shamir, and + B's construction is the textbook FS transcript. Option A needs A-TSP — a new, + signable, currently-unwritten construction assumption — and §6 says I could + not settle its bound. Adding an assumption to a project whose whole thesis is + "one primitive, externally anchored" is the wrong direction. +2. **It removes a gated surface instead of adding one.** `MODE_P = 0` stays + pinned permanently, the 75/75 board keeps covering everything, and the gate + extension is a `tag_word` parameter I already have. A adds a second socket + that needs its own layout, its own KATs, its own gate pass, and deletes the + idx-5 audit in the process. +3. **Cost does not decide between A and B** — 1.2% on `FriToyV0`, and B is + *cheaper* on `TrivialV0`. Anyone choosing A for performance is paying an + assumption for noise. +4. **The sponge is already slated for replacement.** `edsl.rs:6-10` says so in + as many words. B is the scheduled work; A builds a permanent socket to + preserve the shape of a construction that is explicitly provisional. +5. **B is reversible and A is not.** A permute socket, once registered, has a + `program_id`-bearing footprint forever. If the ecosystem later demands a true + 12-felt sponge, A can be added on top of B; B cannot be recovered after A. + +**On option C:** its 13.7% saving is real and it is the only option that would +change my mind on cost grounds — but it puts a second primitive in the trusted +base to save one-seventh of two test programs that the wrap does not even use. +I would revisit C only if a *production* workload turns out to be +transcript-dominated, which today's numbers say it is not (56 compresses to 10 +permutes in the one program that has both). + +**If the user picks B, the next steps are:** sequence it with Phase 3 so the +`program_id` re-bless happens once; add the `TAG_LFMT` allocation beside +`"LFMC"`/`"LFMP"`/`"LFML"`; decide `TrivialV0`'s fate (drop its raw `permute`, or +keep the program as a Test/Poseidon-only fixture); rewrite `SpongeVar` + +`HostSponge` together so the bit-exact mirror property is preserved; and re-run +the gate with the two-tag framing, which is a parameter change rather than new +gate code. + +**If the user picks A instead**, the work is well-understood and my gate extends +cleanly — but A-TSP must be written down and signed *before* the arm is built, +the same way A6R was, and `SOCKET.md` §7's sketch should not be treated as the +spec until it has had the review the compress socket got. + +--- + +## 8. ADDENDUM (2026-08-11) — A-TSP researched: it is citable, with a caveat. +## This UPGRADES option A and I am reporting it against my own recommendation. + +§6 listed A-TSP as ✗ OPEN — cited from memory, unsigned. I went and checked. +The result is better for option A than I represented, and worse in one specific +way that nobody had named. Both directions below. + +### 8.1 The theorem is real and it composes + +- **Sponge indifferentiability holds for a random TRANSFORMATION, not only a + permutation**, up to the birthday-type bound `O(2^{c/2})` — Bertoni, Daemen, + Peeters, Van Assche, *On the Indifferentiability of the Sponge Construction*, + Eurocrypt 2008. This is the load-bearing citation and it directly covers the + fact that `out[0..12]` is non-invertible. +- **Duplex security reduces to sponge indifferentiability**, same `O(2^{c/2})` + bound — *Duplexing the Sponge*, SAC 2011. +- **Overwrite-mode absorb is a known, analysed variant**: the XOR at absorb can + be omitted while maintaining the chosen security level. + +With `c` = 1 cell = 128 bits that is `O(2^64)` — the number already stated in +§2, now with a reference under it rather than my recollection. + +**So A-TSP is no longer an unwritten assumption; it is a composition of three +published results.** That was my first reason for preferring B, and it is +materially weaker than I wrote. Stated plainly because it argues against me. + +### 8.2 The caveat, which is specific to transformation-based sponges + +T-sponges have a **dedicated cryptanalytic literature that P-sponges do not**, +and a broken real-world instance: + +> *Collision Spectrum, Entropy Loss, T-Sponges, and Cryptanalysis of GLUON-64* +> (FSE 2014). Iterating a permutation loses no entropy; iterating a +> **transformation** does — the image shrinks with each application, collision +> trees grow quadratically, and certain collision-spectrum and rate values yield +> **improved preimage attacks on long messages**. GLUON-64 was broken this way. + +Option A's map is a transformation, so it sits in exactly that family. The +attacks bite on *long* iteration counts; our transcript is ~10 applications, at +which the image has shrunk by around a bit. **Quantitatively irrelevant here — +but that is a regime-dependent argument, and it has to be written down and +bounded rather than assumed.** A6R is not regime-dependent; A-TSP would be. A +signer needs to see the iteration-count bound stated as part of the assumption. + +### 8.3 ⚠ The same caveat applies to option B, and I did not say so before + +Being even-handed: option B's squeeze is `state = compress_T(state, DOMAIN)` +with `DOMAIN` **constant**, so a run of consecutive squeezes iterates a fixed +non-injective map exactly as a T-sponge does. ✓ VERIFIED that such runs exist — +`programs.rs:550-551` squeezes twice back-to-back, and the query loop +(`programs.rs:565-567`) squeezes once per query with no absorb between, so runs +of ~4–5 occur. + +The entropy-loss analysis is therefore **the same for A and B**, and equally +negligible at these lengths. B is not immune, and anything above implying it was +should be read as corrected here. **This axis does not separate the options.** + +### 8.4 Does the recommendation change? No — but the margin narrows + +Reason 1 of §7 ("needs no new assumption") must be restated honestly: + +> B needs collision-resistance and RO-behaviour of the compression function — +> already inside A6R. A needs that **plus** the T-sponge + duplex + overwrite +> composition **plus** a written iteration-count bound. Both are defensible; A +> simply has more moving parts, each of which someone must check. + +That is a real difference but a smaller one than §7 implied. Reasons 2–5 — +removes gated surface rather than adding it, cost is a tie, the sponge is +already slated for replacement, and B is reversible where A is not — are +untouched by this research and are what the recommendation now mostly rests on. + +**Recommendation stands: option B.** If the user prefers A, §8.1 means it can +proceed on citations rather than on a novel assumption — provided A-TSP is +written with the iteration bound of §8.2 in it, and signed, *before* the arm is +built. + +**Sources:** +- [On the Indifferentiability of the Sponge Construction (Eurocrypt 2008)](https://keccak.team/files/SpongeIndifferentiability.pdf) +- [Duplexing the Sponge (SAC 2011)](https://link.springer.com/chapter/10.1007/978-3-642-28496-0_19) +- [Collision Spectrum, Entropy Loss, T-Sponges, and Cryptanalysis of GLUON-64](https://link.springer.com/chapter/10.1007/978-3-662-46706-0_5) +- [The sponge and duplex constructions (keccak.team)](https://keccak.team/sponge_duplex.html) + +### 8.5 "For this exact construction" — the part the general theorems do not cover + +The citations in §8.1 are about *idealised* sponges. They say nothing about the +specific map each option instantiates. Three construction-level notes, the last +of which is a concrete difference between A and B that I had not previously +identified. + +**(i) The idealisation step is itself an assumption.** T-sponge results model `T` +as a *random* transformation. Option A's `T` is BLAKE3's compression with a +**fixed** chaining value (`IV`) and a **fixed** tag word — a single public +function, not a random one. Treating it as ideal is the standard move and is the +same move A6R already makes, but it is a step, and A-TSP's text should contain +it rather than leave it implicit. + +**(ii) Rate exceeds capacity, which is fine but worth stating.** Option A's state +is 3 cells = 384 bits, split rate 256 / capacity 128. The bound depends only on +the capacity, so `O(2^{c/2}) = O(2^64)`; the wide rate buys throughput, not +weakness. Same number as the digest's, from the same 128-bit-cell cause. + +**(iii) ⚠ Option A's 12-word output exposes final-state words that option B's +4-word output does not.** ✓ EXECUTED (200 random inputs, exact): + +With `h = IV`, BLAKE3's output is `out[i] = v[i] ^ v[i+8]` and +`out[i+8] = v[i+8] ^ IV[i]`. Taking **twelve** words therefore publishes both +halves of a cross-relation: + +``` +out[i] ^ out[i+8] == v_final[i] ^ IV[i] (i in 0..8) -> v_final[0..4] recoverable +out[8+i] == v_final[8+i] ^ IV[i] (i in 0..4) -> v_final[8..12] recoverable +``` + +So from one option-A permute output a reader recovers **8 of the 16 final state +words directly** — `v_final[0..4]` and `v_final[8..12]` — by XOR with public +constants. + +Option B's socket publishes **four** of the sixteen words, so the same query +gets nothing comparable: `out[0..4] = v[0..4] ^ v[8..12]`, and with no second +output block to cross-XOR against, the two summands cannot be separated +(✓ EXECUTED, 2000/2000 samples). Twelve words stay unpublished. + +**Is (iii) an attack? I do not have one, and I am not claiming one.** The final +state is still a pseudorandom function of the input, so recovering it from the +output is not obviously exploitable — this is a *structural observation*, of the +kind that belongs in a security argument a reviewer signs rather than in a +footnote. But it is a real asymmetry, it points the same way as everything else +in §7, and it is the sort of thing that has historically been the first step of +a T-sponge attack (§8.2's GLUON-64 line began with structure, not with a break). + +Note also that option A's state includes four words BLAKE3's own chaining value +never propagates: standard BLAKE3 chains on `out[0..8]` and uses `out[8..16]` +only as extended output. Option A would make XOF words part of the *chaining +state* — specified, KAT-able (§2), but a role BLAKE3's designers analyse as +output rather than as state. + +**Net effect on the recommendation: unchanged, slightly reinforced.** §8.1 moved +option A's assumption from "unwritten" to "citable"; §8.5(iii) adds a +construction-level reason that points back the other way. Option B remains the +one with fewer moving parts and less exposed structure. diff --git a/thoughts/shared/lfm-real-hash/phase1-report.md b/thoughts/shared/lfm-real-hash/phase1-report.md new file mode 100644 index 000000000..9a11fb5d2 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/phase1-report.md @@ -0,0 +1,328 @@ +# Phase 1 report — a trustworthy 6-round reference and the KATs it pins + +**Date:** 2026-08-10. **Worktree:** `/Users/maurofab/workspace/lambda_vm-blake3-impl` (branch `blake3-real-hash`). +**Status:** complete, uncommitted, nothing pushed. **No cargo was run.** +**Headline:** the 6-round vector set is no longer single-source, and the anchor holds. + +Claims are marked ✓ VERIFIED / ✓ EXECUTED, ? INFERRED, or ✗ OPEN. + +--- + +## 1. Result in one paragraph + +The external anchor is green and the vectors now rest on **two independent +sources**. `thoughts/blake3/`'s reference material was restored from git; +`test_oracle.py` reproduces the official BLAKE3 vectors at 7 rounds (35 cases × +3 modes) and regenerates `canonical_6round_vectors.json` **byte-identical to the +git blob**. A second source — upstream BLAKE3's own portable **C** +implementation, round-parameterised in a 2-hunk diff — independently reproduces +all ten `CANONICAL_VECTORS` at 6 rounds and matches the Rust constants in +`prover/src/lfm/blake3.rs` directly. The 2-to-1 compress socket is now specified +and pinned with its own vectors and nine negative controls, framed so that at 7 +rounds it is literally `blake3::hash(a ‖ b ‖ "LFMC")` truncated. The A6R sheet +is written and recommends declining the assumption. + +## 2. Task 1 — restoration and the anchor + +All nine artifacts restored into the worktree with `git show`, per plan §9. +✓ EXECUTED. + +| file | source commit | +|---|---| +| `blake3-oracle/{blake3_ref.py, ORACLE.md, test_oracle.py}` | `3b9b8137` | +| `blake3-chip/{DESIGN.md, z3_blake_verify.py}` | `3b9b8137` | +| `blake3-oracle/{official_test_vectors.json, canonical_6round_vectors.json}` | `19ed761b` | +| `ground-truth/{Cargo.toml, src/main.rs}` | `19ed761b` | + +`python3 test_oracle.py` (Python 3.14.6, ~4 s): + +``` +[1] Official test_vectors.json : PASS (35/35 cases x 3 modes) +[2] Official `blake3` PyPI pkg : SKIP (package not importable) +[3] Plonky3 blake3-air (direct): PASS (20000 random compressions, flags=0) +[.] Internal self-consistency : PASS (1000 checks) +[4] 6-round variant derivation : PASS (differs from 7r on 2000/2000) +``` + +**The anchor holds.** ✓ EXECUTED. Anchor 2 (the `blake3` PyPI package) is +unavailable in this environment — noted, not a gap, because Task 2 replaced it +with something stronger (§3). + +Two extra checks beyond the brief, both ✓ EXECUTED: + +- The regenerated `canonical_6round_vectors.json` is **byte-identical** to the + git blob. So the oracle is deterministic across the Python version change, and + the recorded vectors were not edited after generation. +- The ten `CANONICAL_VECTORS` in `prover/src/lfm/blake3.rs:151-342` were parsed + and compared field by field against that JSON: **`h`, `m`, `t`, `block_len`, + `flags` and all 16 `out` words match on all ten**. The transcription into Rust + is exact. (The repo asserts the port *reproduces* the vectors; nothing + previously asserted the transcribed *constants* match the JSON.) + +## 3. Task 2 — the second independent source + +**What I used, and why it beats the brief's suggestion.** The brief proposed +vendoring upstream's `reference_impl/reference_impl.rs`. That file is not +shipped in the published `blake3` crate (✓ VERIFIED — the crate's contents are +`src/`, `c/`, `benches/`, `tools/`; no `reference_impl`). But the crate *does* +ship upstream's portable **C** implementation, `c/blake3_portable.c`, together +with the full tree hasher `c/blake3.c`. That is better on every axis that +matters: same authors as the reference impl, a different language from the +Python oracle, and — decisively — **a different construction of the message +schedule**. The C indexes a precomputed `MSG_SCHEDULE[7][16]` table; the Python +oracle and the Rust port iteratively apply one permutation between rounds. A bug +in the iterative composition is exactly the class of error a single source +cannot catch, and this second source catches it. + +Vendored at `thoughts/blake3/reference-impl/` (BLAKE3 is CC0/Apache-2.0; +`LICENSE_CC0` copied alongside). `upstream/` holds `blake3.c`, +`blake3_dispatch.c`, `blake3_impl.h`, `blake3.h`, `blake3_portable.c` +**verbatim**. The single modified file is `blake3_portable_paramrounds.c`, and +the entire diff is in `PARAMETERISATION.diff` — two hunks: + +1. a `BLAKE3_ROUNDS_PARAM` `#define` defaulting to 7, with an `#error` guard at + `> 7` (`MSG_SCHEDULE` has exactly 7 rows); +2. the seven literal `round_fn(state, &block_words[0], 0..6)` calls replaced by + `for (size_t r = 0; r < BLAKE3_ROUNDS_PARAM; r++) round_fn(state, &block_words[0], r);` + +At the default the loop issues the identical seven calls in the identical order, +so the parameterisation is **inert by inspection** — and then re-checked +empirically. NEON is disabled and no x86 SIMD applies, so the dispatcher resolves +every compression to the portable path; the round knob therefore governs the +*whole tree hasher*, not just a directly-called compress. + +`python3 check.py` (after `./build.sh`, ~2 s), all ✓ EXECUTED: + +``` +PASS [A] parameterised C @ rounds=7 vs official vectors (35 cases x 3 modes) +PASS [B] rounds=6 differs from rounds=7 on all 8 probe lengths +PASS [C] MSG_SCHEDULE[r] == permute^r(identity) for r in 0..7 +PASS [C] MSG_SCHEDULE[1] == the repo's BLAKE3_MSG_PERMUTATION +PASS [D] C @ rounds=6 == canonical_6round_vectors.json (all 10, 16 words) +PASS [D] C @ rounds=6 == Rust CANONICAL_VECTORS in prover/src/lfm/blake3.rs +PASS [D] Rust vector INPUTS == JSON vector inputs +PASS [D] negative control: C @ rounds=7 matches none of the 10 vectors +PASS [E] C vs Python oracle @ rounds=7 (5000 random compressions) +PASS [E] C vs Python oracle @ rounds=6 (5000 random compressions) +``` + +**Plan §2.2 step 3's acceptance criterion is met: both sources, at rounds = 6, +reproduce all ten `CANONICAL_VECTORS` byte for byte.** Check [C] is the one that +earns the "independent" label — it proves the two *different* schedule +constructions denote the same function. + +**Deferred to a build phase (✗ OPEN, needs cargo):** the equivalent check +against the Rust `blake3` crate, i.e. plan §2.2 step 4's direct KAT of `f` at +7 rounds via `blake3::hash`. Low risk — the C that was checked *is* upstream +BLAKE3 and passes the official vectors in three modes — but it should still be +written, because it is the form of the check that survives this directory being +deleted. `thoughts/blake3/ground-truth/` (restored, links the real crate) is the +place for it. + +## 4. Task 3 — the socket specification and its KATs + +`thoughts/blake3/socket-kats/` — `SOCKET.md` (the spec), `gen_socket_kats.py` +(the generator and its checks), `socket_kats.json` (the vectors). + +**The decision, as instructed: Option A + domain separation, 128-bit digest.** +Byte-level normative form: + +``` +msg = LE32(a0..a3) ‖ LE32(b0..b3) ‖ "LFMC" (36 bytes) +c = LE32⁻¹( BLAKE3(msg)[0..16] ) (4 u32 lanes, 1 cell) +``` + +Realised as one compression: `h = IV`, `m[0..4] = a`, `m[4..8] = b`, +`m[8] = 0x434D464C`, `m[9..16] = 0`, `t = 0`, `block_len = 36`, +`flags = 0x0B (CHUNK_START|CHUNK_END|ROOT)`, digest = output words `0..4`. + +**The one design choice worth surfacing: the domain tag goes in the message, not +in `flags`.** Plan §5 option D says "domain separation in `flags`", which is +where BLAKE3 itself puts domain bits. But any tag in `flags` (or `t`, or `h`) +makes the socket a *nonstandard* invocation that no library computes, so its +KATs could only ever come from our own oracle — at 7 rounds as well as at 6, +which throws away the main reason to prefer 7. Putting the tag in the message +keeps the socket a standard BLAKE3 hash of a domain-separated byte string, at a +cost of 4 bytes in a block that had 28 spare. Domain separation is equally real. + +Vectors: 10 inputs × 2 round counts, all inputs written out explicitly (no RNG +dependence), each with **9 negative controls** — `swap_a_b`, `tag_changed`, +`tag_omitted`, `truncate_high_half`, `flags_parent`, `block_len_64`, +`counter_one`, `lanes_big_endian`, `other_round_count`. Three computations must +agree per vector (Python word-level, C word-level, C **whole-tree** byte-level); +the generator fails loudly otherwise. All ✓ EXECUTED, both round counts. + +The 7-round cross-check the brief flagged for the build phase is **already +executed** here, against upstream C rather than the Rust crate: every 7-round +vector equals `BLAKE3(a ‖ b ‖ "LFMC")` truncated to 16 bytes. Only the +`blake3`-crate restatement remains ✗ OPEN. + +**A control that fired, and what it taught.** `lanes_big_endian` initially +failed on three of the ten vectors — `zeros`, `all_ones`, `nibble_ramp`. Not a +bug: every lane of those inputs is a byte-palindrome (`0x00000000`, +`0xFFFFFFFF`, `0x11111111`, …), so byte-order cannot be observed on them. Rather +than skip it, the generator now declares applicability per control per vector +**and separately asserts every control is discriminated by at least one +vector** — otherwise a framing degree of freedom would sit unpinned behind a +green run. Worth keeping in mind for the chip tests: three of the five obvious +structural inputs cannot detect a byte-order error. + +## 5. Task 4 — the A6R sheet + +`thoughts/shared/lfm-real-hash/A6R-signoff.md`. One page, with a signature block +offering "decline (recommended)" or "sign, and complete these four record +items". Recommendation: **decline A6R, instantiate 7 rounds.** + +Two findings that changed the sheet relative to the plan: + +**(a) ⚠ `PLAN.md` §7 misquotes the spec, in the strengthening direction.** The +plan renders the external-review note as ending *"variants below 6 rounds are out +of scope and MUST NOT be instantiated."* The actual text +(`git show 783c5a95:spec/blake3.typ`, ✓ VERIFIED by reading) says variants below +6 rounds are *"not formally ruled out, but they are not available on the +project's own authority"* — a procedural bar requiring dedicated external +cryptanalysis, not a prohibition. The sheet quotes the source. The plan should be +corrected. + +Reading the source also surfaced context the plan omits, in both directions: the +precedent argument for A6R (KangarooTwelve; "the margin removed here is one round +of seven"), and the fact that **A6R covers Fiat–Shamir as well as Merkle +compression** — "suitable as a 2-to-1 compression for Merkle hashing *and as a +PRF for Fiat–Shamir*". Both are in the sheet. + +**(b) ⚠ The sheet's recommendation reverses the spec's recorded default,** which +says "the 6-round variant is the primary internal target… the 7-round variant is +the interoperability / zero-assumption fallback". That disagreement is now stated +explicitly in the sheet rather than left implicit, with the note that accepting +the recommendation requires updating `spec/blake3.typ` or the tree will carry two +contradictory statements of intent. + +**Cost numbers re-derived from source, not taken from the plan.** ✓ VERIFIED +`4,946 = MAIN_COLUMNS 3,056 + 3 × aux 630`, `interactions = 11 + 832 + 384 + 32 += 1,259` (`blake3_probe.rs:327-356`). ? INFERRED for 7 rounds: `5,714` per +compression (+15.5%), epoch column `2.903 B` (+5.5%), `3.85×` keccak. The plan's +arithmetic checks out. I added an independent cross-check the plan asserts but +does not show: applying +15.5% to the spec's own table-only figure (5,316 of +7,194 end-to-end) gives +11.5% end-to-end, inside the spec's independently +stated "10–12%". + +**And the finding that most changes the decision's price:** ✓ VERIFIED **the +chip is already round-parameterised.** `NUM_G = BLAKE3_ROUNDS * 8` +(`blake3_chip.rs:98`), the layout derives from `NUM_G` (`:157`), the dataflow +loops `for r in 0..BLAKE3_ROUNDS` (`:280`), and +`NUM_CONSTRAINTS = 16 × NUM_G + 1` (`:1042`). So "build round-parameterised" is +already done; choosing 7 is a constant, plus regenerating four hard-coded test +expectations (`2,880 → 3,360`, `1,259 → 1,451`, `4,946 → 5,714`, `769 → 897`) +and 7-round vectors — which, unlike the 6-round ones, come straight from the +crate. + +## 6. Findings for other phases + +1. **✗ OPEN — the `permute` socket is unspecified, and Phase 5's E1 claim + depends on it.** ✓ VERIFIED: `edsl::merkle_walk` calls `b.compress` + (`edsl.rs:75`) but `edsl::SpongeVar` calls `b.permute` (`edsl.rs:31,43`). So + `FriToyV0`'s Fiat–Shamir sponge rests on `permute`, not `compress`. + Specifying the compress socket makes Merkle authentication real; **it does not + on its own retire the F3.4 disclosure**, which covers the sponge too. + `SOCKET.md` §7 sketches a mapping (12 lanes = 48 bytes fits one block) and + marks it explicitly as a sketch with no vectors and no security argument. +2. **Soundness obligation for Phase 2 (`SOCKET.md` O1).** `merkle_walk`'s sibling + digests are **arena-hinted, i.e. prover-chosen** — the doc comment at + `edsl.rs:63-64` says so. A lane is a Goldilocks felt over `[0, p)`, `p ≈ 2^64`. + If the chip derives message bytes by reduction mod 2^32 instead of a checked + 32-bit decomposition, `v` and `v + 2^32` give the same digest: a + prover-chosen collision, hence a forged Merkle path. Input lanes must be + range-checked in the chip, and the host impl must **reject** rather than + silently reduce. This is plan §3.2's failure mode, on the socket's input side. +3. **⚠ Phase 3 is being written concurrently in this same worktree — see §9.** + `HasherKind` now has explicit `#[repr(u8)]` discriminants and `as_tag()` + (`hash.rs:107-127` in the *working tree*), which is plan §4 step 1. I first + recorded this as "already done"; that was wrong. ✓ VERIFIED by + `git show HEAD:prover/src/lfm/hash.rs`: `as_tag` **does not exist at HEAD**. + It is another agent's **uncommitted** work, along with edits to + `statement.rs`, `registry.rs`, `proof.rs` and `compute_lfm_registry.rs` — + exactly plan §4's file list. **Since resolved:** that work landed as + `2d236786 feat(lfm): bind the hasher into the program digest and registry`, + so `as_tag` and the digest binding are now at `HEAD` and their line numbers + are stable again. The lesson stands — I recorded a concurrent agent's + in-flight edit as pre-existing repo state, which `git show HEAD:` caught. +4. **`compress_iv()` is dead weight under BLAKE3** (`SOCKET.md` O3). BLAKE3's IV + enters through `h` (all 8 words), not through state lanes 8–11, so the arm + overrides `compress` wholesale — explicitly permitted by `hash.rs:25-26`. The + override must be wired into `HasherKind::compress`'s explicit delegation + (`hash.rs:146-151`), whose own doc comment warns about precisely this. + +## 7. What needs a build phase + +| check | why it is deferred | +|---|---| +| `blake3::hash` restatement of the 7-round KAT of `f` (plan §2.2 step 4) | needs cargo; `ground-truth/` is the place | +| `blake3::hash(a ‖ b ‖ "LFMC")` restatement of the socket identity | needs cargo (already executed against upstream C) | +| the four projected 7-round constants (`3,360 / 1,451 / 5,714 / 897`) | compile-time consts; one `cargo test` confirms | +| chip `OUT` columns vs the socket vectors | no chip arm exists yet (Phase 2) | +| plan §2.2 step 6 — a CI job that re-derives the chain | nothing re-derives it today; `check.py` + `gen_socket_kats.py` + `test_oracle.py` are the three commands | + +## 8. Files + +Under `/Users/maurofab/workspace/lambda_vm-blake3-impl/` (all **uncommitted**; +`thoughts/blake3/` is untracked, no repo source was modified): + +- `thoughts/blake3/blake3-oracle/` — restored; `test_oracle.py` is the anchor run +- `thoughts/blake3/blake3-chip/` — restored (`DESIGN.md`, `z3_blake_verify.py`) +- `thoughts/blake3/ground-truth/` — restored; the crate-linked project for the deferred checks +- `thoughts/blake3/reference-impl/` — **new.** `upstream/` verbatim, + `blake3_portable_paramrounds.c` + `PARAMETERISATION.diff` (the 2-hunk edit), + `driver.c`, `build.sh`, `check.py` +- `thoughts/blake3/socket-kats/` — **new.** `SOCKET.md`, `gen_socket_kats.py`, `socket_kats.json` + +Under `/Users/maurofab/workspace/lambda_vm/thoughts/shared/lfm-real-hash/`: +`A6R-signoff.md`, `phase1-report.md` (this file). + +Build products `b3ref6`, `b3ref7` are regenerable (`./build.sh`) and are +`.gitignore`d, along with `__pycache__/`. `PARAMETERISATION.diff` is regenerable +but should be committed — it is the reviewable artifact. + +## 9. Two agents shared this worktree — RESOLVED, kept for the lesson + +> **Resolution (2026-08-10, after the fact).** This warning is historical; the +> hazard did not fire. Both bodies of work landed as separate clean commits — +> `2d236786 feat(lfm): bind the hasher into the program digest and registry` +> (the other agent's), then `65025095 test(blake3): restore the +> round-parameterized reference and add a second independent source` (mine, 25 +> files, all under `thoughts/blake3/`, no `prover/` source swept in). The +> worktree is clean and `as_tag` is now at `HEAD`. ✓ VERIFIED. The section below +> describes the situation as it stood mid-phase; keep it as the record of why +> commits were made by explicit path. + +⚠ **As it stood mid-phase — read before committing:** + +`/Users/maurofab/workspace/lambda_vm-blake3-impl` contains **uncommitted changes +to 12 tracked source files that are not mine**: `hash.rs`, `statement.rs`, +`registry.rs`, `proof.rs`, `compute_lfm_registry.rs`, `mod.rs` and six test +modules (+337 / −87). That is plan §4's file list — another agent is writing +Phase 3 here concurrently. ✓ VERIFIED by `git diff` and by confirming `as_tag` +is absent from `HEAD`. + +**My own changes touch no tracked file.** Everything I produced is untracked and +lives under `thoughts/blake3/` (plus the two docs in the main checkout). So +`git add thoughts/blake3` is safe; `git add -A` or `git commit -a` would sweep up +another agent's half-finished Phase 3 and commit it under a Phase 1 message. + +Two consequences worth acting on: + +- **Committing per phase does not work while the worktree is shared.** Either + give Phase 3 its own worktree, or commit Phase 1 by explicit path. +- **Line citations into `prover/src/lfm/*.rs` are unstable right now.** Mine that + point below `hash.rs:98` (the trait, its default `compress`, the digest + constants) are unaffected — ✓ VERIFIED, the concurrent diff is entirely at + line 98 and after. Citations at or after that point, including + `HasherKind::compress`'s delegation, are working-tree line numbers and will + move. + +**Three commands reproduce everything:** + +``` +python3 thoughts/blake3/blake3-oracle/test_oracle.py +thoughts/blake3/reference-impl/build.sh && python3 thoughts/blake3/reference-impl/check.py +python3 thoughts/blake3/socket-kats/gen_socket_kats.py +``` diff --git a/thoughts/shared/lfm-real-hash/phase2-report.md b/thoughts/shared/lfm-real-hash/phase2-report.md new file mode 100644 index 000000000..d9bbd75a0 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/phase2-report.md @@ -0,0 +1,548 @@ +# Phase 2 — BLAKE3 as a first-class `LFM_HASH` hasher + +**Status:** GREEN. `lfm_prove_with_hasher(…, HasherKind::Blake3)` proves and the production +verifier accepts, at **both** 6 and 7 rounds, with the digest matching the socket KATs at +both. `make lint` and `make fmt` clean. No new test failures. + +**Worktree:** `/Users/maurofab/workspace/lambda_vm-blake3-impl`, branch `blake3-real-hash`, +**all changes uncommitted** as instructed. +**Route:** A (host behind the frozen socket). **Mapping:** Option A + domain separation. +Both as locked by the plan. +**Date:** 2026-08-10. + +Claims are marked ✓ EXECUTED (a test ran and passed, named), ✓ VERIFIED (read the code) or +? INFERRED (arithmetic shown). + +--- + +## 1. Headline numbers + +| | 6 rounds (A6R) | 7 rounds (standard, **default**) | +|---|---|---| +| main (value) columns / compression | **2,956** | **3,436** | +| bus interactions / compression | **1,190** | **1,382** | +| aux columns (`⌈interactions/2⌉`) | **595** | **691** | +| **base-field-equivalent cells** (`main + 3·aux`) | **4,741** | **5,509** | +| constraints | **794** | **922** | +| max degree | 3 | 3 | + +✓ EXECUTED — `blake3_socket_tests::the_socket_budget_is_the_predicted_one_at_both_round_counts` +pins all ten numbers as literals against a closed form; `the_built_layout_matches_the_prediction` +and `the_census_prices_the_blake3_arm` confirm the *built* layout equals the prediction, and +the whole suite was run once per round count (`--features blake3-6round`), so both columns +are measured rather than one measured and one projected. + +> ### ⚠ CORRECTION BOX — this table is a 2026-08-10 snapshot, overtaken TWICE since +> +> **The figures above are what the socket cost on the date of this report.** They are left +> as measured rather than edited, because a dated report that quietly acquires today's +> numbers stops being evidence of anything. What has moved, in order: +> +> | main columns | 6r | 7r | when | +> |---|---:|---:|---| +> | as reported here | 2,956 | 3,436 | 2026-08-10 | +> | + the LEAF mode's canonicity block (`Z`/`GINV` per felt, present on every row) | 2,964 | 3,444 | option C | +> | + the leaf RATE's four extra lanes (COMMIT.md §1.2) | **2,980** | **3,460** | 2026-08-13 | +> +> | cells (`main + 3·aux`) | 6r | 7r | when | +> |---|---:|---:|---| +> | as reported here | 4,741 | 5,509 | 2026-08-10 | +> | + canonicity block | 4,749 | 5,517 | option C | +> | + leaf RATE (which also adds 8 bus interactions) | **4,777** | **5,545** | 2026-08-13 | +> +> ⚠ **So the +16.19% A6R price quoted just below reads +16.07% today**, and the −4.1% / −3.6% +> hosting saving further down reads −3.4% / −3.0%. Neither conclusion moves: A6R is still the +> cheaper round count and hosting is still cheaper than the standalone chip at both. +> +> The current figures are deliberately not restated here as literals to be copied — copying +> is what let the first correction sit unnoticed for two months. They are derived by +> `blake3_socket_tests::predicted_cells`, and `blake3_probe.rs`'s socket-vs-standalone +> comparison now calls that function instead of quoting its output. + +**The A6R price on this socket is +16.19% per compression** (4,741 → 5,509). PLAN §7's paper +estimate for the syscall-shaped chip was +15.5%; the socket pays slightly more because its +constant framing shrinks the round-*independent* part, so the rounds are a larger share. + +**Hosting is cheaper than the standalone chip at both round counts** — and the standalone +chip is now compiled and measured at both, not projected at one: + +| | standalone `LFM_BLAKE3` | `LFM_HASH` BLAKE3 arm | saving | +|---|---|---|---| +| 6 rounds | 4,946 ✓ MEASURED | 4,741 ✓ MEASURED | −4.1% | +| 7 rounds | 5,714 ✓ MEASURED | 5,509 ✓ MEASURED | −3.6% | + +The saving comes from three things the socket framing makes constant: `h = IV` (the entire +initial state is constant, so zero input-state columns), `m[8..16]` (the domain tag and the +zero padding), and the truncation window — twelve of the sixteen output words are never +built. + +### `BLAKE3_ROUNDS` is now 7 by default, one knob for both chips + +Per the A6R sign-off. `BLAKE3_ROUNDS` was 6 and baked into the standalone chip; it is now +`7` unless `--features blake3-6round`, and `blake3_socket::SOCKET_ROUNDS` is an **alias** for +it rather than a second knob — two knobs would let a sweep leave the machine's hash and the +chip it is priced against describing different functions. + +**The four figures you asked me to verify rather than trust are all confirmed by execution** +(`blake3_probe::the_hosted_chip_cell_budget_at_both_round_counts`, +`the_chip_emits_its_constraints_at_degree_3`): + +| standalone chip @ 7 rounds | A6R sheet §4 projected | measured | +|---|---|---| +| G-block region (`cols::OUT − cols::G`) | 3,360 | **3,360** ✓ | +| bus interactions | 1,451 | **1,451** ✓ | +| cell-equivalent | 5,714 | **5,714** ✓ | +| constraints | 897 | **897** ✓ | + +Its full 7-round budget: main 3,536, interactions 1,451, aux 726, cell-equiv 5,714, +constraints 897, BITWISE feed 1,440 per compression (was 1,248 at 6 rounds). Both round +counts' literals are pinned side by side in the probe, so the A6R price stays visible +whichever way the build is compiled. + +**The 6-round vector pin survives the flip.** `CANONICAL_VECTORS` are 6-round data, and +`blake3_probe` asserted the chip's `OUT` columns against them — that assertion would have +become vacuous or wrong at 7. So `blake3.rs` gained `CANONICAL_OUT_7ROUND`: the same ten +inputs at 7 rounds, emitted by the gate-oracle's Python reference and cross-checked +word-for-word against the second in-repo reference (`blake3_ref.py`) — **two implementations +agreeing on all ten** ✓ EXECUTED, with the same run re-deriving the 6-round table and +reproducing it 10/10. Both references' 7-round paths are themselves pinned by the official +BLAKE3 vectors, so this table has an *external* anchor where the 6-round one has an anchor a +step removed. `canonical_expected_out(i)` selects by the knob, and a negative control asserts +the two tables differ on every vector. + +--- + +## 2. What changed — file:line map + +### New files + +| file | lines | what | +|---|---|---| +| `prover/src/lfm/blake3_socket.rs` | 932 | the whole arm: framing constants, host hasher, column layout, wire interpretation, senders, BITWISE mirror, trace filler, constraints | +| `prover/src/lfm/blake3_socket_tests.rs` | 1,198 | 25 tests: KATs, 14 framing controls, layout, degree, O1/O2/O3, prove+verify, tamper, binding | +| `prover/src/lfm/blake3_socket_kats.rs` | 132 | GENERATED — 15 socket vectors × 2 round counts | + +`blake3_socket_kats.rs` is the **union of the two independently produced vector tables**: +`thoughts/blake3/socket-kats/socket_kats.json` (Phase 1) and the gate-oracle's +`socket_kats.json`. They share 5 of the 15 input pairs and **agree on every one of them at +both round counts** — ✓ EXECUTED (checked before generating). The other 10 differ only in +which inputs were sampled, so the union is two sources, not one transcribed twice. The +SOCKET.md §5 worked example (`nibble_ramp`) reproduces exactly. + +### Changed files + +| file:line | change | +|---|---| +| `blake3.rs:63-85` | `BLAKE3_STANDARD_ROUNDS = 7`, `BLAKE3_SIX_ROUNDS = 6`, and **`BLAKE3_ROUNDS` flipped to 7** behind `blake3-6round` | +| `blake3.rs:407` | `CANONICAL_OUT_7ROUND` — the ten canonical inputs at 7 rounds, from two agreeing references | +| `blake3.rs:472` | `canonical_expected_out(i)` — selects the table matching the knob | +| `blake3_chip.rs:720` | `output_words()` follows `BLAKE3_ROUNDS` (was hardwired 6-round) | +| `blake3_probe.rs` | cell budget, constraint count and BITWISE feed all parameterised, both round counts' literals pinned; the `#[ignore]`d census rows de-staled so they cannot mislead if un-ignored | +| `blake3.rs:107` | `blake3_compress_rounds(…, rounds)`; `blake3_compress_6round` now delegates. One loop bound, no second copy | +| `blake3.rs:561` | ★ `seven_rounds_is_the_blake3_crate` — the deferred Phase-1 crate cross-check, over 65 message lengths | +| `blake3.rs:604` | `six_rounds_is_not_the_blake3_crate` — its negative control | +| `blake3_chip.rs:238` | new `FlowConfig { rounds, out_window, full_output }` | +| `blake3_chip.rs:301` | `run_flow(f, cfg)` — the framing decisions that change *which calls happen* moved into the single dataflow | +| `blake3_chip.rs:292,295` | `feed_forward` split into `_low` / `_high` | +| `blake3_chip.rs:394` | `Add3Wire.m` widened `[usize;4]` → `WordRef` (constant message words) | +| `blake3_chip.rs:581` | `ValueFlow::compute_with(…, cfg)` | +| `hash.rs:49` | new `LfmHasher::compress_out` (default = permute-and-truncate); `compress` now derives from it | +| `hash.rs:72` | new `LfmHasher::admits(mode, state)` — the domain-restriction declaration | +| `hash.rs:156` | `HasherKind::Blake3 = 2` | +| `hash.rs:189-215` | explicit delegation of `compress` / `compress_out` / `admits` / `permute` / `compress_iv` | +| `chips.rs:587,683,707` | `num_columns` / `num_constraints` / `eval` arms | +| `chips.rs:603` | **`bus_interactions(kind)` — the signature change.** BLAKE3 appends its BITWISE lookups to the frozen six `LfmMem` tuples (`lfm_mem_interactions`, `chips.rs:612`) | +| `airs.rs:189,429` | the two `bus_interactions` call sites threaded (the census reads `airs.rs:189`) | +| `trace.rs:244` | witness-filling arm | +| `trace.rs:185-196` | BITWISE multiplicities — the one place the shared table's histogram depends on the hash choice | +| `executor.rs:69` | `LfmExecError::HasherRejected(&'static str)` | +| `executor.rs:395-410` | `admits` guard, and Compress now goes through `compress_out` **not** `permute` | +| `blake3_probe.rs:357-375` | the 7-round standalone figures, pinned | +| `blake3_socket.rs:153` | **D9** — `const _: () = assert!(SOCKET_ROUNDS == BLAKE3_ROUNDS)`, the single-knob tripwire | +| `blake3_socket_tests.rs:940` | **D2** — `the_lane_range_check_is_load_bearing_on_its_own` | +| `poseidon_chip_tests.rs:219` | renamed//widened: the *`LfmMem` tuple* contract is hasher-independent; the interaction list is not | +| `prover/Cargo.toml` | `blake3-6round` feature; `blake3 1.8.5` dev-dependency (was already in the local registry cache; resolved `--offline`) | + +**The executor change is load-bearing and easy to miss.** It previously computed *every* +hash row as `hasher.permute(state)`, inlining the trait's default `compress`. An overriding +`compress` was therefore never honoured on the prove path. BLAKE3 must override it +(obligation O3: the IV enters through `h`, not the capacity lanes), so `compress_out` was +added and the executor routed through it. Test/Poseidon behaviour is unchanged by +construction — the default `compress_out` *is* the old expression. + +--- + +## 3. Conformance against `chip_model.py` + +Every `CHIP CONSTRAINT` / `CHIP SENDS` comment in the model, mapped to the Rust that +realizes it. Verdict: **conformant, with one deliberate deviation (row 6) that is provably +equivalent and strictly cheaper, plus four constraints the model does not cover because it +does not model the host socket.** + +| # | `chip_model.py` | obligation | Rust | ✓ | +|---|---|---|---|---| +| 1 | `emit_lane_bytes` — `MU·(LANE_j − Σ MB[j][k]·2^{8k}) = 0`, per lane | eval, mu-gated, deg 2 | `blake3_socket.rs:820` (idx 6–13) | ✓ exact | +| 2 | `emit_lane_bytes` — `AreBytes(MB[j][0],MB[j][1])`, `AreBytes(MB[j][2],MB[j][3])` | 2 sends/lane = 16 | `blake3_socket.rs:609-620` | ✓ exact | +| 3 | `message_words` — m[8..16] carry NO columns and NO range checks | structural | `message_word_ref`, `blake3_socket.rs:402` → `WordRef::Const` for `i ≥ 8` | ✓ exact | +| 4 | `init_state` — all sixteen initial words compile-time constants | structural | `SocketWire::{input_h, iv_const, input_v12}` → `WordRef::Const` | ✓ exact | +| 5 | `emit_xor` — 4 × `ByteAlu[XOR]`, no eval constraint | 4 sends/word | `blake3_socket.rs:584-597` | ✓ exact | +| 6 | `emit_add2` — s bytes, **NO carry column**; `MU·carry·(1−carry)` with `carry := (A+B−s)·2^{−32}` | 4 cells, 1 constraint | `blake3_socket.rs:880-888` | ✓ exact — **the model was revised to match; see §3.1** | +| 7 | `emit_add3` — s + **2 carry columns**; sum identity + two booleanities; NOT a ternary carry | 6 cells, 3 constraints | `blake3_socket.rs:856-878` | ✓ exact | +| 8 | `emit_rotr` — SLL_lo/SLLC_lo/SLL_hi/SLLC_hi (2B each) + Y(4B); 4 mu-gated linear identities | 12 cells, 4 constraints | `blake3_socket.rs:894-930` | ✓ exact | +| 9 | `emit_rotr` — `AreBytes` over the 8 shift bytes = 4 sends | 4 sends/rotation | `blake3_socket.rs:599-607` | ✓ exact | +| 10 | `rotr16`/`rotr8` — FREE byte relabel, no columns | structural | `SocketWire::rotr16/rotr8` permute the `WordRef` byte indices | ✓ exact | +| 11 | `emit_feedforward` — `out[i] = v[i] XOR v[i+8]`, window only | 4 words, via `emit_xor` | `SocketWire::feed_forward_low`; `feed_forward_high` is `unreachable!()` under `FLOW.full_output = false` | ✓ exact | +| 12 | `digest_lane_values` — `MU·(OUT_C[i] − Σ OUTW[i][k]·2^{8k}) = 0`; no range check needed | eval, mu-gated, deg 2 | `blake3_socket.rs:841` (idx 22–25) | ✓ exact | +| 13 | BLOCK 0 — reuse the host's EXISTING cell columns, do not commit a second copy | structural | `cols::{IN0, OUT0, S8}` re-exported from `chips::hash::cols`; no duplicate columns | ✓ exact | +| 14 | MU-GATING — every eval constraint × MU, every send `Multiplicity::Column(MU)`, padding all-zero | structural | `MU = MODE_C` (preprocessed, so prover-unchosen); every BLOCK 1–5 constraint gated; every BITWISE send `Column(MU)` | ✓ exact | +| 15 | MU booleanity + all-zero padding — "NOT BV theorems, checked structurally" | — | emitted as a real constraint (idx 4) and ✓ EXECUTED by `padding_is_satisfied_and_a_real_marked_empty_row_is_not` | ✓ stronger | +| 16 | `tail_truncate` — permitted, off by default | default `False` | not implemented | ✓ conformant | +| 17 | round-0 constant folding — "permitted but must be re-gated" | — | **not done** | ✓ conformant | + +**Sends match the model exactly.** ✓ EXECUTED — running `SocketChip(...).build()` at both +round counts gives `census.sends` = **1,190** (6r) and **1,382** (7r), and +`census.aux_cells()/3` = **595** and **691**: identical to the built chip's, to the unit. The +deviation below costs no sends, only columns. + +*Version note:* the conformance table is against `chip_model.py` as of its 18:12 revision. +Its `CHIP CONSTRAINT` / `CHIP SENDS` / `CHIP COLUMNS` anchor set is unchanged from the +17:46 version I started against; what changed is `ColumnCensus.sends`, which became a +property summing the contract counter and the I/O tuples instead of a hand-incremented field +that omitted the `ByteAlu[XOR]` sends. I had derived the old accounting as an understatement +and was about to report it — executing the current file refuted that, because it had already +been fixed. Recording it only because it is the reason the two send counts now agree. + +### 3.1 The `emit_add2` deviation is RESOLVED — and it went the other way + +I reported this as the one deviation: `chip_model.py` witnessed the add2 carry as a column +and constrained it twice, while the implementation derives it as the expression +`carry := (A + B − s)·2^{−32}` and emits one degree-3 constraint. I recommended re-expressing +the model before Phase 4. + +**That is done — by the oracle side, not by me.** `chip_model.py` (mtime 20:40) now reads +*"CHIP COLUMNS: s[0..4] bytes. **NO carry column.**"* and *"the model follows the chip"*, +citing the implementation's line range. ✓ VERIFIED by reading it. + +The consequence is worth stating precisely, because it closes the gap I flagged: + +| ✓ EXECUTED, current `chip_model.py` | 6 rounds | 7 rounds | +|---|---|---| +| model main columns | 2,956 | 3,436 | +| **implementation main columns** | **2,956** | **3,436** | +| model cell-equivalent | 4,741 | 5,509 | +| **implementation cell-equivalent** | **4,741** | **5,509** | +| model sends | 1,190 | 1,382 | +| **implementation sends** | **1,190** | **1,382** | + +**Zero delta, on every figure, at both round counts.** The earlier −81/−97 column difference +was entirely the carry column plus the frozen-socket prefix accounting, and both are gone: +the model now counts the socket's 28-column shared prefix the way the chip carries it. + +The equivalence argument itself was independently confirmed by the verifier, and by +computation rather than by argument: with `A`, `B`, `s` byte-bound below `2^32` the reachable +integer range of `A + B − s` is `[−4294967295, 8589934590]`, and within that range the field +values `0` and `2^32` have **exactly one** integer preimage each — so a negative difference +cannot alias `2^32 mod p` and the existential really is eliminated by a determined witness. +`INV_SHIFT_32` was confirmed to be `2^{−32} mod p`. The same audit covers add3 and both +rotation identities: only `0` is a multiple of `p` in range, so every field identity in the +arm is an exact integer identity. + +### ⚠ 3.1a The recorded gate verdict is STALE — re-run before task #4 + +`run-gate.log` is **20:09**. `chip_model.py` is **20:40** and `gate.py` is **20:41**. +✓ VERIFIED by `stat`. So the recorded GATE VERDICT: PASS predates the model it is supposed +to certify by half an hour, and it certified the *carry-column* model, not the one now on +disk. **The gate must be re-run before task #4 claims anything about the real chip.** This is +not a defect in either the chip or the model — it is a sequencing artifact of the two sides +converging — but a green log that predates its own inputs is exactly the kind of evidence +that should not be cited. + +`ORACLE.md` (20:16) is stale for the same reason: its §3.2 census table still reports the +carry-column figures (main 3,037/3,533, cell-equiv 4,822/5,606), and its §3.2 reconciliation +against the standalone chip is computed from them. The current model gives 2,956/3,436 and +4,741/5,509. The lead's "expected census targets from the gated model" came from that table +and are superseded — the model and the chip now agree exactly, which is a better outcome than +the "small explainable deltas" that was being aimed at. + +One small thing for whoever owns the oracle: `chip_model.py`'s `emit_add2` docstring cites +`blake3_socket.rs:826-834`, which was correct when written and is now `880-888` — the O5 doc +block, the D9 tripwire and the D10 rewrite moved it. + +### 3.2 Four constraints the model does not cover + +`chip_model.py`'s BLOCK 0 says the socket I/O felts "are not modelled in the BV domain". The +implementation adds four framing constraints there, all additions rather than omissions: + +| idx | constraint | why | +|---|---|---| +| 0–3 | `S_k − (MODE_P·IN_{8+k} + MODE_C·IV_k)` | keeps the shared capacity prefix meaning the same thing under every hasher | +| 4 | `mode_sum·(1 − mode_sum)` | MU booleanity (item 15 above) | +| 5 | **`MODE_P = 0`** | ✗ no permute socket — see §5 | +| 14–21 | `OUT_{4+j} = 0`, j ∈ 0..8 | the digest is one cell; the upper eight lanes carry nothing | + +Total framing constraints 26, hence `NUM_CONSTRAINTS = 26 + 16·NUM_G`. + +--- + +### 3.3 ORACLE.md §7 obligations, and §3.1's degree ledger + +I read ORACLE.md §3, §3.1 and §7 after the 18:12 revision landed. Conformance: + +| | obligation | status | +|---|---|---| +| **O1** | input lanes range-checked to 32 bits; host must **reject**, not reduce | ✓ **DONE** — mu-gated linear identity per lane (idx 6–13) *plus* the 16 `AreBytes` sends; `lanes_of` returns `None` and `admits` turns it into `LfmExecError::HasherRejected`. Both halves tested, with honest controls. ⚠ **the recorded REASON was wrong — see §3.4** | +| **O2** | the socket is closed on its own output | ✓ **DONE** + tested (`the_socket_output_is_always_a_valid_input`), and exercised for real by the 3-compress program feeding `d0`/`d1` back in | +| **O3** | `compress_iv()` does not participate; the override honoured through `HasherKind::compress`'s explicit delegation | ✓ **DONE** — and this is what forced the executor change; `compress_out` is delegated explicitly alongside `compress` | +| **O4** | byte order is the `keccak_host` convention (one felt = one u32 = four LE bytes), **not** `word::pack_digest` | ✓ **DONE** — `lanes_of`/`word_of` are LE u32; `pack_digest` is never called here. The `lanes_big_endian` control fires | +| **O5** | leaf/parent domain separation | ✗ **OPEN — needs a decision, and it is not mine to make.** See below | +| **R1/R2/R3** | reuse the host's cell columns; no columns for `m[8..16]`; build only the four in-window output words | ✓ **DONE** — all three, rows 13/3/11 of the table above | +| §3.1 | degree ledger | ✓ **CONFORMANT** — every constraint lands inside it, worst = 3, and the rejected ternary carry is not used (two summed carry bits instead). The four host-socket constraints §3.1 does not list are degree 2, 2, 1, 1 | +| §3.3 | tail truncation — OPTIONAL, NOT recommended | ✓ **NOT IMPLEMENTED**, as instructed | + +**O5, stated so it does not get lost.** This socket has one tag, so it separates LFM +compressions from other BLAKE3 uses but **not leaves from parents within a tree**. If leaves +ever enter a tree as raw cells rather than through a distinct domain, a variable-depth tree +admits the classic Merkle second-preimage confusion — an internal node replayed as a leaf. +Either fix the tree depth or give leaves the reserved `"LFML"` tag; BLAKE3's own `PARENT` +flag cannot be reused without leaving the standard-hash framing that makes `blake3::hash` a +direct KAT. I have recorded it in `blake3_socket.rs`'s module docs rather than picking an +answer, because it is a protocol decision and nothing in the implementation depends on which +way it goes. It does **not** block anything Phase 2 delivers: the current consumer, +`merkle_walk`, has no leaf-hashing path at all. + +Also on the record, from the same section: the digest is 128 bits, so the socket offers +**64-bit collision resistance** by the birthday bound. That follows from +`HASH_DIGEST_FELTS = 4` and the machine's declared 128-bit target, not from BLAKE3 or from +the truncation. + +--- + +### 3.4 ⚠ D10 — the chip was right, the stated REASON for O1 was wrong + +The verifier found this by pulling on my own case-(c) surprise, and it is the most +interesting thing to come out of the review. **No constraint changes; three doc sites do.** + +**What I had written, in `blake3_socket.rs`, and what ORACLE.md §7 O1 and `chip_model.py`'s +`emit_lane_bytes` docstring also say:** that without the lane check, `v` and `v + 2^32` hash +alike — a free, prover-chosen collision, hence a forged Merkle path. + +**That attack is unconstructible against this chip**, and my own failed assertion is the +proof. The mixing core reads the *same linear form* for `m[lane]` that the identity ties +`IN_lane` to (`message_word_ref` → `word_expr`), so `IN_lane` and `m[lane]` are the same +field element by construction. Move the lane and you move the message word. That is exactly +what case (c) of the D2 test demonstrates: absorbing the carry into `MB[3]` satisfies the +lane identity and then breaks `add3` instead. + +**What the `AreBytes` sends actually buy** ✓ VERIFIED by reading `run_flow`: the message +words reach `add3` at `blake3_chip.rs:327,333` and **nothing else** — never an XOR — so +unlike almost every other word in this design they get no free byte bound from a consuming +lookup. These 16 sends are `m[0..8]`'s only range check. And `add3`'s exactness needs +`m < 2^32`: in round 0 the `a` and `b` operands are compile-time constants (`input_h`, +`input_v12`, `iv_const` all return `WordRef::Const` ✓ VERIFIED) and the output `s` is +byte-bounded by the XOR that consumes it, so with `m` unbounded a prover solves +`m ≡ s + 2^32·k − a − b (mod p)` for any chosen `s`, puts the whole value in `MB[0]` with the +other three bytes zero — satisfying the identity, since nothing bounds them — and hints the +sibling cell to match. The first `add3`'s output, hence the entire compression, is +prover-chosen. + +So the sends are *more* load-bearing than the collision story suggested, not less. Why it +mattered enough to fix rather than wave through: the next auditor reads the O1 bullet, tries +to build the collision, fails exactly as I did, and may reasonably conclude the range check +is redundant. + +Fixed in `blake3_socket.rs`'s module docs, the `idx 6–13` eval comment, and two test doc +comments. + +**Scope correction — ONE out-of-tree site, not two.** Both the verifier and I initially said +`ORACLE.md` §7 O1 carried the same wrong reason. ✓ VERIFIED by reading: it does not, and +neither does anything else in that file — `grep` for the collision story across `ORACLE.md` +returns nothing. Its BLOCK 1 already gives the *correct* argument, and independently of mine: +*"Without the `AreBytes`, the byte columns are full field elements, one linear equation in +four unknowns leaves three of them free, and the prover chooses the message that gets +hashed."* It even records the supporting fact — *"The message enters `f` only through `add3` +— it is never XORed"* — and notes O1's marginal cost over a chip that merely range-checked +its message is 8 linear constraints. §7 O1 is a milder and also-correct statement about host +and chip disagreeing. + +So the only site still carrying the unconstructible attack is **`chip_model.py`'s +`emit_lane_bytes` docstring, lines 153 and 156**. And per the verifier, that file still +*enforces* `are_bytes` (two sends per lane) — the constraint is gated correctly and only the +prose explaining it is wrong, so it is a comment fix, not a re-derivation, and must not be +allowed to become a reason to defer the gate re-run. + +Note the standalone chip already had it right too — `blake3_chip.rs:52` says "all 64 `m` +bytes keep their explicit `AreBytes` (they are never XORed)". So D10 was a localized prose +regression in the socket module plus one stale docstring, not a gap in the design or in the +gate's reasoning. + +--- + +## 4. KAT results + +| check | result | +|---|---| +| 15 socket vectors at **6 rounds** vs `socket_digest_rounds(a,b,6)` | ✓ **15/15** | +| 15 socket vectors at **7 rounds** vs `socket_digest_rounds(a,b,7)` | ✓ **15/15** | +| 7-round socket == `blake3::hash(a ‖ b ‖ "LFMC")[0..16]`, message rebuilt from the byte-level spec | ✓ **15/15** | +| the KAT table itself agrees with the crate | ✓ **15/15** | +| primitive at 7 rounds == `blake3::hash`, message lengths 0..=64 | ✓ **65/65** | +| primitive at 6 rounds ≠ `blake3::hash` (round-count discriminator) | ✓ | +| the chip's `OUT` **and** `OUTW` byte columns == the vectors | ✓ 15/15 (`an_honest_row_satisfies_every_constraint`) | +| public output of a proved 3-compress Merkle program == the reference | ✓ (`the_blake3_socket_proves_and_verifies`) | + +**SOCKET.md §6's one ✗ DEFERRED row is now discharged**: *"the same equality against the +Rust `blake3` crate — DEFERRED to a build phase, needs cargo."* It is +`blake3_socket_tests::seven_rounds_is_blake3_of_the_domain_separated_message`, and it +re-derives the 36-byte message from §2.1's byte-level form rather than calling +`socket_message`, so the word-level and byte-level routes remain two statements that can +disagree. So is the last row (*"the chip's `OUT` columns match these vectors"*). + +### Framing negative controls + +All 14 fire: `swap_a_b`, `tag_changed`, `tag_omitted`, `tag_slot_moved`, +`truncate_high_half`, `flags_parent`, `flags_no_root`, `block_len_64`, `block_len_32`, +`counter_one`, `cv_zero`, `lanes_big_endian`, `msg_perm_swapped`, `other_round_count`. + +Each must change the digest on **every** vector whose effective trace differs from the +honest one, and applicability is *derived* (initial state + the message schedule at every +round + the output window) rather than hand-listed — a hand-list goes stale as controls are +added, and a stale entry is a control that looks covered and is not. Inapplicable cases are +asserted to produce the *same* digest, which checks the applicability derivation itself. + +> Worth recording: writing applicability over the *permutation* instead of the *schedules* +> produced a false failure on the `a_one` vector, whose message has `m[2] = m[6] = 0` — so +> transposing the first two permutation entries yields an identical schedule and the control +> genuinely cannot fire. `socket_ref.py` gets this right for the same reason; I got it wrong +> first and the test caught it. + +--- + +## 5. Deviations from the brief, and why + +### 5.1 ✗ No `permute` socket — the BLAKE3 arm implements `compress` only + +`LFM_HASH` has two modes. SOCKET.md §7 states plainly that the `permute` socket (12 felts +in, 12 out) is **not specified**: no mapping decision, no vectors, and a security argument +that is not the same argument as `compress`'s. Its §7 sketch is labelled "a sketch, not a +decision — unreviewed". Building an unreviewed, un-KAT'd crypto framing is exactly what rule +9 forbids, and it would also roughly double the arm (12 feed-forward words and 12 lane +decompositions instead of 4 and 8). + +So: **the AIR pins `MODE_P = 0`** (idx 5), making a program containing a `permute` +*unprovable* under BLAKE3, and `LfmHasher::admits` refuses it at execution with a message +naming SOCKET.md §7. Defence in depth, both directions ✓ EXECUTED +(`a_permute_row_is_refused_under_blake3`, `a_permute_marked_row_violates_the_air`). + +**Practical consequence, and this is the one thing to carry forward:** `edsl::merkle_walk` +(which compresses) works under BLAKE3; `edsl::SpongeVar` (which permutes) does not. So +**`TrivialV0` and `FriToyV0` cannot be proved under BLAKE3** — both contain a `permute` — +and **the F3.4 disclosure is only half retired**. This matches existing task #8. The +prove/verify acceptance criterion is therefore met with a purpose-built compress-only +program (`compress_program_source`, two leaf merges and a parent merge — the Merkle-parent +shape the socket exists for, which also exercises O2 by feeding socket outputs back in as +inputs) rather than with `trivial_program`. + +### 5.2 `LfmHasher` gained two methods + +`compress_out` and `admits`. Both are defaulted, so no existing implementor changes +behaviour. `compress_out` was unavoidable: without it the executor's inlined +permute-and-truncate silently bypasses any overriding `compress`, and BLAKE3 must override +(O3). `admits` is how "reject, not silently reduce" (PLAN §3.2, SOCKET.md O1) becomes a +returned error rather than a panic. + +`Blake3Permutation::permute` **panics**. It is unreachable — `admits` rejects first and the +AIR pins `MODE_P = 0` — and every value it could return would be a hash the chip does not +prove. Documented as such at `blake3_socket.rs:267`. + +### 5.3 `blake3-6round` is a cargo feature, not a runtime parameter + +The *host* reference is runtime-parameterised (`socket_digest_rounds(a, b, rounds)`), so the +KATs pin both variants in one run. The *chips* cannot be: their layouts are `8·rounds` +G-blocks wide and their width functions are `const fn`. Default is **7 rounds** (standard, +externally anchored, A6R-free), matching the signed decision; `--features blake3-6round` +selects 6, and it drives both chips through the single `BLAKE3_ROUNDS`. + +`blake3_compress_6round` keeps its name and its meaning — it now delegates with +`BLAKE3_SIX_ROUNDS`, not with the knob, so the 6-round vectors it is tested against stay +pinned in every build. The same reasoning applies to the test module's `CANONICAL` +conventions and to `six_rounds_is_not_the_blake3_crate`: both read `BLAKE3_SIX_ROUNDS` +explicitly. Reading the knob in either place would have silently turned a discriminating +control into a tautology at the default — which is exactly what happened on the first run, +and is why those two now name the constant. + +Note `make lint` does not cover the feature — I ran `cargo clippy --features blake3-6round` +separately (clean). Worth adding to CI if the 6-round variant is meant to stay supported. + +### 5.4 Not done, deliberately + +- **Not added to the registry's 6 kinds** — per the brief, a later separate decision. +- **No round-0 constant folding.** The entire initial state is constant, so it is available + and would be a real saving, but `chip_model.py` says a folded round 0 "no longer matches + this model" and must be re-gated. Left on the table. +- **`spec/blake3.typ` not updated** — that is task #7. + +--- + +## 6. Test and lint status + +| | result | +|---|---| +| `lfm::blake3*` at 7 rounds (default) | ✓ **44 passed, 0 failed**, 2 ignored | +| `lfm::blake3*` at 6 rounds (`--features blake3-6round`) | ✓ **44 passed, 0 failed**, 2 ignored | +| full `lfm::` suite | 263 passed, **19 failed — all pre-existing** | +| `make fmt` | ✓ clean | +| `make lint` (fmt check + 4 clippy passes) | ✓ **clean** | +| `cargo clippy --features blake3-6round` | ✓ clean | + +**The 19 failures are the known fixture issue and none is mine.** ✓ VERIFIED by reading each +panic: every one traces to `proof_fixture.rs:73`, `failed to read +executor/program_artifacts/recursion/fibonacci.elf — run make compile-recursion-elfs`, or to +its downstream `ArenaLenMismatch`/epoch-count consequences in `machine_tests`. The set is +`epoch_tests` ×7, `epoch_verify_tests` ×6, `logup_tests` ×1, `machine_tests` ×5. None touches +`LFM_HASH`, and `poseidon_chip_tests` (the closest neighbour, and the one existing test I +edited) passes in full. + +### Honest-path controls + +Per the standing rule that a rejection test passes equally well when the fix rejects +everything, every rejection test here is paired: + +| rejection test | its honest control | +|---|---| +| `tampering_with_the_witness_is_not_accepted` (4 mutations) | `the_blake3_socket_proves_and_verifies` | +| `an_out_of_range_lane_is_rejected_rather_than_reduced` | in-test: the in-range pair is still admitted | +| `a_non_u32_arena_word_fails_execution_under_blake3` | in-test: `execute(&program, &arenas(), …).is_ok()` | +| `a_permute_row_is_refused_under_blake3` | in-test: the same program still executes under `Test` | +| `the_lane_decomposition_binds_the_felt_to_its_bytes` | in-test: `violations(&base)` is empty | +| `the_lane_range_check_is_load_bearing_on_its_own` (D2) | in-test: part (a) asserts the eval set is SILENT, which is what stops the proof-level half from degenerating into a duplicate | +| `padding_is_satisfied_…_empty_row_is_not` | the padding half is itself the control | + +--- + +## 7. What I would look at next + +1. **Re-run the gate.** `run-gate.log` (20:09) predates `chip_model.py` and `gate.py` (20:40, + 20:41), so the recorded PASS certified the superseded carry-column model. §3.1a. Nothing + else stands between the gate and the chip — the `emit_add2` divergence I flagged has been + closed from the oracle side and the two now agree on every census figure. +1b. **Refresh `ORACLE.md` §3.2** — still the superseded carry-column census as of its 21:27 + edit ✓ VERIFIED. Its BLOCK 1 and §7 are correct and need nothing. +1c. **`chip_model.py`**: fix the `emit_lane_bytes` docstring per §3.4 (lines 153, 156 — prose + only, the sends are enforced correctly) and update the `blake3_socket.rs:826-834` + citation, now `880-888`. +1d. *Nit, offered by the verifier and not filed as a finding, recorded so it survives.* + `ORACLE.md` BLOCK 1's intermediate step — "one linear equation in four unknowns leaves + three of them free" — is exactly right for the standalone chip, where `m` has its own + columns. In the **socket** the lane identity pins `m[lane]` to `IN_lane`, so those three + spare byte degrees of freedom buy the prover nothing: the message word is that same linear + form either way. The operative freedom is `IN_lane` itself being an unbounded + prover-hinted felt. ✓ VERIFIED, and the conclusion ("the prover chooses the message that + gets hashed") is correct on both routes — only the route differs. Worth one sentence if + someone is editing BLOCK 1 anyway; not worth a change on its own, and **not** a gate + re-derivation. +2. **The `permute` socket** (task #8) is what stands between this and a fully retired F3.4, + and between BLAKE3 and the two registered toy programs. +3. **Round-0 constant folding** is a real, unclaimed saving — the whole initial state is + constant — but it needs a re-gate first. +4. `make lint` does not build the `blake3-6round` feature; if the 6-round variant is meant + to stay supported, add a lint/test pass for it in CI. diff --git a/thoughts/shared/lfm-real-hash/phase2-verify.md b/thoughts/shared/lfm-real-hash/phase2-verify.md new file mode 100644 index 000000000..809e29687 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/phase2-verify.md @@ -0,0 +1,966 @@ +# Phase 2 — adversarial verification of the `LFM_HASH` BLAKE3 arm + +**Reviewer:** F9 (adversarial). **Date:** 2026-08-10. +**Target:** uncommitted change set in `/Users/maurofab/workspace/lambda_vm-blake3-impl`, +branch `blake3-real-hash`, HEAD `65025095`. +**Report under review:** `thoughts/shared/lfm-real-hash/phase2-report.md`. +**Spec:** `gate-oracle/chip_model.py` + `ORACLE.md` §7. + +> **⚠ READ THE ADDENDUM FIRST.** The worktree was being edited while this review ran, and +> `phase2-report.md` was itself updated at 20:58 — *after* I first read it. The implementer +> has since frozen the tree and confirmed my pinned hashes are final. The addendum below +> records what that changed: **D4, D5 and D8 are WITHDRAWN**, D3's stated *cause* was +> **wrong** and is corrected, and D1 reduces to one precise residual. + +--- + +## Verdict table + +| # | target | verdict | +|---|---|---| +| 1 | executor change | **CONFIRMED-SOUND** | +| 2 | O1 (host reject + chip binding + add2 mod-p algebra) | **CONFIRMED-SOUND** (algebra machine-checked), with **D2** — the load-bearing half is untested | +| 3 | MU-gating, `MODE_P = 0`, #915 admission bounding | **CONFIRMED-SOUND**, with **D5** (a claim, not the code, is wrong) | +| 4 | bus balance / census | **CONFIRMED-SOUND** | +| 5 | claim verification by execution | **CONFIRMED on the final tree**, one count refuted — see **D1**, **D3** | +| 6 | hygiene sweep | **CONFIRMED-SOUND**, with **D4** | +| 7a | knob aliasing — no second rounds knob survives | **CONFIRMED**, with **D9** (invariant unpinned) | +| 7b | `canonical_expected_out` is not vacuously selected | **CONFIRMED-SOUND** | +| 8a | add2 expression-carry equivalence (implementer's challenge) | **CONFIRMED** (machine-checked); premise stale, see **D7** | +| 8b | no permute row can reach the trace filler (implementer's challenge) | **CONFIRMED-SOUND** | +| 8c | tautology sweep — no test reads the knob where it means a fixed count | **CONFIRMED** clean, now enforced (**D9 closed**) | +| 9 | D2/D9 fixes + zero-delta model census | **VERIFIED by execution**; surfaces **D10** | + +--- + +## Addendum 3 — D10 fixed in-tree; two out-of-tree sites remain + +Fixed at hashes `fd19f4c5…` (`blake3_socket.rs`) / `540233bb…` (`blake3_socket_tests.rs`), +other eleven files unchanged, scope still 12 M + 3 ??. + +**The change is doc-only, verified rather than taken.** `NUM_CONSTRAINTS = 26 + 16 * NUM_G` +and `CORE_IDX = 26` are intact, and all three moved anchors land exactly where claimed: +`:267` `panic!(`, `:820` the idx 6–13 loop, `:880` the add2 loop. + +**The rewritten argument is correct on every leg.** I checked it as adversarially as the +original, because a garbled correction to a soundness argument is worse than the wrong one +it replaces: + +1. *"the core reads the same linear form as the message word, so `IN_lane` and `m[lane]` are + the same field element by construction"* — ✓ `message_word_ref(i)` for `i < 8` returns + `word_cols(cols::lane_byte(i, 0))`, the very columns the identity's right-hand side sums. +2. *"the textbook alias … is unconstructible here, not merely prevented"* — ✓. +3. *"(It is real for a chip that derives the message bytes by reduction mod 2^32 instead of + by a checked decomposition … it is not what the `AreBytes` sends buy.)"* — ✓ and this + parenthetical is the right call: it keeps the *design* justification for choosing a + checked decomposition while detaching it from the sends, which is exactly the conflation + that caused D10. +4. *"`m` reaches `add3` and nothing else, never an XOR"* — ✓ (`blake3_socket.rs:428` is the + only site; they independently confirmed `blake3_chip.rs:327, :333`). +5. The round-0 solve-for-any-`s` argument is reproduced correctly, including that `a`, `b` + are compile-time constants, that `s` is byte-bounded by the consuming XOR, and that the + other three `MB` bytes can be zeroed because nothing bounds them. ✓ + +Their observation that the **standalone chip had it right all along** also checks out: +`blake3_chip.rs:52` reads *"all 64 `m` bytes keep their explicit `AreBytes` (they are never +XORed)"*. So D10 was a regression in the socket's prose, not a gap in the design, and the +chip's framing is the one the out-of-tree sites should converge to. + +### ⚠ CORRECTION TO MY OWN FINDING — D10 is ONE site, not three + +I wrote that D10 hit `ORACLE.md` §7 O1 as well. **That was wrong, and I made the claim +without re-reading the file** — the exact failure mode my own claim-verification discipline +exists to prevent. The implementer caught it. Verified now, properly: + +- `grep` for the collision story across the whole of `ORACLE.md` returns **nothing**. +- §7 O1 says only that the host *"must **reject** an out-of-range lane, not silently reduce, + or host and chip disagree about what was proved"* — a correct statement about the host + side, with no collision claim. +- `ORACLE.md` BLOCK 1 already carries the **correct** argument, and reached it + independently of both of us: *"Without the `AreBytes`, the byte columns are full field + elements … and **the prover chooses the message that gets hashed** — every load + authenticated through `compress` becomes forgeable."* It even records the load-bearing + fact I derived: *"The message enters `f` only through `add3` — it is never XORed — so + those 32 bytes needed an explicit `AreBytes` regardless."* + +**So D10's remaining scope is a single site:** `gate-oracle/chip_model.py`'s +`emit_lane_bytes` docstring, lines 152–156 — *"the only thing standing between a +prover-hinted Merkle sibling and a chosen collision … so `v` and `v + 2^32` hash alike."* + +The scoping note survives and matters more now that it is the only item: that file still +**enforces** `self.c.are_bytes(*word)`, two sends per lane, so this is a comment fix, not a +re-derivation, and it must not become a reason to defer the gate re-run. + +Worth remembering: of the four documents discussing this, three — `ORACLE.md` BLOCK 1, +`blake3_chip.rs:52`, and now `blake3_socket.rs` — independently reach the +"`m` is never XORed, so these sends are its only bound" reasoning. The two that drifted to +the collision story were the socket module doc and the model docstring: **the two closest to +the new code.** That the drift happened twice, in exactly those two places, is the +transferable lesson. + +### One clause added after I closed — reviewed, correct + +The implementer added a closing paragraph to the O1 bullet and flagged it as unreviewed +rather than letting it ride on a closed verdict. Correct instinct, and it is reviewed now. +Final hash **`89856eb4…`** (`blake3_socket_tests.rs` unchanged at `540233bb…`); +`NUM_CONSTRAINTS` and `CORE_IDX` sit at `:779`/`:782`, exactly +7 from `:772`/`:775` — the +new paragraph's 6 lines plus a blank — so doc-only is confirmed arithmetically, not +asserted. **44 passed / 0 failed / 2 ignored** re-run at that hash. + +The clause states the mechanism positively: *"what the sends do is **transfer a bound onto +the lane**. Without them the identity is satisfiable for every felt `IN_lane` — put the whole +value in `MB[0]` — so it bounds nothing. With them the four bytes sum to less than `2^32`, +so it is satisfiable exactly when `IN_lane < 2^32`, and then the decomposition is unique."* + +Both directions check, computationally: + +- **Without**: `MB[0] = x`, rest zero, gives `Σ = x` for *any* felt `x`. The identity bounds + nothing. ✓ +- **With**: `Σ ≤ 255·(1 + 2^8 + 2^16 + 2^24) = 4294967295 = 2^32 − 1`, so `Σ < 2^32 ≪ p` and + cannot wrap; the identity is satisfiable exactly when `canonical(IN_lane) < 2^32`, and the + four-byte base-256 representation of such a value is unique. ✓ + +It is also consistent with the paragraph above it rather than a competing story: the sends +bound the *bytes*, the identity transfers that bound to the *lane*, and because `m[lane]` is +the same linear form the message word is bounded by the same step. "Only range check on `m`" +and "transfers a bound onto the lane" are one mechanism seen from two ends. + +*Minor, offered rather than filed:* `ORACLE.md` BLOCK 1's intermediate step — +*"one linear equation in four unknowns leaves three of them free"* — is inherited from the +standalone chip, where `m` has its own columns and it is exactly right. In the **socket** the +lane identity pins `m[lane]` to `IN_lane`, so the three spare byte degrees of freedom buy +the prover nothing; the operative freedom is `IN_lane` itself being an unbounded +prover-hinted felt. The **conclusion is correct either way** — this is a routing nit, not a +second D10, and I flag it only because I have just been burned for over-claiming. + +**Important scoping: the gate's theorem is NOT affected.** I checked that +`emit_lane_bytes` still *enforces* the check — `self.c.are_bytes(*word)`, two sends per lane +— so what is wrong in the model is the prose explaining why, not the constraint being +gated. This is a comment fix on the oracle side, not a re-derivation. + +Freshest mtimes, for D7: `run-gate.log` **20:09:33**, `chip_model.py` **20:40:46**, +`gate.py` **21:16:59**, `ORACLE.md` **21:27:24**. The recorded PASS is now stale against all +three. + +**D7 and the census staleness both survived the 21:27 `ORACLE.md` edit** — I re-checked +rather than assuming the edit swept them up. §3.2 still reads main **3,533 / 3,037**, +cell-equiv **5,606 / 4,822**, and a 7-round breakdown of `add2` **560** + `I/O+MU` **13**, +i.e. the carry-column model's figures, against the executed **3,436 / 2,956** and +**5,509 / 4,741** with `add2` 448 and prefix 28. And `run-gate.log` is still 20:09:33: the +gate has not been re-run. + +--- + +## Addendum 2 — D2 and D9 closed; one NEW finding (D10) + +Tree re-opened and edited to close D2/D9. New hashes, re-verified by me and identical +before and after every run below: + +``` +9d91954dd243b35601ae787ea43a2f1729d800675f208226bff49bfb2c44fafa blake3_socket.rs +cac6348a339a5f129f4f21cf12253796d984ae1ccb492cbaacc1a753dbf32058 blake3_socket_tests.rs +``` +(aggregate of all `prover/src/lfm/*.rs`: `62b13a25…`, unchanged across all three suites.) + +| claim | measured | +|---|---| +| `lfm::blake3` at 7r | **44 passed, 0 failed, 2 ignored** ✓ | +| `lfm::blake3` at 6r | **44 passed, 0 failed, 2 ignored** ✓ | +| full `lfm::` | **263 passed, 19 failed** — the identical pre-existing set ✓ | + +**D9 — CLOSED, and closed better than I asked.** `const _: () = assert!(SOCKET_ROUNDS == BLAKE3_ROUNDS)` +at `blake3_socket.rs:132`, plus `assert_eq!(NUM_G, blake3_chip::NUM_G)` in the layout test. +The second is the one with teeth: it ties the socket's layout to the standalone probe's, so +re-introducing a `cfg` pair fails even if someone edits the alias to match. + +**D2 — CLOSED, and building it produced a genuine correction to my framing.** I proposed two +witnesses; only one behaves as I claimed, and the implementer found this by asserting "no +violations" and getting `[26, 89, 155, 197, 254, 296, 323]`. + +Root cause, which I verified independently: **the lane bytes ARE the message bytes.** +`message_word_ref(i)` for `i < 8` returns `word_cols(cols::lane_byte(i, 0))` — the very +columns the lane-decomposition constraint reads. So absorbing an alias carry into `MB[3]` +satisfies the lane identity *and* moves message word `m[0]` by `2^32`, which the add3 sum +identity rejects. `the_lane_range_check_is_load_bearing_on_its_own` now pins all three cases +and, more usefully, pins *which mechanism* catches each: + +| witness | caught by | pinned as | +|---|---|---| +| (a) `MB[0] += 256, MB[1] −= 1` — weighted sum preserved exactly | **only `AreBytes`** | eval set asserted *silent*, then rejected at proof level | +| (b) `IN0 += 2^32`, bytes untouched | the lane identity | `violations` contains lane index 6 | +| (c) `IN0 += 2^32`, `MB[3] += 256` | the **mixing core** | index 6 explicitly *absent*, all violations ≥ `CORE_IDX` | + +Case (a)'s "eval set is silent" assertion doubles as the honest control for (d): if it ever +starts failing, the proof-level half has silently become a duplicate of +`the_lane_decomposition_binds_the_felt_to_its_bytes`. That is the right shape. + +(d)'s reasoning is sound too: the shuffle leaves `IN0` untouched, so the `LfmMem` receive +token is unchanged and the rejection can only come from the range check. I confirmed the +message bytes have **no** `ByteAlu[XOR]` consumer — `m` is passed only into `add3` +(`blake3_socket.rs:428`), never into `xor` — so the lane `AreBytes` pair is genuinely their +only range check. + +### D10 — the recorded justification for O1 names a hazard that cannot occur (severity MEDIUM, docs/argument) + +Prosecuting the implementer's discovery one step further turns up something neither of us +had. **The chip is correct; the *reason* written down for why it is correct is wrong**, in +three places at once: + +- `blake3_socket.rs` module doc, O1 bullet: *"If the four message bytes of a lane were taken + by reduction mod 2^32, then `v` and `v + 2^32` would hash alike: a free, prover-chosen + collision."* +- `ORACLE.md` §7 O1 and `chip_model.py`'s `emit_lane_bytes` docstring: *"without the + AreBytes the prover picks the bytes — so `v` and `v + 2^32` hash alike."* + +**They cannot hash alike.** The lane identity forces `IN0 = Σ MB[k]·2^{8k}` and the mixing +core reads *the same linear form* as `m[0]`, so `IN0` and `m[0]` are equal as field elements +**by construction**. Move the lane and you move the message word with it — which is exactly +what case (c) demonstrates empirically. There is no configuration in which two different +`IN0` values feed the same message. The stated attack is unconstructible, and the +implementer's failed assertion is the proof. + +**What the sends actually buy is stronger.** The lane `AreBytes` are the *only* bound on +`m[0..8]`, and the mixing core's field identities need that bound to be exact. Concretely, +in round 0 the first add3's other operands are compile-time constants (`input_h` and +`input_v12` return `WordRef::Const`), and its output `s` is byte-bounded because the `X1` +XOR consumes it. So the constraint is `μ·(a + b + m − s − 2^32·(c1+c2)) = 0` with `a, b` +constant, `s ∈ [0, 2^32)`, `c1+c2 ∈ {0,1,2}`. Drop the lane `AreBytes` and `m` becomes an +unbounded field element, so a prover who wants a chosen `s` simply solves + +``` +m ≡ s + 2^32·k − a − b (mod p) +``` + +sets `MB[0][0] = m` with the other three bytes zero (the identity is satisfied, nothing +range-checks them), and hints the sibling cell `IN0 = m` — which `edsl::merkle_walk` lets it +choose. **The first add3's output, and hence the whole compression, becomes prover-chosen.** +That is a real soundness break, it needs no aliasing story, and it is what the sends prevent. + +Why this matters rather than being pedantry: the next person to audit this arm will read the +O1 bullet, try to build the `v` / `v + 2^32` collision, fail exactly as the implementer did, +and may conclude the range check is redundant. The correct one-line statement is *"the lane +`AreBytes` are the only bound on the message words, and the add3 exactness argument needs +`m < 2^32`."* Recommend rewriting the O1 justification in all three places; the constraint +system needs no change. + +## Addendum — after the freeze (supersedes parts of D1/D3/D4/D5/D8) + +The implementer froze the tree, confirmed **all four pinned hashes are the final state**, +and pointed out that `phase2-report.md` was updated at **20:58**, after the round-flip wave +and after I first read it. I re-read the updated report and re-audited. Net effect: + +| finding | status after re-check | +|---|---| +| **D4** — "default build changes, report implies it doesn't" | **WITHDRAWN.** The 20:58 report states it outright: `blake3.rs:63-85` *"**`BLAKE3_ROUNDS` flipped to 7**"*, `blake3_chip.rs:720` *"`output_words()` follows `BLAKE3_ROUNDS` (was hardwired 6-round)"*, and §5.3 *"it drives both chips through the single `BLAKE3_ROUNDS`"*. My objection was against the pre-flip report. | +| **D8** — "report omits O5 and the collision bound" | **WITHDRAWN.** The 20:58 report carries a full ORACLE §7 obligation table with **O5 marked ✗ OPEN** (`:258`), a dedicated paragraph (`:263`) and the 64-bit birthday note (`:274-275`). | +| **D5** — "'every eval constraint × MU' is false" | **WITHDRAWN.** I read a phrase out of its cell. In §3's row 14 the *left* column states the model's requirement; the *right* column — the one describing the Rust — says *"every BLOCK 1–5 constraint gated"*, which is exactly right, and §3.2 declares idx 14–21 separately as BLOCK-0 framing. The report is internally consistent; the code is unchanged and still correct (ungated is strictly stronger). | +| **D3** — the two transient failures | **Conclusion stands, my stated CAUSE was WRONG.** See below. | +| **D1** — moving target | Reduces to **one precise, mechanical residual**. See below. | +| D2, D6, D7, D9 | **Unchanged and still open.** | + +### D3, corrected — they were real assertion failures, not half-applied edits + +I hypothesised the two extra failures were an inconsistent mid-write snapshot. **That was +wrong**, and the implementer's account is better than mine: no file was ever mid-write; +both were genuine assertion failures caused by the round flip landing before the dependent +expectations were updated. + +1. `blake3_probe::the_hosted_chip_proves_and_verifies` asserted a hardcoded `1_248` + BITWISE feed — the **6-round** figure; at 7 rounds it is 1,440. +2. `blake3::tests::six_rounds_is_not_the_blake3_crate` read `BLAKE3_ROUNDS`, which had just + become 7, so it compared 7-round output against `blake3::hash`, they matched, and **the + negative control had become a tautology.** + +Both are fixed in the frozen tree, and §5.3 of the updated report now discloses the +tautology trap by name. This *strengthens* the process point rather than weakening it: a +reviewer sampling an uncommitted tree measured, as real failures, bugs the author had +already found and fixed — and one of them was a control silently ceasing to discriminate, +which is the single worst failure mode for a test suite of this kind. Good catch by the +implementer; my job was to notice it independently and I only got as far as "these two are +new", not "and here is why". + +### D1, reduced — every `blake3_socket.rs` line reference in the report is off by +17 + +The 20:58 refresh updated the counts and the `blake3.rs` / `blake3_chip.rs` / +`blake3_probe.rs` references, but **not** the socket ones — `blake3_socket.rs` gained 17 +lines of module doc at 20:44 and its line numbers were never re-derived. Checked +mechanically, nine for nine: + +| report says | actually at that line | intended construct is at line + 17 | +|---|---|---| +| `:215` (permute panic) | `// The host-side hasher` | `panic!(` | +| `:350` (`message_word_ref`) | `pub const fn out_byte(…)` | `fn message_word_ref(…)` | +| `:532-545` (XOR sends) | a doc line | `for xw in &wires.xors {` | +| `:557-570` (lane `AreBytes`) | `byte_bus_value(xw.b.byte(b))` | `for lane in 0..cols::NUM_LANES {` | +| `:766` (idx 6–13) | `// idx 4: mode sum-boolean` | `for lane in 0..cols::NUM_LANES {` | +| `:787` (idx 22–25) | `b.emit_base(6 + lane, …)` | `for i in 0..OUT_WINDOW {` | +| `:802-824` (add3) | a comment | `for aw in &wires.add3s {` | +| `:826-834` (add2) | `let sum_id = …` | `for aw in &wires.add2s {` | +| `:840-876` (rot) | a comment | `for rw in &wires.rots {` | + +Fix is one `sed`: add 17 to every `blake3_socket.rs:` reference in `phase2-report.md`. +Everything else in the 20:58 report checks out against the frozen tree. + +--- + +**Bottom line: I found no soundness defect and no regression to the existing machine.** +Everything I found is either a process problem (D1, D3), a test-coverage gap on the one +claim that matters most (D2), or a report/claim inaccuracy (D4, D5, D7, D8). The arm +itself holds up under every attack I could construct. + +--- + +## The tree I actually verified + +The review is pinned to these hashes, which were **identical before and after** every +test run reported below: + +``` +f4a61d76100c17438ba29bffc43e5421e4fc5e59ef1c255df735853a2bd88134 prover/src/lfm/blake3_socket.rs +675148529a70404528721ea2d279599482faf1f985c3f864505043e7e16b7280 prover/src/lfm/blake3.rs +8030b0b3c3deae0a6a329dffefb8b5d3655c087dbc3d79ebac417f3154b03f67 prover/src/lfm/blake3_chip.rs +8c14a9057c4ff48e03aa687387e84fd8d91351b3da3006d0f1d05406ebb83349 prover/src/lfm/blake3_probe.rs +d2ecfa5c15d6256196661278df4754558f531a7739d3fef9654e9c7e0cb2cf9a prover/src/lfm/trace.rs +1eb4b8aa573a884e0c46ccc83a7b9c7772b4ce56dfc436d8742d9dc70c848d89 prover/src/lfm/hash.rs +a1f4d19d2a0183b171c8226990919259a11857cb4a83dbe394f7005ea9a1c8ee prover/src/lfm/executor.rs +4a00ec8933a6eb6d0f7f55cea2a6e5662355c0e102d7e73673a3a59565402f02 prover/src/lfm/chips.rs +``` + +Scope matches the brief: **12 modified + 3 new**, nothing outside +(`Cargo.lock`, `prover/Cargo.toml`, `airs.rs`, `blake3.rs`, `blake3_chip.rs`, +`blake3_probe.rs`, `chips.rs`, `executor.rs`, `hash.rs`, `mod.rs`, +`poseidon_chip_tests.rs`, `trace.rs`; new `blake3_socket{,_kats,_tests}.rs`). +Note `mod.rs` and `Cargo.lock` are modified but absent from the report's §2 file map. + +--- + +## Target 1 — the executor change: CONFIRMED-SOUND + +**Claim:** *"Test/Poseidon behaviour is unchanged by construction — the default +`compress_out` IS the old expression."* **✓ CONFIRMED**, and the claim is exactly right. + +Old (`git diff`): `let out_state = hasher.permute(state);` for **both** modes, where +`executor.rs:376-385` had already built `state = [a ‖ b ‖ hasher.compress_iv()]` on the +Compress arm. + +New (`executor.rs:398-409`): Compress goes through `hasher.compress_out(&a, &b)`; the +trait default (`hash.rs:49-57`) is `state[0..4]=a; state[4..8]=b; state[8..12]=self.compress_iv(); self.permute(state)` +— the same expression, reconstructed from the same `compress_iv()`. + +I checked every implementor rather than trusting the default: + +- `TestPermutation` (`hash.rs:100-117`) overrides only `permute` + `compress_iv`. +- `PoseidonGoldilocks` (`poseidon.rs:591-627`) overrides only `permute` + `compress_iv`. +- Neither overrides `compress`, `compress_out` or `admits` ⇒ both take the defaults + ⇒ bit-identical output to the old inline expression. +- `HasherKind`'s explicit delegation (`hash.rs:200-215`) routes `Test`/`Poseidon` to + those same defaults. + +**The Permute arm is untouched** (`executor.rs:408` is still `hasher.permute(state)`), +so the wrap/keccak role-1 path is unaffected. The only new behaviour on that arm is the +`admits` guard at `executor.rs:395-397`, whose default (`hash.rs:72-76`) is `Ok(())` for +every hasher that does not override it. + +The frozen six `LfmMem` tuples are **byte-identical**: `chips.rs` moved the `vec![...]` +body verbatim from `bus_interactions()` into `lfm_mem_interactions()` (the diff hunk +touches only the signature; the six `BusInteraction`s are pure context lines), and +`bus_interactions(kind)` returns exactly that list for `Test`/`Poseidon`. Pinned by +`poseidon_chip_tests.rs:234-239`. + +--- + +## Target 2 — O1: CONFIRMED-SOUND (algebra machine-checked), with a test gap + +### (a) Host side — rejects, never reduces. ✓ VERIFIED + +`lanes_of` (`blake3_socket.rs:198-205`) is the single lane boundary and it uses +`u32::try_from(GoldilocksField::canonical(...)).ok()?` — `try_from`, so ≥ 2^32 yields +`None`. There is **no** `as u32`, no `& 0xFFFF_FFFF` and no `% (1<<32)` anywhere in the +module. `admits` (`:266-282`) turns `None` into `Err`, and `executor.rs:395-397` turns +that into `LfmExecError::HasherRejected`. + +The witness filler (`blake3_socket.rs:658-659`, the `trace.rs:244` arm) also goes through +`lanes_of` and `.expect(...)`s — a panic, not a truncation. Same for `trace.rs:185-196`. +Reaching either means the executor and the filler disagreed; neither can silently reduce. + +### (b) Chip side — the binding is unique. ✓ VERIFIED + +`blake3_socket.rs:783-788` emits, per lane `j ∈ 0..8`, +`MU · (IN_j − Σₖ MB[j][k]·2^{8k}) = 0`, and `:557-568` sends +`AreBytes(MB[j][0], MB[j][1])` and `AreBytes(MB[j][2], MB[j][3])` — all four bytes +covered, `Multiplicity::Column(MU)`. The receiving table +(`tables/bitwise.rs:343-372`, `AreBytes` receiver at `:784`) enumerates exactly +`x, y ∈ [0,256)`, so the bound is the tight one. + +Bytes < 256 ⇒ `Σ ≤ 4294967295 < 2^32 ≪ p` (computed), so the identity cannot wrap: +the felt equals that integer exactly, hence `< 2^32`, and base-256 representation is +unique. Both halves are present and both are needed. + +### (c) The add2 expression-carry deviation — mod-p algebra. ✓ VERIFIED BY COMPUTATION + +`blake3_socket.rs:826-834` (numbering per the report; now `:843-851`) emits only +`MU · c · (1 − c) = 0` with `c := (A + B − s) · 2^{−32}`. + +- `INV_SHIFT_32 = 18446744065119617026` **is** `2^{−32} mod p` — I recomputed it. +- `A, B, s` are `word_expr` recompositions of byte columns whose range checks I traced to + a real consumer (add2's operands are a previous `add2`/`IV` const and a `rotr` relabel + of a `ByteAlu[XOR]` output; its own output `s` is consumed as an XOR operand, including + in the last round via the feed-forward). So all three are in `[0, 2^32)`. +- Integer range of `A + B − s` is `[−4294967295, 8589934590]`. Over that range the field + value `0` has exactly one integer preimage (`0`) and the field value `2^32` has exactly + one (`4294967296`). **A negative difference cannot alias `2^32 mod p`.** + +Same check for the neighbours, all clean: + +| identity | integer range | multiples of `p` in range | +|---|---|---| +| add3 `a+b+m−s−2^32(c1+c2)` | `[−12884901887, 12884901885]` | `{0}` only | +| rot `xlo·2^r − sllc·2^16 − sll`, r=4 | `[−4294967295, 1048560]` | `{0}` only | +| rot, r=9 | `[−4294967295, 33553920]` | `{0}` only | + +So every "field identity" in the arm is an exact integer identity, and add3's +`s` is uniquely pinned because `c1+c2 ∈ {0,1,2}` is the true carry range. + +### D2 — DEFECT (test coverage, severity MEDIUM) + +**`blake3_socket.rs:778-782`** states plainly: *"NEITHER ALONE SUFFICES — without the +sends the bytes are free field elements and this identity holds for arbitrary byte +strings."* **The `AreBytes` half is never exercised adversarially.** + +Every negative control breaks the *linear identity*, which is the half that is not +load-bearing for O1: + +- `the_lane_decomposition_binds_the_felt_to_its_bytes` (`blake3_socket_tests.rs:874-893`) + bumps one byte (identity breaks), then does `IN0 += 2^32` **leaving the bytes alone** + (identity breaks). +- `tampering_with_the_witness_is_not_accepted` (`:1075-1092`): a lane byte `+1`, an add3 + carry `+1`, a digest byte `+1`, a padding row marked real — all identity/mode breaks. + +**The attack that is actually O1 is not in the suite:** set `IN0 = v + 2^32` *and* +`MB[0][0] = v + 2^32` (or, cheaper, `MB[0][0] += 256`, `MB[0][1] -= 1`). The linear +identity is preserved by construction; the only thing standing between that witness and +an accepted proof is the `AreBytes` lookup having no matching table row. That is the +statement the module docs and ORACLE §7 O1 rest on, and it is asserted rather than +executed. + +**I did not run it** — the brief says do not modify the worktree, and the test would have +to live in the crate. **Recommend adding it before the lead commits**; it is ~10 lines in +`tampering_with_the_witness_is_not_accepted` and it is the single highest-value control in +the arm. My reading says it will pass (unmatched send ⇒ LogUp imbalance ⇒ reject), but +"my reading says" is exactly the standard this control exists to replace. + +--- + +## Target 3 — MU-gating and `MODE_P = 0`: CONFIRMED-SOUND + +**MU is preprocessed and prover-unchosen. ✓ VERIFIED, three ways.** + +1. `cols::MU = MODE_C = layout::hash::MODE_C = 6`, and `layout::hash::PREP_WIDTH = 11`, + so MU sits inside the preprocessed prefix. +2. `compiler.rs:329-350` builds the hash group's rows from `Instr::Hash`'s `mode` + (`Compress → (1,0)`, `Permute → (0,1)`) into + `ColumnGroup::from_rows(layout::hash::PREP_WIDTH, hash_rows)`. +3. `airs.rs:426-434` builds the hash AIR with + `.with_preprocessed(roots[5], layout::hash::PREP_WIDTH)` (`airs.rs:331-349`), so the + column is under a committed root that `lfm_program_id` binds. + +**Even if it were not preprocessed, the AIR bounds it.** `blake3_socket.rs:766-774`: +idx 4 is `(MODE_C + MODE_P)·(1 − MODE_C − MODE_P) = 0` and idx 5 is `MODE_P = 0`; +together they force `MODE_C ∈ {0,1}` on **every** row. This matters more than the report +says, because MU is the multiplicity of ~1,382 new BITWISE sends — a field-negative MU +would be the #915 forgery shape again. It is closed in the AIR, not just at admission. + +**`MODE_P = 0` genuinely bites.** ✓ EXECUTED +(`a_permute_marked_row_violates_the_air`, `padding_is_satisfied_and_a_real_marked_empty_row_is_not`). +Belt and braces with `admits` refusing at execution (`a_permute_row_is_refused_under_blake3`, +which has its own honest control under `Test`). + +**Padding rows satisfy everything.** `chip_trace` (`trace.rs:60-75`) fills only +`0..real_rows` and copies the (zero-padded) group prefix for all rows, so a padding row +has `MODE_C = MODE_P = 0` and all-zero values: idx 0–3 reduce to `S = 0` ✓, idx 4/5 ✓, +idx 6–13 and 22–25 are mu-gated ✓, idx 14–21 read zero `OUT` lanes ✓, the whole mixing +core is mu-gated ✓, and every BITWISE send has multiplicity 0 ✓. + +**Every new BITWISE send carries `Multiplicity::Column(MU)`** — all three groups, +`blake3_socket.rs:536, 551, 561`. ✓ + +**#915 (commit `3638b825`) coverage.** `validator.rs:384-403` lists the bounded columns +per chip; for `LFM_HASH` that is `[MULT0, MULT1, MULT2]` — **not** `MODE_C`. So check 9 +does *not* bound the new sends' multiplicity. That is fine, and I confirmed why: the new +sends are gated by MU, which the AIR itself pins to `{0,1}` (above). The #915 attack shape +(a committed group multiplicity holding `p − 1`) is unreachable here. **No gap.** + +### D5 — inaccurate claim (severity LOW, docs only) + +Report §3 item 14 asserts *"every eval constraint × MU"*. **Constraint idx 14–21 are +ungated** (`blake3_socket.rs:794-797` — `b.emit_base(14 + j, out)`, no `mu` factor). The +code is right (ungated is strictly stronger, and padding rows have `OUT = 0` so +completeness holds); the blanket claim is not, and §3.2 lists idx 14–21 without noting +the exception. Fix the sentence, not the code. + +--- + +## Target 4 — bus balance and census: CONFIRMED-SOUND + +`bus_interactions(kind)` has exactly **two** production callers and both thread the same +`hasher`: the census at `airs.rs:189` and the AIR at `airs.rs:429`. Same function, same +argument ⇒ same list in the same order, by construction. (The other three hits are tests.) + +The trace's actual sends are not a separate list — the prover generates them from the +AIR's declared interactions. What must agree is the shared BITWISE table's multiplicity +histogram, and both sides come from the *same* dataflow: +`bitwise_interactions()` (`:527-571`) walks `socket_wires()`, `bitwise_ops_for()` +(`:576-612`) walks `socket_values()`, and both are `run_flow(_, FLOW)` with one shared +`FlowConfig`. Group for group: + +| group | sender tuple | histogram op | +|---|---|---| +| XOR, 4/word | `(XOR, a.byte(b), b.byte(b), out[b])` | `byte_op(ByteAluXor, x>>8b, y>>8b)` | +| rot, 4/rotation | `AreBytes(pair[0], pair[1])` over `sll_lo/sllc_lo/sll_hi/sllc_hi` | `byte_op(AreBytes, hw&0xFF, hw>>8)` | +| lanes, 2/lane | `AreBytes(lane_byte(l,2p), lane_byte(l,2p+1))` | `byte_op(AreBytes, lane>>16p, lane>>(16p+8))` | + +Constant operands (`WordRef::Const`, i.e. `m[8..16]` and the initial state) become +`BusValue::constant` on the sender side and the same literal `u32` on the histogram side. +`fill_socket_witness` writes exactly the 60 cells per G-block that the senders read +(4+4+4+4+16+12+12 = 56 bytes + 4 carries), plus the 32 lane bytes and 16 output bytes — +I checked the arithmetic against `cols::G_SIZE = 60` and `the_layout_assigns_every_column_exactly_once`. + +Empirically, `the_blake3_socket_proves_and_verifies` passing **is** the bus-balance check: +a census/AIR/trace mismatch cannot produce an accepted proof. + +The frozen six are byte-identical for Test/Poseidon before and after (see Target 1), and +`S8` is read by no bus tuple at all — the six read `IN0`, `IN0+4`, `IN0+8`, `OUT0`, +`OUT0+4`, `OUT0+8` only (`chips.rs:612-644`). + +--- + +## Target 5 — claim verification by execution: PARTLY REFUTED + +### ✓ CONFIRMED + +| claim | measured | +|---|---| +| `lfm::blake3` at default (7r) | **43 passed, 0 failed, 2 ignored** | +| `lfm::blake3` at 6r (`--features blake3-6round`) | **43 passed, 0 failed, 2 ignored** | +| the two `#[ignore]`d tests hide nothing | ✓ — both are pre-existing at HEAD (`git show HEAD:…blake3_probe.rs` has `#[ignore]` at 550 and 768). They are cost-model *reporting* tests (`the_blake_column…`, the two-term RSS matrix) that `println!` projections; `#[ignore]`d for runtime, same as `wrap_tests::the_wrap_census_at_blowup_8`. Neither asserts a soundness property. | +| honest-path controls | ✓ `the_blake3_socket_proves_and_verifies` and `tampering_with_the_witness_is_not_accepted` both pass, as do the O1 pair (`an_out_of_range_lane_is_rejected_rather_than_reduced`, `the_lane_decomposition_binds_the_felt_to_its_bytes`) | +| the 7-round external anchors | ✓ `seven_rounds_is_the_blake3_crate`, `seven_rounds_is_blake3_of_the_domain_separated_message`, and the discriminator `six_rounds_is_not_the_blake3_crate` all pass | + +| full `lfm::` suite, final tree | **262 passed, 19 failed, 7 ignored** — the 19 are exactly the report's set (`epoch_tests` ×7, `epoch_verify_tests` ×6, `logup_tests` ×1, `machine_tests` ×5), **no BLAKE3 failures** ✓ | + +### ✗ REFUTED: "40 passed" / "259 passed" + +**The report's passing counts do not describe the current tree.** `lfm::blake3` is **43**, +not 40, and the full suite passes **262**, not 259 — three tests were added after the +report was written. The **19-failure claim is CONFIRMED**; only the pass counts moved. +See D1/D3. + +--- + +## Target 6 — hygiene: CONFIRMED-SOUND + +- **`Blake3Permutation::permute` panics — unreachable, including from a malicious proof.** + ✓ VERIFIED by tracing callers, not by trusting the comment. The only production call is + `executor.rs:408`, guarded by `admits` at `:395-397`. The verifier path + (`proof.rs:192-204`, `verify_against` → `LfmAirs::new_with_hasher`) uses `hasher` only + to select `num_columns(kind)`, `bus_interactions(kind)` and `HashConstraints{kind}` — + **it never calls `permute`, `compress` or `compress_out`.** So no proof, honest or + forged, can reach the panic; a verifier is never in the same call graph. (`edsl.rs:30,41` + are the *builder* emitting `Instr::Permute`, not the hasher; `fixture.rs`/`programs.rs` + name `TestPermutation` explicitly.) Residual: the method is `pub` on a `pub` trait, so a + library consumer calling it directly panics. Documented at the definition; low severity. +- **No debug leftovers** in the new files: zero `println!`/`dbg!`/`eprintln!`/`TODO`/ + `FIXME`/`todo!`/`unimplemented!`. +- **`blake3` crate is dev-only.** ✓ `prover/Cargo.toml` `[dev-dependencies]`, and the only + mention of `blake3::hash` outside test modules is a doc comment (`hash.rs:153`). It + cannot enter the production dependency graph. + +### D4 — the default build DOES change with the feature off (severity LOW) + +`blake3.rs` changed `BLAKE3_ROUNDS` from a hard `6` to `#[cfg(not(feature = "blake3-6round"))] = 7`. +That is not just the socket's knob: `blake3_chip` (`LFM_BLAKE3`) reads the same constant, +so with the feature **off** its `NUM_G` goes 48 → 56, its width and constraint count move +(769 → 897), and `Blake3Operation::output_words` now computes a 7-round compression where +it computed a 6-round one. The report's §5.3 frames the knob as the socket's and its file +map does not flag the change to the existing chip. + +**No production impact** — I checked: `LFM_BLAKE3` is **not** among the 14 registered +chips (`airs.rs:50-66`), so no program digest and no preprocessed root moves. But +"does the default build change at all when the feature is off?" is **yes**, and the report +implies no. `blake3_compress_6round` correctly still pins 6 regardless of the feature. + +--- + +## Target 7a — the knob aliasing: CONFIRMED (one knob), but unpinned + +**There is exactly one rounds knob in the tree.** A full sweep of `prover/src` for +`ROUNDS` / `rounds: usize` / `blake3-6round` finds a single `#[cfg(feature = ...)]` pair, +`blake3.rs:82-85`, defining `BLAKE3_ROUNDS`. Everything downstream derives from it with no +branch of its own: + +``` +blake3.rs:82-85 BLAKE3_ROUNDS = STANDARD(7) | SIX(6) ← the ONLY cfg + blake3_chip.rs:101 NUM_G = BLAKE3_ROUNDS * 8 + blake3_chip.rs:445 run_flow(_, FlowConfig::full(BLAKE3_ROUNDS)) + blake3_chip.rs:580 ValueFlow::compute → FlowConfig::full(BLAKE3_ROUNDS) + blake3_chip.rs:729 Blake3Operation::output_words(_, BLAKE3_ROUNDS) + blake3_socket.rs:120 SOCKET_ROUNDS = BLAKE3_ROUNDS ← a plain alias + blake3_socket.rs:123 NUM_G = SOCKET_ROUNDS * 8 + blake3_socket.rs:150 FLOW.rounds = SOCKET_ROUNDS + blake3_socket.rs:184 socket_digest → SOCKET_ROUNDS +``` + +The three deliberate **non**-knob uses are correct and are what make the tautology fix +real: `blake3_compress_6round` → `BLAKE3_SIX_ROUNDS` (`:115`), the 6-round socket-shaped +check → `BLAKE3_SIX_ROUNDS` (`:770`), and the crate anchors → `BLAKE3_STANDARD_ROUNDS` +(`:664`, `:739`). So `six_rounds_is_not_the_blake3_crate` cannot become a tautology when +the knob is flipped — I confirmed it passes at **both** round counts. + +`blake3-6round` is declared only at `prover/Cargo.toml:20` and is enabled by no crate, no +Makefile target and no CI workflow, so cargo feature unification cannot switch it on +implicitly. + +### D9 — the single-knob invariant is enforced by one line and nothing else (severity LOW) + +Nothing asserts `blake3_socket::SOCKET_ROUNDS == blake3_chip::BLAKE3_ROUNDS`. The existing +assertions are each internally consistent — +`blake3_socket_tests.rs:142` (`NUM_G == 8 * SOCKET_ROUNDS`) and +`blake3_probe.rs:370` (`cols::OUT - cols::G == 60 * NUM_G`) — and would all still pass if +the two chips were compiled for different round counts. + +That is not hypothetical: **the tree had exactly that shape until wave 2.** `SOCKET_ROUNDS` +was its own `#[cfg(feature = "blake3-6round")]` pair before 20:44; wave 2 collapsed it to +the alias. Re-introducing the pair is a one-line regression that no test catches, and its +consequence is precisely the "silent pricing lie" — `blake3_probe`'s matrix would compare a +7-round socket against a 6-round standalone chip and the report's "hosting is 3.6% cheaper" +would be measuring two different hash functions. + +Cheapest fix: a `const { assert!(SOCKET_ROUNDS == super::blake3::BLAKE3_ROUNDS) }` next to +the alias, or one line in `the_built_layout_matches_the_prediction`. + +## Target 7b — `canonical_expected_out` selection: CONFIRMED-SOUND + +Not vacuous in either direction, because three *independent* statements cover it and two of +them are knob-**independent** (they run whichever way the build is compiled): + +| test (`blake3.rs`) | what it pins | knob-dependent? | +|---|---|---| +| `the_compression_matches_the_canonical_vectors_at_seven_rounds` (`:655-670`) | `CANONICAL_OUT_7ROUND` == `blake3_compress_rounds(…, BLAKE3_STANDARD_ROUNDS)`, all 10 | **no** | +| `the_six_and_seven_round_vector_tables_differ_everywhere` (`:674-680`) | `assert_ne!(v.out, CANONICAL_OUT_7ROUND[i])`, all 10 | **no** | +| `canonical_expected_out_follows_the_round_knob` (`:686-698`) | the accessor == the expected table **and** == `blake3_compress_rounds(…, BLAKE3_ROUNDS)` | yes | + +The third is what makes selection non-vacuous: the chosen branch is checked against a +*computation* at the compiled round count, not merely against the table it just selected. +A wrong branch returns the other table, which the second test proves differs on every +vector, so the equality fails. And the second test is the explicit anti-vacuity control the +lead asked about — a generation bug that emitted the 6-round outputs twice is caught even +though it would leave the first test passing. + +Both branches were **executed**: all three tests are inside the 43 that passed at 7 rounds +*and* the 43 that passed at 6 rounds. `blake3_probe.rs:461` is the consumer +(`the_hosted_chip_proves_and_verifies` asserts the chip's `OUT` columns against +`canonical_expected_out(row)`), and it passes at both counts too. + +The socket's own two-table selections (`blake3_socket_tests.rs:396`, `:815`, choosing +`digest_7` vs `digest_6`) are the same shape and were likewise exercised at both counts, +and the `other_round_count` framing control (`:552`) derives its wrong count from the knob +(`if SOCKET_ROUNDS == 7 { 6 } else { 7 }`), so it stays discriminating either way. + +## Prosecuting the implementer's two claims + +They asked me to attack (a) the add2 equivalence and (b) the claim that no permute row can +reach the trace filler. Both survive. + +### (a) "chip_model.py witnesses the carry as a column, the chip derives it — provably equivalent" + +**The equivalence holds** — I machine-checked it (Target 2c): the model's pair asserts +`∃ carry ∈ {0,1}. A + B = s + 2^32·carry`; the chip asserts `(A + B − s)·2^{−32} ∈ {0,1}`, +i.e. `A + B − s ∈ {0, 2^32}` in `F_p`. With `A, B, s` byte-bound below `2^32`, the reachable +integer range is `[−4294967295, 8589934590]`, in which the field values `0` and `2^32` have +**exactly one integer preimage each**. The existential is eliminated because its witness is +determined. Same statement, one fewer column, same degree 3. + +**But the premise is stale, and the direction of fit has inverted.** `chip_model.py` on disk +(mtime **20:40**) no longer witnesses the carry as a column. Its `emit_add2` docstring now +reads *"CHIP COLUMNS: s[0..4] bytes. **NO carry column.** CHIP CONSTRAINT (mu-gated), the +only one — `blake3_socket.rs:826-834`"*, and explicitly *"the gate must certify the chip +that EXISTS, not a stronger cousin, so **the model follows the chip**."* So: + +- The report's §3 row 6 (⚠ DEVIATION) and §3.1's recommendation to *"re-express + `emit_add2` before the Phase-4 gate"* are **already done — by the oracle side, not you.** +- `run-gate.log` is **20:09**, which **predates** both `chip_model.py` and `gate.py` (both + 20:40). **The recorded green verdict does not certify the model now on disk.** +- The model's own line reference (`:826-834`) is stale by the same +17 as the report's. + +That is D7, and it is the one thing here that needs a *re-run*, not an edit: the gate must +be re-executed against the 20:40 model before task #4 can claim anything. Note the model is +honest about the seam — it flags `2^{−32}` as having no faithful BV counterpart and defers +the "only reachable roots" side condition to the field audit `WA7`. **I independently +discharged that side condition** (the table above), so the equivalence is not resting on the +gate to begin with. + +**Executed, independently: the model and the chip now agree to zero.** I ran +`SocketChip(...).build()` at both round counts myself: + +| | model main | model sends | model aux/3 | model cell-equiv | chip | +|---|---:|---:|---:|---:|---| +| 6 rounds | 2,956 | 1,190 | 595 | 4,741 | **identical on all four** | +| 7 rounds | 3,436 | 1,382 | 691 | 5,509 | **identical on all four** | + +Not "small explainable deltas" — **zero**. The old −81/−97 was the carry column (112 at 7r) +net of the prefix accounting (+15), and both are gone: the model's block breakdown now reads +`add2` 448 (was 560) and `frozen_socket_prefix(IN/S/OUT)` 28 (was "I/O+MU 13"). + +**Consequence the lead needs: `ORACLE.md` §3.2 is stale.** Its census table still carries +main 3,533 / 3,037 and cell-equiv 5,606 / 4,822, its 7-round breakdown still says `add2` 560 +and `I/O+MU 13`, and **its whole reconciliation against the standalone chip is computed from +those numbers**. If "expected census targets from the gated model" were taken from that +table, they are superseded by the four figures above. `ORACLE.md` is 20:16, i.e. also older +than the 20:40 model. It is the oracle side's file; I have not touched it. + +(`gate.py` has since moved again — mtime 21:16:59 — so the 20:09 `run-gate.log` is now +stale against both the model *and* the gate.) + +### (b) "no path where a permute row reaches the trace filler" — CONFIRMED + +Traced rather than assumed: + +- **The mode is program-derived, not record-derived.** `trace.rs:132-138` builds + `hash_modes` by filtering `program.instrs` for `Instr::Hash { mode }` — the same source + `compiler.rs:329-350` uses to write the preprocessed `MODE_C`/`MODE_P`. The two cannot + disagree. +- **Only two production callers** of `build_traces_with_hasher`: `trace.rs:116` (the + `build_traces` wrapper, which passes `HasherKind::default()` = `Test`) and `proof.rs:90` + inside `lfm_prove_with_hasher`, which passes the *same* `hasher` it called `execute` with. + (The two other hits are doc comments.) So on the prove path, `admits` has already rejected + any `Permute` row before `records.hash` exists, and the filler cannot see one. +- **Three independent fallbacks if someone hand-built the mismatch** (e.g. calling + `build_traces_with_hasher(prog, records_from_Test, Blake3)`): + 1. `trace.rs:185-196` runs **first**, mapping `lanes_of(...).expect(...)` over *every* + hash record — a permute row's capacity lanes are arbitrary felts, so it panics there, + prover-side, before any witness is written. + 2. If the lanes happened to be `u32`, `fill_socket_witness` would write a BLAKE3 witness + onto a row whose preprocessed `MODE_P = 1`, which violates AIR idx 5 — unprovable. + 3. `MU = MODE_C = 0` on such a row, so every BLAKE3 constraint and every BITWISE send is + vacuous anyway. + + Worst case is a prover-side panic or a rejected proof. **No path produces an accepted + proof**, and none reaches `Blake3Permutation::permute`'s panic. + +## The tautology sweep they asked me to run against them + +The trap class they identified — *a test that reads `BLAKE3_ROUNDS` when it means a fixed +count silently stops discriminating at the default* — is the right thing to audit, so I ran +it wider than the two files they named. **Clean: no surviving tautology.** + +Every `BLAKE3_ROUNDS` read in test code falls into one of two safe shapes: a *parameterised +prediction* (`predicted_main(BLAKE3_ROUNDS)`, `predicted_interactions(…)`, +`predicted_cells(…)`, `predicted_bitwise(…)` — `blake3_probe.rs:376, 379, 382, 488, 494`) or +a *branch selector* (`if BLAKE3_ROUNDS == 6`, `if BLAKE3_ROUNDS == BLAKE3_STANDARD_ROUNDS` — +`blake3_probe.rs:414, 811`, `blake3.rs:688`). The one bare use, `blake3.rs:696`, is the +accessor-vs-primitive cross-check where "the compiled count" is exactly what is meant. + +Everything that means a **fixed** count now names the constant, each with a comment saying +why: `BLAKE3_SIX_ROUNDS` at `blake3.rs:115` (`blake3_compress_6round`), `:495` (the +`CANONICAL` conventions struct) and `:770` (the 6-round socket-shaped check); +`BLAKE3_STANDARD_ROUNDS` at `:664` and `:739` (the crate anchors). + +Extending to `blake3_socket_tests.rs`, which they did not name: `SOCKET_ROUNDS` appears only +as a prediction argument, as a branch selector (`:396`, `:815`), as the honest framing +(`:249`), and — the one worth checking — at `:552` as +`rounds: if SOCKET_ROUNDS == 7 { 6 } else { 7 }`, the `other_round_count` negative control, +which stays a *different* count either way. The explicit-7 KAT rows (`:387`, `:429`) and the +explicit-6 row (`:381`) hardcode their counts rather than reading the knob, which is why +`seven_rounds_is_blake3_of_the_domain_separated_message` still passes under +`--features blake3-6round`. + +**D9 is the residue of this class**: the sweep is clean *today*, but nothing enforces it. +See below. + +## D1 — the review target moved during the review (severity HIGH, process) + +**This is the finding the lead most needs.** `phase2-report.md` describes a tree that no +longer exists. Recorded mtimes: + +``` +18:22–18:24 hash.rs, executor.rs, chips.rs, airs.rs, poseidon_chip_tests.rs +19:27 mod.rs +20:22:43 blake3_socket_tests.rs, blake3_socket_kats.rs, trace.rs +20:37:05 blake3_chip.rs ← after my first test run started +20:44:58 blake3_socket.rs (878 → 895 lines) +20:45:08 blake3.rs, blake3_probe.rs +``` + +Consequences: + +1. **The report's §2 file:line map is off.** e.g. `blake3_socket.rs:215` (the `permute` + panic) is now `:232`; `:826-834` (add2) is now `:843-851`; the file is 895 lines, not + the 878 the report states. +2. **Test counts moved**: 40 → 43. +3. **The implementer's "the socket arm is unchanged in wave 2" claim: ✓ VERIFIED.** + `blake3_socket.rs` is untracked so `git diff` cannot show it; I tested it structurally + instead. All of the +17 lines land **before** line 113, so every declaration from there + on should sit at exactly its old offset + 17. It does, at all eight anchors I checked: + + | line | declaration found | + |---|---| + | 299 | `pub mod cols {` | + | 367 | `fn message_word_ref(i: usize) -> WordRef {` | + | 544 | `pub fn bitwise_interactions() -> Vec {` | + | 593 | `pub fn bitwise_ops_for(rows: &[([u32; 4], [u32; 4])]) -> Vec {` | + | 656 | `pub fn fill_socket_witness(row: &mut [FE]) {` | + | 737 | `pub const NUM_CONSTRAINTS: usize = 26 + 16 * NUM_G;` | + | 750 | `pub fn eval>(b: &mut B) {` | + | 843 | `for aw in &wires.add2s {` | + + Combined with re-reading the entire `eval()` body on the final tree (byte-identical to + what I analysed: same `NUM_CONSTRAINTS`, same `CORE_IDX = 26`, same 26 framing + constraints, same core loops), wave 2's socket-arm delta is **the O5/128-bit module-doc + block plus the `SOCKET_ROUNDS` alias, and nothing else**. Layout, senders, histogram + mirror, trace filler and constraints are untouched. **The analysis above stands for the + pinned hashes.** +4. The lead has since confirmed the tree is idle and wave 2 is final; scope re-checked + after that confirmation is still **12 M + 3 ??** at the same hashes, with wave 2 visible + as the larger per-file deltas (`blake3.rs` +272 vs +107 before, `blake3_probe.rs` +142 + vs +19, `blake3_chip.rs` +158 vs +134). + +**Recommendation: do not commit against the report's numbers.** Have the implementer +regenerate §2's file map and §6's counts against the final tree, or commit first and let +the report describe the commit. The *findings* below need no re-run — every test result in +this document was measured after wave 2 landed. + +## D3 — "19 failures, all pre-existing": CONFIRMED, but only after a rebuild + +My full `lfm::` run against the **20:37 intermediate** tree gave **260 passed / 21 failed**, +not the report's 259 / 19. The two extras were BLAKE3's own: +`blake3::tests::six_rounds_is_not_the_blake3_crate` and +`blake3_probe::the_hosted_chip_proves_and_verifies`. + +**These were artifacts of compiling a half-applied edit, not regressions.** Decisive +evidence: re-running `lfm::blake3` on the final tree (hashes verified unchanged +immediately before and after) gives **43 passed / 0 failed**, with both of those tests +listed as `ok`. The 20:37–20:45 wave was the round-knob unification landing across +`blake3.rs` / `blake3_chip.rs` / `blake3_probe.rs` / `blake3_socket.rs`, and I sampled it +mid-flight. + +**Settled by a clean re-run.** I re-ran the full `lfm::` suite against the frozen final +tree, with the aggregate hash of every `prover/src/lfm/*.rs` (`df061a67…`) verified +identical immediately before and after: **262 passed, 19 failed, 7 ignored**, and the 19 +group exactly as the report says — `epoch_tests` ×7, `epoch_verify_tests` ×6, +`logup_tests` ×1, `machine_tests` ×5, none touching `LFM_HASH`. **The report's +"19 failed, all pre-existing" is CONFIRMED.** Only its pass counts are stale (262 vs 259). + +**Lesson for the record:** the report's "no new test failures" was true of the tree its +author had, but a reviewer sampling the same worktree minutes later measured two new +failures. Uncommitted review targets need a freeze or a commit. + +--- + +## Additional findings outside the numbered targets + +### D6 — underconstrained-but-unread columns (severity INFO, no soundness impact) + +With `FLOW.out_window = 4` the truncated feed-forward reads only `v[0..4]` and `v[8..12]`. +In the **last** round, the four diagonal G's write their `b` slot to `v[4..8]` and their +`d` slot to `v[12..16]` — neither is read by anything. The `d` words are `ByteAlu[XOR]` +outputs so their bytes stay pinned, but the `b` words are `rot` outputs `Y`, and `Y`'s four +byte columns are constrained only by the two half-sums (`Ylo = SLL_hi + SLLC_lo`, +`Yhi = SLL_lo + SLLC_hi`). Their individual bytes get their range check "free from the XOR +that consumes them" (`chip_model.py:emit_rotr`) — and in the last round there is no +consumer. So 16 byte columns carry 2 free degrees of freedom each. + +**Not a soundness issue** — nothing reads them, so no digest, bus token or public value can +move. It is exactly the waste that `chip_model.py`'s optional `tail_truncate` (ORACLE §3.3, +report §3 item 16) would remove, and the model has the same shape, so the chip is +conformant. Worth knowing before someone "optimises" `Y`'s constraints on the assumption +they are tight. + +### D7 — the gate no longer certifies an independently-derived model (severity MEDIUM, process) + +The report's §3.1 headline — *"the one deviation: `emit_add2`'s carry"*, with a +recommendation to re-express the model *before* Phase 4 — is **stale, and the fix went the +wrong way round**. On disk right now: + +- `chip_model.py` (mtime **20:40**) already models the expression-carry form. Its docstring + reads *"CHIP COLUMNS: s[0..4] bytes. **NO carry column.** CHIP CONSTRAINT (mu-gated), the + only one — `blake3_socket.rs:826-834`"* and *"the model follows the chip"*. +- `run-gate.log` is **20:09** — it **predates** both `chip_model.py` and `gate.py` (both + 20:40). **The recorded gate verdict does not cover the model now on disk.** + +So the §3.1 action item is already done, but the spec was retro-fitted to the +implementation and the gate has not been re-run since. Two things follow: (i) the report's +§3 conformance table is against a superseded revision (it says so, but the implication is +understated); (ii) whoever picks up task #4 ("z3 gate on the real chip") must **re-run the +board** — the green log in the directory is not evidence for the current model. The +model's own `emit_add2` docstring is honest about this and points at `WA7` as the field +audit that discharges the aliasing side condition; I independently confirmed that side +condition holds (Target 2c), so the direction of fit is a process problem, not a +correctness one. + +### D8 — the report omits O5 and the 64-bit collision bound (severity MEDIUM, disclosure) + +`ORACLE.md` §7 lists **O5 — leaf/parent domain separation — as ✗ OPEN, needs a decision**, +and states plainly that the socket's 128-bit digest gives **64-bit collision resistance**. +`phase2-report.md` discusses O1, O2 and O3 and never mentions O4, O5 or the collision +bound. A reader of the report alone would conclude the obligation set is discharged. + +The implementer evidently agreed: the 20:44 edit added exactly this to the module doc +(`blake3_socket.rs:59-78`, *"✗ OPEN — O5: leaf/parent domain separation is NOT decided"* +plus the birthday-bound note). **The code is now honest; the report is not.** If the lead +commits from the report, the open obligation is invisible. Recommend a §5.4 entry, and it +should probably become a task alongside #8. + +(O4 — the `keccak_host` one-felt-one-u32 little-endian convention — **is** satisfied: +`word_of`/`lanes_of`/`set_word_bytes` are all LE per-lane, and `lanes_big_endian` is a live +negative control. Just uncalled-out.) + +--- + +## What I could not falsify + +For the record, the attacks I constructed and that the arm survived: + +- Silent reduction of an out-of-range lane anywhere on the host path — no `as u32`, no + mask, no modulus exists. +- A surviving second rounds knob letting the machine's hash and the chip it is priced + against describe different functions — one `cfg` pair in the tree, everything else + derived (D9 notes it is unpinned, not broken). +- A vacuous `canonical_expected_out` branch silently unpinning the chip's `OUT` columns — + two knob-independent controls plus a primitive cross-check close it. +- `A + B − s` negative aliasing to `2^32 mod p` in the expression-carry add2 — ruled out by + exhaustive range arithmetic. +- Choosing MU per row to zero out the BITWISE sends or make them negative — MU is + preprocessed *and* AIR-pinned to `{0,1}`. +- Smuggling a `permute` row under BLAKE3 — refused at execution and unsatisfiable in the + AIR, independently. +- A census/AIR mismatch making the declared and sent interaction lists differ — one + function, one argument, two call sites. +- Reaching the `permute` panic from a verifier — the verifier never calls the hasher. +- A behavioural change to Test/Poseidon from the `compress_out` refactor — the default is + the old expression and neither implementor overrides it. + +--- + +## Appendix — commands run + +All in `/Users/maurofab/workspace/lambda_vm-blake3-impl`, one cargo invocation at a time: + +``` +cargo test --release -p lambda-vm-prover lfm::blake3 + → 43 passed, 0 failed, 2 ignored (final tree, default 7 rounds) + +cargo test --release -p lambda-vm-prover --features blake3-6round lfm::blake3 + → 43 passed, 0 failed, 2 ignored (final tree, 6 rounds) + +cargo test --release -p lambda-vm-prover lfm:: + → 262 passed, 19 failed, 7 ignored (final tree; aggregate source hash + df061a67… verified unchanged + before and after the run) +``` + +An earlier `lfm::` run against the 20:37 intermediate tree gave 260 / 21 — the two extra +failures were the half-applied round-knob edit, see D3. + +Field-arithmetic checks were done in Python against `p = 2^64 − 2^32 + 1`: +`INV_SHIFT_32` recomputed as `2^{−32} mod p`; integer preimage ranges enumerated for the +add2, add3 and rotation identities (Target 2c). diff --git a/thoughts/shared/lfm-real-hash/phase3-report.md b/thoughts/shared/lfm-real-hash/phase3-report.md new file mode 100644 index 000000000..6a99d3dc7 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/phase3-report.md @@ -0,0 +1,264 @@ +# Phase 3 — bind the hasher into the program digest and the registry + +**Status:** BUILT, gates green, **uncommitted** in +`/Users/maurofab/workspace/lambda_vm-blake3-impl` (branch `blake3-real-hash`). +Nothing pushed, nothing committed — the lead reviews and commits. +**Date:** 2026-08-10. Closes review finding **F3-2 / F3.3**; also closes **F3.2** +(see §7, flagged as an extra). + +Claims below are marked ✓ VERIFIED (I read the code and cite `file:line`, or I +ran the thing) or ? INFERRED. Line numbers are post-change unless stated. + +--- + +## 1. The gap, confirmed before fixing + +✓ VERIFIED by reading the pre-change tree, not by trusting the brief: + +- `lfm_program_id`'s preimage was tag ‖ machine version ‖ preset ‖ per-slot + `(index, root, log_height)` ‖ chunk count — **no hasher** + (`statement.rs:40-56`, pre-change). +- `LfmRegistryEntry` had `kind, blowup_factor, roots, log_heights, + keccak_rnd_chunks, program_id` — **no hasher** (`registry.rs:52-60`). +- `LfmArtifacts` likewise (`registry.rs:63-68`). +- `lfm_verify` → `verify_against` → `verify_against_with_hasher(..., + HasherKind::default())` (`proof.rs:135-181`). The registry path could not + reach Poseidon at all. + +So a Test-backed and a Poseidon-backed machine of the same program had +byte-identical roots **and** byte-identical `program_id`. The only separator was +`hash::num_columns` differing (39 vs 623), caught by the framework as a width +mismatch — a layout coincidence, not a binding. + +**The one fact that makes this load-bearing rather than belt-and-braces**, which +I confirmed rather than assumed: no preprocessed root moves with the hasher. +`layout::hash::PREP_WIDTH = 11` under both candidates, so `build_artifacts` +commits identical groups either way. I proved this by measurement, not by +reading — see §5, where the regenerated table shows **all 84 root literals +bit-identical and only the 6 digests moved**. The commitments therefore *cannot* +carry the hasher; a tag in the digest is the only place it can live. + +--- + +## 2. The diff — 12 files, +337 / −87 + +### Source (5 files) + +| File | Change | +|---|---| +| `prover/src/lfm/hash.rs` | `HasherKind` gains `#[repr(u8)]` with written-out discriminants (`Test = 0`, `Poseidon = 1`) and `pub const fn as_tag(self) -> u8`. Doc says why the wire value must not follow declaration order. | +| `prover/src/lfm/statement.rs` | `lfm_program_id` takes `hasher: HasherKind` and folds `h.update([hasher.as_tag()])` in **after `LFM_PRESET_TAG`, before the per-slot loop** — the position §4 of the plan specifies. Everything else in the preimage is untouched and in the same order. | +| `prover/src/lfm/registry.rs` | `hasher: HasherKind` field added to **both** `LfmRegistryEntry` and `LfmArtifacts`. `build_artifacts(program, options)` now delegates to new `build_artifacts_with_hasher(program, options, hasher)`, which derives `program_id` from the hasher and stores it. Registry constants regenerated. | +| `prover/src/lfm/proof.rs` | See §3 — the verify and prove paths. | +| `prover/src/lfm/mod.rs` | Re-exports `HasherKind` and `build_artifacts_with_hasher`. | + +### Generator (1 file) + +`prover/src/bin/compute_lfm_registry.rs` — emits the `hasher:` line, builds via +`build_artifacts_with_hasher` under a named `REGISTRY_HASHER: HasherKind = +HasherKind::Test` constant (so the table's hasher is a stated decision, not an +implicit default), and calls `validate(program)` per program (§7). + +### Tests (6 files) + +`machine_tests.rs` (+129), `poseidon_chip_tests.rs`, `constraint_tests.rs`, +`fri_tests.rs`, `join_tests.rs`, `wrap_tests.rs` — the new tests (§4) plus the +33 mechanical `verify_against` call-site updates (§3). + +--- + +## 3. `lfm_verify` reads the hasher; there is no defaulting path left + +**The fix asked for** (`proof.rs:150-167`): `lfm_verify` resolves the entry and +passes `entry.hasher` into the AIR-set build. ✓ VERIFIED by reading the final +file; the `HasherKind::default()` call is gone from the verify path entirely. + +**One design decision worth the lead's attention.** I **merged** +`verify_against` and `verify_against_with_hasher` into a single +`verify_against(roots, program_id, keccak_rnd_chunks, proof, claimed_public, +options, hasher)` rather than leaving the defaulting wrapper in place. +`grep verify_against_with_hasher` now returns **0** hits. + +Reasoning: the brief asked that `verify_against` "also take/carry the hasher", +and adding a `hasher` parameter to the wrapper would have made it identical to +the function it wrapped. Keeping the wrapper would also have left a live hazard +that this phase creates: `artifacts.program_id` is now derived from the hasher, +so a test that switches to `build_artifacts_with_hasher(..., Poseidon)` and +calls a defaulting `verify_against` would be silently pairing one hasher's +digest with another hasher's AIR set. Passing `artifacts.hasher` explicitly +keeps the two locked together at every call site. + +Cost: **33 call sites** updated, all inside `prover/src/lfm/*` +(`verify_against` was `pub` but never re-exported from `mod.rs`, so there are no +callers outside the module — ✓ VERIFIED by grep across the workspace). They were +patched by a script that reads the receiver off each call's **own** first +argument (`&NAME.roots,`) rather than assuming the binding is called +`artifacts`, and reports anything it cannot parse instead of guessing. The one +"unhandled" report was the function definition itself. + +**What I deliberately did NOT change:** `verify_against` keeps its +piece-by-piece parameter list rather than taking `&LfmArtifacts`. I checked +whether the artifacts-struct signature was viable and it is not: +`wrap_tests.rs:505-512` passes a **deliberately mutated** `program_id` (`other`, +with `other[0] ^= 1`) alongside the real roots, and that falsification is the +point of the test. The loose form has to survive. + +### The prove side, which the brief did not ask about but this change forces + +Adding `hasher` to `LfmArtifacts` creates a new way to be wrong: artifacts built +for one hasher, proved under another, produce a proof whose statement names a +permutation the trace does not use. I closed it rather than leaving it: + +- `lfm_prove` now uses `artifacts.hasher` instead of `HasherKind::default()` + (`proof.rs:51-58`). Same for the test-only `prove_traces`. +- `lfm_prove_with_hasher` **asserts** `artifacts.hasher == hasher` + (`proof.rs:83-88`), documented under a `# Panics` section. It is a caller bug, + not a proof outcome, so it panics rather than returning `Err`. + +This is why `poseidon_chip_tests.rs` needed real changes and not just a rename: +two of its tests previously built default (Test) artifacts and proved under +Poseidon. They now build hasher-matched artifacts. + +--- + +## 4. The soundness property, and the test that proves it + +**The property:** two programs identical except for `HasherKind` now have +**distinct** `program_id`s. + +### The test the lead asked for + +`poseidon_chip_tests::the_hasher_choice_moves_the_program_digest_and_no_root` +✓ VERIFIED PASSING. It replaces the pre-change test +`the_hasher_choice_does_not_move_any_program_digest`, whose name asserted the +exact property this phase inverts. (That old test in fact only checked +`build_artifacts` determinism — it called the same no-hasher function twice — so +its name and doc comment had been describing something it did not test. Worth +noting as a stale-doc finding in its own right.) + +For both `trivial_program` and `fri_toy_program`, over Test vs Poseidon: + +- `assert_eq!(test.roots, pos.roots)` — no preprocessed root moves; +- `assert_eq!` on `log_heights` and `keccak_rnd_chunks` — nothing else moves; +- `assert_ne!(test.program_id, pos.program_id)` — **the digest moves.** + +The first assertion is what makes the third meaningful: with every other input +to `lfm_program_id` held bit-identical, the inequality can only come from the +tag. + +### Three more tests, all ✓ VERIFIED PASSING + +- `machine_tests::every_registry_entry_binds_its_hasher_into_its_digest` — for + each of the six entries: the stored `program_id` **is** what the stored + `(roots, log_heights, chunks, hasher)` derive (honest control, the table is + self-consistent), and recomputing with any *other* `HasherKind` gives a + different digest (the property, at registry level). +- `machine_tests::the_registry_hasher_is_what_verify_builds` — the honest-path + control the standing rule requires. An honest `TrivialV0` proof **verifies** + through `lfm_verify` (which now builds from `entry.hasher`), and the same + proof against the same entry's roots and digest under any other hasher + **rejects**. The accept half is not decoration: a fix that rejected everything + would pass the reject half on its own. +- `poseidon_chip_tests::the_hasher_tags_are_stable_and_distinct` — pins + `Test.as_tag() == 0`, `Poseidon.as_tag() == 1`, `default() == Test`. The tag + is the mechanism, so it is pinned directly and not only through a digest. + +A `const ALL_HASHERS` in `machine_tests.rs` lists every variant by hand, so +adding BLAKE3 in a later phase forces a deliberate edit here rather than +silently narrowing the coverage. + +The six `registry_drift_*` tests also gained +`assert_eq!(entry.hasher, artifacts.hasher, "hasher drifted")`. + +--- + +## 5. Registry regeneration — done, and it moved exactly what it should + +`cargo run --bin compute_lfm_registry --release` ran clean (exit 0). Not +blocked; it needs no ELF or fixture. Output spliced into `registry.rs`, +`cargo fmt` applied. + +**All six `program_id`s moved. No root moved.** ✓ VERIFIED mechanically, not by +eye: I extracted every 32-byte literal from the table before and after (90 per +version = 6 entries × (14 roots + 1 digest)) and compared. Exactly six differ, +at indices 14, 29, 44, 59, 74, 89 — the 15th literal of each entry, i.e. the +`program_id`, and nothing else. + +This is the deliberate re-blessing §4 of the plan calls for, and it needs +calling out in the PR body. New digests (first 4 bytes): + +| kind | new `program_id` | +|---|---| +| `TrivialV0` | `9f 05 37 f5 …` | +| `FriToyV0` | `3b 4e 71 8c …` | +| `KeccakChainV0` | `eb 59 1d e1 …` | +| `KeccakSpongeV0` | `1d 90 d7 b5 …` | +| `TranscriptReplayV0` | `26 03 3a 9e …` | +| `StatementReplayV0` | `af 84 2f d9 …` | + +All six drift tests recompute and match ✓ VERIFIED PASSING — the table is +self-consistent. + +--- + +## 6. Gates + +| Gate | Result | +|---|---| +| `cargo build -p lambda-vm-prover` | ✓ clean | +| `cargo check -p lambda-vm-prover --tests` | ✓ clean | +| `cargo check -p lambda-vm-prover --bin compute_lfm_registry` | ✓ clean | +| `cargo test --release --lib -- registry_drift is_admissible hasher registered_programs` | ✓ **20 passed, 0 failed** | +| `cargo test --release --lib -- wrap_tests constraint_tests fri_tests join_tests poseidon_chip_tests` | ✓ **56 passed, 0 failed**, 4 ignored | +| `make fmt` | ✓ clean | +| `make lint` | ✓ clean — all four clippy feature combos under `-D warnings` | + +`make lint` and `make fmt` were both run from the worktree root; neither was +skipped. + +### The 19 pre-existing failures, checked rather than assumed + +`cargo test --release -p lambda-vm-prover --lib lfm::` reports **233 passed, 19 +failed, 7 ignored**. I did **not** assume those 19 were pre-existing. I stashed +the entire change (`git stash push -- prover/`, after saving a backup patch), +re-ran the same 19 on the pristine tree, and got the **identical failing set**; +then popped the stash and confirmed the diff restored intact. + +They are `lfm::epoch_tests` (7), `lfm::epoch_verify_tests` (6), +`lfm::logup_tests::a_zero_row_fixed_table_carries_some_zero_not_none`, +`arena_filler_reads_real_committed_roots`, +`continuation_fixture_generates_two_epochs`, and three `l2g_binding*` tests. +The visible cause is `Exec(ArenaLenMismatch { arena: 0, expected: 4, found: 2 })` +— fixture-shaped, unrelated to hashing. ✓ VERIFIED pre-existing on +`blake3-real-hash` head `ef13e746`. + +--- + +## 7. One thing I did beyond the four IMPLEMENT items — flag for the lead + +Plan §4 step 6 says to take **F3.2** while in the same file, and I did: +`compute_lfm_registry` now calls `validate(program)` per program before building +artifacts, so the admission gate `validator.rs` declares is mechanically wired +into registry generation instead of resting on the convention that every +registered kind also has a hand-written admissibility test. + +It is 3 lines in one file, and it passed for all six programs on the first run +(the generator would have panicked otherwise) — so this is confirmation, not a +change in what is admitted. It is **not** in the lead's four-item IMPLEMENT +list, so drop it if you want the phase kept to exactly that scope. + +--- + +## 8. State and what is next + +- Worktree `/Users/maurofab/workspace/lambda_vm-blake3-impl`, branch + `blake3-real-hash`, **12 files modified, uncommitted, unpushed.** +- Backup of the diff: `phase3.patch` in this session's scratchpad (insurance for + the stash cycle in §6; the working tree is authoritative). +- `HasherKind` still has exactly two variants. No BLAKE3 arm was added — that is + a later phase, and the binding is hasher-generic so it does not need one. +- The ordering constraint in the plan is now satisfied: the tag is in place + **before** a third candidate exists, so BLAKE3 cannot land on a colliding + identity. The next hasher needs: a variant with the next unused discriminant, + an `ALL_HASHERS` entry in `machine_tests.rs`, and a registry row — the digest + binding itself needs no further work. diff --git a/thoughts/shared/lfm-real-hash/transcript-impl-report.md b/thoughts/shared/lfm-real-hash/transcript-impl-report.md new file mode 100644 index 000000000..3847bcaf2 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-impl-report.md @@ -0,0 +1,529 @@ +# The compress-chain transcript (option B1) — implementation report + +**Status:** GREEN with one named deviation (§7); post-review fixes D1-D4 applied (§9). **Date:** 2026-08-11. +**Ground:** worktree `lambda_vm-blake3-impl`, branch `blake3-real-hash`, parent +`2957c3f9`, uncommitted. **Spec:** `transcript-spec/TRANSCRIPT.md`, treated as +binding; every departure is named in §7 rather than silently taken. + +Claims are ✓ EXECUTED (ran it, output quoted), ✓ VERIFIED (read the code, cited) +or ✗ OPEN. + +--- + +## 0. Board + +| item | result | +|---|---| +| spec KATs — per-op, 6 and 7 rounds | ✓ EXECUTED, PASS | +| spec KAT — end-to-end `FriToyV0`-preamble transcript, 6 and 7 rounds | ✓ EXECUTED, PASS | +| the crate anchor: step == `blake3::hash(state‖operand‖"LFMT")[..16]` @7r | ✓ EXECUTED, PASS | +| M1–M7 pre-committed controls | ✓ EXECUTED, all 7 fire as predicted | +| `TrivialV0` proves + verifies under BLAKE3 | ✓ EXECUTED, PASS | +| `FriToyV0` proves + verifies under BLAKE3 | ✗ **BLOCKED — by O1, not by the sponge.** §7 | +| transcript program proves + verifies under Test / Poseidon / BLAKE3 | ✓ EXECUTED, 3/3 | +| cost claims 16,527 / 369,103 @7r | ✓ EXECUTED, both exact | +| full `lfm::` suite | 291 pass / 19 fail — the 19 are the pre-existing `fibonacci.elf` set, identical at 6r and 7r | +| `make fmt` + `make lint` (4 feature combos) | ✓ EXECUTED, clean | +| `cargo clippy --features blake3-6round` | ✓ EXECUTED, clean | +| keccak wrap path | untouched — the whole diff is inside `prover/src/lfm/` | + +--- + +## 1. What was built + +The Fiat–Shamir sponge is now a **compress chain over one cell**, for every +hasher, and no permute socket exists or ever will. + +``` +absorb(c) state ← T(state, c) 1 step +absorb2(c0, c1) state ← T(T(state, c0), c1) 2 steps +squeeze() out = state ; state ← T(state, SQ(i)) 1 step +``` + +`T` is an ordinary `LFM_HASH` two-to-one row in the **transcript domain**; +`SQ(i) = [SQUEEZE_MARK, i, 0, 0]` with `SQUEEZE_MARK = "SQZ0"` LE. Squeeze +outputs before advancing, mirroring the construction it replaced so the diff +stays reviewable. + +The domain is carried by `m[8] = MODE_C·"LFMC" + MODE_T·"LFMT"` — a linear form +over two **preprocessed** columns. It costs no witness columns, no range checks, +and no degree (`m[8]` went from degree 0 to degree 1 inside an `add3` operand +whose body is degree 1 either way; the arm's max degree is still 3). + +### File:line map + +| what | where | +|---|---| +| `MODE_T` column, `NUM_SELECTORS = 3`, `PREP_WIDTH` 11→12 | `prover/src/lfm/layout.rs:81-113` | +| `HashMode::Transcript` + `is_two_to_one()` | `prover/src/lfm/instr.rs:47-80` | +| `LfmBuilder::transcript_step` / `two_to_one` | `prover/src/lfm/builder.rs:290-316` | +| `SpongeVar` — the chain, `SQUEEZE_MARK` | `prover/src/lfm/edsl.rs:14-131` | +| `HostSponge` — the host mirror, hasher-parameterised | `prover/src/lfm/fixture.rs:46-131` | +| `LfmHasher::transcript` / `transcript_out` (+ `HasherKind` dispatch) | `prover/src/lfm/hash.rs:64-95`, `:238-256` | +| `TAG_LFMT`, `socket_digest_rounds_tagged`, `transcript_digest`, `tag_for_mode` | `prover/src/lfm/blake3_socket.rs:194-300` | +| BLAKE3 `transcript`/`transcript_out`/`step` | `prover/src/lfm/blake3_socket.rs:370-420` | +| `MU_COLUMNS = (MODE_C, MODE_T)` | `prover/src/lfm/blake3_socket.rs:445-455` | +| `TAG_SELECTOR` + `message_word_ref` | `prover/src/lfm/blake3_socket.rs:505-537` | +| the idx 0–5 / MU changes | `prover/src/lfm/blake3_socket.rs:915-975` | +| `WordRef::ModeSelected` + `word_expr` arm + `rotr_bytes` | `prover/src/lfm/blake3_chip.rs:365-407`, `:1017-1042` | +| Test + Poseidon mode-sum widening | `prover/src/lfm/chips.rs:718-765`, `:800-812` | +| `Sum3`→`selector_sum` on the `LfmMem` receives | `prover/src/lfm/chips.rs:614-632` | +| compiler one-hot emission by name | `prover/src/lfm/compiler.rs:326-352` | +| validator one-hot over 3 selectors | `prover/src/lfm/validator.rs:290-311` | +| executor two-to-one arm | `prover/src/lfm/executor.rs:370-425` | +| trace filler: per-row domain tag | `prover/src/lfm/trace.rs:182-203`, `:236-256` | +| `TrivialV0`'s third compress | `prover/src/lfm/programs.rs:62` | +| `permute_coverage_program_source` (`#[cfg(test)]`, unregistered) | `prover/src/lfm/programs.rs:74-108` | +| KAT vectors (generated from the spec JSON) | `prover/src/lfm/transcript_kats.rs` | +| transcript tests (17) | `prover/src/lfm/transcript_tests.rs` | +| M1–M7 + the two F3.4 milestone tests | `prover/src/lfm/blake3_socket_tests.rs:1068-1290`, `:1490-1600` | + +--- + +## 2. KAT results — ✓ EXECUTED + +Every vector in `transcript_kats.json` reproduces. The Rust table +(`transcript_kats.rs`) is **rendered from the spec's JSON**, not hand-copied, so +it cannot drift from the oracle's reference. + +| | check | evidence | +|---|---|---| +| K1 | 6 per-op step vectors, at 6 **and** 7 rounds | `every_step_vector_reproduces_at_both_round_counts` | +| K1′ | the compiled-in entry point matches its own round count's vector | `the_compiled_step_matches_its_round_counts_vectors` | +| K1″ | **the crate anchor**: `blake3::hash(state ‖ operand ‖ "LFMT")[..16]` @7r, message re-derived byte-level | `seven_rounds_is_blake3_of_the_transcript_message` | +| K2 | end-to-end `FriToyV0`-preamble transcript, state after all 10 recorded ops + 3 ext challenges + 4 query-bit vectors, at 6 **and** 7 rounds | `the_end_to_end_vector_reproduces_at_{six,seven}_rounds` | +| K2′ | the same, through `HostSponge` (the mirror property, made checkable) | `the_host_sponge_reproduces_the_end_to_end_vector` | +| K2″ | the same, through the **machine** (`SpongeVar` → `LFM_HASH`, executed under BLAKE3) | `the_machine_reproduces_the_end_to_end_vector` | +| K3 | transcript step ≠ Merkle parent on the same cells, both round counts | `a_transcript_step_is_not_a_merkle_parent` (+ honest control: they differ **only** in the tag) | +| K4 | the squeeze counter is load-bearing | `the_squeeze_counter_is_load_bearing` (+ honest control: squeeze 0 agrees either way, so the test is not passing on noise) | +| K5 | absorb order is load-bearing | `absorb_order_is_load_bearing` (+ honest control: same order ⇒ same state) | +| K6 | the preamble costs 11 compressions, all `Transcript` rows | `the_preamble_costs_eleven_transcript_steps` | + +`transcript_tests`: **17 passed, 0 failed** at 7 rounds; **17 passed, 0 failed** +under `--features blake3-6round`. + +The `blake3-6round` build is not merely lint-clean: the per-op and end-to-end +vectors are pinned at *both* round counts from a single build (the reference +takes `rounds` as an argument), and the compiled-in path is separately checked +against whichever vector its knob selects. + +--- + +## 3. M1–M7 conformance checklist — ✓ EXECUTED, all 7 + +Stated in the spec §5.3 before the chip existed, so these are inherited +obligations. Every one is paired with an honest-path assertion. + +| | spec statement | expected | implemented as | result | +|---|---|---|---|---| +| **M1** | `m[8]` pinned to `TAG_LFMC` while `MODE_T = 1` | SAT (a transcript row computing the Merkle tag) | a `MODE_T` row whose entire witness is the `"LFMC"` computation | **rejected** ✓; honest control (same row, own domain) accepted | +| **M2** | mirror: `TAG_LFMT` while `MODE_C = 1` | SAT | a `MODE_C` row whose witness is the `"LFMT"` computation | **rejected** ✓; honest control accepted | +| **M3** | `MODE_C = MODE_T = 1` on one row | UNSAT via idx 4 | set both, evaluate | **violates exactly idx 4** ✓; clearing it restores acceptance | +| **M4** | `MODE_C = MODE_T = 0` with `MU = 1` | UNSAT — `MU` *is* their sum | `MU_COLUMNS == (MODE_C, MODE_T)` asserted structurally; a garbage row with no mode set is padding | vacuous with no mode ✓; **fails** the moment a mode is restored (so the vacuity is not the set accepting anything) | +| **M5** | drop the mode-sum booleanity ⇒ modes arbitrary ⇒ `m[8]` prover-chosen | **SAT** | see below | **SAT, and it fires** ⚠ | +| **M6** | `MODE_T` as a MAIN column ⇒ prover-chosen | **SAT** | same row as M5 | **SAT, and it fires** ⚠ | +| **M7** | generalised capacity form idx 0–3 | UNSAT present / SAT dropped | tamper each `S8+k` on a **transcript** row | each violates **exactly** constraint `k` ✓; transcript capacity == compress capacity == IV | + +### ⚠ M5/M6 fired, and the finding is sharper than the spec anticipated + +Test: `blake3_socket_tests::m5_m6_the_mode_columns_must_be_preprocessed_or_the_tag_is_prover_chosen`. + +Constraint idx 4 pins the mode **sum** to a bit — it does **not** pin each +selector to a bit. So a row with `MODE_C = x`, `MODE_T = 1 − x` satisfies it for +*every* field element `x`, and `m[8] = x·"LFMC" + (1−x)·"LFMT"`. Solving for `x` +makes that **any 32-bit value the prover likes**. The test picks the tag +`"XXXX"`, derives the `x` that produces it, builds the full honest witness under +that forged domain, and the constraint set **accepts the row with zero +violations**. + +This is not a defect introduced here — it is exactly what M5/M6 were +pre-committed to demonstrate, and it is the same shape as the pre-existing +`MU = MODE_C` argument. Two mechanisms close it, and the test asserts both +rather than asserting them in prose: + +1. **The mode columns are preprocessed** — `MODE_C`, `MODE_T` and `MODE_P` are + all `< PREP_WIDTH`, so a prover supplies none of them; their values are fixed + by the row's position in a trace whose commitment is folded into + `lfm_program_id`. +2. **The admission validator rejects a non-one-hot selector** — the test tampers + a real program's hash group with the same fractional `x` and asserts + `validate` returns `NonOneHotSelector { chip: "LFM_HASH" }`, with an honest + control that the untouched program is admissible. + +**Consequence for review:** the domain separation rests on the preprocessed +binding plus the registrar, *not* on the AIR alone. That was already the design +(the spec §3.3 says "exactly-one-of stays the registrar's job"), but M5/M6 turn +it from a sentence into an executed demonstration, and it should be read as a +standing requirement on any future change that makes a mode selector a main +column. + +--- + +## 4. The programs + +### `TrivialV0` — F3.4 retired for this entry + +Its raw `b.permute` became a third `compress` +(`programs.rs:60-62`). It now **proves and verifies under BLAKE3** +(`the_trivial_program_proves_and_verifies_under_blake3`), which it could not +before. Public output moved from `[d1, permuted_cell, m]` to `[d1, d2, m]` — +still three words; no test asserted the old shape (? INFERRED → ✓ VERIFIED by +the suite being green). + +Permute coverage moved to `programs::permute_coverage_program_source`, a +`#[cfg(test)]` fixture that is **not** a registry entry: two chained permutations +so an output cell is also an input cell. `a_permute_row_is_refused_under_blake3` +now points at it (and keeps its honest control under `Test`). + +### Cost claims — ✓ EXECUTED, both exact + +`transcript_tests::the_programs_cost_what_option_b_priced_them_at`. The per-row +price comes from the census (`main_cols + 3·aux_cols`), not from a literal, and +the row counts are asserted separately — a product that came out right for two +wrong reasons is the failure mode. + +| program | rows | price @7r | total | spec predicted | +|---|---|---:|---:|---:| +| `TrivialV0` | 3 compress + 0 transcript | 5,509 | **16,527** | 16,527 ✓ | +| `FriToyV0` | 56 compress + **11 transcript** | 5,509 | **369,103** | 369,103 ✓ | + +The transcript's share is exactly the spec's 11 compressions (K6). + +--- + +## 5. Regression — the other two hashers + +B1 changed the sponge for **all** hashers, so `Test` and `Poseidon` had to move +and stay green. + +- `transcript_tests::the_transcript_proves_and_verifies_under_every_hasher` — + the preamble program proves and verifies under **Test, Poseidon and BLAKE3**. +- `transcript_tests::the_machine_and_the_host_chain_agree_under_every_hasher` — + `SpongeVar` and `HostSponge` produce identical challenges under all three. +- The full `lfm::` suite has **no new failures** (§6). + +⚠ **Recorded rather than assumed:** under `Test` and `Poseidon` a transcript step +*is* a Merkle parent — those hashers have one domain, so the trait default +(`LfmHasher::transcript_out` = `compress_out`) does not separate them. That is +documented at the default (`hash.rs:64-83`) as a deliberate weakening with the +reason (neither is a production hash) and the standing requirement that a future +production candidate must override it. + +--- + +## 6. Test and lint status + +**Full `lfm::` suite, release:** `290 passed; 19 failed; 7 ignored`. + +All 19 failures are the pre-existing `executor/program_artifacts/recursion/fibonacci.elf` +fixture set — 15 fail directly on the missing ELF and 4 (`arena_filler_reads_real_committed_roots`, +`l2g_binding_holds_on_the_real_bundle`, `l2g_binding_proves_and_verifies`, +`tampered_l2g_binding_rejects`) fail downstream of the resulting one-epoch +fixture. **✓ EXECUTED: the failure set is byte-identical at 7 rounds and under +`--features blake3-6round`** (`diff` of the two sorted lists is empty), so +nothing in this change is round-count-sensitive. + +Baseline was 276 passed / 19 failed; the +14 is this change's new tests. + +| gate | result | +|---|---| +| `make fmt` | clean | +| `make lint` (4 feature combos: default, no-default+debug-checks, disk-spill, cuda) | **clean** | +| `cargo clippy -p lambda-vm-prover --all-targets --features blake3-6round` | **clean** | +| `lfm::blake3_socket_tests` | 35 pass @7r, 35 pass @6r | +| `lfm::transcript_tests` | 17 pass @7r, 17 pass @6r | + +**Keccak wrap path untouched.** ✓ VERIFIED: the entire diff is 19 files under +`prover/src/lfm/` plus `thoughts/blake3/socket-kats/SOCKET.md`. Nothing in +`crypto/`, `syscalls/`, `executor/` or `prover/src/tables/`; `keccak_adapter.rs`, +`keccak_host.rs`, `transcript_replay.rs` and `wrap_tests.rs` are unmodified, and +`wrap_tests` is green. + +### Registry re-bless — once, as planned + +All six `program_id`s moved (`PREP_WIDTH` 11→12 moves the `LFM_HASH` +preprocessed root, which every entry binds): + +| entry | old (first 8 bytes) | new | +|---|---|---| +| `TrivialV0` | `9f0537f570afe0ef` | `998428afa2a39d25` | +| `FriToyV0` | `3b4e718c02077762` | `e527cd58fc9b0c15` | +| `KeccakChainV0` | `eb591de10644b164` | `d5c294dd849e92b5` | +| `KeccakSpongeV0` | `1d90d7b5eb540778` | `421d582b159474ac` | +| `TranscriptReplayV0` | `26033a9e4101fae8` | `3371ec2badbd4c6f` | +| `StatementReplayV0` | `af842fd9b9fe6ebe` | `9f7e67a8d92c7387` | + +One `log_heights` entry moved: `FriToyV0`'s `LFM_CONST` group 4→5 (16→32 rows), +which is the interned `SQ(i)` constants. Regenerated with +`cargo run --release --bin compute_lfm_registry`, pasted whole; the drift tests +pass. + +### Tag tables — NOT touched by this work; one staleness reported instead + +The tag-table pass was already done by the oracle before this build started: +`SOCKET.md` §2.4 and `ORACLE.md` §2.3 both carry `"LFMT" = 0x544D464C`, mark +`"LFMP"` **RETIRED UNUSED** with the reason recorded, note the O5/`"LFML"` +ratification, and put a superseded banner on `SOCKET.md` §7's rejected permute +sketch. + +⚠ **I edited both files before that instruction reached me, and have reverted +those edits.** What I had done: bumped the `"LFMT"` status word from *specified* +to *built* in each table, and rewritten `SOCKET.md` §2.2's `m[8]` row. Both are +backed out; the oracle's pass is byte-for-byte intact (`git diff` on `SOCKET.md` +is now exactly that pass, and the only `gate-oracle/` file I ever opened was +`ORACLE.md`, now reverted). + +**✓ VERIFIED — the tag constants agree, so there is no inconsistency in the +values.** The implementation uses `TAG_LFMC = 0x434D464C` and +`TAG_LFMT = 0x544D464C`, matching both tables exactly, and `"LFMP"` remains +retired-not-deleted so `0x504D464C` cannot be reallocated into an unanalysed +domain. + +**✓ CLOSED (2026-08-11, superseded within the hour).** I had reported two stale +`m[8]` framing rows here — `SOCKET.md` §2.2 and `ORACLE.md` §2.2 still describing +`m[8]` as the bare constant `0x434D464C`, where the built chip has +`MODE_C·TAG_LFMC + MODE_T·TAG_LFMT`. **The oracle's re-transcription pass fixed +both on disk about five minutes after this report was written.** The item is +closed, not outstanding; it is left in the record because the sequence — report +rather than edit, then the owner fixes it — is the one that worked. + +### Pinning instruments — untouched, and the DRIFT is expected + +`artifact_pin.py`, `artifact_pin.json`, `chip_model.py`, `gate.py` and +`CHIP-GATE.md` were not opened by this work. `artifact_pin.py --check` will now +report DRIFT and exit 1 because `blake3_socket.rs` changed; that is the +instrument refusing to vouch for a chip it has not been re-transcribed against, +and re-pinning is the oracle's move, not this one's. + +--- + +## 7. Deviations from the spec — named, with reasons + +### 7.1 ⚠ `FriToyV0` does NOT prove under BLAKE3 — blocked by **O1**, not by the sponge + +This is the one milestone in the brief that was not reached, and the reason is +structural rather than a shortfall in the implementation. + +**✓ EXECUTED.** `execute(fri_toy_program(), fixture_arenas(), Blake3)` returns +`HasherRejected("BLAKE3 compress input lane is not a u32 (SOCKET.md obligation O1)")`. +Measured cause: **124 of the fixture's 128 committed column values are ≥ 2^32.** + +`FriToyV0` hashes **FRI data** — Merkle leaves over LDE evaluations +(`compress(row_even, row_odd)`) and folded ext values — which are arbitrary +Goldilocks elements by construction. The BLAKE3 socket's inputs must be +`u32`-laned (obligation O1, pre-existing and unrelated to the transcript). No +choice of fixture polynomial changes this: the *evaluations* of a low-degree +polynomial over a coset are arbitrary mod `p`. + +**What B1 did deliver here:** the transcript was one of two blockers and it is +gone — the chain runs on the compress socket under every hasher, and +`FriToyV0` now contains **zero** permute instructions. The remaining blocker is +O1 alone. + +**Closing it is a different change:** field elements would have to reach the hash +through a committed `u32`-half decomposition — the shape +`transcript_replay::felt_be_halves` already uses for keccak leaves — which moves +`FriToyV0`'s arena layout and its program identity. That is a decision about the +**leaf convention**, adjacent to obligation O5, and I did not take it +unilaterally. + +**Left as a tripwire, not a silence:** +`blake3_socket_tests::fri_toy_is_still_blocked_by_o1_and_no_longer_by_the_sponge` +asserts (a) no permute remains, (b) the fixture values are not u32-laned, (c) the +refusal is specifically O1 — so a refusal for any *other* reason is a regression +— and (d) the honest control that the same program and arenas still run under +`Test`. Its doc says in as many words that when O1 is closed the test must be +replaced by a prove+verify. + +### 7.2 `MODE_T` sits at index 8, not appended after the multiplicities + +The spec fixes `PREP_WIDTH` 11→12 but not the placement. I first appended +`MODE_T` at index 11 to minimise churn; that was **wrong** and the admission +validator caught it: `one_hot` reads the selectors as a contiguous span +(`NUM_SELECTORS` from `MODE_C`), so a selector parked past the mults would have +been **outside the one-hot check and silently unchecked**. `MODE_T` is now index +8 and `MULT0..2` shifted to 9..11; `layout::hash::NUM_SELECTORS = 3` replaces the +hard-coded `2` at the call site. The reason is recorded at the constant. + +### 7.3 `SQ(i)` is an interned program constant, not a packed word + +The spec says `SQ(i)` is "a constant cell … a program constant either way". The +implementation uses `LfmBuilder::digest_const`, so each distinct index costs one +`LFM_CONST` row and nothing else. (An earlier draft used `pack_word`, which would +have added an `LFM_LANES` row per squeeze — the same value, not the same cost.) + +### 7.4 `WordRef::byte` / `rotr_bytes` panic on `ModeSelected` + +A mode-selected word has no byte decomposition without witnessing one. Since the +whole reason the tag lives in `m[8]` is that message words reach `add3` and +nothing byte-granular, both byte-level accessors `unreachable!` on it rather than +silently acquiring four uncommitted columns. Not in the spec; it is the shape the +new variant needs to be safe. + +### 7.5 Incidental DRY + +`rotr16`/`rotr8` were byte-identical in `blake3_chip::WireFlow` and +`blake3_socket::SocketWire`; adding a third `WordRef` variant would have meant a +third copy in each. They now call one `WordRef::rotr_bytes`. Wire-identical +(the socket and probe suites are green at both round counts). + +--- + +## 8. What is still open + +| item | status | +|---|---| +| `FriToyV0` under BLAKE3 | ✗ OPEN — needs the O1 leaf-convention decision (§7.1) | +| gate extension: `chip_model.py` `MODE_T` role + a `WordRef`-equivalent for the mode-selected tag + `gate.py` B0a/B0b mode-sum widened to `MODE_C + MODE_T + MODE_P` | ✗ OPEN — spec §5.2. Not attempted here: `gate-oracle/` is the oracle's instrument and this build touched none of it. **What the gate needs from the chip is all exposed**: `cols::MODE_T`, `cols::MU_COLUMNS = (MODE_C, MODE_T)`, `TAG_SELECTOR` (the `(column, tag)` pairs verbatim), `tag_for_mode`, and unchanged constraint indices (0–3 capacity, 4 mode-sum, 5 `MODE_P` pin) | +| the two stale `m[8]` framing rows in `SOCKET.md` §2.2 / `ORACLE.md` §2.2 | ✓ CLOSED by the oracle's re-transcription pass | +| ⚠ `others/lfm-hash-matrix-scope.md` cites the pre-B1 rate | ✗ OPEN — §9 below; flagged for the lead rather than edited | +| production hasher overriding `LfmHasher::transcript_out` | ✗ OPEN by design — BLAKE3 does; a future candidate that does not is shipping an unseparated transcript (`hash.rs:64-83`) | +| squeeze-run bound revisit at `k = 2^16` | recorded at `SpongeVar` (`edsl.rs:44-70`); today's max run is `NUM_QUERIES = 4` | + +--- + +## 9. Post-review fixes (b1-verify.md D1–D4) + +The adversarial review found **no soundness defect**. Four items came back to +this workstream; all four are done. No soundness-relevant code changed — D1 is a +cost model, D2 is prose, D4 is filler discipline. + +### D1 — MEDIUM. `LFM_HASH_RATE_FELTS` was derived from the deleted duplex. FIXED, and the projection gets WORSE. + +The review is right and my report missed it: `8` was "2 of 3 state cells", the +rate of the construction B1 deleted. It is a **live** constant driving the epoch +verifier's permutation-axis projection, and it is quoted in the hash decision +record. + +**⚠ This is not a number swap. Two of the model's premises broke with it, and +the corrected projection is materially worse.** + +| | before | after | +|---|---|---| +| `LFM_HASH_RATE_FELTS` | `8` (literal) | `HASH_DIGEST_FELTS` = **4** (derived) | +| candidate/keccak rate ceiling | 17/8 = **2.125×** | 17/4 = **4.25×** | +| FRI-layer leaf (6 felts) | 1 block — rate-INVARIANT | **2 blocks — rate-SENSITIVE** | +| decomposition | "only the leaf term moves" | "only ABSORPTION moves" | + +1. **The constant is now derived, not remembered** (`epoch_verify.rs:428-456`). + The chain absorbs one cell per step, so the rate *is* + `hash::HASH_DIGEST_FELTS`, and it is written as that constant so it cannot + outlive its derivation a second time. +2. **The lever moved, and it is a worse one.** Under the duplex the rate + followed from `HASH_STATE_FELTS = 12`, so widening the state bought + throughput freely. Under the chain it follows from `HASH_DIGEST_FELTS = 4`, + which is *the same constant the socket's 64-bit collision bound rests on* — + **throughput and collision resistance are no longer independent knobs.** That + belongs in the hash decision record. +3. **The FRI-leaf term became rate-sensitive.** The old model folded it into the + invariant remainder on the premise "a layer leaf fits any rate ≥ 6" — true at + 8, false at 4. New `FRI_LEAF_FELTS` + `fri_leaf_permutations_at_rate` + (`epoch_verify.rs:450-510`); `query_permutations_at_rate` now sums four terms + and its doc states the real rule: **absorption is rate-sensitive, compression + is not** (Merkle parents of both kinds compress and do not move). + At rate 17 the new term reduces to `num_committed()`, so the rate-17 + differential against the byte-side `query_permutations` is preserved + unchanged — that check still passes by construction. +4. **The broken assertion is gone, not re-asserted.** `epoch_verify_tests.rs`'s + `6 <= LFM_HASH_RATE_FELTS` is replaced by + `blocks_at_rate(6, 17) == 1` / `blocks_at_rate(6, 4) == 2`, and the + decomposition assert now reads *"only ABSORPTION may move with the rate"*. + The printed banner reports absorbed/compressed instead of leaves/paths, the + ceiling is computed from the constants rather than written `2.125`, and it + carries a ⚠ line naming the change. + +**✗ The corrected epoch ratio is NOT computable in this environment.** The +consuming block lives inside `the_assembled_epoch_verifier_runs`, one of the 19 +`fibonacci.elf`-blocked tests — which is precisely how the constant outlived its +derivation. So I added an **ELF-free** test, +`epoch_verify_tests::the_candidate_rate_model_is_derived_not_remembered`, which +executes here (✓ PASS at both round counts) and pins the correction itself: the +constant's derivation, that a 6-felt leaf is 1 block at 17 / 1 at the old 8 / +**2** at 4, that the new FRI term reduces to `num_committed()` at keccak's rate +and doubles at the candidate's, and that a terminal-only shape contributes zero +at every rate. **That test would have caught the original defect.** + +Illustrative magnitude of the term the old model could not express at all +(ELF-free arithmetic, blowup 8 / 73 queries, **not** the epoch total): + +| `log2_lde` | committed layers | FRI-leaf perms @17 | @8 (old) | @4 (new) | added | +|---:|---:|---:|---:|---:|---:| +| 16 | 9 | 657 | 657 | 1,314 | **+657** | +| 20 | 13 | 949 | 949 | 1,898 | **+949** | +| 22 | 15 | 1,095 | 1,095 | 2,190 | **+1,095** | + +Per sub-proof. The trace-group leaf term worsens separately, from a ≤2.125× +multiplier to ≤4.25×. + +**⚠ FLAGGED, NOT EDITED — `others/lfm-hash-matrix-scope.md` is stale in three +places** (the lead's message offered either; I flagged because the corrected +epoch number cannot be produced here, so any banner I wrote would announce a +wrong number without supplying the right one): + +- **:128** — *"the `LFM_HASH` sponge is 'state = 3 cells (rate 2, capacity 1)' + … **8 felts per permutation**"*. The cited `edsl.rs:16-17` no longer says that. +- **:130** — *"a candidate behind socket 2 pays **2.125×** as many + permutations"*. Now 4.25× on the absorption term. +- **:228 — the one that matters most.** It argues Miden's BlakeG figures + transfer directly because *"State 12, rate 8, digest 4 is exactly our frozen + `LFM_HASH` contract"*, and concludes *"every field-native candidate shares ONE + permutation count"*. **The rate-8 half of that identification is gone**, so + the transfer argument needs re-examining, not just the number. `:1132`'s + ⚠ conservatism note is unaffected (it is about padding, and holds at any rate). + +### D2 — LOW. Two stale `PREP_WIDTH = 11` prose sites. FIXED. + +- `statement.rs:43` — the load-bearing sentence justifying why `lfm_program_id` + folds the hasher tag in. Rewritten to say what is actually true and stable: + the preprocessed group is the **instruction** group, which no candidate + changes, so every hasher commits the same width (12 since `MODE_T`). +- `poseidon_chip_tests.rs:546` — the prose 380 lines below the assertion I had + already updated to 12. + +✓ EXECUTED: `grep -rn "is 11" prover/src/lfm/` is now empty. + +### D3 — my report amended. + +§6's tag-table item and §8's open-items table now record the two `m[8]` framing +rows as **closed by the oracle's re-transcription**, which landed minutes after I +wrote the original claim. The record is no longer self-contradictory. + +### D4 — APPLIED. The filler reads the row, like the Poseidon one. + +It was genuinely small, and it closes a real gap rather than only a stylistic +one: `m[8]` is a linear form over `MODE_C`/`MODE_T`, so a filler that takes the +tag as an argument can be handed a domain the row's own selectors contradict. +`chip_trace` populates the preprocessed columns *before* calling `fill`, so the +row already carries them. + +- `fill_socket_witness(row)` now derives the tag via `tag_from_row`, which panics + if the row selects neither two-to-one domain (`blake3_socket.rs:810-855`). +- `fill_socket_witness_tagged(row, tag)` is retained `pub(crate)` for the M1/M2 + controls, which must build a row whose witness and mode columns deliberately + disagree — production can no longer construct that. +- ✓ VERIFIED asymmetry, stated rather than hidden: the BITWISE **histogram** + (`trace.rs:182-203`) still routes the domain through `tag_for_mode`, because it + runs before any trace row exists. Filler and histogram must agree, and a + disagreement unbalances the `ByteAlu` bus — which the socket's prove+verify + tests cover. + +### One more stale site, not in the review: `blake3_probe.rs`'s rate 8 + +✓ VERIFIED and **corrected as comments only — the arithmetic is right and +untouched.** The probe hard-codes rate `8` in three places. That is BLAKE3's own +rate (its socket absorbs two cells of message per compression) and B1 did not +change it, so every number the probe prints is still correct. But its gloss read +*"@ rate 8 (blake and field-native)"* — the two coincided only while the sponge +was a duplex. The gloss now names BLAKE3 alone and points at +`LFM_HASH_RATE_FELTS` for the field-native chain's 4. + +### Verification of this pass + +| gate | result | +|---|---| +| full `lfm::` suite | **291 passed / 19 failed** — failure set `diff`-identical to before these fixes (the `fibonacci.elf` 19); +1 is the new ELF-free rate-model test | +| `lfm::blake3_socket_tests` | 35 pass (D4 touched every filler call) | +| `lfm::transcript_tests` | 17 pass @7r, 17 pass @6r | +| `the_candidate_rate_model_is_derived_not_remembered` | pass @7r and @6r | +| `make fmt` + `make lint` (4 combos) | clean | +| `clippy --features blake3-6round` | clean | diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/TRANSCRIPT.md b/thoughts/shared/lfm-real-hash/transcript-spec/TRANSCRIPT.md new file mode 100644 index 000000000..c4a28f4f2 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/TRANSCRIPT.md @@ -0,0 +1,473 @@ +# The LFM compress-chain transcript — specification + +**Status:** specification + reference + vectors + gate extension, written +**before any Rust exists**, same discipline as Phase 2. **No chip code exists +for this.** **Date:** 2026-08-11. + +**Decision this implements:** the user ratified **option B, form B1** +(`permute-socket-options.md`): the Fiat–Shamir sponge becomes a compress-based +chain **for all hashers**, **no permute socket is ever built**, and `MODE_P` +stays pinned to 0 permanently. The deciding argument was that B needs no +assumption beyond A6R. + +Claims are ✓ VERIFIED (read the code, cited), ✓ EXECUTED (ran it), ? INFERRED, +or ✗ OPEN. + +--- + +## 0. Board + +| check | result | +|---|---| +| `transcript_kats.py` — K1–K6 | **PASS**, both round counts; the end-to-end vector tracks `FriToyV0`'s CURRENT preamble (`absorb_felts` ×2) | +| `transcript_gate.py` — G1–G5 (executable today) | **PASS 6/6** | +| `squeeze_run_analysis.py` — entropy loss vs run length | executed, §4 | +| M1–M7 against the built chip | **ALL FIRED** (builder, Rust side) | +| **M8** — the eighth control, added after M5/M6 falsified §3.3 | **PASS**, 4 legs, model side | +| post-B1 CHIP-GATE re-gate + re-pin | **PASS 79/79**, `gate-oracle/CHIP-GATE.md` | + +Run order: `python3 transcript_kats.py --write` → `python3 transcript_gate.py` +→ `python3 squeeze_run_analysis.py`. Plain `python3` + `z3`; no cargo. + +--- + +## 1. The construction + +**State: one cell** — 4 lanes × u32 = 128 bits, initially all-zero (mirroring +`SpongeVar::new`, which starts from three zero cells). Down from three cells, +because a chain needs no rate/capacity split. + +Every operation is one ordinary **frozen-socket compress** under the transcript +tag, written `compress_T`: + +| op | definition | compresses | +|---|---|---:| +| `absorb(c)` | `state ← compress_T(state, c)` | 1 | +| `absorb2(c0, c1)` | `state ← compress_T(compress_T(state, c0), c1)` | 2 | +| `squeeze()` | `out = state` **then** `state ← compress_T(state, SQ(i))` | 1 | + +`squeeze` outputs **before** advancing, mirroring `SpongeVar::squeeze_cell` +(`out = state[0]; state = permute(state)`), so the two constructions stay +structurally parallel and the eventual diff is reviewable. + +`squeeze_ext` takes lanes 0–2 of the squeezed cell and `squeeze_bits(n)` the low +`n` bits of lane 0 — unchanged from today, so **`programs.rs` needs no edit**: +`fri_toy_program_source` calls only `absorb`/`absorb2`/`squeeze_ext`/ +`squeeze_bits`, all of which keep their signatures. + +### 1.1 `SQ(i)` — the squeeze counter, and why it is free + +The advance operand is the **constant cell** `SQ(i) = [SQUEEZE_MARK, i, 0, 0]`, +where `i` is the squeeze index and `SQUEEZE_MARK = "SQZ0"` as a little-endian +u32. + +**It costs nothing.** ✓ VERIFIED `edsl.rs:1-4` — the eDSL fully unrolls, +"nothing loop-shaped reaches the machine", so `i` is a compile-time constant and +the operand is a program constant either way, pinned by `program_id`. A constant +cell was going to be emitted regardless; this one just carries a counter. + +**What it buys** is §4's FSE-2014 lesson written into the construction. Without +it, a run of consecutive squeezes iterates **one fixed public non-injective +map**, whose functional graph an adversary can precompute — precisely the +structure the GLUON-64 T-sponge attacks exploit. With it, each step is a +different map and no single functional graph exists to analyse. + +**Absorb/squeeze separation** rests primarily on the operation sequence being a +compile-time constant of the program: a prover cannot perform a squeeze where +the program says absorb, because the sequence is fixed at emission and bound by +`program_id`. `SQUEEZE_MARK` is defence in depth, not the load-bearing argument. + +### 1.2 Framing — identical to the Merkle socket but for one constant + +| input to `f` | value | +|---|---| +| `h` | `IV[0..8]` | +| `m[0..4]` | `state` | +| `m[4..8]` | the operand (absorbed cell, or `SQ(i)`) | +| `m[8]` | **`TAG_LFMT`** ← *the only thing that differs* | +| `m[9..16]`, `t` | `0` | +| `block_len` | `36` | +| `flags` | `0x0B` | +| output | `out[0..4]` | + +**Consequence, and it is the whole point of option B:** the transcript inherits +the compress socket's external anchor unchanged. ✓ EXECUTED (K1): at 7 rounds +every step equals `BLAKE3(LE32(state) ‖ LE32(operand) ‖ "LFMT")[0..16]`, +computed by two separate routes and asserted equal. The implementer must +re-assert this as a one-line `blake3::hash` call. + +--- + +## 2. Tag allocation + +| tag | u32 (LE) | use | status | +|---|---|---|---| +| `"LFMC"` | `0x434D464C` | 2-to-1 compress / Merkle parent | **built** | +| **`"LFMT"`** | **`0x544D464C`** | **transcript step (this document)** | **specified here** | +| `"LFMP"` | `0x504D464C` | ~~permute socket~~ — **retired unused** | never built (B1) | +| `"LFML"` | `0x4C4D464C` | leaf domain | reserved, O5-ratified | + +A tag is never reused for a second purpose. `"LFMP"` is now **permanently +unused**: B1 means no permute socket will ever exist, so the reservation should +be marked retired rather than deleted — deleting it would let a future +allocation reuse the value and silently create a domain nobody analysed. + +**✓ DONE 2026-08-11.** Both tag tables updated: `gate-oracle/ORACLE.md` §2.3 +and `thoughts/blake3/socket-kats/SOCKET.md` §2.4 now carry `"LFMT"`, mark +`"LFMP"` **RETIRED UNUSED** with the reuse-hazard note, and record O5's +ratification on `"LFML"`. `SOCKET.md` §7 — the rejected permute sketch — also +got a superseded banner, so a reader landing there directly cannot mistake it +for a plan. + +--- + +## 3. The `m[8]` mechanism — exact constraint change + +Today `m[8]` is `WordRef::Const(TAG_LFMC)` — a compile-time constant, hence zero +columns and zero range checks. Two tags need `m[8]` to depend on the row's mode +**without becoming prover-chosen**. + +### 3.1 `MODE_T`: a new preprocessed column, not a reuse + +**Recommendation: add a fresh preprocessed `MODE_T`; do NOT repurpose `MODE_P`.** + +Reusing `MODE_P` is tempting — B1 pins it to 0 for BLAKE3, so it looks dead. It +is not: `MODE_P` is in the **shared** preprocessed prefix and the `Test` and +`Poseidon` arms still use it for their permute (and `TrivialV0` still calls +`b.permute` directly — §6). Repurposing it would make one preprocessed column +mean different things under different hashers, which is worse than the column it +saves. + +**Cost:** `PREP_WIDTH` 11 → 12, so the preprocessed roots move and all six +registry entries are re-blessed. ? INFERRED but well-supported: that re-bless is +**already happening** — B1 changes the sponge for every hasher, so every +`program_id` moves regardless. The new column rides along at no marginal +protocol cost, provided it is sequenced into the same re-bless. + +### 3.2 The constraints, before and after + +> **Scope note:** this table records the **B1** change. `MODE_L` has since landed +> and widened the same constraints again — `MU` and the capacity selector are now +> `MODE_C + MODE_T + MODE_L`, `NUM_SELECTORS` is 4 and `PREP_WIDTH` is 13. For +> the current state read `../leaf-spec/LEAF.md` §2 and `../gate-oracle/CHIP-GATE.md` +> §4.7; the "AFTER" column below is B1's after, not today's. + +``` +idx 0-3 BEFORE: S_k − (MODE_P·IN_{8+k} + MODE_C·IV_k) + AFTER: S_k − (MODE_P·IN_{8+k} + (MODE_C + MODE_T)·IV_k) +``` +A transcript row is still a compress, so its capacity prefix is still the IV; +only the selector widens. + +``` +idx 4 BEFORE: mode_sum·(1 − mode_sum), mode_sum = MODE_C + MODE_P + AFTER: mode_sum·(1 − mode_sum), mode_sum = MODE_C + MODE_T + MODE_P +``` +Exactly-one-of stays the registrar's job; this pins the sum to a bit. It is what +excludes `MODE_C = MODE_T = 1` (which would give `mode_sum = 2`, and +`2·(1−2) = −2 ≠ 0`). + +``` +idx 5 BEFORE: MODE_P (pin to zero — no permute socket) + AFTER: MODE_P (unchanged, and now PERMANENT under B1) +``` + +``` +MU BEFORE: MODE_C + AFTER: MODE_C + MODE_T +``` + +``` +m[8] BEFORE: WordRef::Const(TAG_LFMC) + AFTER: MODE_C·TAG_LFMC + MODE_T·TAG_LFMT (a new WordRef variant) +``` + +### 3.3 Soundness argument for the tag + +**The tag stays prover-unchosen because `MODE_C` and `MODE_T` are preprocessed.** +A preprocessed column is fixed by the row's position in the preprocessed trace, +which is bound by the preprocessed commitment, which is folded into +`lfm_program_id`. The prover chooses neither. This is the same argument that +already makes `MU = MODE_C` trustworthy — the existing arm's doc calls it out: +*"a prover cannot choose it"*. + +> **⚠ CORRECTED 2026-08-11 — an earlier revision of this paragraph drew an +> inference that does not hold, and the correction matters more than the +> original claim did.** +> +> It said: *"idx 4 forces the mode sum to a bit, so at most one tag is +> selected."* **The clause after "so" is a non-sequitur.** Over a prime field +> `mode_sum ∈ {0,1}` pins the SUM, not the selectors: `MODE_C = x`, +> `MODE_T = 1 − x` satisfies idx 4 for *any* `x`, and since the two tags are +> distinct, `x = (T − TAG_LFMT)/(TAG_LFMC − TAG_LFMT)` solves for **any** target +> tag `T`. So idx 4 contributes nothing to one-hotness. +> +> ✓ EXECUTED twice, independently: the builder's Rust M5/M6 run forges the tag +> `"XXXX"` by a fractional split and the eval set accepts with zero violations; +> I reproduced it in the gate's own field model — +> `idx 4 alone → sat` with `MODE_C = 4387334679741772800`, +> `MODE_T = 14059409389672811522` (sum ≡ 1, `m[8] = 0x58585858`), and +> `idx 4 + one-hot → unsat`, with both honest tags still reachable under one-hot. + +**What actually closes it, stated correctly.** Two independent mechanisms, and +**idx 4 is neither of them**: + +1. **`MODE_C`/`MODE_T` are preprocessed.** The prover cannot choose them at all — + a row's mode is fixed by its position in the committed instruction group. This + is the primary closure. +2. **The registrar's exactly-one-of check.** This, not idx 4, is what makes the + selectors one-hot. ✓ VERIFIED it is also why `MODE_T` sits at layout index 8 + rather than after the multiplicities: the admission validator reads the + selectors as a **contiguous span** (`NUM_SELECTORS` from `MODE_C` — 3 at B1, **4 since `MODE_L`**), so a + selector parked past the mults would be outside the one-hot check and + silently unchecked. + +**What idx 4 does buy:** it excludes the both-set case `MODE_C = MODE_T = 1` +(sum 2, and `2·(1−2) ≠ 0`). Useful, but strictly weaker than one-hotness. + +**Do not delete the registrar's one-hot check as redundant.** idx 4 would not +save it. **M8 in §5.3 is the control that enforces this**, and it exists because +this paragraph was wrong: a reader who trusted the original sentence could have +removed the load-bearing check while every constraint still passed. M5 and M6 +prove the other two dependencies are real. + +### 3.4 Degree and cost impact: none + +- **Degree unchanged.** `m[8]` was degree 0 (a constant); it becomes degree 1 (a + linear form over preprocessed columns). It appears only as an `add3` operand, + whose body `a + b + m − s − 2^32(c1+c2)` is degree 1 either way; × `MU` = 2. + Max degree stays **3** (the carry booleanities). ✓ VERIFIED against the + committed arm's structure. +- **Zero columns, zero sends.** `m[8]` is used as a whole word value + (`word_expr`), never byte-decomposed, so it needs no byte columns and no + `AreBytes` — exactly as the constant did. +- **One preprocessed column** (`MODE_T`), which is not a main column and does not + enter the census. + +--- + +## 4. Security + +### 4.1 The argument, in one paragraph + +The transcript is a hash chain over a collision-resistant compression function, +domain-separated from Merkle parents by a tag the prover cannot choose. Fiat– +Shamir needs each challenge to be a random-oracle function of everything +committed before it, so that a prover cannot predict or grind it before +committing; it does **not** need a secret capacity, because the protocol is +public-coin — every absorbed value is a public commitment and every squeezed +value a public challenge (✓ VERIFIED against `programs.rs:547-567`). That is +exactly what A6R already asserts: *"suitable as a 2-to-1 compression for Merkle +hashing **and as a PRF for Fiat–Shamir**"*. + +> **New named assumption required: NONE beyond A6R.** This is why option B was +> chosen. Option A would have needed A-TSP (a T-sponge instantiation) on top. + +**Bound:** the state is one cell = 128 bits, so ~**64-bit collision resistance** +by the birthday bound — the same number as the digest's, from the same +`HASH_DIGEST_FELTS = 4` cause, not introduced by this construction. + +### 4.2 ⚠ The squeeze-run analysis — the FSE-2014 lesson, applied to B itself + +A **squeeze run** is a maximal sequence of consecutive squeezes with no absorb +between. Within a run the state advances by repeatedly applying a non-injective +map, so the reachable set shrinks. Option A was not the only construction +exposed to this — **B is too**, and the same rigour demanded of A-TSP's +iteration bound is owed here. + +**(a) The bound.** Model `compress_T(·, operand)` as a random map on `2^128` +points. Image fraction after a run of `k` follows `α_{j+1} = 1 − e^{−α_j}`, +`α_0 = 1`, with `α_k ~ 2/k` (Flajolet–Odlyzko); loss is `−log₂ α_k` bits. +✓ EXECUTED, and the asymptotic verified (`k = 65536`: `α = 3.052e-05`, +`2/k = 3.052e-05`): + +| run `k` | 1 | 4 | 16 | 64 | 256 | 1024 | 65536 | +|---|---:|---:|---:|---:|---:|---:|---:| +| loss (bits) | 0.66 | 1.68 | 3.23 | 5.07 | 7.02 | 9.01 | 15.00 | +| state left | 127.3 | 126.3 | 124.8 | 122.9 | 121.0 | 119.0 | 113.0 | + +**The counter does not change these numbers** — composing distinct random maps +obeys the same recursion. It removes the *attack structure* (one precomputable +functional graph), which is the part that matters. + +**(b) Run lengths as they exist. ✓ VERIFIED** `programs.rs:549-567`: +`FriToyV0`'s runs are **[2, 1, 4]**, so **max run = 4** → **1.68 bits** of 128. +`TrivialV0` has no sponge at all. + +> **⚠ The max run IS `NUM_QUERIES`.** The query loop squeezes once per query with +> no absorb in the body, so the run length **scales with the query count**. +> `NUM_QUERIES = 4` today (`fixture.rs:37`); a production FRI at 100–200 queries +> would have a run that long — 6–7 bits. Still fine, and this is exactly why the +> regime has to be written down rather than left to the current toy shape. + +**(c) Regime and guidance bound.** + +> Runs up to `k ≈ 16` cost under 4 bits; up to `k ≈ 256`, under 8 bits. The +> analysis holds while `k ≪ 2^64`, at which point the birthday bound on the +> 128-bit state dominates anyway. **A program whose squeeze runs exceed +> `k = 2^16` (15 bits of loss) must revisit this section**; below that, the loss +> is dominated by the 64-bit collision bound of §4.1 and changes nothing. + +**(d) Recommendation: keep the counter, do NOT mandate absorb-interleaving.** +Argued rather than asserted. Interleaving a counter-absorb every `K` squeezes +would cost one extra compress per `K` squeezes to buy a *bit-counting* benefit +that is already negligible — at the current `k = 4` it would save 1.68 bits of +128, and even at a production `k = 256` only 7. The counter, by contrast, costs +**zero** (§1.1) and removes the *structural* exposure, which is the part with a +cryptanalytic track record. Paying compressions for the negligible half while +skipping the free fix would be the wrong trade. **Documented bound + free +counter, with the `k = 2^16` revisit trigger, is the right shaping.** + +### 4.3 Exposure profile — the design-level reason this construction stands + +Folded in from `permute-socket-options.md` §8.5 at the lead's request, because it +is construction rationale and belongs in the signed record rather than only in a +decision paper. ✓ EXECUTED (200 and 2000 random inputs respectively, exact). + +With `h = IV`, BLAKE3's output is `out[i] = v[i] ⊕ v[i+8]` and +`out[i+8] = v[i+8] ⊕ IV[i]`. **How many output words you publish therefore +decides how much of the final internal state a reader can reconstruct.** + +**This construction publishes four of sixteen.** `out[0..4] = v[0..4] ⊕ +v[8..12]`, and with no second output block to cross-XOR against the two summands +**cannot be separated** (✓ EXECUTED, 2000/2000 samples). Twelve words stay +unpublished. + +**The rejected option A would have published twelve of sixteen**, which exposes +both halves of a cross-relation: + +``` +out[i] ⊕ out[i+8] == v_final[i] ⊕ IV[i] (i in 0..8) +out[8+i] == v_final[8+i] ⊕ IV[i] (i in 0..4) +``` + +— so one permute output would have revealed **8 of the 16 final state words** +directly, by XOR with public constants. + +**This is not an attack and none is claimed.** The final state is a pseudorandom +function of the input, so recovering it from the output is not obviously +exploitable. It is a *structural* property, of the kind that belongs in a +security argument a reviewer signs — and §4.2's GLUON-64 line began with +structure rather than with a break. It is recorded here because it is the one +argument for this construction that is about the cryptography rather than about +process, reversibility or cost. + +A related note, now moot but worth preserving: option A would have made XOF words +part of the *chaining state*. Standard BLAKE3 chains on `out[0..8]` and uses +`out[8..16]` only as extended output — a role its designers analyse as output, +not as state. This construction stays inside `out[0..8]`, using only `out[0..4]`. + +--- + +## 5. Gate extension + +### 5.1 Executable today — ✓ EXECUTED, PASS 6/6 + +The transcript step is the frozen socket with a different `m[8]` constant, and +`Framing.tag_word` already parameterises exactly that, so the existing theorems +apply **before any Rust exists**. `transcript_gate.py` imports `../gate-oracle/` +rather than editing it — that model is **pinned** to the committed chip and a +spec exercise must not move a pinned instrument. + +| | check | result | +|---|---|---| +| G1 | message schedule under `LFMT`, symbolic, all 7 rounds | UNSAT | +| G2 | full 7-round pipeline == transcript KAT | SAT | +| G3 | the same pipeline **excludes** a wrong digest | UNSAT | +| G4a | Merkle tag used for a transcript step | **SAT** | +| G4b | transcript tag used for a Merkle parent | **SAT** | +| G5 | squeeze counter `i=1` cannot produce squeeze `i=0` | **SAT** | + +G4a/G4b are the domain-separation controls in both directions; G5 makes the +counter load-bearing at the gate level, not only in the vectors. + +### 5.2 What the build must add + +`chip_model.py` needs a `WordRef`-equivalent for the mode-selected tag and a +`MODE_T` role in BLOCK 0; `gate.py`'s BLOCK-0 field audit (`B0a`/`B0b`) needs +its mode-sum widened to `MODE_C + MODE_T + MODE_P` (and again to include +`MODE_L`). Both are small, and the +census is unaffected (§3.4). + +### 5.3 The `MODE_T` controls — ✓ ALL EXECUTED + +> **Status, 2026-08-11:** these were written before the chip existed, as a +> checklist the build would inherit rather than invent. **M1–M7 have since all +> fired against the built chip** (builder, Rust side) and **M8 is executed +> model-side** in the CHIP-GATE board (4 legs, `gate-oracle/CHIP-GATE.md` §4.6.2). +> Kept in full, in the original pre-commitment wording, because a control list +> written *after* seeing the implementation is worth much less than one written +> before it — and because they are now the standing regression set. + +| | control | expected | +|---|---|---| +| M1 | `m[8]` pinned to `TAG_LFMC` while `MODE_T = 1` | **SAT** — a transcript row computing the Merkle tag | +| M2 | `m[8]` pinned to `TAG_LFMT` while `MODE_C = 1` | **SAT** — the mirror image | +| M3 | `MODE_C` and `MODE_T` both 1 on one row | UNSAT — excluded by idx 4 (this **is** what idx 4 buys; see M8 for what it does *not*) | +| M4 | `MODE_C = MODE_T = 0` with `MU = 1` | UNSAT — `MU` *is* their sum | +| M5 | drop the mode-sum booleanity | **SAT** — modes become arbitrary felts, so `m[8]` becomes a prover-chosen combination of both tags and the domain separation evaporates | +| M6 | `MODE_T` as a MAIN (prover-chosen) column | **SAT** — this is the control that proves the preprocessed dependency of §3.3 is real | +| M7 | generalised capacity form idx 0–3 | UNSAT present / SAT dropped | +| **M8** | **idx 4 present, registrar one-hot ABSENT** | **SAT** — a forged `m[8]` is reachable as a fractional blend of the two tags. The control that stops a refactor deleting the one-hot check as redundant. Pair with the honest-path leg: both real tags must stay reachable, or a fix that rejects everything would pass | + +**M5, M6 and M8 are the three that matter.** They are what turn §3.3 from an assertion +into a checked claim, exactly as WA1/WA2 did for obligation O1. + +--- + +## 6. `TrivialV0`'s fate — recommendation + +✓ VERIFIED: `TrivialV0` calls `b.permute` **directly** (`programs.rs`, the +`trivial_program_source` body: two `compress`es then +`b.permute([d1.as_cell(), h[3], d0.as_cell()])`), not through `SpongeVar`. So it +is blocked by B1 independently of the sponge rewrite, and B1 does not touch it. + +**Recommendation: drop the raw `permute` from `TrivialV0` and replace it with a +third `compress`, making the program run under every hasher — and add a +permute-coverage fixture that is NOT a registry entry.** + +Reasoning. Keeping `TrivialV0` as a Test/Poseidon-only fixture would leave the +registry with an entry that cannot run under the machine's real hash, which is +the F3.4 situation in miniature — a registered program whose cryptographic +meaning depends on a placeholder. The registry's six entries should all be +provable under the production hasher. Against that, permute mode does not +disappear: `Test` and `Poseidon` keep it, and it needs *some* test coverage or +the arms rot. But coverage does not require a **registry** entry — a +`#[cfg(test)]` permute fixture exercises the arms without claiming a program +identity, and the cost of the swap is one compress (16,527 vs 16,635 cell-equiv, +✓ EXECUTED — the compress version is marginally *cheaper*). + +? INFERRED and worth checking during the build: whether any test asserts +`TrivialV0`'s public output shape, which the swap would move. Its `program_id` +moves anyway in the B1 re-bless. + +--- + +## 7. What is executed, and what is open + +| claim | status | +|---|---| +| every step == `blake3::hash(state‖operand‖"LFMT")[..16]` at 7 rounds | ✓ EXECUTED (K1, two routes) | +| end-to-end `FriToyV0`-shaped transcript, op by op, both round counts | ✓ EXECUTED (K2) | +| transcript step ≠ Merkle parent on the same cells | ✓ EXECUTED (K3) | +| the squeeze counter is load-bearing | ✓ EXECUTED (K4, G5) | +| absorb order is load-bearing | ✓ EXECUTED (K5) | +| `FriToyV0` transcript costs **13** compressions (11 + 2 leaf rows) | ✓ EXECUTED (K6), re-pointed at the current preamble | +| the frozen socket computes the transcript step correctly under `LFMT` | ✓ EXECUTED (G1–G3) | +| domain-separation controls fire both directions | ✓ EXECUTED (G4a/G4b) | +| squeeze-run entropy bound + the programs' actual runs | ✓ EXECUTED (§4.2) | +| the same identity against the Rust `blake3` **crate** | ✗ DEFERRED — needs cargo | +| `MODE_T` mechanism (M1–M7) | ✗ OPEN — needs the chip | +| tag tables in `ORACLE.md` §2.3 / `SOCKET.md` §2.4 updated | ✗ OPEN — §2, one pass | +| `PREP_WIDTH` 11 → 12 sequenced into the B1 re-bless | ✗ OPEN — build | + +--- + +## 8. Files + +| file | what | +|---|---| +| `transcript_ref.py` | the reference — the future `HostSponge` mirror | +| `transcript_kats.py`, `transcript_kats.json` | K1–K6 + the end-to-end `FriToyV0` vector | +| `squeeze_run_analysis.py` | §4.2's entropy-loss numbers | +| `transcript_gate.py` | G1–G5 executable now + M1–M7 pre-committed | diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/run-gate.log b/thoughts/shared/lfm-real-hash/transcript-spec/run-gate.log new file mode 100644 index 000000000..9916b8348 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/run-gate.log @@ -0,0 +1,30 @@ +============================================================================== +TRANSCRIPT GATE — executable now (tag = LFMT on the frozen socket) +============================================================================== + [PASS ] G1 message schedule @LFMT, 7 rounds, symbolic -> unsat (want unsat) + [PASS ] G2 full 7-round pipeline == transcript KAT -> sat (want sat) 2.9s + [PASS ] G3 same pipeline EXCLUDES a wrong digest -> unsat (want unsat) 3.0s + [PASS ] G4a Merkle tag (LFMC) used for a transcript step -> sat (want sat) 3.1s + [PASS ] G4b transcript tag (LFMT) used for a Merkle parent -> sat (want sat) 3.1s + [PASS ] G5 squeeze counter i=1 cannot produce squeeze i=0 -> sat (want sat) 2.8s + +------------------------------------------------------------------------------ +TRANSCRIPT GATE: PASS (6/6) + +============================================================================== +PRE-COMMITTED CONTROLS (need MODE_T; the build agent must run these) +============================================================================== + M1: m[8] pinned to TAG_LFMC while MODE_T = 1 + expect: SAT — a transcript row computing the Merkle tag is a live confusion bug + M2: m[8] pinned to TAG_LFMT while MODE_C = 1 + expect: SAT — the mirror image; a Merkle parent computing the transcript tag + M3: MODE_C and MODE_T both 1 on one row + expect: UNSAT — excluded by the generalised mode-sum booleanity (idx 4) + M4: MODE_C = MODE_T = 0 on a row with MU = 1 + expect: UNSAT — MU is defined as MODE_C + MODE_T, so this is not a real row + M5: drop the mode-sum booleanity + expect: SAT — modes become arbitrary felts, so m[8] becomes a prover-chosen linear combination of the two tags: the domain separation evaporates + M6: MODE_T treated as a MAIN (prover-chosen) column instead of preprocessed + expect: SAT — the whole soundness argument for the tag rests on MODE_* being preprocessed; this control is what proves that dependency is real + M7: capacity prefix idx 0-3 still pins S_k with the generalised form S_k = MODE_P*IN + (MODE_C + MODE_T)*IV_k + expect: UNSAT with the form present; SAT with it dropped diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/run-kats.log b/thoughts/shared/lfm-real-hash/transcript-spec/run-kats.log new file mode 100644 index 000000000..afe89a471 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/run-kats.log @@ -0,0 +1,13 @@ +========================================================================== +TRANSCRIPT KATs — compress-chain (option B1) +========================================================================== + [PASS] K1 PASS: 6 steps, word route == byte route + [PASS] K2: FriToyV0-shaped transcript, 10 ops, 11 compressions @7r + [PASS] K1 PASS: 6 steps, word route == byte route + [PASS] K2: FriToyV0-shaped transcript, 10 ops, 11 compressions @6r + [PASS] K3 PASS: transcript step != Merkle parent on the same two cells (the LFMT/LFMC tag is load-bearing) + [PASS] K4 PASS: counter-free squeezes diverge from the spec at squeeze #2 (they iterate one fixed map) + [PASS] K5 PASS: swapping two absorbs changes the state + [PASS] K6: FriToyV0 transcript costs 11 compressions (spec claims 11) +-------------------------------------------------------------------------- +TRANSCRIPT KATs: PASS diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/run-squeeze.log b/thoughts/shared/lfm-real-hash/transcript-spec/run-squeeze.log new file mode 100644 index 000000000..c8e65be39 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/run-squeeze.log @@ -0,0 +1,40 @@ +========================================================================== +SQUEEZE-RUN ANALYSIS — entropy loss vs run length +========================================================================== + state = 128 bits (one cell) + + run k | reachable fraction | loss (bits) | state left + -------+--------------------+-------------+----------- + 1 | 0.632121 | 0.66 | 127.34 + 2 | 0.468536 | 1.09 | 126.91 + 4 | 0.312080 | 1.68 | 126.32 + 8 | 0.189050 | 2.40 | 125.60 + 16 | 0.106537 | 3.23 | 124.77 + 64 | 0.029762 | 5.07 | 122.93 + 256 | 0.007703 | 7.02 | 120.98 + 1024 | 0.001945 | 9.01 | 118.99 + 4096 | 0.000488 | 11.00 | 117.00 + 65536 | 0.000031 | 15.00 | 113.00 + + asymptotic check, alpha_k ~ 2/k: + k= 1024: alpha=1.945e-03 2/k=1.953e-03 + k= 65536: alpha=3.052e-05 2/k=3.052e-05 + +-------------------------------------------------------------------------- +(b) RUN LENGTHS IN THE PROGRAMS AS THEY EXIST ✓ VERIFIED + FriToyV0 runs: [2, 1, 4] -> MAX RUN = 4 + loss at k=4: 1.68 bits of 128 + TrivialV0: no sponge (raw permute; see spec) + NOTE: the max run IS NUM_QUERIES — it scales with the query count, + so a production FRI (100-200 queries) would have a run that + long. That is the regime worth stating a bound for. + hypothetical production run k=128: 6.04 bits + hypothetical production run k=256: 7.02 bits + +-------------------------------------------------------------------------- +(c) GUIDANCE BOUND + loss stays under 1 bit(s) for runs up to k ~ 1 + loss stays under 4 bit(s) for runs up to k ~ 16 + loss stays under 8 bit(s) for runs up to k ~ 256 + The birthday bound on a 128-bit state (2^64) dominates long + before image shrinkage matters: even k = 2^16 costs under 16 bits. diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/squeeze_run_analysis.py b/thoughts/shared/lfm-real-hash/transcript-spec/squeeze_run_analysis.py new file mode 100644 index 000000000..507d35a67 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/squeeze_run_analysis.py @@ -0,0 +1,102 @@ +""" +THE SQUEEZE-RUN ANALYSIS — the FSE-2014 lesson, written into this construction's +own spec rather than left in an options-paper appendix. + +A "squeeze run" is a maximal sequence of consecutive squeezes with no absorb +between them. Within a run the transcript advances by repeatedly applying a +non-injective map to a 128-bit state, so the reachable state set shrinks. This +is the same phenomenon that broke GLUON-64 (Collision Spectrum, Entropy Loss, +T-Sponges, FSE 2014), and option A was NOT the only construction exposed to it — +option B is too, which is why it belongs here. + +MODEL. `compress_T(·, operand)` with a fixed operand is a map on 2^128 points; +model it as random. For a random map on N points, the image after one +application has expected size N(1 - e^{-1}); iterating gives the recursion + + alpha_{j+1} = 1 - exp(-alpha_j), alpha_0 = 1 + +with alpha_k ~ 2/k asymptotically (Flajolet-Odlyzko). Entropy loss after a run of +length k is -log2(alpha_k) bits of the state's 128. + +COMPOSING DISTINCT MAPS DOES NOT ESCAPE THIS. With the squeeze counter each step +is a different map, but the same recursion governs the image of a composition of +independent random maps, so the bit-counting is unchanged. What the counter +removes is the *attack structure* — a single fixed public map has ONE functional +graph (rho-shapes, deep nodes, cycles) that an adversary can precompute and that +the T-sponge attacks exploit. That distinction is the whole point and it is why +the counter is in the spec even though the numbers below say the loss is +irrelevant either way. +""" + +from __future__ import annotations + +import math + +STATE_BITS = 128 + + +def alpha(k: int) -> float: + """Fraction of the state space still reachable after a run of length k.""" + a = 1.0 + for _ in range(k): + a = 1.0 - math.exp(-a) + return a + + +def loss_bits(k: int) -> float: + return -math.log2(alpha(k)) if k > 0 else 0.0 + + +# --- (b) run lengths in the ACTUAL programs, ✓ VERIFIED --------------------- +# FriToyV0's sponge sequence (programs.rs:549-567): +# absorb, squeeze, squeeze, absorb, squeeze, absorb2, then NUM_QUERIES +# squeezes with NO absorb in the loop body. +FRI_TOY_RUNS = [2, 1, 4] # NUM_QUERIES = 4 (fixture.rs:37) +FRI_TOY_MAX_RUN = max(FRI_TOY_RUNS) + +# TrivialV0 has no sponge at all (it calls b.permute directly; see the spec's +# TrivialV0 section). +TRIVIAL_RUNS: list[int] = [] + + +def main() -> None: + print("=" * 74) + print("SQUEEZE-RUN ANALYSIS — entropy loss vs run length") + print("=" * 74) + print(f" state = {STATE_BITS} bits (one cell)\n") + print(" run k | reachable fraction | loss (bits) | state left") + print(" -------+--------------------+-------------+-----------") + for k in (1, 2, 4, 8, 16, 64, 256, 1024, 4096, 65536): + a, l = alpha(k), loss_bits(k) + print(f" {k:6d} | {a:18.6f} | {l:11.2f} | {STATE_BITS - l:9.2f}") + + print(f"\n asymptotic check, alpha_k ~ 2/k:") + for k in (1024, 65536): + print(f" k={k:6d}: alpha={alpha(k):.3e} 2/k={2/k:.3e}") + + print("\n" + "-" * 74) + print("(b) RUN LENGTHS IN THE PROGRAMS AS THEY EXIST ✓ VERIFIED") + print(f" FriToyV0 runs: {FRI_TOY_RUNS} -> MAX RUN = {FRI_TOY_MAX_RUN}") + print(f" loss at k={FRI_TOY_MAX_RUN}: {loss_bits(FRI_TOY_MAX_RUN):.2f} bits " + f"of {STATE_BITS}") + print(" TrivialV0: no sponge (raw permute; see spec)") + print(" NOTE: the max run IS NUM_QUERIES — it scales with the query count,") + print(" so a production FRI (100-200 queries) would have a run that") + print(" long. That is the regime worth stating a bound for.") + for k in (128, 256): + print(f" hypothetical production run k={k}: {loss_bits(k):.2f} bits") + + print("\n" + "-" * 74) + print("(c) GUIDANCE BOUND") + for target in (1.0, 4.0, 8.0): + k = 1 + while loss_bits(k) < target: + k *= 2 + print(f" loss stays under {target:.0f} bit(s) for runs up to k ~ {k//2}") + print(f" The birthday bound on a {STATE_BITS}-bit state (2^{STATE_BITS//2}) " + f"dominates long") + print(" before image shrinkage matters: even k = 2^16 costs under 16 bits.") + + +if __name__ == "__main__": + main() diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/transcript_gate.py b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_gate.py new file mode 100644 index 000000000..69cf00a4e --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_gate.py @@ -0,0 +1,135 @@ +""" +GATE EXTENSION for the transcript tag — what is executable NOW, and what is +pre-committed for the build. + +Design note: this harness IMPORTS `../gate-oracle/` rather than editing it. That +directory's model is PINNED to the committed chip (`artifact_pin.json`), and a +costing or spec exercise must not move a pinned instrument. Everything here is +additive. + +WHAT IS EXECUTABLE NOW. The transcript step uses the frozen socket with a +different constant in `m[8]`, and `Framing.tag_word` already parameterises +exactly that — so the existing theorems apply to the transcript step today, +before any Rust exists: + G1 the message schedule under TAG_LFMT, symbolic, all 7 rounds -> UNSAT + G2 the full pipeline, concrete, vs the transcript KATs -> SAT + G3 the same pipeline EXCLUDES a wrong digest -> UNSAT + G4 tag controls: LFMC used where LFMT belongs, and vice versa -> SAT + +WHAT IS PRE-COMMITTED, NOT YET RUNNABLE. The mode-selected tag +`m[8] = MODE_C*TAG_LFMC + MODE_T*TAG_LFMT` needs a chip that has MODE_T. Those +controls are listed in `TRANSCRIPT.md` §5 and stubbed at the bottom of this file +so the build agent inherits them as a checklist rather than inventing them. +""" + +from __future__ import annotations + +import json +import os +import sys +from dataclasses import replace + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "gate-oracle")) + +import gate # noqa: E402 +import socket_ref as sk # noqa: E402 +import transcript_ref as tr # noqa: E402 + +TAG_T = sk.Framing(rounds=7, tag_word=tr.TAG_LFMT) + + +def board(): + rows = [] + + def add(name, got, want, secs=0.0): + ok = str(got) == want + rows.append(ok) + mark = "PASS" if ok else "**FAIL**" + print(f" [{mark:8s}] {name:52s} -> {str(got):6s} (want {want})" + f"{f' {secs:.1f}s' if secs > 0.3 else ''}") + + print("=" * 78) + print("TRANSCRIPT GATE — executable now (tag = LFMT on the frozen socket)") + print("=" * 78) + + # G1 — the schedule under the transcript tag, symbolic. + r, t = gate.theorem_schedule(7, chip_framing=TAG_T, ref_framing=TAG_T) + add("G1 message schedule @LFMT, 7 rounds, symbolic", r, "unsat", t) + + # G2/G3 — the full pipeline against the transcript KATs. + with open(os.path.join(HERE, "transcript_kats.json")) as f: + kats = json.load(f) + vec = kats["rounds"]["7"]["step_vectors"][2] # ramp_state_ramp_operand + a, b, want = vec["state"], vec["operand"], vec["result"] + + r, t = gate.concrete_pipeline(7, a, b, want, chip_framing=TAG_T, + timeout_ms=900_000) + add("G2 full 7-round pipeline == transcript KAT", r, "sat", t) + r, t = gate.concrete_pipeline(7, a, b, want, negate=True, + chip_framing=TAG_T, timeout_ms=900_000) + add("G3 same pipeline EXCLUDES a wrong digest", r, "unsat", t) + + # G4 — tag controls on the NEW surface, both directions. + r, t = gate.concrete_control(7, a, b, want, + chip_framing=sk.Framing(rounds=7), # LFMC + timeout_ms=900_000) + add("G4a Merkle tag (LFMC) used for a transcript step", r, "sat", t) + + merkle_want = sk.socket_digest_wordlevel(a, b, sk.Framing(rounds=7)) + r, t = gate.concrete_control(7, a, b, merkle_want, chip_framing=TAG_T, + timeout_ms=900_000) + add("G4b transcript tag (LFMT) used for a Merkle parent", r, "sat", t) + + # G5 — the squeeze operand is load-bearing at the gate level too. + st = [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10] + want0 = tr.compress_t(st, tr.squeeze_operand(0)) + r, t = gate.concrete_control(7, st, tr.squeeze_operand(1), want0, + chip_framing=TAG_T, timeout_ms=900_000) + add("G5 squeeze counter i=1 cannot produce squeeze i=0", r, "sat", t) + + print("\n" + "-" * 78) + print(f"TRANSCRIPT GATE: {'PASS' if all(rows) else 'FAIL'} " + f"({sum(rows)}/{len(rows)})") + return all(rows) + + +# --------------------------------------------------------------------------- +# PRE-COMMITTED — cannot run until the chip has MODE_T. The build agent must +# make each of these fire before the transcript arm is considered gated. +# --------------------------------------------------------------------------- + +PRECOMMITTED_CONTROLS = [ + ("M1", "m[8] pinned to TAG_LFMC while MODE_T = 1", + "SAT — a transcript row computing the Merkle tag is a live confusion bug"), + ("M2", "m[8] pinned to TAG_LFMT while MODE_C = 1", + "SAT — the mirror image; a Merkle parent computing the transcript tag"), + ("M3", "MODE_C and MODE_T both 1 on one row", + "UNSAT — excluded by the generalised mode-sum booleanity (idx 4)"), + ("M4", "MODE_C = MODE_T = 0 on a row with MU = 1", + "UNSAT — MU is defined as MODE_C + MODE_T, so this is not a real row"), + ("M5", "drop the mode-sum booleanity", + "SAT — modes become arbitrary felts, so m[8] becomes a prover-chosen " + "linear combination of the two tags: the domain separation evaporates"), + ("M6", "MODE_T treated as a MAIN (prover-chosen) column instead of " + "preprocessed", + "SAT — the whole soundness argument for the tag rests on MODE_* being " + "preprocessed; this control is what proves that dependency is real"), + ("M7", "capacity prefix idx 0-3 still pins S_k with the generalised form " + "S_k = MODE_P*IN + (MODE_C + MODE_T)*IV_k", + "UNSAT with the form present; SAT with it dropped"), +] + + +def print_precommitted(): + print("\n" + "=" * 78) + print("PRE-COMMITTED CONTROLS (need MODE_T; the build agent must run these)") + print("=" * 78) + for tag, what, want in PRECOMMITTED_CONTROLS: + print(f" {tag}: {what}\n expect: {want}") + + +if __name__ == "__main__": + ok = board() + print_precommitted() + sys.exit(0 if ok else 1) diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.json b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.json new file mode 100644 index 000000000..8f522814c --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.json @@ -0,0 +1,765 @@ +{ + "construction": "LFM compress-chain transcript (option B1)", + "tag_ascii": "LFMT", + "tag_word": 1414350412, + "squeeze_mark": 811225427, + "state_cells": 1, + "state_bits": 128, + "initial_state": [ + 0, + 0, + 0, + 0 + ], + "framing": "identical to the Merkle socket except m[8] = TAG_LFMT", + "rounds": { + "7": { + "step_vectors": [ + { + "name": "zero_state_zero_operand", + "state": [ + 0, + 0, + 0, + 0 + ], + "operand": [ + 0, + 0, + 0, + 0 + ], + "result": [ + 3789403500, + 341101770, + 2953369136, + 1161295779 + ], + "result_hex": "e1ddb56c1454cccab008d6304537f7a3" + }, + { + "name": "zero_state_main_root", + "state": [ + 0, + 0, + 0, + 0 + ], + "operand": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "result": [ + 3556613968, + 1053918169, + 4128172162, + 3821225588 + ], + "result_hex": "d3fd9f503ed183d9f60ee882e3c34674" + }, + { + "name": "ramp_state_ramp_operand", + "state": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "operand": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "result": [ + 1835636148, + 4200777088, + 394734153, + 743310545 + ], + "result_hex": "6d6995b4fa62c58017872a492c4e04d1" + }, + { + "name": "max_state", + "state": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "operand": [ + 3735928559, + 3405691582, + 2343432205, + 4277009102 + ], + "result": [ + 1798251963, + 487528172, + 527817843, + 3110281407 + ], + "result_hex": "6b2f25bb1d0f16ec1f75dc73b96320bf" + }, + { + "name": "squeeze_operand_0", + "state": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "operand": [ + 811225427, + 0, + 0, + 0 + ], + "result": [ + 628831937, + 1378306807, + 2790322639, + 3670096977 + ], + "result_hex": "257b36c152274af7a650f1cfdac13c51" + }, + { + "name": "squeeze_operand_255", + "state": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "operand": [ + 811225427, + 255, + 0, + 0 + ], + "result": [ + 452758980, + 78889273, + 3950245919, + 1208497044 + ], + "result_hex": "1afc8dc404b3c139eb73f81f48083394" + } + ], + "fri_toy_v0": { + "rounds": 7, + "inputs": { + "main_root": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "l1_root": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "t0w": [ + 3735928559, + 3405691582, + 2343432205, + 4277009102 + ], + "t1w": [ + 195936478, + 3512640997, + 3237998080, + 3131746989 + ] + }, + "shape": { + "num_queries": 4, + "query_bits": 4 + }, + "steps": [ + { + "op": "absorb(main_root)", + "state_after": [ + 3556613968, + 1053918169, + 4128172162, + 3821225588 + ], + "state_after_hex": "d3fd9f503ed183d9f60ee882e3c34674" + }, + { + "op": "squeeze_ext -> alpha", + "state_after": [ + 654458755, + 2704560048, + 2663234482, + 1412507 + ], + "state_after_hex": "27023f83a1344fb09ebdbbb200158d9b", + "output_lanes": [ + 3556613968, + 1053918169, + 4128172162 + ] + }, + { + "op": "squeeze_ext -> zeta0", + "state_after": [ + 1140832608, + 915851117, + 2635096162, + 3936610597 + ], + "state_after_hex": "43ffb9603696c76d9d106062eaa3e925", + "output_lanes": [ + 654458755, + 2704560048, + 2663234482 + ] + }, + { + "op": "absorb(l1_root)", + "state_after": [ + 600953737, + 1072298929, + 2062904039, + 3969085241 + ], + "state_after_hex": "23d1d3893fe9fbb17af56ae7ec936f39" + }, + { + "op": "squeeze_ext -> zeta1", + "state_after": [ + 2484420066, + 2785031031, + 3492343115, + 4092298626 + ], + "state_after_hex": "94153de2a6003377d028ed4bf3eb8582", + "output_lanes": [ + 600953737, + 1072298929, + 2062904039 + ] + }, + { + "op": "absorb_felts(t0w)", + "state_after": [ + 3967948545, + 3440632190, + 2125317775, + 31689816 + ], + "state_after_hex": "ec821701cd13e17e7eadc68f01e38c58" + }, + { + "op": "absorb_felts(t1w)", + "state_after": [ + 243410647, + 506340152, + 2220684167, + 3812539884 + ], + "state_after_hex": "0e8226d71e2e2338845cf387e33ebdec" + }, + { + "op": "squeeze_bits(q=0)", + "state_after": [ + 3059012436, + 1911418129, + 2331981490, + 2798080175 + ], + "state_after_hex": "b654d35471eded118aff36b2a6c750af", + "output": [ + 1, + 1, + 1, + 0 + ], + "output_hex": "00000001000000010000000100000000" + }, + { + "op": "squeeze_bits(q=1)", + "state_after": [ + 111181741, + 2359888466, + 2051596105, + 3384913624 + ], + "state_after_hex": "06a07fad8ca90a527a48df49c9c1aed8", + "output": [ + 0, + 0, + 1, + 0 + ], + "output_hex": "00000000000000000000000100000000" + }, + { + "op": "squeeze_bits(q=2)", + "state_after": [ + 2338234607, + 4062977953, + 3787104146, + 3524048681 + ], + "state_after_hex": "8b5ea0eff22c1fa1e1ba9f92d20cb729", + "output": [ + 1, + 0, + 1, + 1 + ], + "output_hex": "00000001000000000000000100000001" + }, + { + "op": "squeeze_bits(q=3)", + "state_after": [ + 2723948640, + 4275004151, + 1923477743, + 254533939 + ], + "state_after_hex": "a25c2860fecf62f772a5f0ef0f2be133", + "output": [ + 1, + 1, + 1, + 1 + ], + "output_hex": "00000001000000010000000100000001" + } + ], + "challenges": { + "alpha": [ + 3556613968, + 1053918169, + 4128172162 + ], + "zeta0": [ + 654458755, + 2704560048, + 2663234482 + ], + "zeta1": [ + 600953737, + 1072298929, + 2062904039 + ], + "query_bits": [ + [ + 1, + 1, + 1, + 0 + ], + [ + 0, + 0, + 1, + 0 + ], + [ + 1, + 0, + 1, + 1 + ], + [ + 1, + 1, + 1, + 1 + ] + ] + }, + "compressions": 13, + "final_state": [ + 2723948640, + 4275004151, + 1923477743, + 254533939 + ] + } + }, + "6": { + "step_vectors": [ + { + "name": "zero_state_zero_operand", + "state": [ + 0, + 0, + 0, + 0 + ], + "operand": [ + 0, + 0, + 0, + 0 + ], + "result": [ + 3228761638, + 994873871, + 1690118560, + 34858724 + ], + "result_hex": "c072fe263b4c920f64bd29a00213e6e4" + }, + { + "name": "zero_state_main_root", + "state": [ + 0, + 0, + 0, + 0 + ], + "operand": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "result": [ + 2325406339, + 3347237647, + 628425412, + 1899264511 + ], + "result_hex": "8a9ae283c782cb0f257502c4713479ff" + }, + { + "name": "ramp_state_ramp_operand", + "state": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "operand": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "result": [ + 591096368, + 3231223618, + 305482786, + 1486162171 + ], + "result_hex": "233b6a30c0988f4212354c22589508fb" + }, + { + "name": "max_state", + "state": [ + 4294967295, + 4294967295, + 4294967295, + 4294967295 + ], + "operand": [ + 3735928559, + 3405691582, + 2343432205, + 4277009102 + ], + "result": [ + 3430766259, + 3435831081, + 37336044, + 1054867942 + ], + "result_hex": "cc7d56b3ccca9f290239b3ec3ee001e6" + }, + { + "name": "squeeze_operand_0", + "state": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "operand": [ + 811225427, + 0, + 0, + 0 + ], + "result": [ + 926358993, + 1974727727, + 4200707228, + 3970497528 + ], + "result_hex": "37371dd175b3f42ffa61b49ceca8fbf8" + }, + { + "name": "squeeze_operand_255", + "state": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "operand": [ + 811225427, + 255, + 0, + 0 + ], + "result": [ + 1665992089, + 4205659331, + 2459483588, + 360418507 + ], + "result_hex": "634d0599faad44c39298bdc4157b8ccb" + } + ], + "fri_toy_v0": { + "rounds": 6, + "inputs": { + "main_root": [ + 16909060, + 84281096, + 151653132, + 219025168 + ], + "l1_root": [ + 286397204, + 353769240, + 421141276, + 488513312 + ], + "t0w": [ + 3735928559, + 3405691582, + 2343432205, + 4277009102 + ], + "t1w": [ + 195936478, + 3512640997, + 3237998080, + 3131746989 + ] + }, + "shape": { + "num_queries": 4, + "query_bits": 4 + }, + "steps": [ + { + "op": "absorb(main_root)", + "state_after": [ + 2325406339, + 3347237647, + 628425412, + 1899264511 + ], + "state_after_hex": "8a9ae283c782cb0f257502c4713479ff" + }, + { + "op": "squeeze_ext -> alpha", + "state_after": [ + 2295533306, + 3465367076, + 2734988726, + 2469666159 + ], + "state_after_hex": "88d30eface8d4e24a3049db693341d6f", + "output_lanes": [ + 2325406339, + 3347237647, + 628425412 + ] + }, + { + "op": "squeeze_ext -> zeta0", + "state_after": [ + 156489123, + 1294316337, + 1260011018, + 1836937486 + ], + "state_after_hex": "0953d5a34d25b3314b1a3e0a6d7d710e", + "output_lanes": [ + 2295533306, + 3465367076, + 2734988726 + ] + }, + { + "op": "absorb(l1_root)", + "state_after": [ + 1082864478, + 4212261694, + 1322572021, + 1618472488 + ], + "state_after_hex": "408b335efb12033e4ed4d8f56077ee28" + }, + { + "op": "squeeze_ext -> zeta1", + "state_after": [ + 3094637406, + 2580036028, + 1962299373, + 2180724735 + ], + "state_after_hex": "b8746b5e99c839bc74f64fed81fb37ff", + "output_lanes": [ + 1082864478, + 4212261694, + 1322572021 + ] + }, + { + "op": "absorb_felts(t0w)", + "state_after": [ + 3198402399, + 1155156074, + 678881112, + 2989402576 + ], + "state_after_hex": "bea3bf5f44da486a2876e758b22ea9d0" + }, + { + "op": "absorb_felts(t1w)", + "state_after": [ + 2435029396, + 96562807, + 1307669935, + 3342898404 + ], + "state_after_hex": "9123999405c16e774df175afc74094e4" + }, + { + "op": "squeeze_bits(q=0)", + "state_after": [ + 966235550, + 1056262703, + 3616650628, + 1791450194 + ], + "state_after_hex": "3997959e3ef54a2fd791b5846ac75c52", + "output": [ + 0, + 0, + 1, + 0 + ], + "output_hex": "00000000000000000000000100000000" + }, + { + "op": "squeeze_bits(q=1)", + "state_after": [ + 944412110, + 1823519267, + 1783682251, + 2743758201 + ], + "state_after_hex": "384a95ce6cb0b2236a50d4cba38a6d79", + "output": [ + 0, + 1, + 1, + 1 + ], + "output_hex": "00000000000000010000000100000001" + }, + { + "op": "squeeze_bits(q=2)", + "state_after": [ + 1548404354, + 2649680121, + 2960468545, + 4168716706 + ], + "state_after_hex": "5c4ac6829deee8f9b0752a41f87991a2", + "output": [ + 0, + 1, + 1, + 1 + ], + "output_hex": "00000000000000010000000100000001" + }, + { + "op": "squeeze_bits(q=3)", + "state_after": [ + 4146066959, + 3680558935, + 411312343, + 4071015754 + ], + "state_after_hex": "f71ff60fdb60df57188420d7f2a6c54a", + "output": [ + 0, + 1, + 0, + 0 + ], + "output_hex": "00000000000000010000000000000000" + } + ], + "challenges": { + "alpha": [ + 2325406339, + 3347237647, + 628425412 + ], + "zeta0": [ + 2295533306, + 3465367076, + 2734988726 + ], + "zeta1": [ + 1082864478, + 4212261694, + 1322572021 + ], + "query_bits": [ + [ + 0, + 0, + 1, + 0 + ], + [ + 0, + 1, + 1, + 1 + ], + [ + 0, + 1, + 1, + 1 + ], + [ + 0, + 1, + 0, + 0 + ] + ] + }, + "compressions": 13, + "final_state": [ + 4146066959, + 3680558935, + 411312343, + 4071015754 + ] + } + } + } +} \ No newline at end of file diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.py b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.py new file mode 100644 index 000000000..7d0f54371 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_kats.py @@ -0,0 +1,215 @@ +""" +Transcript KATs: per-op vectors, an END-TO-END FriToyV0-shaped transcript, and +the checks that make them meaningful. + +What this establishes, all ✓ EXECUTED: + K1 every transcript step equals `BLAKE3(state ‖ operand ‖ "LFMT")[..16]` at + 7 rounds — computed by two separate routes (word level and byte level) + and asserted equal. This is the crate-KAT identity the implementer must + re-assert with a one-line `blake3::hash` call. + K2 a full FriToyV0-preamble-shaped transcript, op by op, with the state + after every step — so the implementer has an end-to-end vector, not only + per-op ones. + K3 DOMAIN SEPARATION IS REAL: a transcript step and a Merkle parent over the + same two cells produce different digests (the tag is load-bearing). + K4 the squeeze counter is load-bearing: dropping it makes consecutive + squeezes iterate one fixed map, and the vectors change. + K5 ordering is load-bearing: swapping two absorbs changes the transcript. + K6 compression accounting matches the spec's cost claim (11 for FriToyV0). + +Run: python3 transcript_kats.py [--write] +""" + +from __future__ import annotations + +import json +import os +import sys + +import transcript_ref as tr + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "gate-oracle")) +import socket_ref as sk # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) +OUT = os.path.join(HERE, "transcript_kats.json") + +# FriToyV0's shape, ✓ VERIFIED against fixture.rs:26-40. +NUM_QUERIES = 4 +QUERY_BITS = 4 + +# Fixed, written-out inputs — nothing depends on an RNG. +MAIN_ROOT = [0x01020304, 0x05060708, 0x090A0B0C, 0x0D0E0F10] +L1_ROOT = [0x11121314, 0x15161718, 0x191A1B1C, 0x1D1E1F20] +T0W = [0xDEADBEEF, 0xCAFEBABE, 0x8BADF00D, 0xFEEDFACE] +T1W = [0x0BADC0DE, 0xD15EA5E5, 0xC0FFEE00, 0xBAAAAAAD] + + +def hexlanes(c): + return "".join(f"{x:08x}" for x in c) + + +def k1_step_identity(rounds: int) -> tuple[bool, str, list]: + """Every step, both routes. At 7 rounds the byte route IS `blake3::hash`.""" + cases = [ + ("zero_state_zero_operand", [0, 0, 0, 0], [0, 0, 0, 0]), + ("zero_state_main_root", [0, 0, 0, 0], MAIN_ROOT), + ("ramp_state_ramp_operand", MAIN_ROOT, L1_ROOT), + ("max_state", [0xFFFFFFFF] * 4, T0W), + ("squeeze_operand_0", MAIN_ROOT, tr.squeeze_operand(0)), + ("squeeze_operand_255", L1_ROOT, tr.squeeze_operand(255)), + ] + out = [] + for name, st, op in cases: + w = tr.compress_t(st, op, rounds) + b = tr.compress_t_bytelevel(st, op, rounds) + if w != b: + return False, f"K1 route mismatch on {name}@{rounds}", [] + out.append({"name": name, "state": st, "operand": op, + "result": w, "result_hex": hexlanes(w)}) + return True, f"K1 PASS: {len(cases)} steps, word route == byte route", out + + +def k2_end_to_end(rounds: int) -> tuple[dict, list]: + """The FriToyV0 preamble + query loop, op by op. + + ✓ VERIFIED sequence, programs.rs:549-567: + absorb(main_root), squeeze_ext, squeeze_ext, absorb(l1_root), + squeeze_ext, absorb_felts(t0w), absorb_felts(t1w), then + NUM_QUERIES x squeeze_bits. + """ + t = tr.Transcript(rounds=rounds) + steps = [] + + def rec(op, value=None): + steps.append({"op": op, + "state_after": list(t.state), + "state_after_hex": hexlanes(t.state), + **({"output": value, "output_hex": hexlanes(value)} + if value is not None and len(value) == 4 else {}), + **({"output_lanes": value} if value is not None + and len(value) != 4 else {})}) + + t.absorb(MAIN_ROOT); rec("absorb(main_root)") + alpha = t.squeeze_ext(); rec("squeeze_ext -> alpha", alpha) + zeta0 = t.squeeze_ext(); rec("squeeze_ext -> zeta0", zeta0) + t.absorb(L1_ROOT); rec("absorb(l1_root)") + zeta1 = t.squeeze_ext(); rec("squeeze_ext -> zeta1", zeta1) + # ✓ VERIFIED programs.rs: the preamble now calls absorb_felts TWICE, not + # absorb2 — t0/t1 are terminal-polynomial coefficients, i.e. ARBITRARY field + # elements, so each goes leaf-then-absorb. Four compresses where the old + # vector modelled two; this is the 91 -> 93 correction, in the vectors. + t.absorb_felts(T0W); rec("absorb_felts(t0w)") + t.absorb_felts(T1W); rec("absorb_felts(t1w)") + query_bits = [] + for q in range(NUM_QUERIES): + bits = t.squeeze_bits(QUERY_BITS) + query_bits.append(bits) + rec(f"squeeze_bits(q={q})", bits) + + return { + "rounds": rounds, + "inputs": {"main_root": MAIN_ROOT, "l1_root": L1_ROOT, + "t0w": T0W, "t1w": T1W}, + "shape": {"num_queries": NUM_QUERIES, "query_bits": QUERY_BITS}, + "steps": steps, + "challenges": {"alpha": alpha, "zeta0": zeta0, "zeta1": zeta1, + "query_bits": query_bits}, + "compressions": t.compressions, + "final_state": list(t.state), + }, steps + + +def k3_domain_separation(rounds: int) -> tuple[bool, str]: + """A transcript step must NOT equal a Merkle parent over the same cells.""" + a, b = MAIN_ROOT, L1_ROOT + step = tr.compress_t(a, b, rounds) + parent = sk.socket_digest_wordlevel(a, b, sk.Framing(rounds=rounds)) + if step == parent: + return False, ("K3 FAIL: transcript step == Merkle parent — the tag is " + "NOT separating the domains") + return True, ("K3 PASS: transcript step != Merkle parent on the same two " + "cells (the LFMT/LFMC tag is load-bearing)") + + +def k4_counter_is_load_bearing(rounds: int) -> tuple[bool, str]: + """Without SQ(i)'s counter every squeeze advance uses ONE fixed operand, so + a run of squeezes iterates one fixed map. The vectors must notice.""" + t1 = tr.Transcript(rounds=rounds) + t1.absorb(MAIN_ROOT) + with_counter = [t1.squeeze() for _ in range(4)] + + t2 = tr.Transcript(rounds=rounds) + t2.absorb(MAIN_ROOT) + fixed = tr.squeeze_operand(0) + without = [] + for _ in range(4): + without.append(list(t2.state)) + t2.state = tr.compress_t(t2.state, fixed, rounds) + + if with_counter == without: + return False, "K4 FAIL: the squeeze counter changes nothing" + first_diff = next(i for i, (x, y) in enumerate(zip(with_counter, without)) + if x != y) + return True, (f"K4 PASS: counter-free squeezes diverge from the spec at " + f"squeeze #{first_diff} (they iterate one fixed map)") + + +def k5_order_is_load_bearing(rounds: int) -> tuple[bool, str]: + a = tr.Transcript(rounds=rounds); a.absorb(MAIN_ROOT); a.absorb(L1_ROOT) + b = tr.Transcript(rounds=rounds); b.absorb(L1_ROOT); b.absorb(MAIN_ROOT) + if a.state == b.state: + return False, "K5 FAIL: absorb order does not affect the transcript" + return True, "K5 PASS: swapping two absorbs changes the state" + + +def main() -> int: + print("=" * 74) + print("TRANSCRIPT KATs — compress-chain (option B1)") + print("=" * 74) + ok = True + doc = {"construction": "LFM compress-chain transcript (option B1)", + "tag_ascii": tr.TAG_LFMT_ASCII.decode(), + "tag_word": tr.TAG_LFMT, + "squeeze_mark": tr.SQUEEZE_MARK, + "state_cells": 1, + "state_bits": 128, + "initial_state": tr.ZERO_CELL, + "framing": "identical to the Merkle socket except m[8] = TAG_LFMT", + "rounds": {}} + + for rounds in (7, 6): + good, msg, steps = k1_step_identity(rounds) + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] {msg}") + e2e, _ = k2_end_to_end(rounds) + doc["rounds"][str(rounds)] = {"step_vectors": steps, "fri_toy_v0": e2e} + print(f" [PASS] K2: FriToyV0-shaped transcript, {len(e2e['steps'])} " + f"ops, {e2e['compressions']} compressions @{rounds}r") + + for fn in (k3_domain_separation, k4_counter_is_load_bearing, + k5_order_is_load_bearing): + good, msg = fn(7) + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] {msg}") + + # K6 — the cost claim in the spec must match what the reference performed. + n = doc["rounds"]["7"]["fri_toy_v0"]["compressions"] + good = (n == 13) + ok &= good + print(f" [{'PASS' if good else 'FAIL'}] K6: FriToyV0 transcript costs " + f"{n} compressions (13: 11 + the 2 leaf rows)") + + if "--write" in sys.argv: + with open(OUT, "w") as f: + json.dump(doc, f, indent=1) + print(f"\n wrote {OUT}") + + print("-" * 74) + print(f"TRANSCRIPT KATs: {'PASS' if ok else 'FAIL'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/thoughts/shared/lfm-real-hash/transcript-spec/transcript_ref.py b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_ref.py new file mode 100644 index 000000000..66db16a21 --- /dev/null +++ b/thoughts/shared/lfm-real-hash/transcript-spec/transcript_ref.py @@ -0,0 +1,167 @@ +""" +THE COMPRESS-CHAIN TRANSCRIPT — reference implementation (option B1, ratified). + +This is the future `fixture::HostSponge` mirror and the thing the chip's +transcript rows must reproduce. It replaces `edsl::SpongeVar`'s permute-driven +overwrite-rate duplex with a chain over the FROZEN `LFM_HASH` compress socket, +so no permute socket is ever built and `MODE_P` stays pinned to 0. + +STATE: one cell (4 lanes x u32 = 128 bits), initially all-zero — mirroring +`SpongeVar::new`, which starts from three zero cells. + +OPERATIONS (each `compress_T` is one ordinary socket compress under the +TRANSCRIPT tag `"LFMT"`): + + absorb(c) state <- compress_T(state, c) 1 compress + absorb2(c0, c1) state <- compress_T(compress_T(state, c0), c1) 2 compresses + squeeze() out = state ; state <- compress_T(state, SQ(i)) + 1 compress + +`squeeze` outputs BEFORE advancing, mirroring `SpongeVar::squeeze_cell` +(`out = state[0]; state = permute(state)`) so the two constructions stay +structurally parallel and the diff is reviewable. + +SQ(i) — THE SQUEEZE COUNTER, AND WHY IT IS FREE. The advance operand is the +constant cell `[SQUEEZE_MARK, i, 0, 0]`, where `i` is the squeeze index. It costs +NOTHING: the eDSL fully unrolls (`edsl.rs:1-4` — "nothing loop-shaped reaches the +machine"), so `i` is a compile-time constant and the operand is a program +constant either way, pinned by `program_id`. + +What it buys is §8.2's FSE-2014 lesson, written into the construction: without +it, a run of consecutive squeezes iterates ONE fixed public non-injective map, +whose functional graph an attacker can precompute — the structure the GLUON-64 +T-sponge attacks exploit. With it, each step is a different map and no single +functional graph exists to analyse. See `squeeze_run_analysis.py` for the +quantitative side, which is negligible either way; this is about removing the +attack *structure*, not the bit-counting. + +ABSORB/SQUEEZE SEPARATION rests primarily on the operation sequence being a +compile-time constant of the program (so a prover cannot perform a squeeze where +the program says absorb), with `SQUEEZE_MARK` as defence in depth. +""" + +from __future__ import annotations + +import os +import sys + +_GATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "gate-oracle") +sys.path.insert(0, _GATE) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "leaf-spec")) + +import blake3_oracle as ora # noqa: E402 +import socket_ref as sk # noqa: E402 + +MASK32 = 0xFFFFFFFF +LANES = sk.DIGEST_LANES # 4 lanes per cell + +# --- Tag allocation --------------------------------------------------------- +# A tag is never reused for a second purpose. +TAG_LFMT_ASCII = b"LFMT" # transcript step (this construction) +TAG_LFMT = int.from_bytes(TAG_LFMT_ASCII, "little") + +# The squeeze-advance marker. Distinguishes an advance operand from an absorbed +# digest as defence in depth; the load-bearing separation is the fixed sequence. +SQUEEZE_MARK = int.from_bytes(b"SQZ0", "little") + +ZERO_CELL = [0, 0, 0, 0] + + +def squeeze_operand(i: int) -> list[int]: + """SQ(i) — a compile-time constant cell, hence free.""" + return [SQUEEZE_MARK, i & MASK32, 0, 0] + + +def compress_t(state: list[int], operand: list[int], + rounds: int = 7) -> list[int]: + """One transcript step: the FROZEN compress socket under the LFMT tag. + + Identical framing to the Merkle socket in every respect except `m[8]`: + h = IV, m[0..4] = state, m[4..8] = operand, m[8] = TAG_LFMT, m[9..16] = 0, + t = 0, block_len = 36, flags = 0x0B, digest = out[0..4]. + """ + fr = sk.Framing(rounds=rounds, tag_word=TAG_LFMT) + return sk.socket_digest_wordlevel(state, operand, fr) + + +class Transcript: + """The reference. Mirrors the eventual `HostSponge` bit for bit.""" + + def __init__(self, rounds: int = 7): + self.state = list(ZERO_CELL) + self.rounds = rounds + self.squeeze_index = 0 + self.compressions = 0 + self.trace: list[tuple[str, list[int]]] = [] + + def absorb(self, c: list[int]) -> "Transcript": + assert len(c) == LANES + self.state = compress_t(self.state, c, self.rounds) + self.compressions += 1 + self.trace.append(("absorb", list(self.state))) + return self + + def absorb2(self, c0: list[int], c1: list[int]) -> "Transcript": + self.absorb(c0) + self.absorb(c1) + self.trace[-2] = ("absorb2.0", self.trace[-2][1]) + self.trace[-1] = ("absorb2.1", self.trace[-1][1]) + return self + + def absorb_felts(self, felts: list[int]) -> "Transcript": + """Absorb a cell of ARBITRARY field elements: leaf-hash it, then absorb + the resulting digest. ✓ VERIFIED `edsl.rs`: `absorb_felts` is + `let d = b.leaf(c); self.absorb(d)`. + + TWO compresses — one `LFML` leaf row plus one `LFMT` chain step — because + a felt cell cannot enter the socket directly (obligation O1). This is the + step the spec's original 91-row figure missed.""" + import leaf_ref as lr # noqa: PLC0415 (kept local: leaf-spec is optional) + d = lr.leaf_compress(felts, self.rounds) + self.compressions += 1 # the LFML leaf row + self.trace.append(("absorb_felts.leaf", list(d))) + self.absorb(d) # the LFMT chain step + self.trace[-1] = ("absorb_felts.absorb", self.trace[-1][1]) + return self + + def squeeze(self) -> list[int]: + """out = state (pre-advance), then advance with SQ(i).""" + out = list(self.state) + self.state = compress_t(self.state, squeeze_operand(self.squeeze_index), + self.rounds) + self.squeeze_index += 1 + self.compressions += 1 + self.trace.append(("squeeze", list(out))) + return out + + # -- the shapes the eDSL exposes ------------------------------------ + def squeeze_ext(self) -> list[int]: + """`SpongeVar::squeeze_ext`: lanes 0-2 of a squeezed cell.""" + return self.squeeze()[0:3] + + def squeeze_bits(self, nbits: int) -> list[int]: + """`SpongeVar::squeeze_bits`: the low `nbits` of lane 0, LSB first.""" + lane0 = self.squeeze()[0] + return [(lane0 >> k) & 1 for k in range(nbits)] + + +# --------------------------------------------------------------------------- +# The byte-level (library-shaped) form — the external anchor. +# --------------------------------------------------------------------------- + +def compress_t_bytelevel(state: list[int], operand: list[int], + rounds: int = 7) -> list[int]: + """`BLAKE3(LE32(state) ‖ LE32(operand) ‖ "LFMT")[0..16]`, as four u32 lanes. + + At rounds = 7 this is a plain `blake3::hash` call — the property the + 7-round decision was bought for, inherited unchanged because the tag lives + in the message and nothing else about the framing moved. + """ + msg = (b"".join(int(x).to_bytes(4, "little") for x in state) + + b"".join(int(x).to_bytes(4, "little") for x in operand) + + TAG_LFMT_ASCII) + assert len(msg) == 36 + full = ora.hash_bytes(msg, 32, rounds=rounds) + return [int.from_bytes(full[4 * i:4 * i + 4], "little") for i in range(4)]