diff --git a/Cargo.lock b/Cargo.lock index e1cf6044..37f8ac75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5609,6 +5609,7 @@ dependencies = [ "object_store", "serde", "serde_json", + "serde_yaml", "snafu 0.8.9", "tempfile", "tokio", @@ -6637,7 +6638,6 @@ dependencies = [ name = "ndarray" version = "0.17.2" dependencies = [ - "blake3", "fractal", "matrixmultiply", "num-complex", diff --git a/crates/lance-graph/Cargo.toml b/crates/lance-graph/Cargo.toml index 62a3e122..97ad1d70 100644 --- a/crates/lance-graph/Cargo.toml +++ b/crates/lance-graph/Cargo.toml @@ -42,6 +42,12 @@ lancedb = { version = "=0.33.0", optional = true, default-features = false } nom = "7.1" serde = { version = "1", features = ["derive"] } serde_json = "1" +# Boot config ONLY (`soa_config`): the `.config//config.yaml` a deployment +# reads once at startup. Not a hot-path serialization surface — same category as +# lance-graph-contract's build.rs manifest parse, which already pins this +# version. Kept at 0.9 to match that sibling rather than introducing a second +# YAML implementation into one workspace. +serde_yaml = "0.9" bytes = "1" snafu = "0.8" deltalake = { version = "0.32", features = ["datafusion", "s3", "azure", "gcs"], optional = true } diff --git a/crates/lance-graph/examples/soa-config.example.yaml b/crates/lance-graph/examples/soa-config.example.yaml new file mode 100644 index 00000000..96c6cbda --- /dev/null +++ b/crates/lance-graph/examples/soa-config.example.yaml @@ -0,0 +1,186 @@ +# ============================================================================ +# soa-config.example.yaml — example boot config for a lance-graph deployment +# ============================================================================ +# +# WHERE THIS LIVES: a real config object of this shape sits in the object +# store at `.config//config.yaml` — one object per repository +# (see the `ledger_prefix` comment below for why it's per-repo). This example +# file is NOT read by any binary; it exists so the shape is documented next +# to the parser (`crates/lance-graph/src/soa_config.rs`) and so an operator +# can copy it as a starting point. +# +# CREDENTIALS DO NOT GO HERE. Every value in this file is non-secret +# configuration (table names, a prefix, a classid). Access credentials come +# from `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_ENDPOINT_URL` / +# `AWS_DEFAULT_REGION` environment variables — see `dev_s3_env.rs` and +# `docs/S3_LAYOUT.md`. A config object that ever contains a key, a token, or +# a bare bucket-and-endpoint pair meant to substitute for those env vars is +# wrong, full stop. +# +# THIS IS BOOT CONFIG, READ ONCE AT STARTUP — not a hot-path structure. A +# deployment fetches this object at boot, parses it once +# (`soa_config::parse`), and decides which bakes to hydrate to local disk +# versus read remotely on demand. It is the same category of thing as a +# `Cargo.toml` read, not a per-request or per-row concern — see the module +# doc on `soa_config.rs` for why that distinction matters here (the +# workspace's Firewall doctrine bans serialization *between mailboxes during +# cognition*, which this is not). +# +# The bucket name, endpoint, and every other real deployment value below are +# OBVIOUS PLACEHOLDERS ("your-bucket-name", "your-endpoint.example.com") — +# never real infrastructure. This is a public repository. +# ============================================================================ + +# Schema major version this config object declares. The parser refuses +# (loudly, at boot) any version it doesn't understand rather than +# half-reading fields whose meaning may have changed. Bump this only when +# you are also bumping `CONFIG_SCHEMA_VERSION` in `soa_config.rs`. +version: 1 + +# ---------------------------------------------------------------------------- +# ledger_prefix — the per-repository namespace for this deployment's bakes. +# ---------------------------------------------------------------------------- +# Every repository's bakes live under their OWN prefix, e.g. +# "lance-graph/ledger" here, "q2/ledger" for a q2 deployment, and so on. +# This is a convention, not a technical requirement Lance enforces — but it +# is the convention every deployment in this workspace follows, and it buys +# two things: +# 1. Tidy ledgers: `aws s3 ls s3:///lance-graph/ledger/` shows only +# THIS repo's tables, not a mixed bag from every deployment sharing the +# bucket. +# 2. Isolation on refresh: one repo's `on_existing: new_version` refresh +# writes a new table under ITS OWN prefix and repoints ITS OWN config — +# it can never collide with, overwrite, or even be visible to another +# repo's tables, because the prefixes never overlap. +# Combined with a bucket name (supplied separately, e.g. via +# AWS_S3_BUCKET_NAME) and one bake's `table` field, this becomes the full +# table URI — see `SoaConfig::table_uri` in `soa_config.rs`. +ledger_prefix: "your-repo-name/ledger" + +# ---------------------------------------------------------------------------- +# bakes — the declared SoA bakes this deployment knows about. +# ---------------------------------------------------------------------------- +# Each entry is one Lance table written by `soa_to_lance` (see +# `examples/soa_to_lance.rs` and `docs/SOA_BAKE_DEPLOYMENT.md`). Below are +# three example bakes deliberately mixing `hydrate: true` and +# `hydrate: false` so the distinction is visible rather than merely implied +# by the field's default. +bakes: + # --- geo.berlin: hydrated — pulled to local disk once at boot. ----------- + # Use hydrate: true for a bake this deployment reads often enough that + # paying a one-time local copy (see the hydration-cost measurements in + # `docs/SOA_BAKE_DEPLOYMENT.md` §4b) is cheaper than repeated remote reads. + - name: geo-berlin # Unique human key within this file. This is + # ONLY a lookup name (SoaConfig::find) — not a + # filesystem path, and unrelated to `table`. + # Two bakes may not share a `name`. + table: geo.berlin.lance # The Lance table name under `ledger_prefix`. + # The full URI a reader resolves is + # s3:////. Two + # bakes may not share a `table` either — the + # parser rejects both kinds of collision, since + # two entries pointing at one table means one + # of them is either reading garbage or racing + # the other's writes. + classid: "0x0F01" # Hex classid (0x-prefixed) identifying this + # bake's SoA node layout — see the "CANON — + # Minimal SoA node" section of the repo's + # CLAUDE.md. Get this wrong and a reader casts + # bytes through the wrong `ClassView`. + # + # NEVER INVENT A CLASSID. The hi byte is a + # minted concept DOMAIN and the full value is a + # minted CONCEPT — both live in OGAR's + # `ogar-vocab` (`class_ids`), and a plausible- + # looking value you made up will collide with a + # real concept in a different domain. The three + # entries below use real geo concepts: + # 0x0F01 osm_node · 0x0F02 osm_way + # 0x0F03 osm_relation + # (An earlier draft of this file used 0x0D01 + # for an "ontology" bake. 0x0D01 is real — it + # is `hr_employee` in the HR domain. That is + # exactly the failure this note prevents.) + slab_digest: "sha256:REPLACE_WITH_REAL_DIGEST" + # Optional. Pins this bake to one specific + # slab's digest — the pairing that ties a + # table to its `.books` sidecar. Omit this key + # entirely (do not write it as an empty + # string) to mean "trust whatever is at + # `table` right now, unpinned." + hydrate: true # Pull to local disk at boot. What breaks if + # this is wrong: set true on a bake this + # deployment barely touches, and you pay the + # ~2.6s+ fixed hydration cost at every boot + # for nothing; set false on a bake this + # deployment reads constantly, and every read + # after the first pays a remote round trip + # instead of a local file read. + + # --- geo.munich: NOT hydrated — read remotely, on demand. ---------------- + - name: geo-munich + table: geo.munich.lance + classid: "0x0F02" + # slab_digest omitted here on purpose — this bake floats to whatever the + # table currently holds rather than pinning a specific bake. + hydrate: false # Served straight from the object store on + # every access (deployment pattern (a) in + # docs/SOA_BAKE_DEPLOYMENT.md §4). Right choice + # for a bake that is large, rarely read, or + # read by only a small fraction of requests — + # no local disk is spent on it at all. + + # --- geo.relations: NOT hydrated — a reference bake read cold. ----------- + - name: geo-relations + table: geo.relations.lance + classid: "0x0F03" + slab_digest: "sha256:REPLACE_WITH_REAL_DIGEST" + hydrate: false # A reference bake consulted occasionally, not + # on the hot request path — remote reads are + # fine here, and skipping hydration means one + # fewer thing to keep fresh locally when the + # bake is refreshed. + +# ---------------------------------------------------------------------------- +# on_existing — what a refresh does when it finds an already-occupied table. +# ---------------------------------------------------------------------------- +# This is the single most consequential field in this file, so read it in +# full before changing it from the default. +# +# refuse (the default, used if this key is omitted entirely): +# An existing table is NEVER silently overwritten. If a refresh would +# land on a `table` name that already has a dataset at it, that refresh +# is a hard error. Safe-by-default: nothing you didn't explicitly ask +# for gets clobbered. +# +# new_version: +# A refresh instead writes a brand-NEW table, timestamped +# (e.g. `geo.berlin.lance` becomes `geo.berlin..lance` — see +# `versioned_table_name` in `soa_config.rs`), and THIS config file's +# `table` pointer for that bake is updated to name the new table. The +# OLD table is left completely untouched on disk until a separate, +# explicit purge deletes it. +# +# WHY NOT "rename the old table to a .OLD. suffix instead"? +# Because S3 (and any S3-compatible object store) has NO ATOMIC RENAME. +# A Lance dataset is not one object — it is N objects under a shared +# prefix (data files, manifests, transaction files, deletion vectors). +# "Renaming" such a dataset means copying all N objects to new keys and +# then deleting all N originals — two non-atomic bulk operations, not one +# atomic one. A crash or a network failure partway through that sequence +# leaves the dataset split across two prefixes, with objects at BOTH the +# old and new names, and no single pointer that correctly resolves to +# either half. A reader arriving mid-crash could easily open a fragment of +# the wrong dataset with no way to tell. +# +# Flipping one field in THIS file — the `table` value for one bake — +# is, by contrast, a single small-object write. This config file IS the +# pointer: as long as the new table is written COMPLETELY before this +# file is updated (never the other way around), a reader that fetches +# this config always sees a consistent, fully-written table — either the +# old one (this file hasn't been updated yet) or the new one (it has), +# and never a half-migrated mix of the two. Purging the old table is then +# a deliberate, separate, out-of-band operation with no correctness +# deadline — it can happen a minute later or a month later, because +# nothing is still pointing at it once this file's `table` field moved on. +on_existing: refuse diff --git a/crates/lance-graph/src/lib.rs b/crates/lance-graph/src/lib.rs index 21c7ac48..1c8633b9 100644 --- a/crates/lance-graph/src/lib.rs +++ b/crates/lance-graph/src/lib.rs @@ -56,6 +56,7 @@ pub mod query; #[cfg(feature = "planner")] pub mod reasoning; pub mod semantic; +pub mod soa_config; pub mod spark_dialect; pub mod sql_catalog; pub mod sql_query; diff --git a/crates/lance-graph/src/soa_config.rs b/crates/lance-graph/src/soa_config.rs new file mode 100644 index 00000000..a7bb3add --- /dev/null +++ b/crates/lance-graph/src/soa_config.rs @@ -0,0 +1,555 @@ +//! Boot-time configuration for which SoA bakes a deployment knows about. +//! +//! A deployment reads exactly one YAML object from its object store at +//! `.config//config.yaml` (see [`config_key`]) and that object +//! declares which bakes exist and which get hydrated to local disk. The same +//! binary then behaves identically in a dev container and on Railway, +//! because both read the same object at boot — there is no code path that +//! branches on "am I local or am I Railway", only a path that reads whatever +//! config object is sitting at that key. +//! +//! # Why config lives in the bucket, not in env vars or the binary +//! +//! Credentials belong in env vars (they are secrets, and secrets should +//! never live in an object a session's own tooling can list and read). +//! A manifest of *which bakes exist* is a different kind of thing entirely: +//! it changes far more often than code, and it must be readable by both +//! environments (dev container, Railway) without a redeploy. Baking it into +//! the binary means every new bake needs a rebuild+redeploy just to be +//! discoverable; putting it in an env var means it's invisible to `aws s3 +//! ls` and has no place to grow structure. One YAML object in the bucket is +//! the one source of truth both environments already have credentialed +//! access to. +//! +//! # This is boot config, not a hot-path serialization violation +//! +//! This module is read **once, at startup**, exactly the same category as a +//! `build.rs` manifest parse or a `Cargo.toml` read. It is not on the +//! per-request or per-row hot path this workspace's Firewall doctrine +//! (ADR-022/023, "no serialization in the hot path; the IR is wire-truth") +//! protects. A future session should not "fix" this module by stripping +//! serde out of it — the ban is on serialization *between mailboxes during +//! cognition*, not on parsing a startup manifest. +//! +//! # Key names vs. values +//! +//! The struct field names, the YAML keys they map to, and the shapes below +//! are universal and belong in shared/public code — any deployment's config +//! object uses the same keys. The *values* that go in those keys (a bucket +//! name, a ledger prefix, a slab digest) are deployment configuration and +//! must never enter the repository, exactly like the `AWS_*` variables +//! [`crate::dev_s3_env`] reads: the shape is public, the content is not. + +use std::fmt; + +/// The schema version this build of `lance-graph` understands. A config +/// object declaring a different major version is refused outright (see +/// [`parse`]) rather than half-read — a future schema change is a loud +/// failure at boot, not a silent misinterpretation of fields that changed +/// meaning. +pub const CONFIG_SCHEMA_VERSION: u32 = 1; + +/// The file name every deployment's config object uses. +pub const CONFIG_BASENAME: &str = "config.yaml"; + +/// The top-level prefix under which every repo's config object lives. +pub const CONFIG_ROOT: &str = ".config"; + +/// What to do when a bake's `hydrate`/refresh path finds a table that +/// already exists at the target name. +/// +/// # The overwrite doctrine +/// +/// - [`OnExisting::Refuse`] (the default): an existing table is never +/// silently overwritten. A refresh that lands on an occupied name is a +/// hard error, not a clobber. +/// - [`OnExisting::NewVersion`]: a refresh instead writes a **new**, +/// timestamped table (see [`versioned_table_name`]) and the config's +/// pointer (the `table` field naming which one is current) is updated to +/// point at it. The old table is left untouched on disk until an explicit, +/// separate purge. +/// +/// **Why not rename-to-OLD instead of writing-new-then-repointing:** S3 has +/// no atomic rename. A Lance dataset is N objects under a prefix, and +/// "renaming" it is N copies plus N deletes — non-atomic by construction. A +/// crash partway through leaves the dataset split across two prefixes with +/// no single pointer that resolves to either half correctly. Flipping one +/// line of YAML (the `table` field) to name the new, already-fully-written +/// table is O(1) and atomic from a reader's point of view: the config *is* +/// the pointer, so a reader never observes a half-migrated state — it either +/// still sees the old table (config not yet updated) or the new one +/// (config updated), never a mix. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OnExisting { + #[default] + Refuse, + NewVersion, +} + +/// One declared SoA bake: a name a caller looks it up by, the Lance table it +/// lives in, the classid that identifies its node layout, and whether this +/// deployment should pull it to local disk at boot. +#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct BakeEntry { + /// Human-facing key, unique within one config object. Not a filesystem + /// path — just a lookup name (see [`SoaConfig::find`]). + pub name: String, + /// The Lance table name under `ledger_prefix`. + pub table: String, + /// Hex classid, e.g. `"0x0F01"`, identifying this bake's node layout. + pub classid: String, + /// Digest of the bake's slab, when the deployment pins one. Absent + /// means "trust whatever is at `table` right now". + #[serde(default)] + pub slab_digest: Option, + /// Whether this deployment should pull the bake to local disk at boot + /// (`true`) or read it remotely on demand (`false`). + #[serde(default)] + pub hydrate: bool, +} + +/// The parsed, validated contents of one deployment's `config.yaml`. +#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct SoaConfig { + /// Schema major version. [`parse`] refuses anything other than + /// [`CONFIG_SCHEMA_VERSION`]. + pub version: u32, + /// Per-repo ledger prefix, e.g. `"lance-graph/ledger"`. Combined with a + /// bucket and a [`BakeEntry::table`] by [`SoaConfig::table_uri`]. + pub ledger_prefix: String, + /// The declared bakes. Names and tables must each be unique — see + /// [`parse`]. + pub bakes: Vec, + /// What a refresh does when it finds an occupied table name. See + /// [`OnExisting`] for the reasoning. + #[serde(default)] + pub on_existing: OnExisting, +} + +/// Everything that can go wrong turning a YAML string into a validated +/// [`SoaConfig`]. Each variant names the offending value so a bad config +/// object fails loudly with something a human can grep the YAML for, +/// instead of a generic "invalid config" at boot. +#[derive(Debug, Clone, PartialEq)] +pub enum ConfigError { + /// The YAML itself did not parse. Carries `serde_yaml`'s message. + Yaml(String), + /// `version` did not match [`CONFIG_SCHEMA_VERSION`]. + UnsupportedVersion { found: u32, supported: u32 }, + /// Two bakes declared the same `name`. + DuplicateName(String), + /// Two bakes declared the same `table`. + DuplicateTable(String), + /// A required string field was empty. Carries the field's name. + EmptyField(&'static str), + /// `classid` did not parse as a `0x`-prefixed hex value. + BadClassid(String), +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ConfigError::Yaml(msg) => write!(f, "config.yaml did not parse: {msg}"), + ConfigError::UnsupportedVersion { found, supported } => write!( + f, + "config.yaml declares version {found}, this build only supports version {supported}" + ), + ConfigError::DuplicateName(name) => { + write!(f, "duplicate bake name in config.yaml: {name:?}") + } + ConfigError::DuplicateTable(table) => { + write!(f, "duplicate bake table in config.yaml: {table:?}") + } + ConfigError::EmptyField(field) => { + write!(f, "config.yaml field must not be empty: {field}") + } + ConfigError::BadClassid(classid) => { + write!(f, "classid {classid:?} is not a 0x-prefixed hex value") + } + } + } +} + +impl std::error::Error for ConfigError {} + +/// The object-store key a deployment's config object lives at: +/// `.config//config.yaml`. Pure string composition, no I/O — the +/// caller is responsible for actually fetching the object at this key. +pub fn config_key(repo: &str) -> String { + format!("{CONFIG_ROOT}/{repo}/{CONFIG_BASENAME}") +} + +/// Parse and validate a `config.yaml` body into a [`SoaConfig`]. +/// +/// Validation rejects, each with its own [`ConfigError`] variant: +/// - a `version` other than [`CONFIG_SCHEMA_VERSION`] (fail loud on a +/// schema a build doesn't understand, rather than half-reading it), +/// - two bakes sharing a `name`, +/// - two bakes sharing a `table` (two entries pointing at one table on +/// disk is always a mistake — one of them would be reading garbage or +/// racing the other's writes), +/// - an empty `ledger_prefix`, `name`, or `table`, +/// - a `classid` that is not `0x`-prefixed hex. +pub fn parse(yaml: &str) -> Result { + let config: SoaConfig = + serde_yaml::from_str(yaml).map_err(|e| ConfigError::Yaml(e.to_string()))?; + + if config.version != CONFIG_SCHEMA_VERSION { + return Err(ConfigError::UnsupportedVersion { + found: config.version, + supported: CONFIG_SCHEMA_VERSION, + }); + } + + if config.ledger_prefix.is_empty() { + return Err(ConfigError::EmptyField("ledger_prefix")); + } + + let mut seen_names: std::collections::HashSet<&str> = std::collections::HashSet::new(); + let mut seen_tables: std::collections::HashSet<&str> = std::collections::HashSet::new(); + + for bake in &config.bakes { + if bake.name.is_empty() { + return Err(ConfigError::EmptyField("name")); + } + if bake.table.is_empty() { + return Err(ConfigError::EmptyField("table")); + } + if !seen_names.insert(bake.name.as_str()) { + return Err(ConfigError::DuplicateName(bake.name.clone())); + } + if !seen_tables.insert(bake.table.as_str()) { + return Err(ConfigError::DuplicateTable(bake.table.clone())); + } + + let hex = bake + .classid + .strip_prefix("0x") + .or_else(|| bake.classid.strip_prefix("0X")); + match hex { + Some(digits) if !digits.is_empty() && digits.chars().all(|c| c.is_ascii_hexdigit()) => { + } + _ => return Err(ConfigError::BadClassid(bake.classid.clone())), + } + } + + Ok(config) +} + +impl SoaConfig { + /// The full `s3://` URI a bake's table lives at: bucket, this config's + /// `ledger_prefix`, and the entry's own `table`, joined with `/`. + pub fn table_uri(&self, bucket: &str, entry: &BakeEntry) -> String { + format!("s3://{bucket}/{}/{}", self.ledger_prefix, entry.table) + } + + /// The bakes this deployment should pull to local disk at boot. + pub fn hydrate_set(&self) -> impl Iterator { + self.bakes.iter().filter(|b| b.hydrate) + } + + /// Look up a bake by its declared `name`. + pub fn find(&self, name: &str) -> Option<&BakeEntry> { + self.bakes.iter().find(|b| b.name == name) + } +} + +/// Compute the versioned table name an [`OnExisting::NewVersion`] refresh +/// writes to. `berlin.lance` with nanos `N` becomes `berlin..lance`; a +/// table name with no `.lance` suffix gets `.` appended. +/// +/// Takes the timestamp as a **parameter**, deliberately never calling the +/// clock itself — that keeps this function pure and trivially testable, and +/// keeps "what time is it" a decision made once by the caller rather than +/// smeared across every call site that might need a versioned name. +pub fn versioned_table_name(table: &str, unix_nanos: u128) -> String { + match table.strip_suffix(".lance") { + Some(stem) => format!("{stem}.{unix_nanos}.lance"), + None => format!("{table}.{unix_nanos}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn valid_yaml() -> &'static str { + r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +on_existing: new_version +bakes: + - name: berlin + table: berlin.lance + classid: "0x0F01" + slab_digest: "sha256:abc123" + hydrate: true + - name: munich + table: munich.lance + classid: "0x0F02" + hydrate: false +"# + } + + #[test] + fn valid_config_round_trips_with_expected_field_values() { + let config = parse(valid_yaml()).expect("valid config must parse"); + assert_eq!(config.version, 1); + assert_eq!(config.ledger_prefix, "lance-graph/ledger"); + assert_eq!(config.on_existing, OnExisting::NewVersion); + assert_eq!(config.bakes.len(), 2); + + let berlin = config.find("berlin").expect("berlin must be found"); + assert_eq!(berlin.table, "berlin.lance"); + assert_eq!(berlin.classid, "0x0F01"); + assert_eq!(berlin.slab_digest.as_deref(), Some("sha256:abc123")); + assert!(berlin.hydrate); + + let munich = config.find("munich").expect("munich must be found"); + assert!(!munich.hydrate); + assert!(munich.slab_digest.is_none()); + + assert!(config.find("nonexistent").is_none()); + } + + #[test] + fn on_existing_defaults_to_refuse_when_omitted() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: berlin + table: berlin.lance + classid: "0x0F01" +"#; + let config = parse(yaml).expect("must parse"); + assert_eq!(config.on_existing, OnExisting::Refuse); + } + + #[test] + fn rejects_unsupported_version() { + let yaml = r#" +version: 2 +ledger_prefix: "lance-graph/ledger" +bakes: [] +"#; + let err = parse(yaml).expect_err("wrong version must be rejected"); + assert_eq!( + err, + ConfigError::UnsupportedVersion { + found: 2, + supported: CONFIG_SCHEMA_VERSION + } + ); + + // Paired case: same config with the version corrected passes, + // proving the rejection above was specifically about the version + // field and not some other defect in the fixture. + let fixed = yaml.replace("version: 2", "version: 1"); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_duplicate_bake_name() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: berlin + table: berlin.lance + classid: "0x0F01" + - name: berlin + table: berlin2.lance + classid: "0x0F02" +"#; + let err = parse(yaml).expect_err("duplicate name must be rejected"); + assert_eq!(err, ConfigError::DuplicateName("berlin".to_string())); + + let fixed = yaml.replacen( + "name: berlin\n table: berlin2.lance", + "name: berlin2\n table: berlin2.lance", + 1, + ); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_duplicate_bake_table() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: berlin + table: shared.lance + classid: "0x0F01" + - name: munich + table: shared.lance + classid: "0x0F02" +"#; + let err = parse(yaml).expect_err("duplicate table must be rejected"); + assert_eq!(err, ConfigError::DuplicateTable("shared.lance".to_string())); + + let fixed = yaml.replacen( + "name: munich\n table: shared.lance", + "name: munich\n table: munich.lance", + 1, + ); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_empty_ledger_prefix() { + let yaml = r#" +version: 1 +ledger_prefix: "" +bakes: [] +"#; + let err = parse(yaml).expect_err("empty ledger_prefix must be rejected"); + assert_eq!(err, ConfigError::EmptyField("ledger_prefix")); + + let fixed = yaml.replace(r#"ledger_prefix: """#, r#"ledger_prefix: "x""#); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_empty_bake_name() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: "" + table: berlin.lance + classid: "0x0F01" +"#; + let err = parse(yaml).expect_err("empty name must be rejected"); + assert_eq!(err, ConfigError::EmptyField("name")); + + let fixed = yaml.replace(r#"name: """#, r#"name: "berlin""#); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_empty_bake_table() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: berlin + table: "" + classid: "0x0F01" +"#; + let err = parse(yaml).expect_err("empty table must be rejected"); + assert_eq!(err, ConfigError::EmptyField("table")); + + let fixed = yaml.replace(r#"table: """#, r#"table: "berlin.lance""#); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_bad_classid() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: berlin + table: berlin.lance + classid: "F01" +"#; + let err = parse(yaml).expect_err("classid without 0x prefix must be rejected"); + assert_eq!(err, ConfigError::BadClassid("F01".to_string())); + + let fixed = yaml.replace(r#"classid: "F01""#, r#"classid: "0xF01""#); + assert!(parse(&fixed).is_ok()); + } + + #[test] + fn rejects_classid_with_non_hex_digits() { + let yaml = r#" +version: 1 +ledger_prefix: "lance-graph/ledger" +bakes: + - name: berlin + table: berlin.lance + classid: "0xZZ" +"#; + let err = parse(yaml).expect_err("non-hex digits must be rejected"); + assert_eq!(err, ConfigError::BadClassid("0xZZ".to_string())); + } + + #[test] + fn config_key_produces_the_exact_expected_string() { + assert_eq!(config_key("lance-graph"), ".config/lance-graph/config.yaml"); + assert_eq!(config_key("q2"), ".config/q2/config.yaml"); + } + + #[test] + fn versioned_table_name_on_dot_lance_suffix() { + assert_eq!( + versioned_table_name("berlin.lance", 1_700_000_000_000_000_000), + "berlin.1700000000000000000.lance" + ); + } + + #[test] + fn versioned_table_name_on_bare_name() { + assert_eq!(versioned_table_name("berlin", 42), "berlin.42"); + } + + #[test] + fn hydrate_set_returns_only_hydrate_true_entries() { + let config = parse(valid_yaml()).expect("must parse"); + let hydrated: Vec<&str> = config.hydrate_set().map(|b| b.name.as_str()).collect(); + // Fixture carries one true (berlin) and one false (munich) entry — + // the count must be neither 0 (filter over-rejects) nor 2 (filter + // is a no-op that returns everything). + assert_eq!(hydrated.len(), 1); + assert_eq!(hydrated, vec!["berlin"]); + } + + #[test] + fn table_uri_composes_correctly() { + let config = parse(valid_yaml()).expect("must parse"); + let berlin = config.find("berlin").unwrap(); + assert_eq!( + config.table_uri("my-bucket", berlin), + "s3://my-bucket/lance-graph/ledger/berlin.lance" + ); + } + + /// **The shipped example must load through THIS parser.** + /// + /// `examples/soa-config.example.yaml` is what the deployment docs point a + /// human at, and it was authored by reading these structs rather than by + /// running them — so without this test the example and the parser agree + /// only by hand, and the first field rename desynchronises them silently. + /// Documentation that no longer parses is worse than none: it is + /// confidently wrong. + /// + /// `include_str!` resolves at COMPILE time relative to this source file, + /// so a moved or deleted example is a build error rather than a test that + /// quietly stops covering anything. + #[test] + fn the_shipped_example_config_parses_through_this_parser() { + let example = include_str!("../examples/soa-config.example.yaml"); + let config = parse(example) + .unwrap_or_else(|e| panic!("examples/soa-config.example.yaml does not parse: {e}")); + + // Not merely "it parsed" — the example exists to DEMONSTRATE, so + // assert it still demonstrates. A single-entry or all-same-hydrate + // example would parse fine while teaching nothing about the + // distinction it is there to show. + assert!( + config.bakes.len() >= 3, + "the example should show several bakes; found {}", + config.bakes.len() + ); + let hydrated = config.hydrate_set().count(); + assert!( + hydrated > 0 && hydrated < config.bakes.len(), + "the example must carry BOTH hydrate:true and hydrate:false entries \ + or it does not illustrate the choice; {hydrated} of {} are hydrated", + config.bakes.len() + ); + } +} diff --git a/crates/lance-graph/tests/soa_verbatim.rs b/crates/lance-graph/tests/soa_verbatim.rs index ee57a212..fdb93969 100644 --- a/crates/lance-graph/tests/soa_verbatim.rs +++ b/crates/lance-graph/tests/soa_verbatim.rs @@ -4,8 +4,12 @@ //! This is the load-bearing claim for the disk-sink-in deployment pattern: //! **if the row column lands verbatim, then `mmap(file)[off .. off + rows*512]` //! IS the slab** — the process pages in only what it touches, RSS follows the -//! access pattern rather than the dataset size, and (because mmap is -//! page-aligned and `4096 % 64 == 0`) the `&[NodeRow]` cast is always valid. +//! access pattern rather than the dataset size, and because the run is pinned +//! at NATURAL alignment (`off % 512 == 0`, asserted below) the `&[NodeRow]` +//! cast is always valid AND no row ever straddles a 128-bit SIMD lane, a +//! 128-byte cache line, or a 4 KiB page/NVMe-sector boundary — 8 whole rows +//! per page, zero padding. Alignment is a claim about the run's START only; +//! the 512 bytes of each row are one contiguous unit, never fragments. //! //! Measured once by hand on the Iceland bake (335,302,144 slab bytes → //! 335,302,663 file bytes, 100 % identical from offset 0). A single manual run @@ -226,14 +230,30 @@ async fn a_slab_is_written_verbatim_and_contiguously() { // to exist contiguously. A future Lance version could keep the run // contiguous but move it to an odd offset; assertion (2) would still pass // (it re-derives addresses from `off`) while `mmap(file)[off..]` cast to - // `&[NodeRow]` would be undefined behaviour. NODE_ROW_STRIDE (512) is - // itself a multiple of 64, so alignment to the row stride is the same - // condition as alignment to any smaller power-of-two the reader needs. + // `&[NodeRow]` would be undefined behaviour. + // + // Pinned at NATURAL alignment — `off % NODE_ROW_STRIDE == 0` — which is + // deliberately stronger than the three layered requirements it implies, + // because a 512-aligned 512-byte row arithmetically CANNOT straddle any + // boundary that divides 4096: + // - the cast: needs 64 (`align_of::()`, repr(C, align(64))) + // - a 128-bit SIMD lane: needs 16 — 512/16 = 32 whole lanes per row + // - a 128 B cache line (Apple M-series): needs 128 — 4 lines per row + // - a 4 KiB page/NVMe sector: a row at a 512-multiple offset either ends + // exactly on a 4096 boundary or lies wholly inside one; 8 rows per + // sector, zero padding, no half-anything, ever. + // Alignment here is a claim about the START address only — the 512 bytes + // themselves are one contiguous run (that is assertion (1)); nothing is + // ever written as fragments. Measured on the real Iceland bake the run + // sits at offset 0, so this passes with maximal slack; a red here means + // Lance moved the run off natural alignment — re-measure and re-decide, + // never weaken back to 64. assert_eq!( - off % 64, + off % NODE_ROW_STRIDE, 0, - "slab run starts at file offset {off}, which is not 64-byte aligned; \ - mmap(file)[off..] cannot be soundly cast to &[NodeRow]" + "slab run starts at file offset {off}, which is not aligned to the \ + {NODE_ROW_STRIDE}-byte row stride; rows would straddle page/sector \ + boundaries and mmap(file)[off..] loses its straddle-free guarantee" ); // (2) Every row is at its computed address. This is what an mmap reader diff --git a/docs/ALIGNMENT_PRIMER.md b/docs/ALIGNMENT_PRIMER.md new file mode 100644 index 00000000..2ea5643c --- /dev/null +++ b/docs/ALIGNMENT_PRIMER.md @@ -0,0 +1,156 @@ +# Alignment primer — bits, bytes, lanes, and why the 512-byte row never splits + +> **Audience:** technical product level. No Rust knowledge assumed; every claim +> is labelled **[MEASURED]** (we ran it), **[SOURCE]** (read from Lance 9.0.0 +> source, with file:line), or **[ARITHMETIC]** (checkable on paper). +> +> **Why this doc exists:** during the SoA→Lance work the same three questions +> came back repeatedly, and three separate AI research assistants each answered +> them partially wrong in a *different* way. This doc settles the questions and +> names the specific errors, so the next person consulting the same tools can +> tell signal from noise. Companions: `docs/SOA_BAKE_DEPLOYMENT.md` (the +> deployment story), `crates/lance-graph/tests/soa_verbatim.rs` (the pin that +> keeps all of this true). + +## 1. The three numbers that keep getting confused + +Three different "64"s and "512"s circulate in this domain. They are different +quantities at different layers: + +| Number | Equals | What it actually is | Layer | +|---|---|---|---| +| **64 bit** | 8 bytes | Lance's *metadata* word size — offsets in footers and manifests | file-format envelope | +| **64 byte** | 512 bits | One x86 cache line; one AVX-512 register; `align_of::()` | CPU / Rust type | +| **512 byte** | 4096 bits | One whole canonical node row (`key 16 \| edges 16 \| value 480`) | our data model | + +When a document says "Lance uses 64-bit alignment", that is 8 bytes and about +the serialization envelope — it says nothing about where data pages land. +When our code says the row type requires 64, that is 64 **bytes**. And what our +test pins is 512 **bytes** — natural alignment, the strongest of the three. + +## 2. Alignment is about the START address, never about fragmentation + +The most common misreading: "aligned to 64 means written in 64-sized pieces." +**No.** Alignment constrains only where a block *begins*. The 512 bytes of a +row are one contiguous, indivisible run — **[MEASURED]**: the physical-layout +test finds the entire slab as a single unbroken byte run inside the Lance data +file and checks sampled rows at their computed addresses +(`tests/soa_verbatim.rs::a_slab_is_written_verbatim_and_contiguously`, run +against a real 335 MB bake: 335,302,144 slab bytes → 335,302,663 file bytes, +identical from offset 0). + +Nothing ever writes half a row, half a lane, or "64 versions of anything." + +## 3. The straddle arithmetic — why natural alignment ends the discussion + +A 4096-bit row is consumed as 32 × 128-bit SIMD lanes. The worry: could a lane, +a cache line, or a disk sector boundary fall *inside* a row? + +**[ARITHMETIC]** If a 512-byte row starts at a multiple of 512, it cannot +straddle any boundary whose size divides into or is divided by 512: + +- **128-bit (16 B) lane:** 512 / 16 = 32 whole lanes per row. A row starting at + a 512-multiple starts at a 16-multiple; every lane is whole. +- **64 B cache line (x86):** 8 whole lines per row. +- **128 B cache line (Apple M-series):** 4 whole lines per row. +- **4 KiB page / NVMe sector:** a 512-aligned 512-byte row either ends exactly + on a 4096 boundary or lies wholly inside one — 8 rows per page, zero padding. + +And because the stride *is* 512, alignment of row 0 propagates to every row +forever: row *i* sits at `start + i·512`, which is a 512-multiple whenever +`start` is. One aligned start ⇒ an eternally aligned array. + +This is why the test asserts `off % 512 == 0` (natural alignment) rather than +the weaker `off % 64 == 0` the Rust cast minimally needs: natural alignment is +the one condition that excludes **every** straddle at **every** layer at once, +and the measurement (offset 0) already supports it. + +## 4. What Lance actually does with our column — measured, not assumed + +**[SOURCE]** Lance 9 chooses an encoding per column. Values wider than 256 +bytes are "not narrow" (`lance-encoding-9.0.0` +`encodings/logical/primitive.rs:3861`, `MINIBLOCK_MAX_BYTE_LENGTH_PER_VALUE = +256`) and take the **full-zip** path, whose fixed-width branch writes values +verbatim unconditionally (`compression.rs:753`). Our row is 512 > 256, so the +column lands as raw bytes. The `compression = "none"` metadata we also set is +a *backstop*, not the cause — removing it changes nothing **[MEASURED]**; only +the mini-block path (which a 512-byte value never reaches) would read it. + +**[MEASURED]** File layout of the written data object: the data run starts at +**offset 0**; the only other content is a 519-byte footer *after* the run. +There is no header in front of the data. + +## 5. "Versioning will shift your bytes" — why that cannot happen + +One AI dump warned that a Lance version update could "force a 64-byte offset +shift at the file header," splitting rows across pages. This is structurally +impossible, and we have the measurements that show why: + +- **[MEASURED]** A Lance dataset is separate objects: the hydration probe + listed exactly three per dataset — a `.txn`, a `.manifest`, and the data + file. Manifests are **not** prepended to data files. +- **Data files are immutable.** A new version writes a *new* manifest and + *new* data files; existing files are never edited or shifted in place. That + immutability is the entire basis of Lance time-travel — and also of our + config-as-pointer refresh (`docs/S3_LAYOUT.md`): old bytes never move, a + refresh only changes which table the config names. +- Lance's 64-bit "row address" (fragment id + row offset) is a **logical row + index**, not a byte pointer. The reader computes `index × stride`; there is + no bit-stitching path that misalignment could trigger. + +## 6. Cost of guaranteeing alignment: effectively zero + +Even in the worst case, forcing a run to natural alignment costs at most 511 +padding bytes **per file** — not per row — because §3's propagation means only +the start needs fixing. On the 335 MB bake that is < 0.0002 %. There is no +disk-load tradeoff to weigh; the only question is whether the property holds, +and the test answers that on every run. + +## 7. FAQ — the actual questions, with the actual errors named + +**Q: "Is it 64-bit or 64-byte alignment?"** +Both exist, at different layers (§1). Our data-run guarantees are all in +bytes; the "64-bit" figure belongs to Lance's metadata envelope. + +**Q: "Does `align(64)` mean the 4096-bit row is stored as 64×64-bit pieces +with 64 versions?"** +No. Alignment ≠ granularity (§2). The row is one contiguous 512-byte unit, +proven byte-for-byte on real data. + +**Q: "Shouldn't it be at least 128 so lanes don't split?"** +The instinct is right, the fix is stronger: natural alignment (512) is pinned, +which makes lane/line/sector splits arithmetically impossible (§3), not merely +unlikely. + +**Q: "Will S3 chunking or multipart uploads fragment the rows?"** +No. S3 stores an object as an opaque byte sequence; HTTP chunking is +transport, invisible in the stored bytes. **[MEASURED]** the S3 arm of the +verbatim test reads the object back and finds the identical unbroken run — +and the byte-copy hydration path (`examples/hydration_probe.rs`) round-trips +every object byte-identically. + +**Q: "Do we need 4096-bit SIMD registers?"** +They don't exist in current hardware. The widest common register is AVX-512 = +64 bytes; a row is consumed as 8 such loads (or 32 NEON loads). The relevant +hardware alignment is therefore 64 bytes — which natural alignment satisfies +with room to spare. + +## 8. What keeps this true tomorrow + +None of the above is trusted as doctrine. Every load-bearing property is a +red-turning assertion in `tests/soa_verbatim.rs`, run against both a local +write and the real object store: + +| Property | Pinned by | +|---|---| +| Whole slab is one unbroken run | assertion (1), byte search | +| Every row at `off + i·512` | assertion (2), sampled addresses | +| Run at natural alignment | `off % 512 == 0` | +| No second/encoded copy | assertion (3), bounded footer | +| Encoder can't silently compress | `the_narrow_column_falsifier` (two-sided) | +| Same properties through S3 | `a_slab_is_written_verbatim_to_s3_too` | + +A future Lance version that changes any of this turns the suite red — at +which point the answer is *re-measure and re-decide*, never *loosen the +assertion*. If a claim in this document and a red test ever disagree, the test +is right and this document is stale. diff --git a/docs/S3_LAYOUT.md b/docs/S3_LAYOUT.md new file mode 100644 index 00000000..3050457b --- /dev/null +++ b/docs/S3_LAYOUT.md @@ -0,0 +1,124 @@ +# S3 bucket layout — a map, not a tutorial + +This is a short, operator-facing map of what lives where in an object store +shared by lance-graph deployments. It answers "what am I looking at" when +listing a bucket; it does not walk through how to deploy — for that, see +`docs/SOA_BAKE_DEPLOYMENT.md`. + +## 1. The prefix layout + +``` +s3:/// +├── .config/ +│ └── / +│ └── config.yaml ← boot config, one object per repository +│ (schema: crates/lance-graph/examples/soa-config.example.yaml, +│ parser: crates/lance-graph/src/soa_config.rs) +│ +├── / ← e.g. "lance-graph/ledger" — a repo's own +│ ├──
.lance/ namespace, declared by that repo's own +│ ├──
.lance/ config.yaml (see §2) +│ └── ... +│ +├── docs/ ← documentation uploaded alongside the +│ └── S3_LAYOUT.md data it describes (this file), so it is +│ discoverable in a bucket listing without +│ a separate wiki or repo checkout +│ +└── _tests/ ← scratch space for integration tests. + └── ... Safe to purge at any time. Tests clean + up after themselves, including on + failure — nothing under this prefix is + meant to persist. +``` + +| prefix | what it is | who writes it | +|---|---|---| +| `.config//config.yaml` | Boot config: which bakes exist, which get hydrated, the repo's `ledger_prefix`, the `on_existing` policy. | An operator, by hand or by deploy tooling. Read once at startup by the deployment it names. | +| `/
.lance/` | The actual Lance datasets — one directory-shaped object tree per table. | `soa_to_lance` (see `docs/SOA_BAKE_DEPLOYMENT.md` §2). | +| `docs/` | Documentation, uploaded to sit next to the data it describes. | Whoever ships a doc change; not written by any running deployment. | +| `_tests/` | Scratch space integration tests write to and clean up. | Test suites only. | + +## 2. Why `.config//` is per-repository + +Each repository owns exactly one config object, at `.config//config.yaml` +(`soa_config::config_key`). That object in turn names one `ledger_prefix`, +and every table that repo's deployment refreshes or creates lives under +*that* prefix and no other. Two consequences: + +- **Tidy ledgers.** Listing `s3:////` shows exactly + one repository's tables — never a mixed bag from every deployment that + happens to share the bucket. +- **No cross-repo interference on refresh.** A repo's `on_existing: new_version` + refresh (see §4) writes a new table under its OWN prefix and repoints its + OWN config. It has no way to collide with, shadow, or overwrite another + repository's tables, because the prefixes are disjoint by convention and + every repo's boot config only ever reads and writes its own. + +## 3. The environment contract + +Credentials and the endpoint are **never** part of any object in the +bucket — they come from environment variables, read through the one shared +helper in `dev_s3_env.rs`: + +| variable | required | purpose | +|---|---|---| +| `AWS_ACCESS_KEY_ID` | yes | credential | +| `AWS_SECRET_ACCESS_KEY` | yes | credential | +| `AWS_ENDPOINT_URL` | yes | the S3-compatible endpoint | +| `AWS_DEFAULT_REGION` | no (defaults to `"auto"`) | region | +| `AWS_S3_BUCKET_NAME` | yes, for the probes | bucket | + +These are the **same variable names a Railway deployment already sets** — +nothing here invents a new naming scheme; a dev container and a Railway +deployment read the identical variable names and behave identically once +both are populated. + +**The `AWS_ENDPOINT` vs `AWS_ENDPOINT_URL` trap.** `object_store`'s own +built-in environment discovery reads a variable named `AWS_ENDPOINT` — which +this environment does **not** set. If a caller skips `dev_s3_env::s3_options()` +and instead lets `object_store` fall back to its own default credential/endpoint +discovery, it silently resolves to AWS proper (or whatever that default +discovery lands on) instead of the configured S3-compatible endpoint. The +failure this produces does not look like a naming mistake — it looks like a +permissions error (auth failures, "bucket not found," timeouts against the +wrong host), because the request goes somewhere that plausibly *could* deny +it for unrelated reasons. `dev_s3_env::s3_options()` exists specifically to +map `AWS_ENDPOINT_URL` (the variable actually set here) into the +`aws_endpoint` option Lance consumes, and to return `None` — a hard error a +caller must not silently swallow — if any required variable is missing, +rather than letting the call fall through to that wrong-host default. + +## 4. Refresh / purge lifecycle + +**An existing table is never overwritten in place.** A refresh under +`on_existing: new_version` writes a brand-new, timestamped table +(`versioned_table_name` in `soa_config.rs`) and then flips the owning +repo's `config.yaml` `table` pointer to name it. The old table is left +completely untouched on disk. See the `on_existing` field's comment in +`crates/lance-graph/examples/soa-config.example.yaml` for the full +S3-has-no-atomic-rename reasoning behind this design. + +**Purging an old table is a separate, deliberate action** — never automatic, +never triggered by a refresh, never on any deadline. It is the *only* +destructive operation this layout has. Nothing in this bucket's normal +read/write/refresh path deletes data; only an explicit purge does, and only +once nothing — no config's `table` pointer — still names the table being +removed. + +## 5. How to inspect what's there, without opening a data file + +- **`aws s3 ls s3:///.config/`** — the top-level list of directories + here tells you which repositories have a deployment against this bucket at + all, with zero data reads. +- **`aws s3 ls s3:///.config//config.yaml` + fetch it** — + a small YAML object; tells you that repo's `ledger_prefix`, every bake it + declares (`name`, `table`, `classid`, whether it hydrates), and its + `on_existing` policy. Still zero reads of any actual Lance data. +- **`aws s3 ls s3:////`** — lists every bake (current + and, if any refreshes have happened and not yet been purged, superseded) + under one repository's namespace, again without opening any of them. + +For the deployment walkthrough — writing a bake, the physical-layout +guarantee it relies on, and the two ways a deployment can serve one — see +`docs/SOA_BAKE_DEPLOYMENT.md`. diff --git a/docs/SOA_BAKE_DEPLOYMENT.md b/docs/SOA_BAKE_DEPLOYMENT.md new file mode 100644 index 00000000..b8837f21 --- /dev/null +++ b/docs/SOA_BAKE_DEPLOYMENT.md @@ -0,0 +1,259 @@ +# SoA bake deployment — from `.soa` slab to a queryable Lance table + +This is an operator-facing walkthrough of the one-time write-back, the +physical-layout guarantee it depends on, the two deployment patterns it +enables, and the environment contract a deployment must satisfy. Every +measured claim below cites the source or plan section that measured it; +every convention is labelled as a convention, not a measurement. + +> For the bits-vs-bytes / lanes / straddle questions this design keeps +> raising ("64 bit or 64 byte?", "does alignment fragment the row?"), +> see **`docs/ALIGNMENT_PRIMER.md`** — the product-level deep dive with +> every claim labelled measured / source / arithmetic. + +## 1. What a bake is + +The canonical node row is **512 bytes**: `key(16) | edges(16) | value(480)` +(`CLAUDE.md`, "CANON — Minimal SoA node"). `crates/lance-graph-contract` +exposes this as `canonical_node::NODE_ROW_STRIDE = 512`. A `.soa` slab file +is nothing more than a whole number of these 512-byte rows, back to back — +`soa_to_lance` refuses to even open a file whose length is not a multiple of +`NODE_ROW_STRIDE`, treating that as a truncated bake rather than something to +tolerate. + +The 512-byte stride is load-bearing for everything that follows: it is what +keeps the row column uncompressed inside Lance (§3), which is what makes the +mmap-serving deployment pattern possible at all (§4b). + +## 2. The one-time write-back — `soa_to_lance` + +```text +soa_to_lance
+``` + +`uri` is either a local directory or an `s3://bucket/prefix` — the same call +either way. There is no separate export step: writing to the object store +**is** the write (`crates/lance-graph/examples/soa_to_lance.rs`, module doc). + +The binary: + +1. Reads the slab file into one `Vec`, asserts its length is a multiple + of `NODE_ROW_STRIDE`. +2. Wraps that `Vec` in an Arrow `FixedSizeBinaryArray` via `Buffer::from_vec` + — which **adopts** the allocation rather than copying it. The code asserts + this at runtime by comparing the buffer's pointer against the `Vec`'s + original pointer; `FixedSizeBinaryArray::try_from_iter` would have copied + the data chunk-by-chunk instead, which was measured on this branch's + history (commit `a27b06a`). +3. Writes one `RecordBatch` (a single `row` column) via `Dataset::write`, + with the SoA contract carried as Arrow schema metadata — which Lance + persists into its own manifest. +4. Re-opens what it just wrote and asserts the persisted header matches the + **compiled** contract, the same shape as `SoaEnvelope::verify_layout`. + +### The header keys + +Every value is imported from `lance-graph-contract` rather than restated as +a literal — a restated constant is a second source of truth that drifts. + +| key | value | source | +|---|---|---| +| `soa:envelope_layout_version` | `2` | `soa_envelope::ENVELOPE_LAYOUT_VERSION` | +| `soa:row_stride` | `512` | `canonical_node::NODE_ROW_STRIDE` | +| `soa:row_carving` | `key:0..16\|edges:16..32\|value:32..512` | canon, locked 2026-06-13 | +| `soa:endianness` | `le` | the LE contract | +| `soa:classid` | per bake (CLI arg) | the bake's own report | +| `soa:slab_digest` | per bake (CLI arg) | the bake's own report — pairs the table with its `.books` sidecar | +| `soa:source` | filename + row count | provenance | + +A reader is expected to verify `soa:envelope_layout_version` against its own +compiled `ENVELOPE_LAYOUT_VERSION` before casting anything read back — the +same discipline `soa_to_lance` applies to itself on re-open. + +## 3. Why the row column lands verbatim — the STRIDE, not the metadata + +This is the part most likely to be misunderstood, because an earlier version +of this module's own doc got it wrong and was corrected in place rather than +silently edited (`soa_to_lance.rs` module doc; mirrored in +`tests/soa_verbatim.rs`). + +**What is false:** that the `lance-encoding:compression = "none"` field +metadata is what keeps the row column byte-for-byte. It is not — removing +the key leaves the written file byte-identical, and so does setting it to +`"zstd"`. The key is spelled correctly and genuinely parsed +(`lance-encoding-9.0.0` `compression.rs:576`); it simply never reaches this +column. + +**What is true**, read from lance 9 source: + +- `is_narrow` (`encodings/logical/primitive.rs:3861`) classifies a column as + narrow when its value length is below `MINIBLOCK_MAX_BYTE_LENGTH_PER_VALUE + = 256` bytes. `NODE_ROW_STRIDE` is 512, so a canonical row is **not** + narrow and the column takes the **full-zip** path, not mini-block. +- Full-zip's `create_per_value` returns `ValueEncoder::default()` + unconditionally for `DataBlock::FixedWidth` (`compression.rs:753`) — it + computes the merged field params one line earlier and then ignores them on + that branch. +- The only branch that honours the compression metadata, + `build_fixed_width_compressor` (`compression.rs:624`), lives on the + mini-block path — which a 512-byte value never reaches. + +So it is `NODE_ROW_STRIDE = 512 > 256` (the mini-block cutoff) that buys the +verbatim write. The `lance-encoding:compression = "none"` metadata is kept in +the writer only as a **defensive pin**: if that 256-byte threshold ever rose +above 512 in a future Lance version, the mini-block path would start +honouring the metadata and would still refuse compression — but today it does +no work. + +`tests/soa_verbatim.rs` pins both halves of this claim: + +- The physical-layout assertions (`a_slab_is_written_verbatim_and_contiguously` + and its S3 twin) — the slab's bytes are found as one contiguous run in the + data file, at a 64-byte-aligned offset, with every sampled row at its + computed address, and total file overhead bounded under 64 KiB (so it is + not the slab plus a second encoded copy). +- `the_narrow_column_falsifier` — the sensitivity proof that the byte search + used above can actually **see** compression at all: at a 64-byte stride + (below the mini-block cutoff) the same metadata key *is* honoured, and + `"zstd"` makes the rows vanish from the file while `"none"` still leaves + them verbatim. + +**Why it matters:** this is what makes the mmap-serving deployment pattern +(§4b) sound. If the row column were chunked or compressed, `mmap(file)[off .. +off + rows*512]` would not be the slab at all. + +## 4. The two deployment patterns + +### (a) Read from S3 directly + +A deployment opens the Lance dataset straight from `s3://…` and lets Lance's +S3 object-store client serve reads. No local copy exists. Simplest to +operate; every read after the first for a given byte range costs whatever +Lance's own caching does (see §7 — this is one of the open questions). + +### (b) Hydrate to local disk once, then serve locally + +A deployment copies the dataset's objects to local disk once and serves +subsequent reads from there — the pattern the mmap-serving argument in §3 +targets, and the one `.claude/plans/idle-flush-dataset-eviction-v1.md` +proposes eviction over. + +**Measured hydration cost** (`examples/hydration_probe.rs`; plan §8a, +measured 2026-08-07, against the configured endpoint): + +| MB | hydrate | implied MB/s | +|---|---|---| +| 0.3 | 2.64 s | 0.1 | +| 6.7 | 2.83 s | 2.4 | +| 33.5 | 3.33 s | 10.1 | + +A linear fit over these three points gives **≈2.63 s fixed cost + ≈0.021 +s/MB** (≈48 MB/s marginal). The dominant term is size-independent: roughly +2.6 s of every hydration is fixed overhead, not bytes moved. + +**Stated scope of these numbers, do not over-read them:** one endpoint, one +day, single-fragment datasets, 0.3–33.5 MB, no concurrency, and eviction +itself is not implemented — this measured only the hydration primitive a +future eviction policy would call. The plan explicitly retires the earlier +"~1.4 s" figure as a *constant* (it was a single, unre-run observation on a +different endpoint) while keeping the *decomposition* (fixed cost + a small +marginal rate) as the finding that survives. + +Hydration here means a raw byte copy of every object under the dataset's +root — not a scan-and-rewrite through `Dataset::write`. The probe verified +this (`T10`, plan's own acceptance-criterion name) by comparing every copied +object's raw bytes against its local original, which is strictly stronger +than a column checksum: it would catch a dropped `.txn` file, a deletion +vector, or any other non-row artifact that a logical re-export would silently +lose. + +## 5. Environment contract + +Both the write-back binary and its verification arms read S3 credentials +through one shared reader, `dev_s3_env.rs`, specifically to avoid the write +path and the verification path independently reading two different option +maps: + +| variable | required | purpose | +|---|---|---| +| `AWS_ACCESS_KEY_ID` | yes | credential | +| `AWS_SECRET_ACCESS_KEY` | yes | credential | +| `AWS_ENDPOINT_URL` | yes | the S3-compatible endpoint | +| `AWS_DEFAULT_REGION` | no (defaults to `"auto"`) | region | +| `AWS_S3_BUCKET_NAME` | yes, for the probes | bucket | + +These are the **same variable names a Railway deployment already sets** +(`dev_s3_env.rs` module doc) — no separate deployment-specific naming exists. + +**The `AWS_ENDPOINT` vs `AWS_ENDPOINT_URL` trap:** `object_store`'s own +environment discovery reads `AWS_ENDPOINT`, which this environment does not +set. `dev_s3_env::s3_options()` explicitly maps `AWS_ENDPOINT_URL` (the +variable this deployment actually sets) into the `aws_endpoint` option Lance +consumes. Relying on `object_store`'s own discovery instead — by passing +`store_params: None` and letting it fall back to default credential +resolution — silently addresses AWS proper (or whatever the default resolves +to) rather than the configured S3-compatible endpoint. `s3_options()` +returns `None` if any *required* variable is missing, and callers on a path +that has already committed to being remote are expected to treat `None` as a +hard error, never a silent fallback to default discovery — a prior defect on +this branch had `soa_to_lance` write with `store_params: None` (silently +falling back) while its own re-open used `s3_options().expect(...)`, which +could panic only *after* the dataset had already been written to the wrong +place. + +`dev_s3_env::env()` also strips wrapping quotes from an environment value — +this sandbox's exporter can wrap a value in literal `"…"`, and a quoted +credential authenticates as garbage while pointing the resulting error at the +credential rather than at the quoting. + +## 6. Boot config + +A deployment reads `.config//config.yaml` from the same bucket, +declaring which bakes exist and which get hydrated at boot. The schema itself +is defined in `crates/lance-graph/examples/soa-config.example.yaml` — refer +there for the full shape rather than duplicating it here. + +**Doctrine, not a measurement:** an existing table is never silently +overwritten. A refresh writes a **new, timestamped table** and flips the +config's pointer to it, because S3 has no atomic rename — an in-place +overwrite of an existing table's objects is exactly the kind of +half-written-directory hazard §5a of the idle-flush plan warns about for +local hydration, and the same reasoning applies to a refresh in place on the +object store. + +## 7. What is NOT measured + +Stated plainly rather than presented as risks that are probably fine +(`.claude/knowledge/lance-cache-surface.md`; plan §8a, §9): + +- **Request count per hydration is still open.** The plan originally claimed + this was closed at "3 remote objects per dataset," but that number came + from `dir_stats()` — a **local** pre-upload file count, never wired to any + object-store request instrumentation. How many actual requests a hydration + issues, and what that costs against a real provider's pricing (request + count, retrieval class, egress), remains unmeasured. Closing it needs a + `WrappingObjectStore` request counter or equivalent instrumentation, not a + local `read_dir`. + +- **P-CACHE-1 — whether decoded data enters `LanceCache` at all.** `moka` is + an unconditional dependency of `lance-core` (no feature gate) and its + `MokaCacheBackend` is byte-weighed (`.max_capacity` + a weigher over key and + entry size), with a `no_cache()` constructor available. But **it is not yet + known what the cache actually holds** on the data path — decoded column + pages / record batches, versus only manifests, schemas, and index metadata. + This is the load-bearing open question: if decoded data does *not* go + through `LanceCache`, capping its capacity bounds only metadata, and the RAM + that shows up on a bill lives in whatever the *caller* holds — a failure + mode already caught once on this branch (a `read_batch` that collected and + concat-copied a whole table, commit `a27b06a`). If decoded data *does* go + through the cache, `with_capacity(n)` is a hard ceiling on exactly the + memory being billed. Two further companion questions are open alongside it + and equally unrun: whether the capacity knob (or `no_cache`) is reachable + from public API (`lancedb::connect`, `DatasetBuilder`, env, session object) + at all, and whether measured RSS actually tracks capacity in practice. None + of the three has a probe run yet. + +Neither gap blocks the write-back path described in §2–3, which is measured +and pinned by tests. Both gaps block any claim about steady-state RAM cost +under deployment pattern (b), and about the true dollar cost of pattern (a) +under request-metered billing.