Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/vfs-store/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ serde_json = "1.0"
vfs = { workspace = true }

[dev-dependencies]
agentos-runtime = { workspace = true }
aws-config = "1"
aws-credential-types = "1"
tempfile = "3"
Expand Down
73 changes: 69 additions & 4 deletions crates/vfs-store/src/local/sqlite_metadata_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,26 @@ const LOCAL_FS_MIGRATIONS: &[LocalFsMigration] = &[
DROP TABLE agentos_fs_inodes_v1;
"#,
},
LocalFsMigration {
version: 3,
statements: r#"
DROP INDEX agentos_fs_dentries_parent;
ALTER TABLE agentos_fs_dentries RENAME TO agentos_fs_dentries_v2;
CREATE TABLE agentos_fs_dentries (
parent_ino INTEGER NOT NULL CHECK (parent_ino > 0),
name TEXT NOT NULL CHECK (length(name) > 0),
child_ino INTEGER NOT NULL CHECK (child_ino > 0),
kind INTEGER NOT NULL CHECK (kind IN (0, 1, 2, 3, 4, 5)),
PRIMARY KEY (parent_ino, name)
) STRICT;
INSERT INTO agentos_fs_dentries (parent_ino, name, child_ino, kind)
SELECT parent_ino, name, child_ino, kind
FROM agentos_fs_dentries_v2;
DROP TABLE agentos_fs_dentries_v2;
CREATE INDEX agentos_fs_dentries_parent
ON agentos_fs_dentries(parent_ino);
"#,
},
];

pub struct SqliteMetadataStore {
Expand Down Expand Up @@ -267,13 +287,19 @@ impl SqliteMetadataStore {
upsert_inode(connection, meta)?;
if storage_changed {
connection
.execute("DELETE FROM agentos_fs_chunks WHERE ino = ?", params![meta.ino])
.execute(
"DELETE FROM agentos_fs_chunks WHERE ino = ?",
params![meta.ino],
)
.map_err(|err| VfsError::eio(format!("delete setattr chunks: {err}")))?;
for key in affected_keys {
let refcount = self.inner.refcount(&key);
if refcount == 0 {
connection
.execute("DELETE FROM agentos_fs_block_refs WHERE block_key = ?", params![key.0])
.execute(
"DELETE FROM agentos_fs_block_refs WHERE block_key = ?",
params![key.0],
)
.map_err(|err| {
VfsError::eio(format!("delete setattr block ref {}: {err}", key.0))
})?;
Expand Down Expand Up @@ -399,7 +425,9 @@ impl SqliteMetadataStore {

if new_size < old_size {
connection
.prepare_cached("DELETE FROM agentos_fs_chunks WHERE ino = ? AND chunk_index >= ?")
.prepare_cached(
"DELETE FROM agentos_fs_chunks WHERE ino = ? AND chunk_index >= ?",
)
.map_err(|err| VfsError::eio(format!("prepare truncated chunk delete: {err}")))?
.execute(params![ino, keep_chunks])
.map_err(|err| {
Expand All @@ -425,7 +453,10 @@ impl SqliteMetadataStore {
let refcount = self.inner.refcount(&key);
if refcount == 0 {
connection
.execute("DELETE FROM agentos_fs_block_refs WHERE block_key = ?", params![key.0])
.execute(
"DELETE FROM agentos_fs_block_refs WHERE block_key = ?",
params![key.0],
)
.map_err(|err| {
VfsError::eio(format!("delete SQLite block ref {}: {err}", key.0))
})?;
Expand Down Expand Up @@ -1031,4 +1062,38 @@ mod tests {
.expect("inspect database");
assert_eq!(table_count, 0);
}

#[test]
fn upgrades_v2_dentries_without_losing_existing_rows() {
let mut connection = Connection::open_in_memory().expect("open database");
install_schema_migrations(&mut connection, &LOCAL_FS_MIGRATIONS[..2])
.expect("install schema v2");
connection
.execute(
"INSERT INTO agentos_fs_dentries (parent_ino, name, child_ino, kind)
VALUES (1, 'existing', 2, 0)",
[],
)
.expect("seed v2 dentry");

install_schema_migrations(&mut connection, LOCAL_FS_MIGRATIONS)
.expect("upgrade schema to v3");

let existing: (u64, i64) = connection
.query_row(
"SELECT child_ino, kind FROM agentos_fs_dentries
WHERE parent_ino = 1 AND name = 'existing'",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.expect("read preserved dentry");
assert_eq!(existing, (2, 0));
connection
.execute(
"INSERT INTO agentos_fs_dentries (parent_ino, name, child_ino, kind)
VALUES (1, 'character', 3, 3)",
[],
)
.expect("insert character-device dentry after upgrade");
}
}
55 changes: 48 additions & 7 deletions crates/vfs-store/tests/local.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
use agentos_runtime::{RuntimeConfig, SidecarRuntime};
use agentos_vfs::{FileBlockStore, SqliteMetadataStore};
use rusqlite::Connection;
use vfs::adapter::MountedEngineFileSystem;
use vfs::engine::engines::{ChunkedFs, ChunkedFsOptions};
use vfs::engine::mem::MemoryBlockStore;
use vfs::engine::{BlockKey, BlockStore, VirtualFileSystem};
use vfs::engine::{BlockKey, BlockStore, InodeType, VirtualFileSystem, S_IFBLK, S_IFCHR, S_IFIFO};
use vfs::posix::MountedFileSystem;
use vfs::posix::{MemoryFileSystem, MountOptions, MountTable};

fn test_runtime_context() -> agentos_runtime::RuntimeContext {
SidecarRuntime::process(&RuntimeConfig::default())
.expect("create test runtime")
.context()
}

#[tokio::test]
async fn file_block_store_persists_blocks() {
let temp = tempfile::tempdir().unwrap();
Expand Down Expand Up @@ -35,7 +42,7 @@ async fn sqlite_store_installs_canonical_schema() {
|row| row.get(0),
)
.unwrap();
assert_eq!(version, 2);
assert_eq!(version, 3);

let mut statement = connection
.prepare(
Expand Down Expand Up @@ -133,7 +140,7 @@ fn sqlite_store_rejects_future_schema_versions() {
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
schema_version INTEGER NOT NULL CHECK (schema_version >= 0)
) STRICT;
INSERT INTO agentos_fs_schema_version (singleton, schema_version) VALUES (1, 3);",
INSERT INTO agentos_fs_schema_version (singleton, schema_version) VALUES (1, 4);",
)
.unwrap();
drop(connection);
Expand All @@ -143,7 +150,41 @@ fn sqlite_store_rejects_future_schema_versions() {
.expect("future schema must be rejected");
assert!(error
.message()
.contains("version 3; latest supported version is 2"));
.contains("version 4; latest supported version is 3"));
}

#[tokio::test]
async fn sqlite_store_persists_special_inode_kinds_and_device_ids() {
let temp = tempfile::tempdir().unwrap();
let db = temp.path().join("special-inodes.sqlite");
let blocks = MemoryBlockStore::new();

{
let fs = ChunkedFs::new(SqliteMetadataStore::open(&db).unwrap(), blocks.clone());
fs.create_dir("/devices").await.unwrap();
fs.mknod("/devices/char", S_IFCHR | 0o600, (1 << 8) | 3)
.await
.unwrap();
fs.mknod("/devices/block", S_IFBLK | 0o640, (8 << 8) | 1)
.await
.unwrap();
fs.mknod("/devices/fifo", S_IFIFO | 0o600, 0).await.unwrap();
}

let fs = ChunkedFs::new(SqliteMetadataStore::open(&db).unwrap(), blocks);
let entries = fs.read_dir_with_types("/devices").await.unwrap();
assert!(entries
.iter()
.any(|entry| entry.name == "char" && entry.kind == InodeType::CharacterDevice));
assert!(entries
.iter()
.any(|entry| entry.name == "block" && entry.kind == InodeType::BlockDevice));
assert!(entries
.iter()
.any(|entry| entry.name == "fifo" && entry.kind == InodeType::Fifo));
assert_eq!(fs.stat("/devices/char").await.unwrap().rdev, (1 << 8) | 3);
assert_eq!(fs.stat("/devices/block").await.unwrap().rdev, (8 << 8) | 1);
assert_eq!(fs.stat("/devices/fifo").await.unwrap().rdev, 0);
}

#[tokio::test]
Expand Down Expand Up @@ -440,7 +481,7 @@ fn chunked_local_mounted_adapter_creates_exclusive_files_with_modes() {
SqliteMetadataStore::open(temp.path().join("metadata.sqlite")).unwrap(),
FileBlockStore::new(temp.path().join("blocks")).unwrap(),
);
let mut mounted = MountedEngineFileSystem::new(fs).unwrap();
let mut mounted = MountedEngineFileSystem::with_runtime_context(fs, test_runtime_context());

mounted.mkdir("/nested", false).unwrap();
mounted
Expand All @@ -461,7 +502,7 @@ fn chunked_local_mounted_adapter_preserves_sparse_pwrite_allocation() {
SqliteMetadataStore::open(temp.path().join("metadata.sqlite")).unwrap(),
FileBlockStore::new(temp.path().join("blocks")).unwrap(),
);
let mut mounted = MountedEngineFileSystem::new(fs).unwrap();
let mut mounted = MountedEngineFileSystem::with_runtime_context(fs, test_runtime_context());

mounted.write_file("/sparse", Vec::new()).unwrap();
mounted
Expand All @@ -480,7 +521,7 @@ fn chunked_local_mount_table_preserves_sparse_pwrite_allocation() {
SqliteMetadataStore::open(temp.path().join("metadata.sqlite")).unwrap(),
FileBlockStore::new(temp.path().join("blocks")).unwrap(),
);
let mounted = MountedEngineFileSystem::new(fs).unwrap();
let mounted = MountedEngineFileSystem::with_runtime_context(fs, test_runtime_context());
let mut table = MountTable::new(MemoryFileSystem::new());
vfs::posix::VirtualFileSystem::mkdir(&mut table, "/mnt", false).unwrap();
table
Expand Down
3 changes: 3 additions & 0 deletions crates/vfs-store/tests/s3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ async fn s3_block_store_round_trips_and_cleans_blocks() {
}

#[tokio::test]
#[ignore = "ObjectS3 is dormant; retain this unsupported whole-object target for its return"]
async fn object_s3_round_trips_native_objects() {
let server = MockS3Server::start();
let backend = S3ObjectBackend::with_options(
Expand Down Expand Up @@ -104,6 +105,7 @@ async fn object_s3_round_trips_native_objects() {
}

#[tokio::test]
#[ignore = "ObjectS3 is dormant; retain this unsupported whole-object target for its return"]
async fn object_s3_resolves_explicit_directory_markers_for_new_files() {
let server = MockS3Server::start();
let backend = S3ObjectBackend::with_options(
Expand Down Expand Up @@ -143,6 +145,7 @@ async fn object_s3_resolves_explicit_directory_markers_for_new_files() {
}

#[tokio::test]
#[ignore = "ObjectS3 is dormant; retain this unsupported whole-object target for its return"]
async fn object_s3_preserves_and_cleans_special_nodes_under_directory_markers() {
let server = MockS3Server::start();
let backend = S3ObjectBackend::with_options(
Expand Down
Loading