diff --git a/.config/hakari.toml b/.config/hakari.toml new file mode 100644 index 00000000..bb5d464e --- /dev/null +++ b/.config/hakari.toml @@ -0,0 +1,27 @@ +# This file contains settings for `cargo hakari`. +# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options. + +hakari-package = "workspace-hack" + +# Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above. +dep-format-version = "4" + +# Setting workspace.resolver = "2" or higher in the root Cargo.toml is HIGHLY recommended. +# Hakari works much better with the v2 resolver. (The v2 and v3 resolvers are identical from +# hakari's perspective, so you're welcome to set either.) +# +# For more about the new feature resolver, see: +# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver +resolver = "3" + +# Add triples corresponding to platforms commonly used by developers here. +# https://doc.rust-lang.org/rustc/platform-support.html +platforms = [ + "x86_64-unknown-linux-gnu", + # "x86_64-apple-darwin", + # "aarch64-apple-darwin", + # "x86_64-pc-windows-msvc", +] + +# Write out exact versions rather than a semver range. (Defaults to false.) +# exact-versions = true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..07be7bcf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks +# Cargo. +Cargo.lock merge=binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1073282e..ef0db1a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,15 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 - - # c.f. https://github.com/extractions/setup-just/issues/23 - - uses: extractions/setup-crate@4993624604c307fbca528d28a3c8b60fa5ecc859 # v1.4.0 - with: - repo: casey/just - version: 1.42.4 - - uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 - with: - bins: cargo-nextest, cargo-insta@1.28.0 + - uses: ./.github/actions/setup-nix-direnv-rust - run: just test-unit test-integration: name: Integration test (& build all in dev-mode) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 250aa835..435cd524 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -3,8 +3,8 @@ name: "Opsqueue CI" agent: machine: - type: "f1-standard-2" - os_image: "ubuntu2204" + type: "f1-standard-4" + os_image: "ubuntu2404" # If we push a new build to some branch that isn't master, and another build is # already running, we cancel it. @@ -60,17 +60,14 @@ blocks: when: "branch = 'master'" task: jobs: - - name: "Build Opsqueue binary (in release mode) & run unit tests" + - name: "Build Opsqueue binary & Python library (in release mode) & run unit tests" commands: - - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-bin)) | grep -v '\\.drv$' | cachix push channable" + - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build)) | grep -v '\\.drv$' | cachix push channable" # Once a day, we update the devenv cache. # Except when fresh, this takes < 1 second # but it cannot live in the prologue # as that might corrupt it - cache store nix-store-$(date -u -Idate) /nix - - name: "Build Python library (in release mode)" - commands: - - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-python)) | grep -v '\\.drv$' | cachix push channable" - name: "Integration" dependencies: - "Build" diff --git a/Cargo.lock b/Cargo.lock index 4ea740a9..4867868e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,15 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloca" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" -dependencies = [ - "cc", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -50,62 +41,12 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - [[package]] name = "anyhow" version = "1.0.104" @@ -121,12 +62,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "assert_matches" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" - [[package]] name = "async-channel" version = "2.5.0" @@ -288,7 +223,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", - "gloo-timers", "tokio", ] @@ -367,12 +301,6 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.3.0" @@ -415,10 +343,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-link", ] @@ -465,10 +391,8 @@ version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ - "anstream", "anstyle", "clap_lex", - "strsim", ] [[package]] @@ -504,12 +428,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - [[package]] name = "combine" version = "4.6.7" @@ -529,17 +447,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "console" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" -dependencies = [ - "encode_unicode", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "const-oid" version = "0.10.2" @@ -595,42 +502,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" -[[package]] -name = "criterion" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" -dependencies = [ - "alloca", - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "itertools 0.13.0", - "num-traits", - "oorandom", - "page_size", - "plotters", - "rayon", - "regex", - "serde", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" -dependencies = [ - "cast", - "itertools 0.13.0", -] - [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -640,16 +511,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -818,12 +679,6 @@ dependencies = [ "serde", ] -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - [[package]] name = "equivalent" version = "1.0.2" @@ -1101,18 +956,6 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "h2" version = "0.4.15" @@ -1498,7 +1341,6 @@ version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ - "console", "once_cell", "similar", "tempfile", @@ -1510,21 +1352,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2076,18 +1903,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -2194,14 +2009,12 @@ version = "0.36.0" dependencies = [ "anyhow", "arc-swap", - "assert_matches", "axum", "axum-prometheus", "backon", "chrono", "ciborium", "clap", - "criterion", "crossbeam-skiplist", "dashmap", "either", @@ -2225,7 +2038,6 @@ dependencies = [ "sentry", "sentry-tracing", "serde", - "serde-error", "serde_json", "snowflaked", "sqlformat", @@ -2241,6 +2053,7 @@ dependencies = [ "url", "uuid", "ux", + "workspace-hack", ] [[package]] @@ -2260,6 +2073,7 @@ dependencies = [ "tracing", "uuid", "ux", + "workspace-hack", ] [[package]] @@ -2278,16 +2092,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking" version = "2.2.1" @@ -2355,34 +2159,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.14.0" @@ -2724,26 +2500,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -3139,15 +2895,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-error" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342110fb7a5d801060c885da03bf91bfa7c7ca936deafcc64bb6706375605d47" -dependencies = [ - "serde", -] - [[package]] name = "serde_core" version = "1.0.229" @@ -3390,6 +3137,7 @@ dependencies = [ "log", "memchr", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -3399,6 +3147,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots", ] [[package]] @@ -3545,12 +3294,6 @@ dependencies = [ "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "subtle" version = "2.6.1" @@ -3693,16 +3436,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.12.0" @@ -3974,10 +3707,8 @@ checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", - "smallvec", "tracing", "tracing-core", - "tracing-log", "tracing-subscriber", "web-time", ] @@ -4145,12 +3876,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.24.0" @@ -4521,6 +4246,61 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "workspace-hack" +version = "0.1.0" +dependencies = [ + "anyhow", + "aws-lc-rs", + "base64", + "bitflags", + "chrono", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "event-listener", + "fastrand", + "futures-channel", + "futures-io", + "futures-sink", + "futures-util", + "hyper", + "hyper-util", + "libc", + "libsqlite3-sys", + "log", + "num-traits", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "rand 0.10.2", + "rand 0.9.5", + "regex-automata", + "regex-syntax", + "reqwest", + "rustls-pki-types", + "serde", + "serde_core", + "serde_json", + "sha2 0.10.9", + "slab", + "smallvec", + "sqlx-core", + "sqlx-sqlite", + "syn 3.0.2", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tracing-core", + "typenum", + "url", + "uuid", + "zerocopy", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index a16b5baa..205fd1d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,11 +6,73 @@ resolver = "3" members = [ "opsqueue/", "libs/opsqueue_python", + "workspace-hack", ] +[workspace.metadata.hakari] +hakari-package = "workspace-hack" + [workspace.package] +edition = "2024" version = "0.36.0" +[workspace.dependencies] +anyhow = { version = "1.0.102", default-features = false } +arc-swap = { version = "1.9.1", default-features = false } +assert_matches = { version = "1.5.0", default-features = false } +axum = { version = "0.8.9", default-features = false, features = ["ws", "macros"] } +axum-prometheus = { version = "0.10.0", default-features = false, features = ["prometheus"] } +backon = { version = "1.3.0", default-features = false, features = ["tokio-sleep"] } +chrono = { version = "0.4.44", default-features = false, features = ["serde"] } +ciborium = { version = "0.2.2", default-features = false, features = ["std"] } +clap = { version = "4.5.60", default-features = false, features = ["derive", "std"] } +criterion = { version = "0.8", default-features = false, features = ["async_tokio"] } +crossbeam-skiplist = { version = "0.1.3", default-features = false, features = ["std"] } +dashmap = { version = "6.1.0", default-features = false } +either = { version = "1.13.0", default-features = false } +futures = { version = "0.3.32", default-features = false } +http = { version = "1.4.0", default-features = false } +humantime = { version = "2.1.0", default-features = false } +insta = { version = "1.47.2", default-features = false } +itertools = { version = "0.15.0", default-features = false } +moka = { version = "0.12.15", default-features = false, features = ["sync"] } +moro-local = { version = "0.4.0", default-features = false } +object_store = { version = "0.14.0", default-features = false, features = ["gcp", "http", "fs"] } +once_cell = { version = "1.21.4", default-features = false } +opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } +opentelemetry-http = { version = "0.32", default-features = false } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace", "http-proto", "reqwest-client"] } +opentelemetry-resource-detectors = { version = "0.11.0", default-features = false } +opentelemetry-semantic-conventions = { version = "0.32.0", default-features = false, features = ["semconv_experimental"] } +opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } +opsqueue = { path = "opsqueue", default-features = false } +pyo3 = { version = "0.28.3", default-features = false, features = ["chrono"] } +pyo3-async-runtimes = { version = "0.28.0", default-features = false, features = ["tokio-runtime", "unstable-streams"] } +pyo3-log = { version = "0.13.3", default-features = false } +rand = { version = "0.10.1", default-features = false } +reqwest = { version = "0.13.2", default-features = false, features = ["json"] } +rustc-hash = { version = "2.1.2", default-features = false, features = ["std"] } +sentry = { version = "0.48", default-features = false, features = ["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"] } +sentry-tracing = { version = "0.48", default-features = false } +serde = { version = "1.0.203", default-features = false, features = ["derive"] } +serde-error = { version = "0.1.3", default-features = false } +serde_json = { version = "1.0.149", default-features = false } +snowflaked = { version = "1.0.3", default-features = false, features = ["sync"] } +sqlformat = { version = "0.5.0", default-features = false } +sqlx = { version = "0.9.0", default-features = false, features = ["chrono", "derive", "json", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls-aws-lc-rs"] } +thiserror = { version = "2.0.18", default-features = false } +tokio = { version = "1.52.1", default-features = false, features = ["macros", "signal", "rt-multi-thread"] } +tokio-tungstenite = { version = "0.30.0", default-features = false, features = ["connect", "stream"] } +tokio-util = { version = "0.7.18", default-features = false, features = ["io", "rt", "time"] } +tower-http = { version = "0.7.0", default-features = false, features = ["trace", "catch-panic"] } +tracing = { version = "0.1.41", default-features = false, features = ["log"] } +tracing-opentelemetry = { version = "0.33.0", default-features = false } +tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "env-filter", "fmt"] } +url = { version = "2.5.2", default-features = false } +uuid = { version = "1.23.1", default-features = false, features = ["v7", "fast-rng", "serde"] } +ux = { version = "0.1.6", default-features = false } +workspace-hack = { version = "0.1", path = "workspace-hack" } + [workspace.lints.clippy] cargo = { level = "deny", priority = -1 } cargo_common_metadata = "allow" diff --git a/clippy.toml b/clippy.toml index 274bb7be..b2d17c78 100644 --- a/clippy.toml +++ b/clippy.toml @@ -5,13 +5,13 @@ allowed-duplicate-crates = [ "digest", "getrandom", "hashbrown", + "itertools", "r-efi", "rand", "rand_core", "sha1", "sha2", "syn", - "thiserror", "tokio-tungstenite", "tower-http", "tungstenite", diff --git a/default.nix b/default.nix index 2da1a739..ef1e7aed 100644 --- a/default.nix +++ b/default.nix @@ -48,9 +48,12 @@ let pkgs.cargo-audit pkgs.cargo-edit pkgs.cargo-insta + pkgs.cargo-hakari pkgs.cargo-nextest pkgs.maturin + # Resolve native sqlite from Nix for libsqlite3-sys + pkgs.pkg-config # sqlite3 binary, for easy debugging/introspection pkgs.sqlite ]; @@ -63,6 +66,12 @@ let # For the shell, libpython needs to be in the search path. pythonEnv ]; + shellHook = '' + # We need Cargo to re-run if the sqlite3 dev package changes, so we set this env var to point to it. + export LIBSQLITE3_SYS_USE_PKG_CONFIG="${pkgs.sqlite.dev}" + # This is needed for the libsqlite3-sys crate to find the correct sqlite3. + export PKG_CONFIG_PATH="${pkgs.sqlite.dev}/lib/pkgconfig''${PKG_CONFIG_PATH:+:''${PKG_CONFIG_PATH}}" + ''; }; }; in diff --git a/justfile b/justfile index 0cba2f34..7c8ad88e 100644 --- a/justfile +++ b/justfile @@ -81,7 +81,12 @@ lint-light *ARGS: # Run the slow linters/static analysers that need to look at everything [group('lint')] -lint-heavy: clippy mypy +lint-heavy: clippy hakari mypy + +# Verify the workspace-hack crate is up to date +[group('lint')] +hakari: + cargo hakari verify # Rust static analysis [group('lint')] diff --git a/libs/opsqueue_python/Cargo.toml b/libs/opsqueue_python/Cargo.toml index 67d10548..4cd01f26 100644 --- a/libs/opsqueue_python/Cargo.toml +++ b/libs/opsqueue_python/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "opsqueue_python" version.workspace = true -edition = "2024" +edition.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] @@ -11,32 +11,31 @@ crate-type = ["cdylib"] [dependencies] # Opsqueue itself: # NOTE: We compile without the default `sqlx` feature, reducing the amount of dependencies -opsqueue = {path = "../../opsqueue/", default-features = false, features = ["client-logic"] } +opsqueue = { workspace = true, features = ["client-logic"] } # General-purpose datatypes: -uuid = {version = "1.23.1", features = [ - "v7", # Temporally orderable UUIDs with random component - "fast-rng", # Use a faster (but still sufficiently random) RNG - ]} -chrono = { version = "0.4.44"} -ux = "0.1.6" +uuid.workspace = true +chrono.workspace = true +ux.workspace = true # Concurrency: -tokio = {version = "1.52", features = ["macros", "rt-multi-thread"]} -futures = "0.3.32" +tokio.workspace = true +futures.workspace = true # Error handling: -anyhow = "1.0.102" -thiserror = "2.0.18" +anyhow.workspace = true +thiserror.workspace = true # Python FFI: -pyo3 = { version = "0.28.3", features = ["chrono"] } -pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime", "unstable-streams"] } -once_cell = "1.21.4" # Only currently used for `unsync::OnceCell` as part of PyO3 async helpers. +pyo3.workspace = true +pyo3-async-runtimes.workspace = true +once_cell.workspace = true # Only currently used for `unsync::OnceCell` as part of PyO3 async helpers. # Logging/tracing: -pyo3-log = "0.13.3" -tracing = { version = "0.1.41", features = ["log"] } +pyo3-log.workspace = true +tracing.workspace = true + +workspace-hack.workspace = true [lints] workspace = true diff --git a/libs/opsqueue_python/opsqueue_python.nix b/libs/opsqueue_python/opsqueue_python.nix index 57a3e89f..3645f38e 100644 --- a/libs/opsqueue_python/opsqueue_python.nix +++ b/libs/opsqueue_python/opsqueue_python.nix @@ -32,6 +32,7 @@ let sources = import ../../nix/sources.nix; crane = import sources.crane { pkgs = pkgs; }; craneLib = crane.overrideToolchain (pkgs: rustToolchain); + sqlitePkgConfigPath = lib.makeSearchPathOutput "dev" "lib/pkgconfig" [ pkgs.sqlite ]; # Only the files necessary to build the Rust-side and cache dependencies sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; @@ -59,7 +60,11 @@ let inherit version pname; src = depsSrc; strictDeps = true; - nativeBuildInputs = [ python ]; + nativeBuildInputs = [ + python + pkgs.pkg-config + ]; + buildInputs = [ pkgs.sqlite ]; cargoExtraArgs = "--package opsqueue_python"; doCheck = false; }; @@ -77,6 +82,9 @@ let // { inherit cargoArtifacts; src = wheelSrc; + env.DATABASE_URL = "sqlite://${depsSrc}/opsqueue/opsqueue_example_database_schema.db"; + env.LIBSQLITE3_SYS_USE_PKG_CONFIG = "${pkgs.sqlite.dev}"; + env.PKG_CONFIG_PATH = sqlitePkgConfigPath; } )).overrideAttrs (old: { diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 59e3fbae..310eb645 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "opsqueue" version.workspace = true -edition = "2024" +edition.workspace = true description = "lightweight batch processing queue for heavy loads" repository = "https://github.com/channable/opsqueue" license = "MIT" @@ -17,65 +17,61 @@ required-features = ["server-logic"] [dependencies] # Datatypes and concurrency: -itertools = "0.15.0" -arc-swap = {version = "1.9.1", optional = true} -moka = { version = "0.12.15", features = ["sync"], optional = true } -chrono = { version = "0.4.44", features = ["serde"]} -futures = "0.3.32" -tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } -uuid = {version = "1.23.1", features = [ - "v7", # Timestamp-sortable UUIDs with random component - "fast-rng", # Use a faster (but still sufficiently random) RNG - "serde", - ]} -ux = "0.1.6" +itertools.workspace = true +arc-swap = { workspace = true, optional = true } +moka = { workspace = true, optional = true } +chrono.workspace = true +futures.workspace = true +tokio.workspace = true +uuid.workspace = true +ux.workspace = true # Error handling: -anyhow = "1.0.102" +anyhow.workspace = true # Database: -sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono"], optional = true } +sqlx = { workspace = true, optional = true } # Serialization: -serde = { version = "1.0.203", features = ["derive"] } -serde_json = "1.0.149" -ciborium = "0.2.2" +serde.workspace = true +serde_json.workspace = true +ciborium.workspace = true # Webservers/clients -http = "1.4.0" -object_store = {version = "0.14.0", features = ["gcp", "http"]} -snowflaked = {version = "1.0.3", features = ["sync"] } -tokio-tungstenite = {version = "0.30.0", optional = true} -axum = { version = "0.8.9", features = ["ws", "macros"], optional = true } -reqwest = { version = "0.13.2", default-features = false, features = ["json"], optional = true } -url = {version = "2.5.2"} -tokio-util = { version = "0.7.18", features = ["io", "rt", "time"] } -tower-http = { version = "0.7.0", features = ["trace", "catch-panic"], optional = true } +http.workspace = true +object_store.workspace = true +snowflaked.workspace = true +tokio-tungstenite = { workspace = true, optional = true } +axum = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } +url.workspace = true +tokio-util.workspace = true +tower-http = { workspace = true, optional = true } # Logging and tracing: -tracing = {version = "0.1", features = ["log"] } -tracing-subscriber = {version = "0.3", features = ["std", "env-filter"] } -sentry = {version = "0.48", optional = true, default-features=false, features=["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"]} -sentry-tracing = {version = "0.48", optional = true} +tracing.workspace = true +tracing-subscriber.workspace = true +sentry = { workspace = true, optional = true } +sentry-tracing = { workspace = true, optional = true } # Exporting traces to Opentelemetry: -opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } -opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } -opentelemetry-resource-detectors = "0.11.0" -opentelemetry-http = { version = "0.32" } -opentelemetry-otlp = { version = "0.32", optional = true, features = ["grpc-tonic"] } -tracing-opentelemetry = "0.33.0" -opentelemetry-semantic-conventions = {version = "0.32.0", features = ["semconv_experimental"], optional = true} -moro-local = "0.4.0" -thiserror = "2.0.18" -either = "1.13.0" -serde-error = "0.1.3" -backon = { version = "1.3.0", features = ["tokio-sleep"] } -rand = "0.10.1" -rustc-hash = "2.1.2" -axum-prometheus = {version = "0.10.0", optional = true} +opentelemetry.workspace = true +opentelemetry_sdk.workspace = true +opentelemetry-resource-detectors.workspace = true +opentelemetry-http.workspace = true +opentelemetry-otlp = { workspace = true, optional = true } +tracing-opentelemetry.workspace = true +opentelemetry-semantic-conventions = { workspace = true, optional = true } +moro-local.workspace = true +thiserror.workspace = true +either.workspace = true +backon.workspace = true +rand.workspace = true +rustc-hash.workspace = true +axum-prometheus = { workspace = true, optional = true } # Configuration: -clap = { version = "4.5.60", features = ["derive"] } -humantime = "2.1.0" +clap.workspace = true +humantime.workspace = true -dashmap = "6.1.0" -crossbeam-skiplist = "0.1.3" -sqlformat = "0.5.0" +dashmap.workspace = true +crossbeam-skiplist.workspace = true +sqlformat.workspace = true +workspace-hack.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -83,17 +79,7 @@ sqlformat = "0.5.0" workspace = true [dev-dependencies] -criterion = {version = "0.8", features = ["async_tokio"]} -insta = { version = "1.47.2" } -assert_matches = { version = "1.5.0" } - -# [[bench]] -# name = "chunks_select" -# harness = false - -# [[bench]] -# name = "submissions_insert" -# harness = false +insta.workspace = true [features] # Dependencies only in use by the server-logic: diff --git a/opsqueue/opsqueue.nix b/opsqueue/opsqueue.nix index 95e8ac77..c63c5610 100644 --- a/opsqueue/opsqueue.nix +++ b/opsqueue/opsqueue.nix @@ -16,6 +16,7 @@ let sources = import ../nix/sources.nix; crane = import sources.crane { pkgs = pkgs; }; craneLib = crane.overrideToolchain (pkgs: rustToolchain); + sqlitePkgConfigPath = lib.makeSearchPathOutput "dev" "lib/pkgconfig" [ pkgs.sqlite ]; # Only the files necessary to build the Rust-side and cache dependencies sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; @@ -36,7 +37,8 @@ let commonArgs = { inherit src version pname; strictDeps = true; - nativeBuildInputs = [ ]; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.sqlite ]; cargoExtraArgs = "--package opsqueue"; doCheck = true; }; @@ -52,7 +54,9 @@ craneLib.buildPackage ( # Needed for the SQLx macros: env = { - DATABASE_URL = "sqlite:///build/opsqueue/opsqueue/opsqueue_example_database_schema.db"; + DATABASE_URL = "sqlite://${src}/opsqueue/opsqueue_example_database_schema.db"; + LIBSQLITE3_SYS_USE_PKG_CONFIG = "${pkgs.sqlite.dev}"; + PKG_CONFIG_PATH = sqlitePkgConfigPath; }; } diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 17906e89..a725d138 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -309,13 +309,13 @@ pub mod test { 8, 2, CORRELATED SCALAR SUBQUERY 4 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 22, 8, LIST SUBQUERY 2 - 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 1, UNION ALL - 61, 58, SCAN chunks - 64, 58, CORRELATED SCALAR SUBQUERY 6 - 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 78, 64, LIST SUBQUERY 2 - 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: + 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 33, 22, CREATE BLOOM FILTER + 62, 1, UNION ALL + 65, 62, SCAN chunks + 68, 62, CORRELATED SCALAR SUBQUERY 6 + 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 80, 68, REUSE LIST SUBQUERY 2 "); } @@ -399,13 +399,13 @@ pub mod test { 8, 2, CORRELATED SCALAR SUBQUERY 4 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 22, 8, LIST SUBQUERY 2 - 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 1, UNION ALL - 61, 58, SCAN chunks - 64, 58, CORRELATED SCALAR SUBQUERY 6 - 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 78, 64, LIST SUBQUERY 2 - 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: + 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 33, 22, CREATE BLOOM FILTER + 62, 1, UNION ALL + 65, 62, SCAN chunks + 68, 62, CORRELATED SCALAR SUBQUERY 6 + 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 80, 68, REUSE LIST SUBQUERY 2 "); } @@ -498,27 +498,25 @@ pub mod test { 16, 4, CORRELATED SCALAR SUBQUERY 5 20, 16, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 30, 16, LIST SUBQUERY 3 - 32, 30, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 3, UNION ALL - 61, 58, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 127, 115, CORRELATED SCALAR SUBQUERY 7 - 131, 127, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 141, 127, LIST SUBQUERY 3 - 143, 141, SCAN json_each VIRTUAL TABLE INDEX 0: - 169, 114, UNION ALL - 172, 169, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 121, 109, CORRELATED SCALAR SUBQUERY 7 + 125, 121, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 133, 121, REUSE LIST SUBQUERY 3 + 153, 108, UNION ALL + 156, 153, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 197, 185, CORRELATED SCALAR SUBQUERY 7 - 201, 197, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 211, 197, LIST SUBQUERY 3 - 213, 211, SCAN json_each VIRTUAL TABLE INDEX 0: - 231, 185, CORRELATED SCALAR SUBQUERY 11 - 235, 231, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 245, 231, LIST SUBQUERY 9 - 247, 245, SCAN json_each VIRTUAL TABLE INDEX 0: - 273, 184, UNION ALL - 276, 273, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 378, 366, CORRELATED SCALAR SUBQUERY 5 - 382, 378, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 392, 378, LIST SUBQUERY 3 - 394, 392, SCAN json_each VIRTUAL TABLE INDEX 0: - 412, 366, CORRELATED SCALAR SUBQUERY 13 - 416, 412, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 426, 412, LIST SUBQUERY 9 - 428, 426, SCAN json_each VIRTUAL TABLE INDEX 0: - 454, 365, UNION ALL - 457, 454, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 557, 545, CORRELATED SCALAR SUBQUERY 7 - 561, 557, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 571, 557, LIST SUBQUERY 3 - 573, 571, SCAN json_each VIRTUAL TABLE INDEX 0: - 591, 545, CORRELATED SCALAR SUBQUERY 13 - 595, 591, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 605, 591, LIST SUBQUERY 9 - 607, 605, SCAN json_each VIRTUAL TABLE INDEX 0: - 633, 544, UNION ALL - 636, 633, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 213, 201, CORRELATED SCALAR SUBQUERY 7 + 217, 213, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 227, 213, LIST SUBQUERY 3 + 230, 227, SCAN json_each VIRTUAL TABLE INDEX 0: + 238, 227, CREATE BLOOM FILTER + 251, 201, CORRELATED SCALAR SUBQUERY 11 + 255, 251, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 265, 251, LIST SUBQUERY 9 + 268, 265, SCAN json_each VIRTUAL TABLE INDEX 0: + 276, 265, CREATE BLOOM FILTER + 297, 200, UNION ALL + 300, 297, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 410, 398, CORRELATED SCALAR SUBQUERY 5 + 414, 410, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 424, 410, LIST SUBQUERY 3 + 427, 424, SCAN json_each VIRTUAL TABLE INDEX 0: + 435, 424, CREATE BLOOM FILTER + 448, 398, CORRELATED SCALAR SUBQUERY 13 + 452, 448, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 462, 448, LIST SUBQUERY 9 + 465, 462, SCAN json_each VIRTUAL TABLE INDEX 0: + 473, 462, CREATE BLOOM FILTER + 494, 397, UNION ALL + 497, 494, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 605, 593, CORRELATED SCALAR SUBQUERY 7 + 609, 605, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 619, 605, LIST SUBQUERY 3 + 622, 619, SCAN json_each VIRTUAL TABLE INDEX 0: + 630, 619, CREATE BLOOM FILTER + 643, 593, CORRELATED SCALAR SUBQUERY 13 + 647, 643, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 657, 643, LIST SUBQUERY 9 + 660, 657, SCAN json_each VIRTUAL TABLE INDEX 0: + 668, 657, CREATE BLOOM FILTER + 689, 592, UNION ALL + 692, 689, SEARCH chunks USING INDEX random_chunks_order (random_order