diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a01744937dd..7e77246e316 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,25 +70,22 @@ jobs: echo "Merge queue commit ${GITHUB_SHA} differs from PR #${pr_number} head ${pr_head_sha}; running CI normally." fi - smoketests: + upload-build-artifacts-linux: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} - name: Smoketests (${{ matrix.name }}) - strategy: - matrix: - include: - - name: Linux - runner: spacetimedb-new-runner-2 - - name: Windows - runner: spacetimedb-windows-runner - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 + name: Upload build artifacts (Linux) + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 15 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full - SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp - steps: - - name: Find Git ref + ARTIFACT_SUFFIX: linux + EXE_SUFFIX: "" + TARGET_TRIPLE: x86_64-unknown-linux-gnu + OPENSSL_CACHE_SUFFIX: "" + steps: &upload-build-artifact-steps + - &find-git-ref + name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash @@ -101,24 +98,225 @@ jobs: fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - name: Checkout sources + - &checkout-sources + name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain + - &set-default-rust-toolchain + name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 + - &verify-openssl-assembler + name: Verify OpenSSL assembler + if: runner.os == 'Windows' + shell: pwsh + run: nasm -v + + # v1 is a manual schema for invalidating this cache if its layout changes. + # The v8 version is fixed and needs to be manually kept in sync with Cargo.lock. + # It should hardly ever be updated, so prefer this over more auto-derivation machinery. + # The target triple identifies OS and ABI. + # CI always uses release with default features, so no other inputs are needed. + - &cache-rusty-v8 + name: Cache rusty_v8 + uses: actions/cache@v4 with: - workspaces: ${{ github.workspace }} - shared-key: spacetimedb - cache-on-failure: false - cache-all-crates: true - cache-workspace-crates: true - prefix-key: v1 + path: ${{ env.CARGO_TARGET_DIR }}/release/gn_out/obj + key: rusty-v8-v1-145.0.0-${{ env.TARGET_TRIPLE }} + + # v1 is a manual schema for invalidating this cache if its layout changes. + # The openssl-src version in fixed and needs to be manually kept in sync with Cargo.lock. + # It should hardly ever be updated, so prefer this over more auto-derivation machinery. + # The Windows jobs force NASM; all other OpenSSL options are fixed by openssl-src/CI, + # so these inputs fully identify the native installation. + - &cache-openssl + name: Cache OpenSSL + id: cache-openssl + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.ci-cache/openssl + key: openssl-v1-300.5.3+3.5.4-${{ env.TARGET_TRIPLE }}${{ env.OPENSSL_CACHE_SUFFIX }} + + - &configure-cached-openssl + name: Configure cached OpenSSL + if: steps.cache-openssl.outputs.cache-hit == 'true' + shell: bash + run: | + # Reuse the cached vendored output as a prebuilt OpenSSL installation. + echo "OPENSSL_NO_VENDOR=1" >>"$GITHUB_ENV" + echo "OPENSSL_DIR=${{ github.workspace }}/.ci-cache/openssl" >>"$GITHUB_ENV" + + - name: Build CLI and standalone + shell: bash + run: | + cargo build --timings --release \ + -p spacetimedb-cli \ + -p spacetimedb-standalone \ + --features spacetimedb-standalone/allow_loopback_http_for_tests + + - &prepare-openssl-cache + name: Prepare OpenSSL cache + if: steps.cache-openssl.outputs.cache-hit != 'true' + shell: bash + run: | + shopt -s nullglob + openssl_installs=(target/release/build/openssl-sys-*/out/openssl-build/install) + if (( ${#openssl_installs[@]} != 1 )); then + echo "Expected one vendored OpenSSL installation, found ${#openssl_installs[@]}." + exit 1 + fi + mkdir -p .ci-cache/openssl + cp -R "${openssl_installs[0]}/." .ci-cache/openssl/ + + - name: Package build artifacts + shell: bash + run: | + tar -czf build-support.tar.gz \ + "target/release/spacetimedb-cli${EXE_SUFFIX}" \ + "target/release/spacetimedb-standalone${EXE_SUFFIX}" + + - name: Upload Cargo timing reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: build-artifacts-cargo-timings-${{ env.ARTIFACT_SUFFIX }} + path: ${{ github.workspace }}/target/cargo-timings/ + if-no-files-found: warn + overwrite: true + retention-days: 14 + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts-${{ env.ARTIFACT_SUFFIX }} + path: build-support.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 14 + + upload-build-artifacts-windows: + needs: [merge_queue_noop, lints] + if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + name: Upload build artifacts (Windows) + runs-on: spacetimedb-windows-runner + timeout-minutes: 15 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: windows + EXE_SUFFIX: .exe + TARGET_TRIPLE: x86_64-pc-windows-msvc + OPENSSL_CACHE_SUFFIX: -nasm + OPENSSL_RUST_USE_NASM: "1" + steps: *upload-build-artifact-steps + + smoketest_build_linux: + needs: [upload-build-artifacts-linux] + name: Build smoketests (Linux) + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 15 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + EXE_SUFFIX: "" + TARGET_TRIPLE: x86_64-unknown-linux-gnu + OPENSSL_CACHE_SUFFIX: "" + steps: &smoketest-build-steps + - *find-git-ref + + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + + - *verify-openssl-assembler + - *cache-rusty-v8 + - *cache-openssl + - *configure-cached-openssl + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Build smoketest dependencies and archive test binaries + shell: bash + run: | + cargo ci smoketests archive --archive-file smoketest-nextest.tar.zst + + shopt -s nullglob + precompiled_modules=(target/wasm32-unknown-unknown/release/smoketest_module_*.wasm) + if (( ${#precompiled_modules[@]} == 0 )); then + echo "No precompiled smoketest modules were produced." + exit 1 + fi + + tar -czf smoketest-support.tar.gz \ + "target/debug/ci${EXE_SUFFIX}" \ + "${precompiled_modules[@]}" + + - *prepare-openssl-cache + + - name: Upload Cargo timing reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoketest-build-cargo-timings-${{ env.ARTIFACT_SUFFIX }} + path: ${{ github.workspace }}/target/cargo-timings/ + if-no-files-found: warn + overwrite: true + retention-days: 14 + + - name: Upload smoketest build + uses: actions/upload-artifact@v4 + with: + name: smoketest-build-${{ env.ARTIFACT_SUFFIX }} + path: | + smoketest-nextest.tar.zst + smoketest-support.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 14 + + smoketest_build_windows: + needs: [upload-build-artifacts-windows] + name: Build smoketests (Windows) + runs-on: spacetimedb-windows-runner + timeout-minutes: 15 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: windows + EXE_SUFFIX: .exe + TARGET_TRIPLE: x86_64-pc-windows-msvc + OPENSSL_CACHE_SUFFIX: -nasm + OPENSSL_RUST_USE_NASM: "1" + steps: *smoketest-build-steps + + smoketest_partitions_linux: + needs: [smoketest_build_linux] + name: Smoketests (Linux ${{ matrix.partition }}/2) + strategy: + fail-fast: false + matrix: + partition: [1, 2] + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 30 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp + ARTIFACT_SUFFIX: linux + PARTITION_COUNT: 2 + steps: &smoketest-partition-steps + - *find-git-ref + + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain - uses: actions/setup-dotnet@v4 with: @@ -153,7 +351,7 @@ jobs: .\emsdk install 4.0.21 .\emsdk activate 4.0.21 - - name: Install psql (Windows) + - name: Install psql if: runner.os == 'Windows' shell: pwsh run: | @@ -167,6 +365,7 @@ jobs: - name: Update dotnet workloads if: runner.os == 'Windows' + shell: pwsh run: | # Fail properly if any individual command fails $ErrorActionPreference = 'Stop' @@ -191,53 +390,77 @@ jobs: cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. - # ChatGPT suspects that this could be due to different build invocations using the same target dir, - # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with - # `cargo build --manifest-path` (which apparently build different dependency trees). - # However, we've been unable to fix it so... /shrug - - name: Check v8 outputs + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-artifacts-${{ env.ARTIFACT_SUFFIX }} + path: build-artifacts + + - name: Extract build artifacts shell: bash run: | - find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true - if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean --release -p v8 || true - cargo build --release -p v8 + tar -xzf build-artifacts/build-support.tar.gz + exe_suffix="" + if [[ "${RUNNER_OS}" == "Windows" ]]; then + exe_suffix=".exe" fi + test -f "${CARGO_TARGET_DIR}/release/spacetimedb-cli${exe_suffix}" + test -f "${CARGO_TARGET_DIR}/release/spacetimedb-standalone${exe_suffix}" - - name: Install cargo-nextest - uses: taiki-e/install-action@nextest + - name: Download smoketest build + uses: actions/download-artifact@v4 + with: + name: smoketest-build-${{ env.ARTIFACT_SUFFIX }} - # --test-threads=1 eliminates contention in the C# tests where they fight over bindings - # build artifacts. - # It also seemed to improve performance a fair amount (11m -> 6m) - - name: Run smoketests (Linux) + - name: Extract smoketest support files + shell: bash + run: tar -xzf smoketest-support.tar.gz + + # Serial execution avoids contention between the C# tests over generated bindings. + - name: Run smoketest partition (Linux) if: runner.os == 'Linux' shell: bash run: | if [ -f ~/emsdk/emsdk_env.sh ]; then source ~/emsdk/emsdk_env.sh fi - cargo ci smoketests -- --test-threads=1 + ./target/debug/ci smoketests run-archive \ + --archive-file smoketest-nextest.tar.zst \ + -- \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} - # Due to Emscripten PATH issues this was separated to make sure OpenSSL still builds correctly - - name: Run smoketests (Windows) + # Due to Emscripten PATH issues this remains separate from the Linux command. + - name: Run smoketest partition (Windows) if: runner.os == 'Windows' shell: pwsh run: | if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") { & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null } - cargo ci smoketests -- --test-threads=1 - - - name: Upload Cargo timing reports - if: always() - uses: actions/upload-artifact@v4 - with: - name: cargo-timings-smoketests-${{ matrix.name }} - path: ${{ github.workspace }}/target/cargo-timings/ - retention-days: 30 + .\target\debug\ci.exe smoketests run-archive ` + --archive-file smoketest-nextest.tar.zst ` + -- ` + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} + + smoketest_partitions_windows: + needs: [smoketest_build_windows] + name: Smoketests (Windows ${{ matrix.partition }}/4) + strategy: + fail-fast: false + matrix: + partition: [1, 2, 3, 4] + runs-on: spacetimedb-windows-runner + timeout-minutes: 30 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp + ARTIFACT_SUFFIX: windows + PARTITION_COUNT: 4 + steps: *smoketest-partition-steps # this is a no-op version of the above check with a trivially-passing body. # we can't just let the check be entirely skipped because each matrix target is a required check, @@ -249,8 +472,12 @@ jobs: strategy: matrix: include: - - name: Linux - - name: Windows + - name: Linux 1/2 + - name: Linux 2/2 + - name: Windows 1/4 + - name: Windows 2/4 + - name: Windows 3/4 + - name: Windows 4/4 runs-on: ubuntu-latest steps: - name: Skip duplicate merge queue smoketest @@ -451,7 +678,7 @@ jobs: with: name: cargo-timings-public-lints path: ${{ github.workspace }}/target/cargo-timings/ - retention-days: 30 + retention-days: 14 codeowners_check: if: ${{ github.event_name == 'pull_request' }} diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 5b90de38a03..5c90ae988d2 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -56,7 +56,6 @@ pub mod modules; use anyhow::{bail, Context, Result}; use regex::Regex; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; -use std::env; use std::fs; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; diff --git a/crates/smoketests/tests/smoketests/default_module_clippy.rs b/crates/smoketests/tests/smoketests/default_module_clippy.rs index f32b0fd94e2..73e76f40357 100644 --- a/crates/smoketests/tests/smoketests/default_module_clippy.rs +++ b/crates/smoketests/tests/smoketests/default_module_clippy.rs @@ -1,17 +1,8 @@ //! These tests verify that the Rust module templates have no clippy warnings. -use std::path::PathBuf; +use spacetimedb_smoketests::workspace_root; use std::process::Command; -fn workspace_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf() -} - /// Run clippy on a template's spacetimedb module directory. /// Both templates use workspace dependencies, so they can be checked in place. fn check_template_clippy(template_name: &str) { diff --git a/crates/smoketests/tests/smoketests/namespaces.rs b/crates/smoketests/tests/smoketests/namespaces.rs index 3d632edd7e2..971de3ae027 100644 --- a/crates/smoketests/tests/smoketests/namespaces.rs +++ b/crates/smoketests/tests/smoketests/namespaces.rs @@ -1,15 +1,6 @@ -use spacetimedb_smoketests::Smoketest; +use spacetimedb_smoketests::{workspace_root, Smoketest}; use std::fs; -use std::path::{Path, PathBuf}; - -fn workspace_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf() -} +use std::path::Path; /// Count occurrences of a needle string in all .cs files under a directory fn count_matches(dir: &Path, needle: &str) -> usize { diff --git a/crates/smoketests/tests/smoketests/permissions.rs b/crates/smoketests/tests/smoketests/permissions.rs index 35a9f8a7c37..36e0b0dcd1e 100644 --- a/crates/smoketests/tests/smoketests/permissions.rs +++ b/crates/smoketests/tests/smoketests/permissions.rs @@ -1,14 +1,4 @@ -use spacetimedb_smoketests::Smoketest; -use std::path::PathBuf; - -fn workspace_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf() -} +use spacetimedb_smoketests::{workspace_root, Smoketest}; /// Ensure that anyone has the permission to call any standard reducer #[test] diff --git a/tools/ci/src/smoketest.rs b/tools/ci/src/smoketest.rs index 2a35b91ca59..9c7bd69e7db 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/src/smoketest.rs @@ -4,7 +4,7 @@ use clap::{Args, Subcommand}; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; use std::ffi::OsStr; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::{env, fs}; use tempfile::TempDir; @@ -36,6 +36,7 @@ pub struct SmoketestsArgs { #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] auth_host: Option, + /// Run .NET smoketests. #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] dotnet: bool, @@ -46,11 +47,29 @@ pub struct SmoketestsArgs { #[derive(Subcommand)] enum SmoketestCmd { - /// Only build binaries without running tests + /// Local helper: only build binaries without running tests. /// /// Use this before running `cargo test --all` to ensure binaries are built. Prepare, CheckModList, + + /// CI build job: build dependencies and archive the smoketest binaries. + Archive { + /// Path to the nextest archive to create. + #[arg(long)] + archive_file: PathBuf, + }, + + /// CI partition job: run smoketests from an existing nextest archive. + RunArchive { + /// Path to the nextest archive to run. + #[arg(long)] + archive_file: PathBuf, + + /// Additional arguments to pass to nextest. + #[arg(trailing_var_arg = true)] + args: Vec, + }, } pub fn run(args: SmoketestsArgs) -> Result<()> { @@ -66,6 +85,8 @@ pub fn run(args: SmoketestsArgs) -> Result<()> { eprintln!("smoketests/mod.rs is up to date."); Ok(()) } + Some(SmoketestCmd::Archive { archive_file }) => archive_smoketests(&archive_file), + Some(SmoketestCmd::RunArchive { archive_file, args }) => run_smoketest_archive(&archive_file, args), None => run_smoketest(args.server, args.dotnet, args.auth_host.as_deref(), args.args), } } @@ -142,6 +163,58 @@ fn build_precompiled_modules() -> Result<()> { Ok(()) } +fn archive_smoketests(archive_file: &Path) -> Result<()> { + build_precompiled_modules()?; + + let status = Command::new("cargo") + .args([ + "nextest", + "archive", + "--release", + "--timings", + "-p", + "spacetimedb-smoketests", + "--archive-file", + ]) + .arg(archive_file) + .status()?; + ensure!(status.success(), "Failed to archive smoketests"); + Ok(()) +} + +// TODO: Share smoketest setup and cleanup with `run_smoketest` so the archive +// and local execution paths cannot drift. +fn run_smoketest_archive(archive_file: &Path, args: Vec) -> Result<()> { + let workspace_root = env::current_dir()?; + let archive_file = if archive_file.is_absolute() { + archive_file.to_path_buf() + } else { + workspace_root.join(archive_file) + }; + + // CI supplies the release CLI and standalone through the shared build artifact. + let cli_path = ensure_binaries_built(); + let base_config_dir = prepare_base_config(&cli_path, None, None)?; + let base_config_path = base_config_dir.path().join("config.toml"); + + let mut cmd = Command::new("cargo"); + set_env(&mut cmd, None, true, false, &base_config_path); + cmd.args(["nextest", "run", "--archive-file"]) + .arg(archive_file) + .args(["--workspace-remap"]) + .arg(&workspace_root) + .args(["--no-fail-fast", "--no-tests", "pass", "-j", "1"]) + .args(args); + + ensure!(cmd.status()?.success(), "Tests failed"); + let diff_status = cmd!("bash", "tools/check-diff.sh", "crates/smoketests").run()?; + ensure!( + diff_status.status.success(), + "There is a diff in the smoketests directory." + ); + Ok(()) +} + /// Default parallelism for smoketests. /// 16 was found to be optimal - higher values cause OS scheduler overhead. const DEFAULT_PARALLELISM: &str = "16";