From 332bc6e590a302858d63b1f893756240f3c2434e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:16:44 -0700 Subject: [PATCH 01/93] lite implementation of validation infra with placeholders for image names --- .github/workflows/Build.Linux.Job.yml | 47 +- .github/workflows/Build.MacOS.Job.yml | 12 + .github/workflows/Build.Windows.Job.yml | 43 +- .github/workflows/Build.yml | 38 ++ .github/workflows/Scheduled.E2E.yml | 41 ++ .github/workflows/Test.Matrix.Job.yml | 189 ++++++ .github/workflows/hyperlight-e2e.yml | 2 + .github/workflows/microvm-e2e.yml | 2 + scripts/ci/resolve-test-matrix.mjs | 284 +++++++++ scripts/ci/test-matrix.json | 565 ++++++++++++++++++ scripts/ci/tests/resolve-test-matrix.test.mjs | 124 ++++ tests/scripts/README.md | 34 +- tests/scripts/run_ci_backend_tests.ps1 | 141 +++++ tests/scripts/run_ci_backend_tests.sh | 59 ++ 14 files changed, 1572 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/Scheduled.E2E.yml create mode 100644 .github/workflows/Test.Matrix.Job.yml create mode 100644 scripts/ci/resolve-test-matrix.mjs create mode 100644 scripts/ci/test-matrix.json create mode 100644 scripts/ci/tests/resolve-test-matrix.test.mjs create mode 100644 tests/scripts/run_ci_backend_tests.ps1 create mode 100644 tests/scripts/run_ci_backend_tests.sh diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index cdb4108b2..c9a3d7d77 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -13,9 +13,11 @@ jobs: - arch: x64 runner: ubuntu-latest target: x86_64-unknown-linux-gnu + features: hyperlight microvm - arch: arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu + features: '' runs-on: ${{ matrix.runner }} defaults: run: @@ -54,12 +56,26 @@ jobs: run: sudo apt-get update && sudo apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu - name: Build lxc - run: cargo build --locked --release --target ${{ matrix.target }} - --no-default-features --features hyperlight + shell: bash + run: | + set -euo pipefail + features=() + if [[ -n "${{ matrix.features }}" ]]; then + features=(--features "${{ matrix.features }}") + fi + cargo build --locked --release --target "${{ matrix.target }}" \ + --no-default-features "${features[@]}" - name: Test lxc - run: cargo test --locked --release --target ${{ matrix.target }} - --no-default-features --features hyperlight + shell: bash + run: | + set -euo pipefail + features=() + if [[ -n "${{ matrix.features }}" ]]; then + features=(--features "${{ matrix.features }}") + fi + cargo test --locked --release --target "${{ matrix.target }}" \ + --no-default-features "${features[@]}" # Bubblewrap is required to run the executor characterization tests in # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec @@ -103,6 +119,25 @@ jobs: run: cargo build --locked --release --target ${{ matrix.target }} -p unix_test_proxy + - name: Verify artifact payload + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + bin_dir="src/target/${{ matrix.target }}/release" + required=(lxc-exec unix-test-proxy) + if [[ "${{ matrix.arch }}" == "x64" ]]; then + required+=(nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf) + fi + missing=() + for file in "${required[@]}"; do + [[ -f "$bin_dir/$file" ]] || missing+=("$file") + done + if (( ${#missing[@]} > 0 )); then + printf 'Missing artifact files: %s\n' "${missing[*]}" >&2 + exit 1 + fi + - name: Upload binaries uses: actions/upload-artifact@v4 with: @@ -110,5 +145,9 @@ jobs: path: | src/target/${{ matrix.target }}/release/lxc-exec src/target/${{ matrix.target }}/release/unix-test-proxy + src/target/${{ matrix.target }}/release/nanvixd.elf + src/target/${{ matrix.target }}/release/nanvix_rootfs.img + src/target/${{ matrix.target }}/release/python3.initrd + src/target/${{ matrix.target }}/release/bin/kernel.elf if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/Build.MacOS.Job.yml b/.github/workflows/Build.MacOS.Job.yml index 8e36298b1..bba76bc11 100644 --- a/.github/workflows/Build.MacOS.Job.yml +++ b/.github/workflows/Build.MacOS.Job.yml @@ -55,6 +55,18 @@ jobs: run: cargo test --locked --release --target aarch64-apple-darwin -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests + - name: Verify artifact payload + shell: bash + run: | + set -euo pipefail + bin_dir="target/aarch64-apple-darwin/release" + for file in mxc-exec-mac unix-test-proxy; do + if [[ ! -f "$bin_dir/$file" ]]; then + echo "Missing artifact file: $file" >&2 + exit 1 + fi + done + - name: Upload binaries uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index fb06cea3c..523e42c8b 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -18,8 +18,8 @@ jobs: - arch: arm64 runner: windows-11-arm target: aarch64-pc-windows-msvc - # nanvixd.exe is x64-only, so microvm tests can't run on arm64. - features: hyperlight isolation_session wslc + # Hyperlight and MicroVM runtimes are x64-only. + features: isolation_session wslc runs-on: ${{ matrix.runner }} defaults: run: @@ -71,6 +71,35 @@ jobs: --no-default-features --features "${{ matrix.features }}" + - name: Verify artifact payload + shell: pwsh + run: | + $binDir = "target\${{ matrix.target }}\release" + $required = @( + "wxc-exec.exe", + "wxc-host-prep.exe", + "winhttp-proxy-shim.exe", + "wxc-windows-sandbox-daemon.exe", + "wxc-windows-sandbox-guest.exe", + "wxc-test-proxy.exe", + "wxc-test-driver.exe", + "wxc-ui-probe.exe", + "mxc-diagnostic-console.exe", + "wslcsdk.dll" + ) + if ("${{ matrix.arch }}" -eq "x64") { + $required += @( + "nanvixd.exe", + "nanvix_rootfs.img", + "python3.initrd", + "bin\kernel.elf" + ) + } + $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } + if ($missing) { + throw "Missing artifact files: $($missing -join ', ')" + } + - name: Upload binaries uses: actions/upload-artifact@v4 with: @@ -82,7 +111,15 @@ jobs: src/target/${{ matrix.target }}/release/wxc-windows-sandbox-daemon.exe src/target/${{ matrix.target }}/release/wxc-windows-sandbox-guest.exe src/target/${{ matrix.target }}/release/wxc-test-proxy.exe + src/target/${{ matrix.target }}/release/wxc-test-driver.exe + src/target/${{ matrix.target }}/release/wxc-ui-probe.exe src/target/${{ matrix.target }}/release/mxc-diagnostic-console.exe src/target/${{ matrix.target }}/release/wslcsdk.dll - if-no-files-found: warn + src/target/${{ matrix.target }}/release/nanvixd.exe + src/target/${{ matrix.target }}/release/nanvix_rootfs.img + src/target/${{ matrix.target }}/release/python3.initrd + src/target/${{ matrix.target }}/release/bin/kernel.elf + src/target/${{ matrix.target }}/release/snapshots/kernel.vmem + src/target/${{ matrix.target }}/release/snapshots/kernel.whp.cbor + if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index fbcd36f0a..87e0c2f9b 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -18,9 +18,40 @@ concurrency: cancel-in-progress: true permissions: + actions: read contents: read jobs: + pr-test-eligibility: + name: Check PR test eligibility + if: github.event_name == 'pull_request' && github.base_ref == 'main' + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.changes.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check for non-documentation changes + id: changes + shell: bash + run: | + set -euo pipefail + should_run=false + while IFS= read -r file; do + case "$file" in + docs/*|*.md) ;; + *) + should_run=true + break + ;; + esac + done < <(git diff --name-only \ + '${{ github.event.pull_request.base.sha }}' \ + '${{ github.event.pull_request.head.sha }}') + echo "should_run=$should_run" >> "$GITHUB_OUTPUT" + # Dependency feed check: resolve the locked graph through the public MxcDependencies # feed so a crate not yet in the feed fails at PR time. dependency-feed-check: @@ -45,6 +76,13 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Build.MacOS.Job.yml + pr-test-matrix: + needs: [pr-test-eligibility, windows, linux, macos] + if: needs.pr-test-eligibility.outputs.should_run == 'true' + uses: ./.github/workflows/Test.Matrix.Job.yml + with: + plan: pr + sdk-unit-tests: uses: ./.github/workflows/SDK.Unit.Test.Job.yml diff --git a/.github/workflows/Scheduled.E2E.yml b/.github/workflows/Scheduled.E2E.yml new file mode 100644 index 000000000..6e36b2eb0 --- /dev/null +++ b/.github/workflows/Scheduled.E2E.yml @@ -0,0 +1,41 @@ +name: Scheduled E2E + +on: + schedule: + - cron: '0 8 * * 1-6' + - cron: '0 8 * * 0' + workflow_dispatch: + inputs: + plan: + description: Test plan to run. + required: true + default: nightly + type: choice + options: + - pr + - nightly + - weekly + +concurrency: + group: test-matrix-e2e-${{ github.ref }}-${{ github.event.schedule || inputs.plan }} + cancel-in-progress: false + +permissions: + actions: read + contents: read + +jobs: + windows: + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + uses: ./.github/workflows/Build.MacOS.Job.yml + + test: + needs: [windows, linux, macos] + uses: ./.github/workflows/Test.Matrix.Job.yml + with: + plan: ${{ github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0' && 'weekly' || github.event_name == 'schedule' && 'nightly' || inputs.plan }} diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml new file mode 100644 index 000000000..12f64b4b0 --- /dev/null +++ b/.github/workflows/Test.Matrix.Job.yml @@ -0,0 +1,189 @@ +name: Test Matrix + +on: + workflow_call: + inputs: + plan: + description: Test plan to resolve. + required: true + type: string + +permissions: + actions: read + contents: read + +jobs: + resolve: + name: Resolve ${{ inputs.plan }} matrix + runs-on: ubuntu-latest + outputs: + windows: ${{ steps.matrix.outputs.windows }} + linux: ${{ steps.matrix.outputs.linux }} + macos: ${{ steps.matrix.outputs.macos }} + has_windows: ${{ steps.matrix.outputs.has_windows }} + has_linux: ${{ steps.matrix.outputs.has_linux }} + has_macos: ${{ steps.matrix.outputs.has_macos }} + steps: + - uses: actions/checkout@v4 + + - name: Resolve test combinations + id: matrix + run: node scripts/ci/resolve-test-matrix.mjs --plan "${{ inputs.plan }}" + + windows: + name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + needs: resolve + if: needs.resolve.outputs.has_windows == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} + runs-on: + - self-hosted + - "1ES.Pool=${{ matrix.pool }}" + - "JobId=mxc-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Download ${{ matrix.target }} artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: artifacts/bin + + - name: Prepare Process Container host + if: matrix.command == 'process-container' + shell: pwsh + run: | + $hostPrep = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin\wxc-host-prep.exe' + & $hostPrep prepare-system-drive + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & $hostPrep prepare-null-device --no-sacl + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Run backend tests + shell: pwsh + run: | + $arguments = @{ + Backend = '${{ matrix.command }}' + BinaryDirectory = (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') + Architecture = '${{ matrix.architecture }}' + } + if ('${{ matrix.backend }}' -eq 'process-t1') { + $arguments.ExpectedTier = 'T1' + } elseif ('${{ matrix.backend }}' -eq 'process-t3') { + $arguments.ExpectedTier = 'T3' + } + & ./tests/scripts/run_ci_backend_tests.ps1 @arguments *>&1 | + Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') + + - name: Upload failure logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/mxc-ci.log + ${{ runner.temp }}/mxc-wpc-tests/logs + ${{ runner.temp }}/WinProcessContainer-Tests.results.* + ${{ runner.temp }}/mxc_concurrent_oneshot + if-no-files-found: ignore + retention-days: 7 + + linux: + name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + needs: resolve + if: needs.resolve.outputs.has_linux == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} + runs-on: + - self-hosted + - "1ES.Pool=${{ matrix.pool }}" + - "JobId=mxc-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Download ${{ matrix.target }} artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: artifacts/bin + + - name: Verify backend prerequisites + shell: bash + run: | + set -euo pipefail + chmod +x artifacts/bin/lxc-exec + case '${{ matrix.command }}' in + bubblewrap) + command -v bwrap + if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + ;; + lxc) + command -v lxc-start + sudo -n true + ;; + esac + + - name: Run backend tests + shell: bash + run: | + set -euo pipefail + if [[ '${{ matrix.command }}' == 'lxc' ]]; then + sudo --preserve-env=RUNNER_TEMP bash tests/scripts/run_ci_backend_tests.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee "$RUNNER_TEMP/mxc-ci.log" + else + bash tests/scripts/run_ci_backend_tests.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee "$RUNNER_TEMP/mxc-ci.log" + fi + + - name: Upload failure logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/mxc-ci.log + if-no-files-found: ignore + retention-days: 7 + + macos: + name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + needs: resolve + if: needs.resolve.outputs.has_macos == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.macos) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Download ${{ matrix.target }} artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: artifacts/bin + + - name: Run backend tests + shell: bash + run: | + set -euo pipefail + chmod +x artifacts/bin/mxc-exec-mac artifacts/bin/unix-test-proxy + bash tests/scripts/run_ci_backend_tests.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee "$RUNNER_TEMP/mxc-ci.log" + + - name: Upload failure logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/mxc-ci.log + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/hyperlight-e2e.yml b/.github/workflows/hyperlight-e2e.yml index 203612765..7e66db8c1 100644 --- a/.github/workflows/hyperlight-e2e.yml +++ b/.github/workflows/hyperlight-e2e.yml @@ -1,5 +1,7 @@ name: Hyperlight E2E Tests +# Retained until the unified test matrix has equivalent Hyperlight coverage. + on: push: branches: [main] diff --git a/.github/workflows/microvm-e2e.yml b/.github/workflows/microvm-e2e.yml index d53ffed00..eca8c12da 100644 --- a/.github/workflows/microvm-e2e.yml +++ b/.github/workflows/microvm-e2e.yml @@ -1,5 +1,7 @@ name: Integration Tests +# Retained until the unified test matrix has equivalent MicroVM coverage. + on: push: branches: [main] diff --git a/scripts/ci/resolve-test-matrix.mjs b/scripts/ci/resolve-test-matrix.mjs new file mode 100644 index 000000000..02d49a6f7 --- /dev/null +++ b/scripts/ci/resolve-test-matrix.mjs @@ -0,0 +1,284 @@ +#!/usr/bin/env node + +// Validates the declarative test catalog and emits GitHub Actions matrices. +// Keeping expansion here makes the workflow YAML small and lets CI reject an +// invalid catalog before allocating any specialized test runners. + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const FAMILIES = ['windows', 'linux', 'macos']; +const PLANS = ['pr', 'nightly', 'weekly']; +const ARM64_UNSUPPORTED_BACKENDS = new Set(['hyperlight', 'microvm']); + +function combinationKey(plan, os, architecture, backend) { + return `${plan}|${os}|${architecture}|${backend}`; +} + +function assertNonEmptyString(value, label) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`${label} must be a non-empty string`); + } +} + +export function readCatalog(catalogPath) { + return JSON.parse(fs.readFileSync(catalogPath, 'utf8')); +} + +export function validateCatalog(catalog) { + if (catalog.schemaVersion !== 1) { + throw new Error(`unsupported catalog schemaVersion: ${catalog.schemaVersion}`); + } + + const platforms = new Map(); + const targets = new Set(); + for (const platform of catalog.platforms ?? []) { + assertNonEmptyString(platform.id, 'platform.id'); + assertNonEmptyString(platform.displayName, `${platform.id}.displayName`); + if (!FAMILIES.includes(platform.family)) { + throw new Error(`${platform.id} has unsupported family ${platform.family}`); + } + if (platforms.has(platform.id)) { + throw new Error(`duplicate platform id: ${platform.id}`); + } + if (platform.private === true) { + // Private platforms use neutral IDs in public matrix fields. + if (!/^windows-private-[a-z]$/.test(platform.id)) { + throw new Error(`${platform.id} must use a neutral private-platform alias`); + } + } + + const architectures = Object.entries(platform.architectures ?? {}); + if (architectures.length === 0) { + throw new Error(`${platform.id} has no architectures`); + } + + for (const [architecture, details] of architectures) { + if (!['x64', 'arm64'].includes(architecture)) { + throw new Error(`${platform.id} has unsupported architecture ${architecture}`); + } + assertNonEmptyString(details.target, `${platform.id}.${architecture}.target`); + assertNonEmptyString(details.artifact, `${platform.id}.${architecture}.artifact`); + targets.add(details.target); + + if (platform.family === 'macos') { + assertNonEmptyString(details.runner, `${platform.id}.${architecture}.runner`); + } else { + assertNonEmptyString(details.pool, `${platform.id}.${architecture}.pool`); + } + + const backends = new Set(); + for (const backend of details.backends ?? []) { + assertNonEmptyString(backend, `${platform.id}.${architecture}.backend`); + if (backends.has(backend)) { + throw new Error(`duplicate backend ${backend} on ${platform.id}/${architecture}`); + } + if (architecture === 'arm64' && ARM64_UNSUPPORTED_BACKENDS.has(backend)) { + throw new Error(`${backend} cannot be scheduled on arm64 (${platform.id})`); + } + if (!catalog.handlers?.[platform.family]?.[backend]) { + throw new Error(`missing ${platform.family} handler entry for ${backend}`); + } + backends.add(backend); + } + } + platforms.set(platform.id, platform); + } + + const expectedTargets = new Set([ + 'aarch64-apple-darwin', + 'aarch64-pc-windows-msvc', + 'aarch64-unknown-linux-gnu', + 'x86_64-pc-windows-msvc', + 'x86_64-unknown-linux-gnu' + ]); + if (targets.size !== expectedTargets.size + || [...expectedTargets].some(target => !targets.has(target))) { + throw new Error(`catalog targets do not match the five required build targets`); + } + + // Trigger entries name an OS/backend pair. Architecture expansion happens + // later, so a backend is valid here when at least one OS architecture has it. + for (const plan of PLANS) { + const seenRequests = new Set(); + for (const request of catalog.triggers?.[plan] ?? []) { + const platform = platforms.get(request.os); + if (!platform) { + throw new Error(`${plan} references unknown platform ${request.os}`); + } + for (const backend of request.backends ?? []) { + const requestKey = `${request.os}|${backend}`; + if (seenRequests.has(requestKey)) { + throw new Error(`duplicate ${plan} request ${requestKey}`); + } + const supported = Object.values(platform.architectures) + .some(details => details.backends.includes(backend)); + if (!supported) { + throw new Error(`${plan} requests unsupported ${request.os}/${backend}`); + } + seenRequests.add(requestKey); + } + } + } + + const enabled = new Set(); + for (const entry of catalog.enabled ?? []) { + if (!PLANS.includes(entry.plan)) { + throw new Error(`enabled entry has unsupported plan ${entry.plan}`); + } + const platform = platforms.get(entry.os); + const architecture = platform?.architectures?.[entry.architecture]; + if (!architecture?.backends?.includes(entry.backend)) { + throw new Error( + `enabled entry is unsupported: ${entry.os}/${entry.architecture}/${entry.backend}` + ); + } + const requested = (catalog.triggers?.[entry.plan] ?? []) + .some(request => request.os === entry.os && request.backends.includes(entry.backend)); + if (!requested) { + throw new Error( + `enabled entry is not present in ${entry.plan}: ${entry.os}/${entry.backend}` + ); + } + const handler = catalog.handlers[platform.family][entry.backend]; + // Capability entries may describe future coverage, but enabled entries + // must have an executable dispatcher command for their architecture. + if (handler.status !== 'wired' || !handler.command) { + throw new Error( + `enabled entry has no wired handler: ${entry.os}/${entry.architecture}/${entry.backend}` + ); + } + if (handler.architectures && !handler.architectures.includes(entry.architecture)) { + throw new Error( + `enabled entry handler does not support ${entry.architecture}: ` + + `${entry.os}/${entry.backend}` + ); + } + const key = combinationKey(entry.plan, entry.os, entry.architecture, entry.backend); + if (enabled.has(key)) { + throw new Error(`duplicate enabled entry: ${key}`); + } + enabled.add(key); + } + + return { platforms, enabled }; +} + +export function expandPlan(catalog, plan) { + if (!PLANS.includes(plan)) { + throw new Error(`unsupported plan: ${plan}`); + } + const { platforms } = validateCatalog(catalog); + // Sunday is one run containing the normal nightly set plus weekly additions. + const planNames = plan === 'weekly' ? ['nightly', 'weekly'] : [plan]; + const combinations = []; + + for (const planName of planNames) { + for (const request of catalog.triggers[planName]) { + const platform = platforms.get(request.os); + // A trigger is architecture-neutral. Expand it only where the platform's + // capability declaration supports the requested backend. + for (const [architecture, details] of Object.entries(platform.architectures)) { + for (const backend of request.backends) { + if (!details.backends.includes(backend)) { + continue; + } + combinations.push({ + plan: planName, + os: platform.id, + os_name: platform.displayName, + family: platform.family, + architecture, + target: details.target, + artifact: details.artifact, + pool: details.pool, + runner: details.runner, + backend, + command: catalog.handlers[platform.family][backend].command, + handler_status: catalog.handlers[platform.family][backend].status + }); + } + } + } + } + + return combinations; +} + +export function resolvePlan(catalog, plan) { + const { enabled } = validateCatalog(catalog); + const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); + + for (const combination of expandPlan(catalog, plan)) { + const key = combinationKey( + combination.plan, + combination.os, + combination.architecture, + combination.backend + ); + if (enabled.has(key)) { + // family selects the workflow job and handler_status is validation-only; + // neither belongs in the matrix consumed by the runner. + const { family, handler_status: _, ...matrixEntry } = combination; + matrices[family].push(matrixEntry); + } + } + + for (const family of FAMILIES) { + // Stable ordering keeps local output and workflow diagnostics reproducible. + matrices[family].sort((left, right) => ( + `${left.os}|${left.architecture}|${left.backend}` + .localeCompare(`${right.os}|${right.architecture}|${right.backend}`) + )); + } + return matrices; +} + +function parseArguments(argv) { + const args = { plan: undefined, catalog: undefined }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--plan') { + args.plan = argv[++index]; + } else if (argument === '--catalog') { + args.catalog = argv[++index]; + } else { + throw new Error(`unknown argument: ${argument}`); + } + } + if (!args.plan) { + throw new Error('--plan is required'); + } + return args; +} + +function writeOutputs(matrices) { + const lines = []; + for (const family of FAMILIES) { + // Empty-matrix flags let the reusable workflow skip an OS-family job + // instead of asking GitHub Actions to evaluate an empty matrix. + lines.push(`${family}=${JSON.stringify({ include: matrices[family] })}`); + lines.push(`has_${family}=${matrices[family].length > 0}`); + } + + if (process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`); + } else { + process.stdout.write(`${lines.join('\n')}\n`); + } +} + +const currentFile = fileURLToPath(import.meta.url); +if (process.argv[1] && path.resolve(process.argv[1]) === currentFile) { + try { + const args = parseArguments(process.argv.slice(2)); + const defaultCatalog = path.join(path.dirname(currentFile), 'test-matrix.json'); + const catalog = readCatalog(path.resolve(args.catalog ?? defaultCatalog)); + writeOutputs(resolvePlan(catalog, args.plan)); + } catch (error) { + process.stderr.write(`resolve-test-matrix: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json new file mode 100644 index 000000000..54c0b8f3b --- /dev/null +++ b/scripts/ci/test-matrix.json @@ -0,0 +1,565 @@ +{ + "schemaVersion": 1, + "platforms": [ + { + "id": "windows-private-a", + "displayName": "Windows Pre-release A", + "family": "windows", + "private": true, + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-private-b", + "displayName": "Windows Pre-release Isolation Session", + "family": "windows", + "private": true, + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-canary", + "displayName": "Windows canary", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session" + ] + } + } + }, + { + "id": "windows-25h2", + "displayName": "Windows 25H2", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-24h2", + "displayName": "Windows 24H2", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-23h2", + "displayName": "Windows 23H2", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "ubuntu-26.04", + "displayName": "Ubuntu 26.04", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "ubuntu-24.04", + "displayName": "Ubuntu 24.04", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "microvm", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "rhel-10.2", + "displayName": "RHEL 10.2", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "debian-13", + "displayName": "Debian 13", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "macos-26", + "displayName": "macOS 26", + "family": "macos", + "architectures": { + "arm64": { + "target": "aarch64-apple-darwin", + "artifact": "mxc-binaries-aarch64-apple-darwin", + "runner": "macos-26-xlarge", + "backends": [ + "seatbelt" + ] + } + } + }, + { + "id": "macos-15", + "displayName": "macOS 15", + "family": "macos", + "architectures": { + "arm64": { + "target": "aarch64-apple-darwin", + "artifact": "mxc-binaries-aarch64-apple-darwin", + "runner": "macos-15-xlarge", + "backends": [ + "seatbelt" + ] + } + } + } + ], + "handlers": { + "windows": { + "process-t1": { + "command": "process-container", + "status": "wired" + }, + "process-t3": { + "command": "process-container", + "status": "wired" + }, + "isolation-session": { + "command": "isolation-session", + "status": "wired" + }, + "wslc": { + "command": "wslc", + "status": "wired", + "architectures": [ + "x64" + ] + }, + "windows-sandbox": { + "command": "windows-sandbox", + "status": "wired" + }, + "microvm": { + "command": "microvm", + "status": "wired" + }, + "hyperlight": { + "status": "placeholder" + } + }, + "linux": { + "bubblewrap": { + "command": "bubblewrap", + "status": "wired" + }, + "lxc": { + "command": "lxc", + "status": "wired" + }, + "microvm": { + "status": "placeholder" + }, + "hyperlight": { + "status": "placeholder" + } + }, + "macos": { + "seatbelt": { + "status": "placeholder" + } + } + }, + "triggers": { + "pr": [ + { + "os": "windows-canary", + "backends": [ + "process-t1" + ] + }, + { + "os": "windows-25h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "macos-26", + "backends": [ + "seatbelt" + ] + }, + { + "os": "ubuntu-26.04", + "backends": [ + "bubblewrap" + ] + } + ], + "nightly": [ + { + "os": "windows-private-b", + "backends": [ + "isolation-session" + ] + }, + { + "os": "windows-private-a", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "windows-canary", + "backends": [ + "process-t1" + ] + }, + { + "os": "windows-25h2", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "windows-24h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "windows-23h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "macos-26", + "backends": [ + "seatbelt" + ] + }, + { + "os": "ubuntu-26.04", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + { + "os": "ubuntu-24.04", + "backends": [ + "microvm" + ] + }, + { + "os": "rhel-10.2", + "backends": [ + "bubblewrap" + ] + } + ], + "weekly": [ + { + "os": "windows-canary", + "backends": [ + "isolation-session" + ] + }, + { + "os": "windows-24h2", + "backends": [ + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "windows-23h2", + "backends": [ + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "macos-15", + "backends": [ + "seatbelt" + ] + }, + { + "os": "ubuntu-24.04", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + { + "os": "rhel-10.2", + "backends": [ + "hyperlight", + "lxc" + ] + }, + { + "os": "debian-13", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + } + ] + }, + "enabled": [ + { + "plan": "pr", + "os": "windows-canary", + "architecture": "x64", + "backend": "process-t1" + }, + { + "plan": "pr", + "os": "ubuntu-26.04", + "architecture": "x64", + "backend": "bubblewrap" + }, + { + "plan": "nightly", + "os": "windows-private-b", + "architecture": "x64", + "backend": "isolation-session" + }, + { + "plan": "nightly", + "os": "ubuntu-26.04", + "architecture": "x64", + "backend": "lxc" + }, + { + "plan": "weekly", + "os": "windows-24h2", + "architecture": "x64", + "backend": "windows-sandbox" + }, + { + "plan": "weekly", + "os": "debian-13", + "architecture": "x64", + "backend": "bubblewrap" + } + ] +} diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs new file mode 100644 index 000000000..cc4cd715e --- /dev/null +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + expandPlan, + readCatalog, + resolvePlan, + validateCatalog +} from '../resolve-test-matrix.mjs'; + +// Tests load a fresh catalog for each case so negative mutations cannot leak +// into later assertions. +const testDirectory = path.dirname(fileURLToPath(import.meta.url)); +const catalogPath = path.resolve(testDirectory, '..', 'test-matrix.json'); + +function catalog() { + return readCatalog(catalogPath); +} + +function clone(value) { + // Catalog values are JSON data, so a JSON round-trip is sufficient here. + return JSON.parse(JSON.stringify(value)); +} + +test('catalog validates and contains all five build targets', () => { + assert.doesNotThrow(() => validateCatalog(catalog())); +}); + +test('phase one enables two PR, two nightly, and four Sunday jobs', () => { + const pr = resolvePlan(catalog(), 'pr'); + const nightly = resolvePlan(catalog(), 'nightly'); + const weekly = resolvePlan(catalog(), 'weekly'); + + assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 2); + assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 2); + assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 4); +}); + +test('weekly includes all enabled nightly combinations', () => { + const nightly = resolvePlan(catalog(), 'nightly'); + const weekly = resolvePlan(catalog(), 'weekly'); + + for (const family of ['windows', 'linux', 'macos']) { + // The weekly matrix must be a superset, not a separate replacement plan. + const weeklyKeys = new Set( + weekly[family].map(entry => `${entry.os}|${entry.architecture}|${entry.backend}`) + ); + for (const entry of nightly[family]) { + assert.ok( + weeklyKeys.has(`${entry.os}|${entry.architecture}|${entry.backend}`), + `weekly is missing ${family} nightly entry ${entry.os}/${entry.backend}` + ); + } + } +}); + +test('full plan expands supported backends to both architectures', () => { + const expanded = expandPlan(catalog(), 'pr'); + const ubuntuBubblewrap = expanded + .filter(entry => entry.os === 'ubuntu-26.04' && entry.backend === 'bubblewrap'); + assert.deepEqual( + ubuntuBubblewrap.map(entry => entry.architecture).sort(), + ['arm64', 'x64'] + ); +}); + +test('arm64 never expands Hyperlight or MicroVM', () => { + for (const plan of ['pr', 'nightly', 'weekly']) { + const invalid = expandPlan(catalog(), plan) + .filter(entry => ( + entry.architecture === 'arm64' + && ['hyperlight', 'microvm'].includes(entry.backend) + )); + assert.deepEqual(invalid, []); + } +}); + +test('enabled placeholder handlers are rejected', () => { + // Seatbelt is declared in the capability map but intentionally not wired yet. + const modified = clone(catalog()); + modified.enabled.push({ + plan: 'weekly', + os: 'macos-15', + architecture: 'arm64', + backend: 'seatbelt' + }); + assert.throws( + () => validateCatalog(modified), + /enabled entry has no wired handler/ + ); +}); + +test('enabled handlers must support the selected architecture', () => { + // WSLC remains in the arm64 capability catalog while its current test + // dispatcher is explicitly restricted to x64. + const modified = clone(catalog()); + modified.enabled.push({ + plan: 'weekly', + os: 'windows-24h2', + architecture: 'arm64', + backend: 'wslc' + }); + assert.throws( + () => validateCatalog(modified), + /enabled entry handler does not support arm64/ + ); +}); + +test('duplicate enabled combinations are rejected', () => { + const modified = clone(catalog()); + modified.enabled.push(clone(modified.enabled[0])); + assert.throws( + () => validateCatalog(modified), + /duplicate enabled entry/ + ); +}); + +test('private platforms use neutral public aliases', () => { + for (const platform of catalog().platforms.filter(entry => entry.private === true)) { + assert.match(platform.id, /^windows-private-[a-z]$/); + } +}); diff --git a/tests/scripts/README.md b/tests/scripts/README.md index bc1fdeabe..a9f93129f 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -1,7 +1,7 @@ # Test Scripts -This directory contains PowerShell convenience scripts for running MXC end-to-end -tests locally on Windows. The primary Rust executor E2E path is +This directory contains convenience scripts for running MXC end-to-end tests +locally and in CI. The primary Rust executor E2E path is `cargo test -p wxc_e2e_tests`, which invokes the MXC binaries directly instead of shelling through these scripts. @@ -37,6 +37,36 @@ These scripts are local helpers. Not every script is run by CI because several depend on local OS features such as Windows Sandbox, WHP, proxy setup, or stress test duration. +## CI test matrix + +[`run_ci_backend_tests.ps1`](run_ci_backend_tests.ps1) and +[`run_ci_backend_tests.sh`](run_ci_backend_tests.sh) are thin dispatchers used +by the reusable test workflow. They consume downloaded target artifacts and +delegate to the existing backend test scripts. + +The capability map and PR, nightly, and weekly triggers are defined in +[`../../scripts/ci/test-matrix.json`](../../scripts/ci/test-matrix.json). +[`../../scripts/ci/resolve-test-matrix.mjs`](../../scripts/ci/resolve-test-matrix.mjs) +validates that catalog and emits one matrix per OS family. Weekly resolution +includes both nightly coverage and weekly additions. + +The initial rollout enables six combinations: + +- PR: Windows x64 Process Container and Ubuntu 26.04 x64 Bubblewrap. +- Nightly: Windows Pre-release Isolation Session x64 and Ubuntu 26.04 x64 LXC. +- Weekly additions: Windows 24H2 x64 Windows Sandbox and Debian 13 x64 + Bubblewrap. + +Other requested combinations remain in the catalog as disabled entries. +Backends without an artifact-only test entry point are explicit placeholders; +the resolver rejects any attempt to enable one. The standalone MicroVM and +Hyperlight workflows remain active until this matrix provides equivalent +coverage. + +PR matrix tests run for non-documentation changes targeting `main`. Nightly runs +start at 08:00 UTC Monday through Saturday. The Sunday 08:00 UTC run resolves +the weekly plan, which includes the nightly set. + ### Manual smoke tests Manual smokes are visual-inspection scripts for rendering and event-propagation diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 new file mode 100644 index 000000000..4b04df3c3 --- /dev/null +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -0,0 +1,141 @@ +<# +.SYNOPSIS +Runs a Windows backend test from a downloaded CI artifact. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateSet('process-container', 'isolation-session', 'windows-sandbox', 'wslc', 'microvm', 'hyperlight')] + [string]$Backend, + + [Parameter(Mandatory)] + [string]$BinaryDirectory, + + [Parameter(Mandatory)] + [ValidateSet('x64', 'arm64')] + [string]$Architecture, + + [ValidateSet('T1', 'T3')] + [string]$ExpectedTier +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path +$binaryDirectoryPath = (Resolve-Path -LiteralPath $BinaryDirectory).Path +$wxc = Join-Path $binaryDirectoryPath 'wxc-exec.exe' + +function Assert-File { + param([Parameter(Mandatory)][string]$Path) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Required CI artifact file is missing: $Path" + } +} + +function Invoke-TestScript { + param( + [Parameter(Mandatory)][string]$Path, + [string[]]$Arguments = @() + ) + + # PowerShell scripts do not always replace a previous native exit code. + # Reset it so a successful script cannot inherit a stale failure. + $global:LASTEXITCODE = 0 + & $Path @Arguments + if ($LASTEXITCODE -ne 0) { + throw "Backend test failed with exit code $LASTEXITCODE`: $Path" + } +} + +Assert-File -Path $wxc + +switch ($Backend) { + 'process-container' { + if (-not $ExpectedTier) { + throw 'Process Container tests require -ExpectedTier.' + } + + $expectedProbeTier = @{ + T1 = 'base-container' + T3 = 'appcontainer-dacl' + }[$ExpectedTier] + + # The existing harness expects separate debug and release layouts. CI + # intentionally tests one release artifact, so stage it in both slots. + $debugDirectory = Join-Path $binaryDirectoryPath 'debug' + $releaseDirectory = Join-Path $binaryDirectoryPath 'release' + New-Item -ItemType Directory -Force -Path $debugDirectory, $releaseDirectory | Out-Null + Copy-Item -LiteralPath $wxc -Destination (Join-Path $debugDirectory 'wxc-exec.exe') -Force + Copy-Item -LiteralPath $wxc -Destination (Join-Path $releaseDirectory 'wxc-exec.exe') -Force + + $uiProbe = Join-Path $binaryDirectoryPath 'wxc-ui-probe.exe' + Assert-File -Path $uiProbe + Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $debugDirectory 'wxc-ui-probe.exe') -Force + Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force + + # Fail before running the suite if the selected image does not provide + # the Process Container tier promised by the matrix entry. + $probe = (& $wxc --probe | Out-String) | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { + throw "Process Container probe failed with exit code $LASTEXITCODE." + } + if ($probe.tier -ne $expectedProbeTier) { + throw "Expected $ExpectedTier ($expectedProbeTier), but the host selected '$($probe.tier)'." + } + + $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1' + # Skip build and Cargo phases because this job consumes a previously + # built artifact; retain the host and containment behavior phases. + $phases = @( + 'Probes', + 'T3Forced', + 'T1DenyForced', + 'UiMitigationMatrix', + 'GlobalAtomIsolation', + 'DaclDisabled', + 'CrashRecovery' + ) + $global:LASTEXITCODE = 0 + & $script ` + -SkipBuild ` + -SkipReleaseLane ` + -WxcDebug (Join-Path $debugDirectory 'wxc-exec.exe') ` + -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') ` + -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') ` + -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') ` + -Phases $phases + if ($LASTEXITCODE -ne 0) { + throw "Process Container tests failed with exit code $LASTEXITCODE." + } + } + 'isolation-session' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @( + '-WxcExePath', $wxc + ) + } + 'windows-sandbox' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @( + '-BinDir', $binaryDirectoryPath + ) + } + 'wslc' { + # The current WSLC helper hardcodes the x64 target when locating assets. + if ($Architecture -ne 'x64') { + throw 'The existing WSLC test harness is not architecture-portable yet.' + } + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @( + '-WxcExecPath', $wxc + ) + } + 'microvm' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @( + '-BinDir', $binaryDirectoryPath + ) + } + 'hyperlight' { + # Keep unwired commands explicit so accidental activation fails loudly. + throw 'The Hyperlight CI handler is not wired to an existing backend test entry point yet.' + } +} diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh new file mode 100644 index 000000000..0980557ec --- /dev/null +++ b/tests/scripts/run_ci_backend_tests.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dispatches a downloaded Unix artifact to the repository's existing backend +# test suites. Unsupported handlers fail explicitly rather than reporting a +# false-success placeholder job. + +usage() { + echo "Usage: $0 " >&2 +} + +if [[ $# -ne 2 ]]; then + usage + exit 2 +fi + +backend="$1" +binary_directory="$(cd "$2" && pwd)" +script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_root/../.." && pwd)" +release_directory="$repo_root/src/target/release" + +case "$backend" in + bubblewrap|lxc) + ;; + microvm) + echo "The MicroVM CI handler is not wired to an artifact-only Linux test entry point yet." >&2 + exit 2 + ;; + hyperlight) + echo "The Hyperlight CI handler is not wired to an existing backend test entry point yet." >&2 + exit 2 + ;; + seatbelt) + echo "The Seatbelt CI handler is not wired to an existing backend test entry point yet." >&2 + exit 2 + ;; + *) + usage + exit 2 + ;; +esac + +# Existing shell tests locate binaries under src/target/release. Recreate that +# layout from the downloaded artifact, including adjacent runtime assets. +test -x "$binary_directory/lxc-exec" +test -f "$binary_directory/unix-test-proxy" +mkdir -p "$release_directory" +cp -a "$binary_directory/." "$release_directory/" +chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" + +case "$backend" in + bubblewrap) + "$script_root/run_bwrap_all_tests.sh" + ;; + lxc) + "$script_root/run_lxc_all_tests.sh" + ;; +esac From 03fc97e3c5b8bb778994d7dee29ebaf14f976012 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:31:57 -0700 Subject: [PATCH 02/93] aliging with similar projects over naming of prerelease --- scripts/ci/resolve-test-matrix.mjs | 8 ++++---- scripts/ci/test-matrix.json | 14 +++++++------- scripts/ci/tests/resolve-test-matrix.test.mjs | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/ci/resolve-test-matrix.mjs b/scripts/ci/resolve-test-matrix.mjs index 02d49a6f7..2be65887f 100644 --- a/scripts/ci/resolve-test-matrix.mjs +++ b/scripts/ci/resolve-test-matrix.mjs @@ -43,10 +43,10 @@ export function validateCatalog(catalog) { if (platforms.has(platform.id)) { throw new Error(`duplicate platform id: ${platform.id}`); } - if (platform.private === true) { - // Private platforms use neutral IDs in public matrix fields. - if (!/^windows-private-[a-z]$/.test(platform.id)) { - throw new Error(`${platform.id} must use a neutral private-platform alias`); + if (platform.prerelease === true) { + // Prerelease platforms use neutral IDs in public matrix fields. + if (!/^windows-prerelease-[a-z-]+$/.test(platform.id)) { + throw new Error(`${platform.id} must use a neutral prerelease-platform alias`); } } diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 54c0b8f3b..988ce9f3b 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -2,10 +2,10 @@ "schemaVersion": 1, "platforms": [ { - "id": "windows-private-a", + "id": "windows-prerelease-a", "displayName": "Windows Pre-release A", "family": "windows", - "private": true, + "prerelease": true, "architectures": { "x64": { "target": "x86_64-pc-windows-msvc", @@ -34,10 +34,10 @@ } }, { - "id": "windows-private-b", + "id": "windows-prerelease-isolation-session", "displayName": "Windows Pre-release Isolation Session", "family": "windows", - "private": true, + "prerelease": true, "architectures": { "x64": { "target": "x86_64-pc-windows-msvc", @@ -397,13 +397,13 @@ ], "nightly": [ { - "os": "windows-private-b", + "os": "windows-prerelease-isolation-session", "backends": [ "isolation-session" ] }, { - "os": "windows-private-a", + "os": "windows-prerelease-a", "backends": [ "process-t1", "isolation-session", @@ -539,7 +539,7 @@ }, { "plan": "nightly", - "os": "windows-private-b", + "os": "windows-prerelease-isolation-session", "architecture": "x64", "backend": "isolation-session" }, diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs index cc4cd715e..45922767a 100644 --- a/scripts/ci/tests/resolve-test-matrix.test.mjs +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -117,8 +117,8 @@ test('duplicate enabled combinations are rejected', () => { ); }); -test('private platforms use neutral public aliases', () => { - for (const platform of catalog().platforms.filter(entry => entry.private === true)) { - assert.match(platform.id, /^windows-private-[a-z]$/); +test('prerelease platforms use neutral public aliases', () => { + for (const platform of catalog().platforms.filter(entry => entry.prerelease === true)) { + assert.match(platform.id, /^windows-prerelease-[a-z-]+$/); } }); From cc38e548c7d940d5945f52b7feb9e50dd44265fe Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:19:26 -0700 Subject: [PATCH 03/93] changing what's enabled for testing --- scripts/ci/test-matrix.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 988ce9f3b..6adef0926 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -527,9 +527,9 @@ "enabled": [ { "plan": "pr", - "os": "windows-canary", + "os": "windows-25h2", "architecture": "x64", - "backend": "process-t1" + "backend": "process-t3" }, { "plan": "pr", From a8040c1d41a281c600a30b90f673dc67cc156512 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:07:36 -0700 Subject: [PATCH 04/93] test run with one windows and one linux runner --- .../workflows/Validation.Infra.PR.Tests.yml | 28 +++++++++++++++++++ scripts/ci/test-matrix.json | 28 ++----------------- scripts/ci/tests/resolve-test-matrix.test.mjs | 6 ++-- 3 files changed, 33 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/Validation.Infra.PR.Tests.yml diff --git a/.github/workflows/Validation.Infra.PR.Tests.yml b/.github/workflows/Validation.Infra.PR.Tests.yml new file mode 100644 index 000000000..5dc195b7b --- /dev/null +++ b/.github/workflows/Validation.Infra.PR.Tests.yml @@ -0,0 +1,28 @@ +name: Validation Infrastructure PR Tests + +on: + push: + branches: + - user/emichlin/validation-infra-lite + +concurrency: + group: validation-infra-pr-tests-${{ github.ref }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + +jobs: + windows: + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + uses: ./.github/workflows/Build.Linux.Job.yml + + test: + needs: [windows, linux] + uses: ./.github/workflows/Test.Matrix.Job.yml + with: + # The resolver selects only explicitly enabled entries in the PR trigger. + plan: pr diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 6adef0926..2799a88df 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -98,7 +98,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "1es-mxc-e2e-windows-25h2-pro-x64", "backends": [ "process-t3", "wslc", @@ -185,7 +185,7 @@ "x64": { "target": "x86_64-unknown-linux-gnu", "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "1es-mxc-e2e-ubuntu-26.04-x64", "backends": [ "bubblewrap", "hyperlight", @@ -536,30 +536,6 @@ "os": "ubuntu-26.04", "architecture": "x64", "backend": "bubblewrap" - }, - { - "plan": "nightly", - "os": "windows-prerelease-isolation-session", - "architecture": "x64", - "backend": "isolation-session" - }, - { - "plan": "nightly", - "os": "ubuntu-26.04", - "architecture": "x64", - "backend": "lxc" - }, - { - "plan": "weekly", - "os": "windows-24h2", - "architecture": "x64", - "backend": "windows-sandbox" - }, - { - "plan": "weekly", - "os": "debian-13", - "architecture": "x64", - "backend": "bubblewrap" } ] } diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs index 45922767a..154b4c56e 100644 --- a/scripts/ci/tests/resolve-test-matrix.test.mjs +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -28,14 +28,14 @@ test('catalog validates and contains all five build targets', () => { assert.doesNotThrow(() => validateCatalog(catalog())); }); -test('phase one enables two PR, two nightly, and four Sunday jobs', () => { +test('current rollout enables two PR jobs and no scheduled jobs', () => { const pr = resolvePlan(catalog(), 'pr'); const nightly = resolvePlan(catalog(), 'nightly'); const weekly = resolvePlan(catalog(), 'weekly'); assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 2); - assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 2); - assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 4); + assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 0); + assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 0); }); test('weekly includes all enabled nightly combinations', () => { From 04a6cf8e70dca09417132559d00771b24185c79c Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:19:05 -0700 Subject: [PATCH 05/93] removed tests from build workflows --- .github/workflows/Build.Linux.Job.yml | 52 ++++++++++++------------- .github/workflows/Build.MacOS.Job.yml | 6 +-- .github/workflows/Build.Windows.Job.yml | 8 ++-- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index c9a3d7d77..9d3158cc2 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -66,37 +66,37 @@ jobs: cargo build --locked --release --target "${{ matrix.target }}" \ --no-default-features "${features[@]}" - - name: Test lxc - shell: bash - run: | - set -euo pipefail - features=() - if [[ -n "${{ matrix.features }}" ]]; then - features=(--features "${{ matrix.features }}") - fi - cargo test --locked --release --target "${{ matrix.target }}" \ - --no-default-features "${features[@]}" + # - name: Test lxc + # shell: bash + # run: | + # set -euo pipefail + # features=() + # if [[ -n "${{ matrix.features }}" ]]; then + # features=(--features "${{ matrix.features }}") + # fi + # cargo test --locked --release --target "${{ matrix.target }}" \ + # --no-default-features "${features[@]}" # Bubblewrap is required to run the executor characterization tests in # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec # always includes the Bubblewrap backend (bwrap_common is a non-optional # dependency), so the binary built above can drive it. - - name: Install Bubblewrap - working-directory: ${{ github.workspace }} - run: | - sudo apt-get update - sudo apt-get install -y bubblewrap - # Ubuntu 24.04 runners restrict unprivileged user namespaces via - # AppArmor, which blocks `bwrap --unshare-user`. Relax it so the - # sandbox can start (no-op on kernels without this knob). - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + # - name: Install Bubblewrap + # working-directory: ${{ github.workspace }} + # run: | + # sudo apt-get update + # sudo apt-get install -y bubblewrap + # # Ubuntu 24.04 runners restrict unprivileged user namespaces via + # # AppArmor, which blocks `bwrap --unshare-user`. Relax it so the + # # sandbox can start (no-op on kernels without this knob). + # sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true # Runs the Bubblewrap executor characterization tests. lxc-exec was built # into src/target//release above, where find_binary() locates it. - - name: Test executor characterization (wxc_e2e_tests) - working-directory: src - run: cargo test --locked --release --target ${{ matrix.target }} - -p wxc_e2e_tests + # - name: Test executor characterization (wxc_e2e_tests) + # working-directory: src + # run: cargo test --locked --release --target ${{ matrix.target }} + # -p wxc_e2e_tests # PLM (Permissive Learning Mode) is functionally Windows-only, but the # crate builds cross-platform: the lib's helper modules compile on every @@ -109,9 +109,9 @@ jobs: working-directory: src run: cargo build --locked --release --target ${{ matrix.target }} -p plm - - name: Test plm (cross-platform modules) - working-directory: src - run: cargo test --locked --release --target ${{ matrix.target }} -p plm + # - name: Test plm (cross-platform modules) + # working-directory: src + # run: cargo test --locked --release --target ${{ matrix.target }} -p plm # unix_test_proxy is a separate workspace member, not a dep of lxc. - name: Build unix-test-proxy diff --git a/.github/workflows/Build.MacOS.Job.yml b/.github/workflows/Build.MacOS.Job.yml index bba76bc11..1ed447840 100644 --- a/.github/workflows/Build.MacOS.Job.yml +++ b/.github/workflows/Build.MacOS.Job.yml @@ -51,9 +51,9 @@ jobs: # mxc-exec-mac (built above via -p mxc_darwin) is what they drive; the # tests skip via has_platform_exec() if it is missing. sandbox-exec needs # no elevation, so they run in this standard macOS job. - - name: Test - run: cargo test --locked --release --target aarch64-apple-darwin - -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests + # - name: Test + # run: cargo test --locked --release --target aarch64-apple-darwin + # -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests - name: Verify artifact payload shell: bash diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index 523e42c8b..6c7a1d5d3 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -66,10 +66,10 @@ jobs: --no-default-features --features "${{ matrix.features }}" - - name: Test - run: cargo test --locked --release --target ${{ matrix.target }} - --no-default-features - --features "${{ matrix.features }}" + # - name: Test + # run: cargo test --locked --release --target ${{ matrix.target }} + # --no-default-features + # --features "${{ matrix.features }}" - name: Verify artifact payload shell: pwsh From 1b88616e02c97e62d882c52bb4781181d18f2527 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:16:44 -0700 Subject: [PATCH 06/93] lite implementation of validation infra with placeholders for image names --- .github/workflows/Build.Linux.Job.yml | 47 +- .github/workflows/Build.MacOS.Job.yml | 12 + .github/workflows/Build.Windows.Job.yml | 43 +- .github/workflows/Build.yml | 38 ++ .github/workflows/Scheduled.E2E.yml | 41 ++ .github/workflows/Test.Matrix.Job.yml | 189 ++++++ .github/workflows/hyperlight-e2e.yml | 2 + .github/workflows/microvm-e2e.yml | 2 + scripts/ci/resolve-test-matrix.mjs | 284 +++++++++ scripts/ci/test-matrix.json | 565 ++++++++++++++++++ scripts/ci/tests/resolve-test-matrix.test.mjs | 124 ++++ tests/scripts/README.md | 34 +- tests/scripts/run_ci_backend_tests.ps1 | 141 +++++ tests/scripts/run_ci_backend_tests.sh | 59 ++ 14 files changed, 1572 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/Scheduled.E2E.yml create mode 100644 .github/workflows/Test.Matrix.Job.yml create mode 100644 scripts/ci/resolve-test-matrix.mjs create mode 100644 scripts/ci/test-matrix.json create mode 100644 scripts/ci/tests/resolve-test-matrix.test.mjs create mode 100644 tests/scripts/run_ci_backend_tests.ps1 create mode 100644 tests/scripts/run_ci_backend_tests.sh diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index cdb4108b2..c9a3d7d77 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -13,9 +13,11 @@ jobs: - arch: x64 runner: ubuntu-latest target: x86_64-unknown-linux-gnu + features: hyperlight microvm - arch: arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu + features: '' runs-on: ${{ matrix.runner }} defaults: run: @@ -54,12 +56,26 @@ jobs: run: sudo apt-get update && sudo apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu - name: Build lxc - run: cargo build --locked --release --target ${{ matrix.target }} - --no-default-features --features hyperlight + shell: bash + run: | + set -euo pipefail + features=() + if [[ -n "${{ matrix.features }}" ]]; then + features=(--features "${{ matrix.features }}") + fi + cargo build --locked --release --target "${{ matrix.target }}" \ + --no-default-features "${features[@]}" - name: Test lxc - run: cargo test --locked --release --target ${{ matrix.target }} - --no-default-features --features hyperlight + shell: bash + run: | + set -euo pipefail + features=() + if [[ -n "${{ matrix.features }}" ]]; then + features=(--features "${{ matrix.features }}") + fi + cargo test --locked --release --target "${{ matrix.target }}" \ + --no-default-features "${features[@]}" # Bubblewrap is required to run the executor characterization tests in # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec @@ -103,6 +119,25 @@ jobs: run: cargo build --locked --release --target ${{ matrix.target }} -p unix_test_proxy + - name: Verify artifact payload + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + bin_dir="src/target/${{ matrix.target }}/release" + required=(lxc-exec unix-test-proxy) + if [[ "${{ matrix.arch }}" == "x64" ]]; then + required+=(nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf) + fi + missing=() + for file in "${required[@]}"; do + [[ -f "$bin_dir/$file" ]] || missing+=("$file") + done + if (( ${#missing[@]} > 0 )); then + printf 'Missing artifact files: %s\n' "${missing[*]}" >&2 + exit 1 + fi + - name: Upload binaries uses: actions/upload-artifact@v4 with: @@ -110,5 +145,9 @@ jobs: path: | src/target/${{ matrix.target }}/release/lxc-exec src/target/${{ matrix.target }}/release/unix-test-proxy + src/target/${{ matrix.target }}/release/nanvixd.elf + src/target/${{ matrix.target }}/release/nanvix_rootfs.img + src/target/${{ matrix.target }}/release/python3.initrd + src/target/${{ matrix.target }}/release/bin/kernel.elf if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/Build.MacOS.Job.yml b/.github/workflows/Build.MacOS.Job.yml index 8e36298b1..bba76bc11 100644 --- a/.github/workflows/Build.MacOS.Job.yml +++ b/.github/workflows/Build.MacOS.Job.yml @@ -55,6 +55,18 @@ jobs: run: cargo test --locked --release --target aarch64-apple-darwin -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests + - name: Verify artifact payload + shell: bash + run: | + set -euo pipefail + bin_dir="target/aarch64-apple-darwin/release" + for file in mxc-exec-mac unix-test-proxy; do + if [[ ! -f "$bin_dir/$file" ]]; then + echo "Missing artifact file: $file" >&2 + exit 1 + fi + done + - name: Upload binaries uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index fb06cea3c..523e42c8b 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -18,8 +18,8 @@ jobs: - arch: arm64 runner: windows-11-arm target: aarch64-pc-windows-msvc - # nanvixd.exe is x64-only, so microvm tests can't run on arm64. - features: hyperlight isolation_session wslc + # Hyperlight and MicroVM runtimes are x64-only. + features: isolation_session wslc runs-on: ${{ matrix.runner }} defaults: run: @@ -71,6 +71,35 @@ jobs: --no-default-features --features "${{ matrix.features }}" + - name: Verify artifact payload + shell: pwsh + run: | + $binDir = "target\${{ matrix.target }}\release" + $required = @( + "wxc-exec.exe", + "wxc-host-prep.exe", + "winhttp-proxy-shim.exe", + "wxc-windows-sandbox-daemon.exe", + "wxc-windows-sandbox-guest.exe", + "wxc-test-proxy.exe", + "wxc-test-driver.exe", + "wxc-ui-probe.exe", + "mxc-diagnostic-console.exe", + "wslcsdk.dll" + ) + if ("${{ matrix.arch }}" -eq "x64") { + $required += @( + "nanvixd.exe", + "nanvix_rootfs.img", + "python3.initrd", + "bin\kernel.elf" + ) + } + $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } + if ($missing) { + throw "Missing artifact files: $($missing -join ', ')" + } + - name: Upload binaries uses: actions/upload-artifact@v4 with: @@ -82,7 +111,15 @@ jobs: src/target/${{ matrix.target }}/release/wxc-windows-sandbox-daemon.exe src/target/${{ matrix.target }}/release/wxc-windows-sandbox-guest.exe src/target/${{ matrix.target }}/release/wxc-test-proxy.exe + src/target/${{ matrix.target }}/release/wxc-test-driver.exe + src/target/${{ matrix.target }}/release/wxc-ui-probe.exe src/target/${{ matrix.target }}/release/mxc-diagnostic-console.exe src/target/${{ matrix.target }}/release/wslcsdk.dll - if-no-files-found: warn + src/target/${{ matrix.target }}/release/nanvixd.exe + src/target/${{ matrix.target }}/release/nanvix_rootfs.img + src/target/${{ matrix.target }}/release/python3.initrd + src/target/${{ matrix.target }}/release/bin/kernel.elf + src/target/${{ matrix.target }}/release/snapshots/kernel.vmem + src/target/${{ matrix.target }}/release/snapshots/kernel.whp.cbor + if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index fbcd36f0a..87e0c2f9b 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -18,9 +18,40 @@ concurrency: cancel-in-progress: true permissions: + actions: read contents: read jobs: + pr-test-eligibility: + name: Check PR test eligibility + if: github.event_name == 'pull_request' && github.base_ref == 'main' + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.changes.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check for non-documentation changes + id: changes + shell: bash + run: | + set -euo pipefail + should_run=false + while IFS= read -r file; do + case "$file" in + docs/*|*.md) ;; + *) + should_run=true + break + ;; + esac + done < <(git diff --name-only \ + '${{ github.event.pull_request.base.sha }}' \ + '${{ github.event.pull_request.head.sha }}') + echo "should_run=$should_run" >> "$GITHUB_OUTPUT" + # Dependency feed check: resolve the locked graph through the public MxcDependencies # feed so a crate not yet in the feed fails at PR time. dependency-feed-check: @@ -45,6 +76,13 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Build.MacOS.Job.yml + pr-test-matrix: + needs: [pr-test-eligibility, windows, linux, macos] + if: needs.pr-test-eligibility.outputs.should_run == 'true' + uses: ./.github/workflows/Test.Matrix.Job.yml + with: + plan: pr + sdk-unit-tests: uses: ./.github/workflows/SDK.Unit.Test.Job.yml diff --git a/.github/workflows/Scheduled.E2E.yml b/.github/workflows/Scheduled.E2E.yml new file mode 100644 index 000000000..6e36b2eb0 --- /dev/null +++ b/.github/workflows/Scheduled.E2E.yml @@ -0,0 +1,41 @@ +name: Scheduled E2E + +on: + schedule: + - cron: '0 8 * * 1-6' + - cron: '0 8 * * 0' + workflow_dispatch: + inputs: + plan: + description: Test plan to run. + required: true + default: nightly + type: choice + options: + - pr + - nightly + - weekly + +concurrency: + group: test-matrix-e2e-${{ github.ref }}-${{ github.event.schedule || inputs.plan }} + cancel-in-progress: false + +permissions: + actions: read + contents: read + +jobs: + windows: + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + uses: ./.github/workflows/Build.MacOS.Job.yml + + test: + needs: [windows, linux, macos] + uses: ./.github/workflows/Test.Matrix.Job.yml + with: + plan: ${{ github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0' && 'weekly' || github.event_name == 'schedule' && 'nightly' || inputs.plan }} diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml new file mode 100644 index 000000000..12f64b4b0 --- /dev/null +++ b/.github/workflows/Test.Matrix.Job.yml @@ -0,0 +1,189 @@ +name: Test Matrix + +on: + workflow_call: + inputs: + plan: + description: Test plan to resolve. + required: true + type: string + +permissions: + actions: read + contents: read + +jobs: + resolve: + name: Resolve ${{ inputs.plan }} matrix + runs-on: ubuntu-latest + outputs: + windows: ${{ steps.matrix.outputs.windows }} + linux: ${{ steps.matrix.outputs.linux }} + macos: ${{ steps.matrix.outputs.macos }} + has_windows: ${{ steps.matrix.outputs.has_windows }} + has_linux: ${{ steps.matrix.outputs.has_linux }} + has_macos: ${{ steps.matrix.outputs.has_macos }} + steps: + - uses: actions/checkout@v4 + + - name: Resolve test combinations + id: matrix + run: node scripts/ci/resolve-test-matrix.mjs --plan "${{ inputs.plan }}" + + windows: + name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + needs: resolve + if: needs.resolve.outputs.has_windows == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} + runs-on: + - self-hosted + - "1ES.Pool=${{ matrix.pool }}" + - "JobId=mxc-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Download ${{ matrix.target }} artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: artifacts/bin + + - name: Prepare Process Container host + if: matrix.command == 'process-container' + shell: pwsh + run: | + $hostPrep = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin\wxc-host-prep.exe' + & $hostPrep prepare-system-drive + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & $hostPrep prepare-null-device --no-sacl + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Run backend tests + shell: pwsh + run: | + $arguments = @{ + Backend = '${{ matrix.command }}' + BinaryDirectory = (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') + Architecture = '${{ matrix.architecture }}' + } + if ('${{ matrix.backend }}' -eq 'process-t1') { + $arguments.ExpectedTier = 'T1' + } elseif ('${{ matrix.backend }}' -eq 'process-t3') { + $arguments.ExpectedTier = 'T3' + } + & ./tests/scripts/run_ci_backend_tests.ps1 @arguments *>&1 | + Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') + + - name: Upload failure logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/mxc-ci.log + ${{ runner.temp }}/mxc-wpc-tests/logs + ${{ runner.temp }}/WinProcessContainer-Tests.results.* + ${{ runner.temp }}/mxc_concurrent_oneshot + if-no-files-found: ignore + retention-days: 7 + + linux: + name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + needs: resolve + if: needs.resolve.outputs.has_linux == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} + runs-on: + - self-hosted + - "1ES.Pool=${{ matrix.pool }}" + - "JobId=mxc-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Download ${{ matrix.target }} artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: artifacts/bin + + - name: Verify backend prerequisites + shell: bash + run: | + set -euo pipefail + chmod +x artifacts/bin/lxc-exec + case '${{ matrix.command }}' in + bubblewrap) + command -v bwrap + if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + ;; + lxc) + command -v lxc-start + sudo -n true + ;; + esac + + - name: Run backend tests + shell: bash + run: | + set -euo pipefail + if [[ '${{ matrix.command }}' == 'lxc' ]]; then + sudo --preserve-env=RUNNER_TEMP bash tests/scripts/run_ci_backend_tests.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee "$RUNNER_TEMP/mxc-ci.log" + else + bash tests/scripts/run_ci_backend_tests.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee "$RUNNER_TEMP/mxc-ci.log" + fi + + - name: Upload failure logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/mxc-ci.log + if-no-files-found: ignore + retention-days: 7 + + macos: + name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + needs: resolve + if: needs.resolve.outputs.has_macos == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.macos) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Download ${{ matrix.target }} artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact }} + path: artifacts/bin + + - name: Run backend tests + shell: bash + run: | + set -euo pipefail + chmod +x artifacts/bin/mxc-exec-mac artifacts/bin/unix-test-proxy + bash tests/scripts/run_ci_backend_tests.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee "$RUNNER_TEMP/mxc-ci.log" + + - name: Upload failure logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/mxc-ci.log + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/hyperlight-e2e.yml b/.github/workflows/hyperlight-e2e.yml index 203612765..7e66db8c1 100644 --- a/.github/workflows/hyperlight-e2e.yml +++ b/.github/workflows/hyperlight-e2e.yml @@ -1,5 +1,7 @@ name: Hyperlight E2E Tests +# Retained until the unified test matrix has equivalent Hyperlight coverage. + on: push: branches: [main] diff --git a/.github/workflows/microvm-e2e.yml b/.github/workflows/microvm-e2e.yml index d53ffed00..eca8c12da 100644 --- a/.github/workflows/microvm-e2e.yml +++ b/.github/workflows/microvm-e2e.yml @@ -1,5 +1,7 @@ name: Integration Tests +# Retained until the unified test matrix has equivalent MicroVM coverage. + on: push: branches: [main] diff --git a/scripts/ci/resolve-test-matrix.mjs b/scripts/ci/resolve-test-matrix.mjs new file mode 100644 index 000000000..02d49a6f7 --- /dev/null +++ b/scripts/ci/resolve-test-matrix.mjs @@ -0,0 +1,284 @@ +#!/usr/bin/env node + +// Validates the declarative test catalog and emits GitHub Actions matrices. +// Keeping expansion here makes the workflow YAML small and lets CI reject an +// invalid catalog before allocating any specialized test runners. + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const FAMILIES = ['windows', 'linux', 'macos']; +const PLANS = ['pr', 'nightly', 'weekly']; +const ARM64_UNSUPPORTED_BACKENDS = new Set(['hyperlight', 'microvm']); + +function combinationKey(plan, os, architecture, backend) { + return `${plan}|${os}|${architecture}|${backend}`; +} + +function assertNonEmptyString(value, label) { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`${label} must be a non-empty string`); + } +} + +export function readCatalog(catalogPath) { + return JSON.parse(fs.readFileSync(catalogPath, 'utf8')); +} + +export function validateCatalog(catalog) { + if (catalog.schemaVersion !== 1) { + throw new Error(`unsupported catalog schemaVersion: ${catalog.schemaVersion}`); + } + + const platforms = new Map(); + const targets = new Set(); + for (const platform of catalog.platforms ?? []) { + assertNonEmptyString(platform.id, 'platform.id'); + assertNonEmptyString(platform.displayName, `${platform.id}.displayName`); + if (!FAMILIES.includes(platform.family)) { + throw new Error(`${platform.id} has unsupported family ${platform.family}`); + } + if (platforms.has(platform.id)) { + throw new Error(`duplicate platform id: ${platform.id}`); + } + if (platform.private === true) { + // Private platforms use neutral IDs in public matrix fields. + if (!/^windows-private-[a-z]$/.test(platform.id)) { + throw new Error(`${platform.id} must use a neutral private-platform alias`); + } + } + + const architectures = Object.entries(platform.architectures ?? {}); + if (architectures.length === 0) { + throw new Error(`${platform.id} has no architectures`); + } + + for (const [architecture, details] of architectures) { + if (!['x64', 'arm64'].includes(architecture)) { + throw new Error(`${platform.id} has unsupported architecture ${architecture}`); + } + assertNonEmptyString(details.target, `${platform.id}.${architecture}.target`); + assertNonEmptyString(details.artifact, `${platform.id}.${architecture}.artifact`); + targets.add(details.target); + + if (platform.family === 'macos') { + assertNonEmptyString(details.runner, `${platform.id}.${architecture}.runner`); + } else { + assertNonEmptyString(details.pool, `${platform.id}.${architecture}.pool`); + } + + const backends = new Set(); + for (const backend of details.backends ?? []) { + assertNonEmptyString(backend, `${platform.id}.${architecture}.backend`); + if (backends.has(backend)) { + throw new Error(`duplicate backend ${backend} on ${platform.id}/${architecture}`); + } + if (architecture === 'arm64' && ARM64_UNSUPPORTED_BACKENDS.has(backend)) { + throw new Error(`${backend} cannot be scheduled on arm64 (${platform.id})`); + } + if (!catalog.handlers?.[platform.family]?.[backend]) { + throw new Error(`missing ${platform.family} handler entry for ${backend}`); + } + backends.add(backend); + } + } + platforms.set(platform.id, platform); + } + + const expectedTargets = new Set([ + 'aarch64-apple-darwin', + 'aarch64-pc-windows-msvc', + 'aarch64-unknown-linux-gnu', + 'x86_64-pc-windows-msvc', + 'x86_64-unknown-linux-gnu' + ]); + if (targets.size !== expectedTargets.size + || [...expectedTargets].some(target => !targets.has(target))) { + throw new Error(`catalog targets do not match the five required build targets`); + } + + // Trigger entries name an OS/backend pair. Architecture expansion happens + // later, so a backend is valid here when at least one OS architecture has it. + for (const plan of PLANS) { + const seenRequests = new Set(); + for (const request of catalog.triggers?.[plan] ?? []) { + const platform = platforms.get(request.os); + if (!platform) { + throw new Error(`${plan} references unknown platform ${request.os}`); + } + for (const backend of request.backends ?? []) { + const requestKey = `${request.os}|${backend}`; + if (seenRequests.has(requestKey)) { + throw new Error(`duplicate ${plan} request ${requestKey}`); + } + const supported = Object.values(platform.architectures) + .some(details => details.backends.includes(backend)); + if (!supported) { + throw new Error(`${plan} requests unsupported ${request.os}/${backend}`); + } + seenRequests.add(requestKey); + } + } + } + + const enabled = new Set(); + for (const entry of catalog.enabled ?? []) { + if (!PLANS.includes(entry.plan)) { + throw new Error(`enabled entry has unsupported plan ${entry.plan}`); + } + const platform = platforms.get(entry.os); + const architecture = platform?.architectures?.[entry.architecture]; + if (!architecture?.backends?.includes(entry.backend)) { + throw new Error( + `enabled entry is unsupported: ${entry.os}/${entry.architecture}/${entry.backend}` + ); + } + const requested = (catalog.triggers?.[entry.plan] ?? []) + .some(request => request.os === entry.os && request.backends.includes(entry.backend)); + if (!requested) { + throw new Error( + `enabled entry is not present in ${entry.plan}: ${entry.os}/${entry.backend}` + ); + } + const handler = catalog.handlers[platform.family][entry.backend]; + // Capability entries may describe future coverage, but enabled entries + // must have an executable dispatcher command for their architecture. + if (handler.status !== 'wired' || !handler.command) { + throw new Error( + `enabled entry has no wired handler: ${entry.os}/${entry.architecture}/${entry.backend}` + ); + } + if (handler.architectures && !handler.architectures.includes(entry.architecture)) { + throw new Error( + `enabled entry handler does not support ${entry.architecture}: ` + + `${entry.os}/${entry.backend}` + ); + } + const key = combinationKey(entry.plan, entry.os, entry.architecture, entry.backend); + if (enabled.has(key)) { + throw new Error(`duplicate enabled entry: ${key}`); + } + enabled.add(key); + } + + return { platforms, enabled }; +} + +export function expandPlan(catalog, plan) { + if (!PLANS.includes(plan)) { + throw new Error(`unsupported plan: ${plan}`); + } + const { platforms } = validateCatalog(catalog); + // Sunday is one run containing the normal nightly set plus weekly additions. + const planNames = plan === 'weekly' ? ['nightly', 'weekly'] : [plan]; + const combinations = []; + + for (const planName of planNames) { + for (const request of catalog.triggers[planName]) { + const platform = platforms.get(request.os); + // A trigger is architecture-neutral. Expand it only where the platform's + // capability declaration supports the requested backend. + for (const [architecture, details] of Object.entries(platform.architectures)) { + for (const backend of request.backends) { + if (!details.backends.includes(backend)) { + continue; + } + combinations.push({ + plan: planName, + os: platform.id, + os_name: platform.displayName, + family: platform.family, + architecture, + target: details.target, + artifact: details.artifact, + pool: details.pool, + runner: details.runner, + backend, + command: catalog.handlers[platform.family][backend].command, + handler_status: catalog.handlers[platform.family][backend].status + }); + } + } + } + } + + return combinations; +} + +export function resolvePlan(catalog, plan) { + const { enabled } = validateCatalog(catalog); + const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); + + for (const combination of expandPlan(catalog, plan)) { + const key = combinationKey( + combination.plan, + combination.os, + combination.architecture, + combination.backend + ); + if (enabled.has(key)) { + // family selects the workflow job and handler_status is validation-only; + // neither belongs in the matrix consumed by the runner. + const { family, handler_status: _, ...matrixEntry } = combination; + matrices[family].push(matrixEntry); + } + } + + for (const family of FAMILIES) { + // Stable ordering keeps local output and workflow diagnostics reproducible. + matrices[family].sort((left, right) => ( + `${left.os}|${left.architecture}|${left.backend}` + .localeCompare(`${right.os}|${right.architecture}|${right.backend}`) + )); + } + return matrices; +} + +function parseArguments(argv) { + const args = { plan: undefined, catalog: undefined }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--plan') { + args.plan = argv[++index]; + } else if (argument === '--catalog') { + args.catalog = argv[++index]; + } else { + throw new Error(`unknown argument: ${argument}`); + } + } + if (!args.plan) { + throw new Error('--plan is required'); + } + return args; +} + +function writeOutputs(matrices) { + const lines = []; + for (const family of FAMILIES) { + // Empty-matrix flags let the reusable workflow skip an OS-family job + // instead of asking GitHub Actions to evaluate an empty matrix. + lines.push(`${family}=${JSON.stringify({ include: matrices[family] })}`); + lines.push(`has_${family}=${matrices[family].length > 0}`); + } + + if (process.env.GITHUB_OUTPUT) { + fs.appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`); + } else { + process.stdout.write(`${lines.join('\n')}\n`); + } +} + +const currentFile = fileURLToPath(import.meta.url); +if (process.argv[1] && path.resolve(process.argv[1]) === currentFile) { + try { + const args = parseArguments(process.argv.slice(2)); + const defaultCatalog = path.join(path.dirname(currentFile), 'test-matrix.json'); + const catalog = readCatalog(path.resolve(args.catalog ?? defaultCatalog)); + writeOutputs(resolvePlan(catalog, args.plan)); + } catch (error) { + process.stderr.write(`resolve-test-matrix: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json new file mode 100644 index 000000000..54c0b8f3b --- /dev/null +++ b/scripts/ci/test-matrix.json @@ -0,0 +1,565 @@ +{ + "schemaVersion": 1, + "platforms": [ + { + "id": "windows-private-a", + "displayName": "Windows Pre-release A", + "family": "windows", + "private": true, + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-private-b", + "displayName": "Windows Pre-release Isolation Session", + "family": "windows", + "private": true, + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-canary", + "displayName": "Windows canary", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t1", + "isolation-session" + ] + } + } + }, + { + "id": "windows-25h2", + "displayName": "Windows 25H2", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-24h2", + "displayName": "Windows 24H2", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "windows-23h2", + "displayName": "Windows 23H2", + "family": "windows", + "architectures": { + "x64": { + "target": "x86_64-pc-windows-msvc", + "artifact": "wxc-binaries-x86_64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + "arm64": { + "target": "aarch64-pc-windows-msvc", + "artifact": "wxc-binaries-aarch64-pc-windows-msvc", + "pool": "placeholder", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox" + ] + } + } + }, + { + "id": "ubuntu-26.04", + "displayName": "Ubuntu 26.04", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "ubuntu-24.04", + "displayName": "Ubuntu 24.04", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "microvm", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "rhel-10.2", + "displayName": "RHEL 10.2", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "debian-13", + "displayName": "Debian 13", + "family": "linux", + "architectures": { + "x64": { + "target": "x86_64-unknown-linux-gnu", + "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + "arm64": { + "target": "aarch64-unknown-linux-gnu", + "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", + "pool": "placeholder", + "backends": [ + "bubblewrap", + "lxc" + ] + } + } + }, + { + "id": "macos-26", + "displayName": "macOS 26", + "family": "macos", + "architectures": { + "arm64": { + "target": "aarch64-apple-darwin", + "artifact": "mxc-binaries-aarch64-apple-darwin", + "runner": "macos-26-xlarge", + "backends": [ + "seatbelt" + ] + } + } + }, + { + "id": "macos-15", + "displayName": "macOS 15", + "family": "macos", + "architectures": { + "arm64": { + "target": "aarch64-apple-darwin", + "artifact": "mxc-binaries-aarch64-apple-darwin", + "runner": "macos-15-xlarge", + "backends": [ + "seatbelt" + ] + } + } + } + ], + "handlers": { + "windows": { + "process-t1": { + "command": "process-container", + "status": "wired" + }, + "process-t3": { + "command": "process-container", + "status": "wired" + }, + "isolation-session": { + "command": "isolation-session", + "status": "wired" + }, + "wslc": { + "command": "wslc", + "status": "wired", + "architectures": [ + "x64" + ] + }, + "windows-sandbox": { + "command": "windows-sandbox", + "status": "wired" + }, + "microvm": { + "command": "microvm", + "status": "wired" + }, + "hyperlight": { + "status": "placeholder" + } + }, + "linux": { + "bubblewrap": { + "command": "bubblewrap", + "status": "wired" + }, + "lxc": { + "command": "lxc", + "status": "wired" + }, + "microvm": { + "status": "placeholder" + }, + "hyperlight": { + "status": "placeholder" + } + }, + "macos": { + "seatbelt": { + "status": "placeholder" + } + } + }, + "triggers": { + "pr": [ + { + "os": "windows-canary", + "backends": [ + "process-t1" + ] + }, + { + "os": "windows-25h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "macos-26", + "backends": [ + "seatbelt" + ] + }, + { + "os": "ubuntu-26.04", + "backends": [ + "bubblewrap" + ] + } + ], + "nightly": [ + { + "os": "windows-private-b", + "backends": [ + "isolation-session" + ] + }, + { + "os": "windows-private-a", + "backends": [ + "process-t1", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "windows-canary", + "backends": [ + "process-t1" + ] + }, + { + "os": "windows-25h2", + "backends": [ + "process-t3", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "windows-24h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "windows-23h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "macos-26", + "backends": [ + "seatbelt" + ] + }, + { + "os": "ubuntu-26.04", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + { + "os": "ubuntu-24.04", + "backends": [ + "microvm" + ] + }, + { + "os": "rhel-10.2", + "backends": [ + "bubblewrap" + ] + } + ], + "weekly": [ + { + "os": "windows-canary", + "backends": [ + "isolation-session" + ] + }, + { + "os": "windows-24h2", + "backends": [ + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "windows-23h2", + "backends": [ + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" + ] + }, + { + "os": "macos-15", + "backends": [ + "seatbelt" + ] + }, + { + "os": "ubuntu-24.04", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + }, + { + "os": "rhel-10.2", + "backends": [ + "hyperlight", + "lxc" + ] + }, + { + "os": "debian-13", + "backends": [ + "bubblewrap", + "hyperlight", + "lxc" + ] + } + ] + }, + "enabled": [ + { + "plan": "pr", + "os": "windows-canary", + "architecture": "x64", + "backend": "process-t1" + }, + { + "plan": "pr", + "os": "ubuntu-26.04", + "architecture": "x64", + "backend": "bubblewrap" + }, + { + "plan": "nightly", + "os": "windows-private-b", + "architecture": "x64", + "backend": "isolation-session" + }, + { + "plan": "nightly", + "os": "ubuntu-26.04", + "architecture": "x64", + "backend": "lxc" + }, + { + "plan": "weekly", + "os": "windows-24h2", + "architecture": "x64", + "backend": "windows-sandbox" + }, + { + "plan": "weekly", + "os": "debian-13", + "architecture": "x64", + "backend": "bubblewrap" + } + ] +} diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs new file mode 100644 index 000000000..cc4cd715e --- /dev/null +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + expandPlan, + readCatalog, + resolvePlan, + validateCatalog +} from '../resolve-test-matrix.mjs'; + +// Tests load a fresh catalog for each case so negative mutations cannot leak +// into later assertions. +const testDirectory = path.dirname(fileURLToPath(import.meta.url)); +const catalogPath = path.resolve(testDirectory, '..', 'test-matrix.json'); + +function catalog() { + return readCatalog(catalogPath); +} + +function clone(value) { + // Catalog values are JSON data, so a JSON round-trip is sufficient here. + return JSON.parse(JSON.stringify(value)); +} + +test('catalog validates and contains all five build targets', () => { + assert.doesNotThrow(() => validateCatalog(catalog())); +}); + +test('phase one enables two PR, two nightly, and four Sunday jobs', () => { + const pr = resolvePlan(catalog(), 'pr'); + const nightly = resolvePlan(catalog(), 'nightly'); + const weekly = resolvePlan(catalog(), 'weekly'); + + assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 2); + assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 2); + assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 4); +}); + +test('weekly includes all enabled nightly combinations', () => { + const nightly = resolvePlan(catalog(), 'nightly'); + const weekly = resolvePlan(catalog(), 'weekly'); + + for (const family of ['windows', 'linux', 'macos']) { + // The weekly matrix must be a superset, not a separate replacement plan. + const weeklyKeys = new Set( + weekly[family].map(entry => `${entry.os}|${entry.architecture}|${entry.backend}`) + ); + for (const entry of nightly[family]) { + assert.ok( + weeklyKeys.has(`${entry.os}|${entry.architecture}|${entry.backend}`), + `weekly is missing ${family} nightly entry ${entry.os}/${entry.backend}` + ); + } + } +}); + +test('full plan expands supported backends to both architectures', () => { + const expanded = expandPlan(catalog(), 'pr'); + const ubuntuBubblewrap = expanded + .filter(entry => entry.os === 'ubuntu-26.04' && entry.backend === 'bubblewrap'); + assert.deepEqual( + ubuntuBubblewrap.map(entry => entry.architecture).sort(), + ['arm64', 'x64'] + ); +}); + +test('arm64 never expands Hyperlight or MicroVM', () => { + for (const plan of ['pr', 'nightly', 'weekly']) { + const invalid = expandPlan(catalog(), plan) + .filter(entry => ( + entry.architecture === 'arm64' + && ['hyperlight', 'microvm'].includes(entry.backend) + )); + assert.deepEqual(invalid, []); + } +}); + +test('enabled placeholder handlers are rejected', () => { + // Seatbelt is declared in the capability map but intentionally not wired yet. + const modified = clone(catalog()); + modified.enabled.push({ + plan: 'weekly', + os: 'macos-15', + architecture: 'arm64', + backend: 'seatbelt' + }); + assert.throws( + () => validateCatalog(modified), + /enabled entry has no wired handler/ + ); +}); + +test('enabled handlers must support the selected architecture', () => { + // WSLC remains in the arm64 capability catalog while its current test + // dispatcher is explicitly restricted to x64. + const modified = clone(catalog()); + modified.enabled.push({ + plan: 'weekly', + os: 'windows-24h2', + architecture: 'arm64', + backend: 'wslc' + }); + assert.throws( + () => validateCatalog(modified), + /enabled entry handler does not support arm64/ + ); +}); + +test('duplicate enabled combinations are rejected', () => { + const modified = clone(catalog()); + modified.enabled.push(clone(modified.enabled[0])); + assert.throws( + () => validateCatalog(modified), + /duplicate enabled entry/ + ); +}); + +test('private platforms use neutral public aliases', () => { + for (const platform of catalog().platforms.filter(entry => entry.private === true)) { + assert.match(platform.id, /^windows-private-[a-z]$/); + } +}); diff --git a/tests/scripts/README.md b/tests/scripts/README.md index bc1fdeabe..a9f93129f 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -1,7 +1,7 @@ # Test Scripts -This directory contains PowerShell convenience scripts for running MXC end-to-end -tests locally on Windows. The primary Rust executor E2E path is +This directory contains convenience scripts for running MXC end-to-end tests +locally and in CI. The primary Rust executor E2E path is `cargo test -p wxc_e2e_tests`, which invokes the MXC binaries directly instead of shelling through these scripts. @@ -37,6 +37,36 @@ These scripts are local helpers. Not every script is run by CI because several depend on local OS features such as Windows Sandbox, WHP, proxy setup, or stress test duration. +## CI test matrix + +[`run_ci_backend_tests.ps1`](run_ci_backend_tests.ps1) and +[`run_ci_backend_tests.sh`](run_ci_backend_tests.sh) are thin dispatchers used +by the reusable test workflow. They consume downloaded target artifacts and +delegate to the existing backend test scripts. + +The capability map and PR, nightly, and weekly triggers are defined in +[`../../scripts/ci/test-matrix.json`](../../scripts/ci/test-matrix.json). +[`../../scripts/ci/resolve-test-matrix.mjs`](../../scripts/ci/resolve-test-matrix.mjs) +validates that catalog and emits one matrix per OS family. Weekly resolution +includes both nightly coverage and weekly additions. + +The initial rollout enables six combinations: + +- PR: Windows x64 Process Container and Ubuntu 26.04 x64 Bubblewrap. +- Nightly: Windows Pre-release Isolation Session x64 and Ubuntu 26.04 x64 LXC. +- Weekly additions: Windows 24H2 x64 Windows Sandbox and Debian 13 x64 + Bubblewrap. + +Other requested combinations remain in the catalog as disabled entries. +Backends without an artifact-only test entry point are explicit placeholders; +the resolver rejects any attempt to enable one. The standalone MicroVM and +Hyperlight workflows remain active until this matrix provides equivalent +coverage. + +PR matrix tests run for non-documentation changes targeting `main`. Nightly runs +start at 08:00 UTC Monday through Saturday. The Sunday 08:00 UTC run resolves +the weekly plan, which includes the nightly set. + ### Manual smoke tests Manual smokes are visual-inspection scripts for rendering and event-propagation diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 new file mode 100644 index 000000000..4b04df3c3 --- /dev/null +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -0,0 +1,141 @@ +<# +.SYNOPSIS +Runs a Windows backend test from a downloaded CI artifact. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateSet('process-container', 'isolation-session', 'windows-sandbox', 'wslc', 'microvm', 'hyperlight')] + [string]$Backend, + + [Parameter(Mandatory)] + [string]$BinaryDirectory, + + [Parameter(Mandatory)] + [ValidateSet('x64', 'arm64')] + [string]$Architecture, + + [ValidateSet('T1', 'T3')] + [string]$ExpectedTier +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$scriptRoot = Split-Path -Parent $MyInvocation.MyCommand.Path +$binaryDirectoryPath = (Resolve-Path -LiteralPath $BinaryDirectory).Path +$wxc = Join-Path $binaryDirectoryPath 'wxc-exec.exe' + +function Assert-File { + param([Parameter(Mandatory)][string]$Path) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Required CI artifact file is missing: $Path" + } +} + +function Invoke-TestScript { + param( + [Parameter(Mandatory)][string]$Path, + [string[]]$Arguments = @() + ) + + # PowerShell scripts do not always replace a previous native exit code. + # Reset it so a successful script cannot inherit a stale failure. + $global:LASTEXITCODE = 0 + & $Path @Arguments + if ($LASTEXITCODE -ne 0) { + throw "Backend test failed with exit code $LASTEXITCODE`: $Path" + } +} + +Assert-File -Path $wxc + +switch ($Backend) { + 'process-container' { + if (-not $ExpectedTier) { + throw 'Process Container tests require -ExpectedTier.' + } + + $expectedProbeTier = @{ + T1 = 'base-container' + T3 = 'appcontainer-dacl' + }[$ExpectedTier] + + # The existing harness expects separate debug and release layouts. CI + # intentionally tests one release artifact, so stage it in both slots. + $debugDirectory = Join-Path $binaryDirectoryPath 'debug' + $releaseDirectory = Join-Path $binaryDirectoryPath 'release' + New-Item -ItemType Directory -Force -Path $debugDirectory, $releaseDirectory | Out-Null + Copy-Item -LiteralPath $wxc -Destination (Join-Path $debugDirectory 'wxc-exec.exe') -Force + Copy-Item -LiteralPath $wxc -Destination (Join-Path $releaseDirectory 'wxc-exec.exe') -Force + + $uiProbe = Join-Path $binaryDirectoryPath 'wxc-ui-probe.exe' + Assert-File -Path $uiProbe + Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $debugDirectory 'wxc-ui-probe.exe') -Force + Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force + + # Fail before running the suite if the selected image does not provide + # the Process Container tier promised by the matrix entry. + $probe = (& $wxc --probe | Out-String) | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { + throw "Process Container probe failed with exit code $LASTEXITCODE." + } + if ($probe.tier -ne $expectedProbeTier) { + throw "Expected $ExpectedTier ($expectedProbeTier), but the host selected '$($probe.tier)'." + } + + $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1' + # Skip build and Cargo phases because this job consumes a previously + # built artifact; retain the host and containment behavior phases. + $phases = @( + 'Probes', + 'T3Forced', + 'T1DenyForced', + 'UiMitigationMatrix', + 'GlobalAtomIsolation', + 'DaclDisabled', + 'CrashRecovery' + ) + $global:LASTEXITCODE = 0 + & $script ` + -SkipBuild ` + -SkipReleaseLane ` + -WxcDebug (Join-Path $debugDirectory 'wxc-exec.exe') ` + -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') ` + -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') ` + -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') ` + -Phases $phases + if ($LASTEXITCODE -ne 0) { + throw "Process Container tests failed with exit code $LASTEXITCODE." + } + } + 'isolation-session' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @( + '-WxcExePath', $wxc + ) + } + 'windows-sandbox' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @( + '-BinDir', $binaryDirectoryPath + ) + } + 'wslc' { + # The current WSLC helper hardcodes the x64 target when locating assets. + if ($Architecture -ne 'x64') { + throw 'The existing WSLC test harness is not architecture-portable yet.' + } + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @( + '-WxcExecPath', $wxc + ) + } + 'microvm' { + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @( + '-BinDir', $binaryDirectoryPath + ) + } + 'hyperlight' { + # Keep unwired commands explicit so accidental activation fails loudly. + throw 'The Hyperlight CI handler is not wired to an existing backend test entry point yet.' + } +} diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh new file mode 100644 index 000000000..0980557ec --- /dev/null +++ b/tests/scripts/run_ci_backend_tests.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dispatches a downloaded Unix artifact to the repository's existing backend +# test suites. Unsupported handlers fail explicitly rather than reporting a +# false-success placeholder job. + +usage() { + echo "Usage: $0 " >&2 +} + +if [[ $# -ne 2 ]]; then + usage + exit 2 +fi + +backend="$1" +binary_directory="$(cd "$2" && pwd)" +script_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_root/../.." && pwd)" +release_directory="$repo_root/src/target/release" + +case "$backend" in + bubblewrap|lxc) + ;; + microvm) + echo "The MicroVM CI handler is not wired to an artifact-only Linux test entry point yet." >&2 + exit 2 + ;; + hyperlight) + echo "The Hyperlight CI handler is not wired to an existing backend test entry point yet." >&2 + exit 2 + ;; + seatbelt) + echo "The Seatbelt CI handler is not wired to an existing backend test entry point yet." >&2 + exit 2 + ;; + *) + usage + exit 2 + ;; +esac + +# Existing shell tests locate binaries under src/target/release. Recreate that +# layout from the downloaded artifact, including adjacent runtime assets. +test -x "$binary_directory/lxc-exec" +test -f "$binary_directory/unix-test-proxy" +mkdir -p "$release_directory" +cp -a "$binary_directory/." "$release_directory/" +chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" + +case "$backend" in + bubblewrap) + "$script_root/run_bwrap_all_tests.sh" + ;; + lxc) + "$script_root/run_lxc_all_tests.sh" + ;; +esac From 94cf6d7ee6be32da7e3f3ae7e287a9891ee406ca Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:31:57 -0700 Subject: [PATCH 07/93] aliging with similar projects over naming of prerelease --- scripts/ci/resolve-test-matrix.mjs | 8 ++++---- scripts/ci/test-matrix.json | 14 +++++++------- scripts/ci/tests/resolve-test-matrix.test.mjs | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/ci/resolve-test-matrix.mjs b/scripts/ci/resolve-test-matrix.mjs index 02d49a6f7..2be65887f 100644 --- a/scripts/ci/resolve-test-matrix.mjs +++ b/scripts/ci/resolve-test-matrix.mjs @@ -43,10 +43,10 @@ export function validateCatalog(catalog) { if (platforms.has(platform.id)) { throw new Error(`duplicate platform id: ${platform.id}`); } - if (platform.private === true) { - // Private platforms use neutral IDs in public matrix fields. - if (!/^windows-private-[a-z]$/.test(platform.id)) { - throw new Error(`${platform.id} must use a neutral private-platform alias`); + if (platform.prerelease === true) { + // Prerelease platforms use neutral IDs in public matrix fields. + if (!/^windows-prerelease-[a-z-]+$/.test(platform.id)) { + throw new Error(`${platform.id} must use a neutral prerelease-platform alias`); } } diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 54c0b8f3b..988ce9f3b 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -2,10 +2,10 @@ "schemaVersion": 1, "platforms": [ { - "id": "windows-private-a", + "id": "windows-prerelease-a", "displayName": "Windows Pre-release A", "family": "windows", - "private": true, + "prerelease": true, "architectures": { "x64": { "target": "x86_64-pc-windows-msvc", @@ -34,10 +34,10 @@ } }, { - "id": "windows-private-b", + "id": "windows-prerelease-isolation-session", "displayName": "Windows Pre-release Isolation Session", "family": "windows", - "private": true, + "prerelease": true, "architectures": { "x64": { "target": "x86_64-pc-windows-msvc", @@ -397,13 +397,13 @@ ], "nightly": [ { - "os": "windows-private-b", + "os": "windows-prerelease-isolation-session", "backends": [ "isolation-session" ] }, { - "os": "windows-private-a", + "os": "windows-prerelease-a", "backends": [ "process-t1", "isolation-session", @@ -539,7 +539,7 @@ }, { "plan": "nightly", - "os": "windows-private-b", + "os": "windows-prerelease-isolation-session", "architecture": "x64", "backend": "isolation-session" }, diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs index cc4cd715e..45922767a 100644 --- a/scripts/ci/tests/resolve-test-matrix.test.mjs +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -117,8 +117,8 @@ test('duplicate enabled combinations are rejected', () => { ); }); -test('private platforms use neutral public aliases', () => { - for (const platform of catalog().platforms.filter(entry => entry.private === true)) { - assert.match(platform.id, /^windows-private-[a-z]$/); +test('prerelease platforms use neutral public aliases', () => { + for (const platform of catalog().platforms.filter(entry => entry.prerelease === true)) { + assert.match(platform.id, /^windows-prerelease-[a-z-]+$/); } }); From 01996fe94f5a23b085e4b007dc9377e5426c81f0 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:19:26 -0700 Subject: [PATCH 08/93] changing what's enabled for testing --- scripts/ci/test-matrix.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 988ce9f3b..6adef0926 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -527,9 +527,9 @@ "enabled": [ { "plan": "pr", - "os": "windows-canary", + "os": "windows-25h2", "architecture": "x64", - "backend": "process-t1" + "backend": "process-t3" }, { "plan": "pr", From 5de032ef7088e3cbf69efba79659f5525e23f484 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:07:36 -0700 Subject: [PATCH 09/93] test run with one windows and one linux runner --- .../workflows/Validation.Infra.PR.Tests.yml | 28 +++++++++++++++++++ scripts/ci/test-matrix.json | 28 ++----------------- scripts/ci/tests/resolve-test-matrix.test.mjs | 6 ++-- 3 files changed, 33 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/Validation.Infra.PR.Tests.yml diff --git a/.github/workflows/Validation.Infra.PR.Tests.yml b/.github/workflows/Validation.Infra.PR.Tests.yml new file mode 100644 index 000000000..5dc195b7b --- /dev/null +++ b/.github/workflows/Validation.Infra.PR.Tests.yml @@ -0,0 +1,28 @@ +name: Validation Infrastructure PR Tests + +on: + push: + branches: + - user/emichlin/validation-infra-lite + +concurrency: + group: validation-infra-pr-tests-${{ github.ref }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + +jobs: + windows: + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + uses: ./.github/workflows/Build.Linux.Job.yml + + test: + needs: [windows, linux] + uses: ./.github/workflows/Test.Matrix.Job.yml + with: + # The resolver selects only explicitly enabled entries in the PR trigger. + plan: pr diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 6adef0926..2799a88df 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -98,7 +98,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "1es-mxc-e2e-windows-25h2-pro-x64", "backends": [ "process-t3", "wslc", @@ -185,7 +185,7 @@ "x64": { "target": "x86_64-unknown-linux-gnu", "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "1es-mxc-e2e-ubuntu-26.04-x64", "backends": [ "bubblewrap", "hyperlight", @@ -536,30 +536,6 @@ "os": "ubuntu-26.04", "architecture": "x64", "backend": "bubblewrap" - }, - { - "plan": "nightly", - "os": "windows-prerelease-isolation-session", - "architecture": "x64", - "backend": "isolation-session" - }, - { - "plan": "nightly", - "os": "ubuntu-26.04", - "architecture": "x64", - "backend": "lxc" - }, - { - "plan": "weekly", - "os": "windows-24h2", - "architecture": "x64", - "backend": "windows-sandbox" - }, - { - "plan": "weekly", - "os": "debian-13", - "architecture": "x64", - "backend": "bubblewrap" } ] } diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs index 45922767a..154b4c56e 100644 --- a/scripts/ci/tests/resolve-test-matrix.test.mjs +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -28,14 +28,14 @@ test('catalog validates and contains all five build targets', () => { assert.doesNotThrow(() => validateCatalog(catalog())); }); -test('phase one enables two PR, two nightly, and four Sunday jobs', () => { +test('current rollout enables two PR jobs and no scheduled jobs', () => { const pr = resolvePlan(catalog(), 'pr'); const nightly = resolvePlan(catalog(), 'nightly'); const weekly = resolvePlan(catalog(), 'weekly'); assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 2); - assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 2); - assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 4); + assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 0); + assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 0); }); test('weekly includes all enabled nightly combinations', () => { From 39c63f851caa2f198a59eba3b5b647ee3eb6314d Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:19:05 -0700 Subject: [PATCH 10/93] removed tests from build workflows --- .github/workflows/Build.Linux.Job.yml | 52 ++++++++++++------------- .github/workflows/Build.MacOS.Job.yml | 6 +-- .github/workflows/Build.Windows.Job.yml | 8 ++-- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index c9a3d7d77..9d3158cc2 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -66,37 +66,37 @@ jobs: cargo build --locked --release --target "${{ matrix.target }}" \ --no-default-features "${features[@]}" - - name: Test lxc - shell: bash - run: | - set -euo pipefail - features=() - if [[ -n "${{ matrix.features }}" ]]; then - features=(--features "${{ matrix.features }}") - fi - cargo test --locked --release --target "${{ matrix.target }}" \ - --no-default-features "${features[@]}" + # - name: Test lxc + # shell: bash + # run: | + # set -euo pipefail + # features=() + # if [[ -n "${{ matrix.features }}" ]]; then + # features=(--features "${{ matrix.features }}") + # fi + # cargo test --locked --release --target "${{ matrix.target }}" \ + # --no-default-features "${features[@]}" # Bubblewrap is required to run the executor characterization tests in # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec # always includes the Bubblewrap backend (bwrap_common is a non-optional # dependency), so the binary built above can drive it. - - name: Install Bubblewrap - working-directory: ${{ github.workspace }} - run: | - sudo apt-get update - sudo apt-get install -y bubblewrap - # Ubuntu 24.04 runners restrict unprivileged user namespaces via - # AppArmor, which blocks `bwrap --unshare-user`. Relax it so the - # sandbox can start (no-op on kernels without this knob). - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + # - name: Install Bubblewrap + # working-directory: ${{ github.workspace }} + # run: | + # sudo apt-get update + # sudo apt-get install -y bubblewrap + # # Ubuntu 24.04 runners restrict unprivileged user namespaces via + # # AppArmor, which blocks `bwrap --unshare-user`. Relax it so the + # # sandbox can start (no-op on kernels without this knob). + # sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true # Runs the Bubblewrap executor characterization tests. lxc-exec was built # into src/target//release above, where find_binary() locates it. - - name: Test executor characterization (wxc_e2e_tests) - working-directory: src - run: cargo test --locked --release --target ${{ matrix.target }} - -p wxc_e2e_tests + # - name: Test executor characterization (wxc_e2e_tests) + # working-directory: src + # run: cargo test --locked --release --target ${{ matrix.target }} + # -p wxc_e2e_tests # PLM (Permissive Learning Mode) is functionally Windows-only, but the # crate builds cross-platform: the lib's helper modules compile on every @@ -109,9 +109,9 @@ jobs: working-directory: src run: cargo build --locked --release --target ${{ matrix.target }} -p plm - - name: Test plm (cross-platform modules) - working-directory: src - run: cargo test --locked --release --target ${{ matrix.target }} -p plm + # - name: Test plm (cross-platform modules) + # working-directory: src + # run: cargo test --locked --release --target ${{ matrix.target }} -p plm # unix_test_proxy is a separate workspace member, not a dep of lxc. - name: Build unix-test-proxy diff --git a/.github/workflows/Build.MacOS.Job.yml b/.github/workflows/Build.MacOS.Job.yml index bba76bc11..1ed447840 100644 --- a/.github/workflows/Build.MacOS.Job.yml +++ b/.github/workflows/Build.MacOS.Job.yml @@ -51,9 +51,9 @@ jobs: # mxc-exec-mac (built above via -p mxc_darwin) is what they drive; the # tests skip via has_platform_exec() if it is missing. sandbox-exec needs # no elevation, so they run in this standard macOS job. - - name: Test - run: cargo test --locked --release --target aarch64-apple-darwin - -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests + # - name: Test + # run: cargo test --locked --release --target aarch64-apple-darwin + # -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests - name: Verify artifact payload shell: bash diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index 523e42c8b..6c7a1d5d3 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -66,10 +66,10 @@ jobs: --no-default-features --features "${{ matrix.features }}" - - name: Test - run: cargo test --locked --release --target ${{ matrix.target }} - --no-default-features - --features "${{ matrix.features }}" + # - name: Test + # run: cargo test --locked --release --target ${{ matrix.target }} + # --no-default-features + # --features "${{ matrix.features }}" - name: Verify artifact payload shell: pwsh From c8db517a320b7a95025051ad50f6eb61124dcf71 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:07:09 -0700 Subject: [PATCH 11/93] Fix naming + re-running failed action --- .github/workflows/Test.Matrix.Job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index 12f64b4b0..005a8d6b9 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -31,7 +31,7 @@ jobs: run: node scripts/ci/resolve-test-matrix.mjs --plan "${{ inputs.plan }}" windows: - name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + name: ${{ matrix.backend || 'Windows tests' }} needs: resolve if: needs.resolve.outputs.has_windows == 'true' strategy: @@ -91,7 +91,7 @@ jobs: retention-days: 7 linux: - name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + name: ${{ matrix.backend || 'Linux tests' }} needs: resolve if: needs.resolve.outputs.has_linux == 'true' strategy: @@ -153,7 +153,7 @@ jobs: retention-days: 7 macos: - name: ${{ matrix.os_name }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + name: ${{ matrix.backend || 'macOS tests' }} needs: resolve if: needs.resolve.outputs.has_macos == 'true' strategy: From d951ef9d5467db6775c7bb9e4e814b6ec041d5ff Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 14:27:57 -0700 Subject: [PATCH 12/93] ci: add throwaway 1ES connectivity smoke workflow Verifies microsoft/mxc can dispatch a job to the 1ES Hosted GitHub Runner pool 1es-mxc-e2e-windows-25h2-pro-x64 (org-level, runner group 145). Pickup-only; nested-virt/WHP checks intentionally omitted. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/1es-smoke.yml diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml new file mode 100644 index 000000000..49737c486 --- /dev/null +++ b/.github/workflows/1es-smoke.yml @@ -0,0 +1,57 @@ +# ============================================================================ +# 1ES connectivity smoke test — proves microsoft/mxc can dispatch a job to the +# 1ES Hosted GitHub Runner pool and that runner group 145 grants this repo. +# +# THROWAWAY / PROOF workflow. Not part of the real CI matrix (Build.yml). +# Nested-virt/WHP checks intentionally omitted — this ONLY verifies pickup. +# +# Pool values from the created pool: +# 1ES.Pool = 1es-mxc-e2e-windows-25h2-pro-x64 +# 1ES.ImageOverride = windows-25h2-pro-x64 +# ============================================================================ +name: 1ES Connectivity Smoke + +on: + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + smoke: + name: 1ES pickup + runs-on: + - self-hosted + - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64" + - "1ES.ImageOverride=windows-25h2-pro-x64" + timeout-minutes: 15 + + steps: + - name: Hello from the 1ES runner + shell: pwsh + run: | + Write-Host "=== Job picked up on the 1ES pool ===" + Write-Host "Hostname : $(hostname)" + Write-Host "User : $(whoami)" + Write-Host "OS : $([System.Environment]::OSVersion.VersionString)" + Write-Host "RUNNER_NAME: $env:RUNNER_NAME" + Write-Host "Cores : $env:NUMBER_OF_PROCESSORS" + + - name: Confirm this is the expected Azure pool VM (IMDS) + shell: pwsh + run: | + try { + $c = (Invoke-RestMethod -Headers @{Metadata="true"} -NoProxy ` + -Uri "http://169.254.169.254/metadata/instance?api-version=2021-02-01").compute + Write-Host "vmSize : $($c.vmSize)" + Write-Host "location : $($c.location)" + Write-Host "vmId : $($c.vmId)" + if ($env:GITHUB_STEP_SUMMARY) { + "## 1ES runner picked up`n- **vmSize**: $($c.vmSize)`n- **location**: $($c.location)`n- **host**: $(hostname)" | + Add-Content -Path $env:GITHUB_STEP_SUMMARY + } + } catch { + Write-Host "::warning::IMDS unreachable: $($_.Exception.Message)" + } From 81fc3677b63c951c754d3b5ce67127c8688c10da Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 14:42:30 -0700 Subject: [PATCH 13/93] ci: restructure 1ES smoke to push-trigger, drop ImageOverride - Trigger only on push to this throwaway branch (not pull_request) so it doesn't re-run the full mxc CI matrix. - Omit 1ES.ImageOverride (single-image pool defaults) to isolate an image-match failure from an org-level authorization failure. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index 49737c486..93c5f9840 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -1,20 +1,26 @@ # ============================================================================ # 1ES connectivity smoke test — proves microsoft/mxc can dispatch a job to the -# 1ES Hosted GitHub Runner pool and that runner group 145 grants this repo. +# 1ES Hosted GitHub Runner pool and that its runner group grants this repo. # # THROWAWAY / PROOF workflow. Not part of the real CI matrix (Build.yml). -# Nested-virt/WHP checks intentionally omitted — this ONLY verifies pickup. # -# Pool values from the created pool: -# 1ES.Pool = 1es-mxc-e2e-windows-25h2-pro-x64 -# 1ES.ImageOverride = windows-25h2-pro-x64 +# Design notes: +# * Triggers ONLY on push to this throwaway branch (NOT pull_request), so it +# does not re-run the full mxc CI matrix on every push. +# * 1ES.ImageOverride is intentionally OMITTED. The pool has a single image +# (windows-25h2-pro-x64), so it defaults. If the job now picks up, an image +# override was the culprit; if it still queues, the cause is org-level +# authorization (1ES GitHub App repo access and/or runner-group grant for a +# PUBLIC repo) or the 1ES scaler — NOT the labels. +# +# Pool: 1es-mxc-e2e-windows-25h2-pro-x64 (org-level, runner group 145) # ============================================================================ name: 1ES Connectivity Smoke on: - pull_request: - branches: [main] - workflow_dispatch: + push: + branches: + - user/modanish/1es-github-runner-e2e permissions: contents: read @@ -25,7 +31,6 @@ jobs: runs-on: - self-hosted - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64" - - "1ES.ImageOverride=windows-25h2-pro-x64" timeout-minutes: 15 steps: From c4fbae49c27207f465a648ea90554b3517546ce7 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 14:54:18 -0700 Subject: [PATCH 14/93] ci: add unique JobId label to 1ES runs-on Multiple recent microsoft-org migrations (vscode, msquic, netperf, ebpf-for-windows) add a unique JobId= label to their 1ES runs-on. Some pool configs require it to route/schedule the job; a missing JobId can leave the job queued with no runner. Testing this cheaper hypothesis before escalating the org-level authorization theory. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index 93c5f9840..5bba12f7e 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -31,6 +31,7 @@ jobs: runs-on: - self-hosted - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64" + - "JobId=smoke-${{ github.run_id }}-${{ github.run_attempt }}" timeout-minutes: 15 steps: From c52e17dc27cb50beb992f6500e2a476370913563 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:39:08 -0700 Subject: [PATCH 15/93] migrating APIs --- .github/workflows/Test.Matrix.Job.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index 005a8d6b9..d616b91e1 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -37,10 +37,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} - runs-on: - - self-hosted - - "1ES.Pool=${{ matrix.pool }}" - - "JobId=mxc-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + runs-on: ${{ matrix.pool }} timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -97,10 +94,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} - runs-on: - - self-hosted - - "1ES.Pool=${{ matrix.pool }}" - - "JobId=mxc-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + runs-on: ${{ matrix.pool }} timeout-minutes: 90 steps: - uses: actions/checkout@v4 From 11c7cd77996d148ad6b04c6b40fdb9bb28282c2a Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 15:41:55 -0700 Subject: [PATCH 16/93] ci: use scale-set targeting (pool name only) for 1ES pool Root cause of the eternal queue: the pool has tag UseScaleSet=true, so it uses the 1ES Scale Set API, which is targeted by the pool NAME ALONE. The prior webhook-mode labels (self-hosted + 1ES.Pool= + JobId=) never match a scale-set pool, so jobs queued forever. Per 1ES docs, scale-set pools use: runs-on: with no other labels. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index 5bba12f7e..fd2d48865 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -1,19 +1,18 @@ # ============================================================================ # 1ES connectivity smoke test — proves microsoft/mxc can dispatch a job to the -# 1ES Hosted GitHub Runner pool and that its runner group grants this repo. +# 1ES Hosted GitHub Runner pool. # # THROWAWAY / PROOF workflow. Not part of the real CI matrix (Build.yml). # -# Design notes: -# * Triggers ONLY on push to this throwaway branch (NOT pull_request), so it -# does not re-run the full mxc CI matrix on every push. -# * 1ES.ImageOverride is intentionally OMITTED. The pool has a single image -# (windows-25h2-pro-x64), so it defaults. If the job now picks up, an image -# override was the culprit; if it still queues, the cause is org-level -# authorization (1ES GitHub App repo access and/or runner-group grant for a -# PUBLIC repo) or the 1ES scaler — NOT the labels. +# IMPORTANT: this pool has tag UseScaleSet=true, so it uses the 1ES **Scale Set +# API** integration (NOT webhooks). Per the 1ES docs, scale-set pools are +# targeted by the POOL NAME ALONE — no "self-hosted", no "1ES.Pool=", no +# "JobId=" labels (those are the webhook-mode format and DO NOT MATCH a +# scale-set pool, leaving the job queued forever). # -# Pool: 1es-mxc-e2e-windows-25h2-pro-x64 (org-level, runner group 145) +# runs-on: +# +# Pool: 1es-mxc-e2e-windows-25h2-pro-x64 # ============================================================================ name: 1ES Connectivity Smoke @@ -28,10 +27,8 @@ permissions: jobs: smoke: name: 1ES pickup - runs-on: - - self-hosted - - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64" - - "JobId=smoke-${{ github.run_id }}-${{ github.run_attempt }}" + # Scale-set pool → target by pool name ONLY (no other labels). + runs-on: 1es-mxc-e2e-windows-25h2-pro-x64 timeout-minutes: 15 steps: From 157df5fe55f087028e27c7209dafa691619240d3 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 15:52:54 -0700 Subject: [PATCH 17/93] ci: revert to webhook multi-label format after removing UseScaleSet Pool tag UseScaleSet=true was removed (reverts pool to webhook mode), so target it with the standard proven format: self-hosted + 1ES.Pool= + JobId=. Benchmark: ebpf 1ES pool picks up in ~2.5-3 min. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index fd2d48865..2a3e2ae85 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -4,13 +4,10 @@ # # THROWAWAY / PROOF workflow. Not part of the real CI matrix (Build.yml). # -# IMPORTANT: this pool has tag UseScaleSet=true, so it uses the 1ES **Scale Set -# API** integration (NOT webhooks). Per the 1ES docs, scale-set pools are -# targeted by the POOL NAME ALONE — no "self-hosted", no "1ES.Pool=", no -# "JobId=" labels (those are the webhook-mode format and DO NOT MATCH a -# scale-set pool, leaving the job queued forever). -# -# runs-on: +# Pool is in WEBHOOK mode (UseScaleSet tag removed), so it uses the standard +# multi-label targeting that every working example (SbomToolkit, ebpf, vscode, +# openvmm) uses. Benchmark: ebpf's 1ES pool picks up jobs in ~2.5-3 min, so a +# healthy pickup here should occur well under ~3-4 min. # # Pool: 1es-mxc-e2e-windows-25h2-pro-x64 # ============================================================================ @@ -27,8 +24,10 @@ permissions: jobs: smoke: name: 1ES pickup - # Scale-set pool → target by pool name ONLY (no other labels). - runs-on: 1es-mxc-e2e-windows-25h2-pro-x64 + runs-on: + - self-hosted + - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64" + - "JobId=smoke-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" timeout-minutes: 15 steps: From b004f5e33113b33adda66962f10a47f240078148 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:02:24 -0700 Subject: [PATCH 18/93] changing format of runs-on to match similar projects --- .github/workflows/Test.Matrix.Job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index d616b91e1..e8d20951b 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} - runs-on: ${{ matrix.pool }} + runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -94,7 +94,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} - runs-on: ${{ matrix.pool }} + runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 90 steps: - uses: actions/checkout@v4 From 2750006ee751b8757b193a6206df10db0ea96159 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 16:04:59 -0700 Subject: [PATCH 19/93] ci: retrigger 1ES smoke after pool re-save (webhook mode) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> From afd00da55906aa9d4451726985e08448b40fdcaa Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Mon, 3 Aug 2026 16:37:32 -0700 Subject: [PATCH 20/93] ci: target fresh webhook-mode pool ...-wh (clean experiment) New pool 1es-mxc-e2e-windows-25h2-pro-x64-wh created cleanly via the 1ES Hub wizard: webhook mode (no UseScaleSet tag), SKU Standard_D8s_v7, its own auto-created runner group 146. Tests whether the original pool's eternal queue was due to its scale-set/half-registered state. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index 2a3e2ae85..06cd25bc9 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -26,7 +26,7 @@ jobs: name: 1ES pickup runs-on: - self-hosted - - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64" + - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64-wh" - "JobId=smoke-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" timeout-minutes: 15 From 7d99dc7ab1f82546dfc73984af6ef234803f1094 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Tue, 4 Aug 2026 10:00:05 -0700 Subject: [PATCH 21/93] ci: target repo-level pool 1es-mxc-test-2 (bypasses org group gate) 1es-mxc-test-2 is a REPOSITORY-level 1ES pool (organizationProfile level=Repository, repositoryName=mxc) bound directly to microsoft/mxc. Repo-level runners register straight to the repo, bypassing the org runner-group public-repo gate that org-level pools depend on. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index 06cd25bc9..dad7a70b2 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -26,7 +26,7 @@ jobs: name: 1ES pickup runs-on: - self-hosted - - "1ES.Pool=1es-mxc-e2e-windows-25h2-pro-x64-wh" + - "1ES.Pool=1es-mxc-test-2" - "JobId=smoke-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" timeout-minutes: 15 From e7e0822090eeba2d7635f162a370135bae21f8f0 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Tue, 4 Aug 2026 10:22:12 -0700 Subject: [PATCH 22/93] ci: fix shell to powershell (client image has no pwsh); repo-level pool CONFIRMED working Fresh-eyes RCA: the repo-level pool 1es-mxc-test-2 DID pick up run 30931840357 (runner 21 / bf347e45c000000, group 1/Default) after a ~10.5-min cold start. Prior 'eternal queue' was a too-short (3-4 min) patience window vs ebpf's ~2.5-min WARM-pool benchmark. Only failure was pwsh not found: the raw windows-25h2-pro-x64 client image ships PS 5.1 (powershell.exe), not PS7 (pwsh.exe). Switched both steps to powershell. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/1es-smoke.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml index dad7a70b2..55f5ef668 100644 --- a/.github/workflows/1es-smoke.yml +++ b/.github/workflows/1es-smoke.yml @@ -4,12 +4,13 @@ # # THROWAWAY / PROOF workflow. Not part of the real CI matrix (Build.yml). # -# Pool is in WEBHOOK mode (UseScaleSet tag removed), so it uses the standard -# multi-label targeting that every working example (SbomToolkit, ebpf, vscode, -# openvmm) uses. Benchmark: ebpf's 1ES pool picks up jobs in ~2.5-3 min, so a -# healthy pickup here should occur well under ~3-4 min. +# Pool: 1es-mxc-test-2 — REPOSITORY-level pool bound directly to microsoft/mxc +# (runner group 1 / Default). Confirmed working: run 30931840357 registered +# runner 21 (bf347e45c000000) after a ~10.5-min COLD start on the idle pool. # -# Pool: 1es-mxc-e2e-windows-25h2-pro-x64 +# NOTE: uses `shell: powershell` (Windows PowerShell 5.1) because the raw +# windows-25h2-pro-x64 CLIENT image ships powershell.exe, NOT pwsh.exe (PS7). +# Cold-start pickup on an idle Stateless pool is ~10-11 min — allow a full 15. # ============================================================================ name: 1ES Connectivity Smoke @@ -32,20 +33,21 @@ jobs: steps: - name: Hello from the 1ES runner - shell: pwsh + shell: powershell run: | Write-Host "=== Job picked up on the 1ES pool ===" Write-Host "Hostname : $(hostname)" Write-Host "User : $(whoami)" Write-Host "OS : $([System.Environment]::OSVersion.VersionString)" + Write-Host "PSVersion : $($PSVersionTable.PSVersion)" Write-Host "RUNNER_NAME: $env:RUNNER_NAME" Write-Host "Cores : $env:NUMBER_OF_PROCESSORS" - name: Confirm this is the expected Azure pool VM (IMDS) - shell: pwsh + shell: powershell run: | try { - $c = (Invoke-RestMethod -Headers @{Metadata="true"} -NoProxy ` + $c = (Invoke-RestMethod -Headers @{Metadata="true"} ` -Uri "http://169.254.169.254/metadata/instance?api-version=2021-02-01").compute Write-Host "vmSize : $($c.vmSize)" Write-Host "location : $($c.location)" From af27bedc489edb0128ca67f3f744a36b66a83d80 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:38:59 -0700 Subject: [PATCH 23/93] updated powershell to use 5.1 --- .github/workflows/Test.Matrix.Job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index e8d20951b..f544344bc 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -50,7 +50,7 @@ jobs: - name: Prepare Process Container host if: matrix.command == 'process-container' - shell: pwsh + shell: powershell run: | $hostPrep = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin\wxc-host-prep.exe' & $hostPrep prepare-system-drive @@ -59,7 +59,7 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run backend tests - shell: pwsh + shell: powershell run: | $arguments = @{ Backend = '${{ matrix.command }}' From 1903465133a7ebec2cf31b2b126873cead3db5ee Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:11:35 -0700 Subject: [PATCH 24/93] adding bubblewrap install for linux test prereq. for windows, reverting to powershell 7 commands. --- .github/workflows/Test.Matrix.Job.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index f544344bc..af606d97c 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -49,8 +49,8 @@ jobs: path: artifacts/bin - name: Prepare Process Container host - if: matrix.command == 'process-container' - shell: powershell + if: matrix.backend == 'process-t3' + shell: pwsh run: | $hostPrep = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin\wxc-host-prep.exe' & $hostPrep prepare-system-drive @@ -59,7 +59,7 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run backend tests - shell: powershell + shell: pwsh run: | $arguments = @{ Backend = '${{ matrix.command }}' @@ -105,13 +105,17 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin - - name: Verify backend prerequisites + - name: Prepare backend prerequisites shell: bash run: | set -euo pipefail chmod +x artifacts/bin/lxc-exec case '${{ matrix.command }}' in bubblewrap) + if ! command -v bwrap >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends bubblewrap + fi command -v bwrap if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 From 095417262305bcbe0af72f92b36bec22f236392a Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:59:59 -0700 Subject: [PATCH 25/93] triggering linux test scripts via bash --- tests/scripts/run_ci_backend_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index 0980557ec..52f753738 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -51,9 +51,9 @@ chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" case "$backend" in bubblewrap) - "$script_root/run_bwrap_all_tests.sh" + bash "$script_root/run_bwrap_all_tests.sh" ;; lxc) - "$script_root/run_lxc_all_tests.sh" + bash "$script_root/run_lxc_all_tests.sh" ;; esac From 2c0531692956a88b803f8b4b63ff2fd9a3cc3dd2 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:23:48 -0700 Subject: [PATCH 26/93] bubblewrap test now owns files it delegates --- tests/scripts/run_bwrap_denied_masking_test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/run_bwrap_denied_masking_test.sh b/tests/scripts/run_bwrap_denied_masking_test.sh index b80db0b8f..90058582e 100644 --- a/tests/scripts/run_bwrap_denied_masking_test.sh +++ b/tests/scripts/run_bwrap_denied_masking_test.sh @@ -49,6 +49,8 @@ sudo mkdir -p "$DIR" echo "VISIBLE_SECRET" | sudo tee "$VISIBLE" > /dev/null echo "FILE_SECRET" | sudo tee "$FILE" > /dev/null echo "DIR_SECRET" | sudo tee "$DIR/inner.txt" > /dev/null +# The parent is delegated as read-write, so the invoking user must own it. +sudo chown -R "$(id -u):$(id -g)" "$BASE" # Sanity: all fixtures are readable on the host. if ! sudo cat "$VISIBLE" | grep -q "VISIBLE_SECRET"; then @@ -116,6 +118,8 @@ echo "DIR_TARGET_SECRET" | sudo tee "$SYMBASE/real_dir/inner.txt" > /dev/null echo "FILE_TARGET_SECRET" | sudo tee "$SYMBASE/real_file.txt" > /dev/null sudo ln -s "$SYMBASE/real_dir" "$SYMBASE/link_to_dir" sudo ln -s "$SYMBASE/real_file.txt" "$SYMBASE/link_to_file" +# Keep the canonical symlink targets and their read-write parent delegable. +sudo chown -R "$(id -u):$(id -g)" "$SYMBASE" echo "Running Bubblewrap denied-symlink -> dir masking test..." DIR_OUT=$("$LXC_EXEC" --experimental \ From d932a4275e2142c8b2c4b82dcc199c87bcac0872 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:48:05 -0700 Subject: [PATCH 27/93] diagnosing interactive desktop issues --- .github/workflows/Test.Matrix.Job.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index af606d97c..d33a3fdf9 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -58,6 +58,19 @@ jobs: & $hostPrep prepare-null-device --no-sacl if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Diagnose interactive desktop + shell: pwsh + run: | + Write-Host "UserInteractive=$([Environment]::UserInteractive)" + Write-Host "SessionId=$((Get-Process -Id $PID).SessionId)" + query user + query session + Get-Process explorer, dwm, ShellExperienceHost -ErrorAction SilentlyContinue | + Select-Object Name, Id, SessionId + Get-Process | + Where-Object { $_.SessionId -eq (Get-Process -Id $PID).SessionId } | + Select-Object Name, Id, SessionId + - name: Run backend tests shell: pwsh run: | From 6af23d88c17da2ef532d3639f892a31f468659a9 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:59:31 -0700 Subject: [PATCH 28/93] diagnostics should continue on error --- .github/workflows/Test.Matrix.Job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index d33a3fdf9..1a2b245ef 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -59,6 +59,7 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Diagnose interactive desktop + continue-on-error: true shell: pwsh run: | Write-Host "UserInteractive=$([Environment]::UserInteractive)" From 8a223798e89a5dcb35a4d2bf75b03852de4fd39b Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:19:18 -0700 Subject: [PATCH 29/93] Removing diagnostics - intended behavior. --- .github/workflows/Test.Matrix.Job.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index 1a2b245ef..af606d97c 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -58,20 +58,6 @@ jobs: & $hostPrep prepare-null-device --no-sacl if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - name: Diagnose interactive desktop - continue-on-error: true - shell: pwsh - run: | - Write-Host "UserInteractive=$([Environment]::UserInteractive)" - Write-Host "SessionId=$((Get-Process -Id $PID).SessionId)" - query user - query session - Get-Process explorer, dwm, ShellExperienceHost -ErrorAction SilentlyContinue | - Select-Object Name, Id, SessionId - Get-Process | - Where-Object { $_.SessionId -eq (Get-Process -Id $PID).SessionId } | - Select-Object Name, Id, SessionId - - name: Run backend tests shell: pwsh run: | From ba25dbafba48153b6d26cbd77ce9d1bc04d149a4 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:50:08 -0700 Subject: [PATCH 30/93] Testing MacOS runners with sample e2e test script --- .github/workflows/Build.Linux.Job.yml | 33 +- .github/workflows/Build.MacOS.Job.yml | 9 +- .github/workflows/Build.Windows.Job.yml | 8 +- .github/workflows/Build.yml | 40 +- .github/workflows/Lint.Job.yml | 2 +- .../workflows/SDK.Integration.Test.Job.yml | 2 +- .../workflows/Validation.Infra.PR.Tests.yml | 10 +- scripts/ci/resolve-test-matrix.mjs | 44 +- scripts/ci/test-matrix.json | 25 +- scripts/ci/tests/resolve-test-matrix.test.mjs | 52 +- tests/scripts/run_ci_backend_tests.sh | 41 +- tests/scripts/run_seatbelt_all_tests.sh | 563 ++++++++++++++++++ 12 files changed, 736 insertions(+), 93 deletions(-) create mode 100644 tests/scripts/run_seatbelt_all_tests.sh diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index 9d3158cc2..295a8ccd0 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -66,16 +66,20 @@ jobs: cargo build --locked --release --target "${{ matrix.target }}" \ --no-default-features "${features[@]}" - # - name: Test lxc - # shell: bash - # run: | - # set -euo pipefail - # features=() - # if [[ -n "${{ matrix.features }}" ]]; then - # features=(--features "${{ matrix.features }}") - # fi - # cargo test --locked --release --target "${{ matrix.target }}" \ - # --no-default-features "${features[@]}" + - name: Test lxc + shell: bash + run: | + set -euo pipefail + features=() + if [[ -n "${{ matrix.features }}" ]]; then + features=(--features "${{ matrix.features }}") + fi + cargo test --locked --release --target "${{ matrix.target }}" \ + --no-default-features "${features[@]}" + + #################### + # E2E Tests are being moved to the Test.Matrix.Job.yml workflow, which runs cross-platform validation + ############### # Bubblewrap is required to run the executor characterization tests in # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec @@ -98,6 +102,9 @@ jobs: # run: cargo test --locked --release --target ${{ matrix.target }} # -p wxc_e2e_tests + ############### + #################### + # PLM (Permissive Learning Mode) is functionally Windows-only, but the # crate builds cross-platform: the lib's helper modules compile on every # target, and the binary has a no-op stub `fn main()` for non-Windows so @@ -109,9 +116,9 @@ jobs: working-directory: src run: cargo build --locked --release --target ${{ matrix.target }} -p plm - # - name: Test plm (cross-platform modules) - # working-directory: src - # run: cargo test --locked --release --target ${{ matrix.target }} -p plm + - name: Test plm (cross-platform modules) + working-directory: src + run: cargo test --locked --release --target ${{ matrix.target }} -p plm # unix_test_proxy is a separate workspace member, not a dep of lxc. - name: Build unix-test-proxy diff --git a/.github/workflows/Build.MacOS.Job.yml b/.github/workflows/Build.MacOS.Job.yml index 1ed447840..0928cf26a 100644 --- a/.github/workflows/Build.MacOS.Job.yml +++ b/.github/workflows/Build.MacOS.Job.yml @@ -6,8 +6,7 @@ on: jobs: build: name: arm64 - # macos-14 / macos-latest are Apple Silicon (arm64); older labels are Intel. - runs-on: macos-14 + runs-on: macos-26 defaults: run: working-directory: src @@ -51,9 +50,9 @@ jobs: # mxc-exec-mac (built above via -p mxc_darwin) is what they drive; the # tests skip via has_platform_exec() if it is missing. sandbox-exec needs # no elevation, so they run in this standard macOS job. - # - name: Test - # run: cargo test --locked --release --target aarch64-apple-darwin - # -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests + - name: Test + run: cargo test --locked --release --target aarch64-apple-darwin + -p mxc_darwin -p seatbelt_common -p wxc_common -p wxc_e2e_tests - name: Verify artifact payload shell: bash diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index 6c7a1d5d3..523e42c8b 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -66,10 +66,10 @@ jobs: --no-default-features --features "${{ matrix.features }}" - # - name: Test - # run: cargo test --locked --release --target ${{ matrix.target }} - # --no-default-features - # --features "${{ matrix.features }}" + - name: Test + run: cargo test --locked --release --target ${{ matrix.target }} + --no-default-features + --features "${{ matrix.features }}" - name: Verify artifact payload shell: pwsh diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 87e0c2f9b..6820133b9 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -5,9 +5,13 @@ name: Build on: push: + paths-ignore: + - 'docs/**' branches: [main] pull_request: + paths-ignore: + - 'docs/**' branches: [main, 'feature/*', 'user/*'] workflow_call: @@ -22,36 +26,6 @@ permissions: contents: read jobs: - pr-test-eligibility: - name: Check PR test eligibility - if: github.event_name == 'pull_request' && github.base_ref == 'main' - runs-on: ubuntu-latest - outputs: - should_run: ${{ steps.changes.outputs.should_run }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Check for non-documentation changes - id: changes - shell: bash - run: | - set -euo pipefail - should_run=false - while IFS= read -r file; do - case "$file" in - docs/*|*.md) ;; - *) - should_run=true - break - ;; - esac - done < <(git diff --name-only \ - '${{ github.event.pull_request.base.sha }}' \ - '${{ github.event.pull_request.head.sha }}') - echo "should_run=$should_run" >> "$GITHUB_OUTPUT" - # Dependency feed check: resolve the locked graph through the public MxcDependencies # feed so a crate not yet in the feed fails at PR time. dependency-feed-check: @@ -76,9 +50,9 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Build.MacOS.Job.yml - pr-test-matrix: - needs: [pr-test-eligibility, windows, linux, macos] - if: needs.pr-test-eligibility.outputs.should_run == 'true' + pr-e2e-test-matrix: + needs: [windows, linux, macos] + if: github.event_name == 'pull_request' && github.base_ref == 'main' uses: ./.github/workflows/Test.Matrix.Job.yml with: plan: pr diff --git a/.github/workflows/Lint.Job.yml b/.github/workflows/Lint.Job.yml index b26ca6c41..b6a2a38a8 100644 --- a/.github/workflows/Lint.Job.yml +++ b/.github/workflows/Lint.Job.yml @@ -24,7 +24,7 @@ jobs: clippy-args: --locked --all-targets --all-features --release - os_label: macos component: MAC - runner: macos-14 + runner: macos-26 target: aarch64-apple-darwin working-directory: src # Workspace has Windows-only crates, so --all-features won't build on macOS. diff --git a/.github/workflows/SDK.Integration.Test.Job.yml b/.github/workflows/SDK.Integration.Test.Job.yml index aad3532a8..09a57911e 100644 --- a/.github/workflows/SDK.Integration.Test.Job.yml +++ b/.github/workflows/SDK.Integration.Test.Job.yml @@ -15,7 +15,7 @@ jobs: - os_label: linux runner: ubuntu-latest - os_label: macos - runner: macos-14 + runner: macos-26 runs-on: ${{ matrix.runner }} defaults: run: diff --git a/.github/workflows/Validation.Infra.PR.Tests.yml b/.github/workflows/Validation.Infra.PR.Tests.yml index 5dc195b7b..923eec762 100644 --- a/.github/workflows/Validation.Infra.PR.Tests.yml +++ b/.github/workflows/Validation.Infra.PR.Tests.yml @@ -20,9 +20,13 @@ jobs: linux: uses: ./.github/workflows/Build.Linux.Job.yml + macos: + uses: ./.github/workflows/Build.MacOS.Job.yml + test: - needs: [windows, linux] + needs: [windows, linux, macos] uses: ./.github/workflows/Test.Matrix.Job.yml with: - # The resolver selects only explicitly enabled entries in the PR trigger. - plan: pr + # Exercise every combination currently rolled out, independent of its + # normal PR/nightly/weekly schedule. + plan: enabled diff --git a/scripts/ci/resolve-test-matrix.mjs b/scripts/ci/resolve-test-matrix.mjs index 2be65887f..75c7a97a0 100644 --- a/scripts/ci/resolve-test-matrix.mjs +++ b/scripts/ci/resolve-test-matrix.mjs @@ -11,6 +11,7 @@ import { fileURLToPath } from 'node:url'; const FAMILIES = ['windows', 'linux', 'macos']; const PLANS = ['pr', 'nightly', 'weekly']; +const RESOLUTION_PLANS = [...PLANS, 'enabled']; const ARM64_UNSUPPORTED_BACKENDS = new Set(['hyperlight', 'microvm']); function combinationKey(plan, os, architecture, backend) { @@ -208,6 +209,43 @@ export function expandPlan(catalog, plan) { } export function resolvePlan(catalog, plan) { + if (!RESOLUTION_PLANS.includes(plan)) { + throw new Error(`unsupported plan: ${plan}`); + } + + if (plan === 'enabled') { + const { platforms } = validateCatalog(catalog); + const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); + const seen = new Set(); + + for (const entry of catalog.enabled ?? []) { + const key = `${entry.os}|${entry.architecture}|${entry.backend}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + + const platform = platforms.get(entry.os); + const details = platform.architectures[entry.architecture]; + const handler = catalog.handlers[platform.family][entry.backend]; + matrices[platform.family].push({ + plan, + os: platform.id, + os_name: platform.displayName, + architecture: entry.architecture, + target: details.target, + artifact: details.artifact, + pool: details.pool, + runner: details.runner, + backend: entry.backend, + command: handler.command + }); + } + + sortMatrices(matrices); + return matrices; + } + const { enabled } = validateCatalog(catalog); const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); @@ -226,6 +264,11 @@ export function resolvePlan(catalog, plan) { } } + sortMatrices(matrices); + return matrices; +} + +function sortMatrices(matrices) { for (const family of FAMILIES) { // Stable ordering keeps local output and workflow diagnostics reproducible. matrices[family].sort((left, right) => ( @@ -233,7 +276,6 @@ export function resolvePlan(catalog, plan) { .localeCompare(`${right.os}|${right.architecture}|${right.backend}`) )); } - return matrices; } function parseArguments(argv) { diff --git a/scripts/ci/test-matrix.json b/scripts/ci/test-matrix.json index 2799a88df..4e8fad871 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/test-matrix.json @@ -290,7 +290,7 @@ "arm64": { "target": "aarch64-apple-darwin", "artifact": "mxc-binaries-aarch64-apple-darwin", - "runner": "macos-26-xlarge", + "runner": "macos-26", "backends": [ "seatbelt" ] @@ -305,7 +305,7 @@ "arm64": { "target": "aarch64-apple-darwin", "artifact": "mxc-binaries-aarch64-apple-darwin", - "runner": "macos-15-xlarge", + "runner": "macos-15", "backends": [ "seatbelt" ] @@ -364,7 +364,8 @@ }, "macos": { "seatbelt": { - "status": "placeholder" + "command": "seatbelt", + "status": "wired" } } }, @@ -536,6 +537,24 @@ "os": "ubuntu-26.04", "architecture": "x64", "backend": "bubblewrap" + }, + { + "plan": "pr", + "os": "macos-26", + "architecture": "arm64", + "backend": "seatbelt" + }, + { + "plan": "nightly", + "os": "macos-26", + "architecture": "arm64", + "backend": "seatbelt" + }, + { + "plan": "weekly", + "os": "macos-15", + "architecture": "arm64", + "backend": "seatbelt" } ] } diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/tests/resolve-test-matrix.test.mjs index 154b4c56e..6031ebd90 100644 --- a/scripts/ci/tests/resolve-test-matrix.test.mjs +++ b/scripts/ci/tests/resolve-test-matrix.test.mjs @@ -28,14 +28,51 @@ test('catalog validates and contains all five build targets', () => { assert.doesNotThrow(() => validateCatalog(catalog())); }); -test('current rollout enables two PR jobs and no scheduled jobs', () => { +test('current rollout enables three PR jobs and scheduled macOS coverage', () => { const pr = resolvePlan(catalog(), 'pr'); const nightly = resolvePlan(catalog(), 'nightly'); const weekly = resolvePlan(catalog(), 'weekly'); - assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 2); - assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 0); - assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 0); + assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 3); + assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 1); + assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 2); +}); + +test('macOS rollout uses 26 for PR and nightly with 15 added weekly', () => { + const project = entry => ({ + plan: entry.plan, + os: entry.os, + runner: entry.runner, + backend: entry.backend + }); + + assert.deepEqual(resolvePlan(catalog(), 'pr').macos.map(project), [ + { plan: 'pr', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } + ]); + assert.deepEqual(resolvePlan(catalog(), 'nightly').macos.map(project), [ + { plan: 'nightly', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } + ]); + assert.deepEqual(resolvePlan(catalog(), 'weekly').macos.map(project), [ + { plan: 'weekly', os: 'macos-15', runner: 'macos-15', backend: 'seatbelt' }, + { plan: 'nightly', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } + ]); +}); + +test('enabled plan deduplicates and runs both macOS versions', () => { + const enabled = resolvePlan(catalog(), 'enabled'); + assert.equal(enabled.windows.length + enabled.linux.length + enabled.macos.length, 4); + assert.deepEqual( + enabled.macos.map(entry => ({ + plan: entry.plan, + os: entry.os, + runner: entry.runner, + backend: entry.backend + })), + [ + { plan: 'enabled', os: 'macos-15', runner: 'macos-15', backend: 'seatbelt' }, + { plan: 'enabled', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } + ] + ); }); test('weekly includes all enabled nightly combinations', () => { @@ -78,13 +115,12 @@ test('arm64 never expands Hyperlight or MicroVM', () => { }); test('enabled placeholder handlers are rejected', () => { - // Seatbelt is declared in the capability map but intentionally not wired yet. const modified = clone(catalog()); modified.enabled.push({ plan: 'weekly', - os: 'macos-15', - architecture: 'arm64', - backend: 'seatbelt' + os: 'ubuntu-24.04', + architecture: 'x64', + backend: 'hyperlight' }); assert.throws( () => validateCatalog(modified), diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index 52f753738..992154ef5 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -21,8 +21,6 @@ repo_root="$(cd "$script_root/../.." && pwd)" release_directory="$repo_root/src/target/release" case "$backend" in - bubblewrap|lxc) - ;; microvm) echo "The MicroVM CI handler is not wired to an artifact-only Linux test entry point yet." >&2 exit 2 @@ -31,29 +29,30 @@ case "$backend" in echo "The Hyperlight CI handler is not wired to an existing backend test entry point yet." >&2 exit 2 ;; - seatbelt) - echo "The Seatbelt CI handler is not wired to an existing backend test entry point yet." >&2 - exit 2 - ;; - *) - usage - exit 2 - ;; -esac - -# Existing shell tests locate binaries under src/target/release. Recreate that -# layout from the downloaded artifact, including adjacent runtime assets. -test -x "$binary_directory/lxc-exec" -test -f "$binary_directory/unix-test-proxy" -mkdir -p "$release_directory" -cp -a "$binary_directory/." "$release_directory/" -chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" - -case "$backend" in bubblewrap) + # Existing Linux shell tests locate binaries under src/target/release. + test -x "$binary_directory/lxc-exec" + test -f "$binary_directory/unix-test-proxy" + mkdir -p "$release_directory" + cp -a "$binary_directory/." "$release_directory/" + chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" bash "$script_root/run_bwrap_all_tests.sh" ;; lxc) + test -x "$binary_directory/lxc-exec" + test -f "$binary_directory/unix-test-proxy" + mkdir -p "$release_directory" + cp -a "$binary_directory/." "$release_directory/" + chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" bash "$script_root/run_lxc_all_tests.sh" ;; + seatbelt) + test -x "$binary_directory/mxc-exec-mac" + test -x "$binary_directory/unix-test-proxy" + bash "$script_root/run_seatbelt_all_tests.sh" "$binary_directory" + ;; + *) + usage + exit 2 + ;; esac diff --git a/tests/scripts/run_seatbelt_all_tests.sh b/tests/scripts/run_seatbelt_all_tests.sh new file mode 100644 index 000000000..52deced70 --- /dev/null +++ b/tests/scripts/run_seatbelt_all_tests.sh @@ -0,0 +1,563 @@ +#!/bin/bash +# Artifact-only macOS Seatbelt executor tests. +set -uo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +BINARY_DIRECTORY="$(cd "$1" && pwd)" +MXC_EXEC="$BINARY_DIRECTORY/mxc-exec-mac" +UNIX_TEST_PROXY="$BINARY_DIRECTORY/unix-test-proxy" + +if [[ ! -x "$MXC_EXEC" ]]; then + echo "Error: executable mxc-exec-mac not found in $BINARY_DIRECTORY" >&2 + exit 1 +fi +if [[ ! -x "$UNIX_TEST_PROXY" ]]; then + echo "Error: executable unix-test-proxy not found in $BINARY_DIRECTORY" >&2 + exit 1 +fi +for command_name in curl python3; do + if ! command -v "$command_name" >/dev/null; then + echo "Error: required command not found: $command_name" >&2 + exit 1 + fi +done + +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/mxc-seatbelt-tests.XXXXXX")" +PASSED=0 +FAILED=0 +FAILURES="" +INFO_PASSED=0 +INFO_FAILED=0 +INFO_FAILURES="" +SERVER_PID="" + +cleanup() { + if [[ -n "$SERVER_PID" ]]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +run_test() { + local name="$1" + local function_name="$2" + echo "=== $name ===" + if "$function_name"; then + echo "PASS: $name" + PASSED=$((PASSED + 1)) + else + echo "FAIL: $name" + FAILED=$((FAILED + 1)) + FAILURES="${FAILURES}\n - ${name}" + fi + echo "" +} + +run_info_test() { + local name="$1" + local function_name="$2" + echo "=== $name (INFORMATION ONLY) ===" + if "$function_name"; then + echo "INFO-PASS: $name" + INFO_PASSED=$((INFO_PASSED + 1)) + else + echo "INFO-FAIL: $name" + INFO_FAILED=$((INFO_FAILED + 1)) + INFO_FAILURES="${INFO_FAILURES}\n - ${name}" + fi + echo "" +} + +seed_host_clipboard() { + python3 - "$1" <<'PYTHON' +import subprocess +import sys + +token = sys.argv[1].encode() +subprocess.run(["/usr/bin/pbcopy"], input=token, timeout=5, check=True) +result = subprocess.run( + ["/usr/bin/pbpaste"], + capture_output=True, + timeout=5, + check=True, +) +sys.exit(0 if result.stdout == token else 1) +PYTHON +} + +host_clipboard_matches() { + python3 - "$1" <<'PYTHON' +import subprocess +import sys + +result = subprocess.run( + ["/usr/bin/pbpaste"], + capture_output=True, + timeout=5, + check=True, +) +sys.exit(0 if result.stdout == sys.argv[1].encode() else 1) +PYTHON +} + +test_execution() { + local config="$TEST_ROOT/execution.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-execution", + "containment": "seatbelt", + "process": { "commandLine": "printf 'SEATBELT_EXEC_OK\\n'" } +} +JSON + + local output + output=$("$MXC_EXEC" "$config" 2>&1) || { + echo "$output" + return 1 + } + grep -q "SEATBELT_EXEC_OK" <<<"$output" +} + +test_exit_code() { + local config="$TEST_ROOT/exit-code.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-exit-code", + "containment": "seatbelt", + "process": { "commandLine": "exit 7" } +} +JSON + + local output status + if output=$("$MXC_EXEC" "$config" 2>&1); then + echo "Expected exit code 7, got 0" + return 1 + else + status=$? + fi + if [[ $status -ne 7 ]]; then + echo "$output" + echo "Expected exit code 7, got $status" + return 1 + fi +} + +test_filesystem_policy() { + local allowed="$TEST_ROOT/allowed" + local denied="$TEST_ROOT/denied" + local config="$TEST_ROOT/filesystem.json" + mkdir -p "$allowed" "$denied" + printf 'host secret\n' >"$denied/secret.txt" + + cat >"$config" < '$allowed/output.txt'; if cat '$denied/secret.txt' >/dev/null 2>&1; then echo DENIED_PATH_LEAK; exit 1; fi; echo FILESYSTEM_OK" + }, + "filesystem": { + "readwritePaths": ["$allowed"], + "deniedPaths": ["$denied"] + } +} +JSON + + local output + output=$("$MXC_EXEC" "$config" 2>&1) || { + echo "$output" + return 1 + } + grep -q "FILESYSTEM_OK" <<<"$output" && + ! grep -q "DENIED_PATH_LEAK" <<<"$output" && + grep -q "sandbox write" "$allowed/output.txt" +} + +test_environment_policy() { + local config="$TEST_ROOT/environment.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-environment", + "containment": "seatbelt", + "process": { + "commandLine": "printf 'HOST=[%s] CONFIG=[%s]\\n' \"$MXC_HOST_MARKER\" \"$MXC_CONFIG_MARKER\"", + "env": ["MXC_CONFIG_MARKER=from_config"] + } +} +JSON + + local output + output=$(MXC_HOST_MARKER=must_not_leak "$MXC_EXEC" "$config" 2>&1) || { + echo "$output" + return 1 + } + grep -q "HOST=\[\] CONFIG=\[from_config\]" <<<"$output" && + ! grep -q "must_not_leak" <<<"$output" +} + +test_timeout() { + local config="$TEST_ROOT/timeout.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-timeout", + "containment": "seatbelt", + "process": { + "commandLine": "echo TIMEOUT_STARTED; /bin/sleep 10; echo TIMEOUT_LEAK", + "timeout": 1000 + } +} +JSON + + local output status start elapsed + start=$SECONDS + if output=$("$MXC_EXEC" "$config" 2>&1); then + echo "$output" + echo "Expected timed-out execution to fail" + return 1 + else + status=$? + fi + elapsed=$((SECONDS - start)) + if [[ $status -eq 0 || $elapsed -ge 8 ]]; then + echo "$output" + echo "Timeout did not terminate promptly (status=$status elapsed=${elapsed}s)" + return 1 + fi + grep -q "TIMEOUT_STARTED" <<<"$output" && + ! grep -q "TIMEOUT_LEAK" <<<"$output" +} + +start_host_server() { + local server_script="$TEST_ROOT/server.py" + local port_file="$TEST_ROOT/server.port" + cat >"$server_script" <<'PYTHON' +import http.server +import socketserver +import sys + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = b"HOST_SERVER_OK\n" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + pass + +with socketserver.TCPServer(("127.0.0.1", 0), Handler) as server: + with open(sys.argv[1], "w", encoding="utf-8") as port_file: + port_file.write(str(server.server_address[1])) + server.serve_forever() +PYTHON + + python3 "$server_script" "$port_file" >"$TEST_ROOT/server.log" 2>&1 & + SERVER_PID=$! + for _ in {1..50}; do + [[ -s "$port_file" ]] && break + sleep 0.1 + done + [[ -s "$port_file" ]] || { + cat "$TEST_ROOT/server.log" + return 1 + } + HOST_SERVER_PORT="$(cat "$port_file")" + curl --fail --silent --max-time 2 "http://127.0.0.1:$HOST_SERVER_PORT" | + grep -q "HOST_SERVER_OK" +} + +test_network_default_deny() { + start_host_server || return 1 + + local config="$TEST_ROOT/network-deny.json" + cat >"$config" </dev/null 2>&1; then echo NETWORK_LEAK; exit 1; else echo NETWORK_BLOCKED; fi" + }, + "network": { "defaultPolicy": "block" } +} +JSON + + local output + output=$("$MXC_EXEC" "$config" 2>&1) || { + echo "$output" + return 1 + } + grep -q "NETWORK_BLOCKED" <<<"$output" && + ! grep -q "NETWORK_LEAK" <<<"$output" +} + +test_builtin_proxy_startup() { + local config="$TEST_ROOT/proxy.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-proxy", + "containment": "seatbelt", + "process": { + "commandLine": "printf 'HTTP_PROXY=%s HTTPS_PROXY=%s\\n' \"$HTTP_PROXY\" \"$HTTPS_PROXY\"" + }, + "network": { + "defaultPolicy": "block", + "proxy": { "builtinTestServer": true } + } +} +JSON + + local output + output=$("$MXC_EXEC" --experimental --allow-testing-features "$config" 2>&1) || { + echo "$output" + return 1 + } + grep -Eq "HTTP_PROXY=http://127\.0\.0\.1:[0-9]+ HTTPS_PROXY=http://127\.0\.0\.1:[0-9]+" \ + <<<"$output" +} + +test_clipboard_allow() { + local token="mxc_seatbelt_clipboard_$$_${RANDOM}" + local allow_config="$TEST_ROOT/clipboard-allow.json" + + cat >"$allow_config" <&1) || { + echo "$output" + echo "Clipboard allow probe failed" + return 1 + } + if ! grep -q "$token" <<<"$output"; then + echo "$output" + echo "Clipboard allow probe did not return its token" + return 1 + fi +} + +test_clipboard_deny() { + local token="mxc_seatbelt_clipboard_deny_$$_${RANDOM}" + local deny_config="$TEST_ROOT/clipboard-deny.json" + if ! seed_host_clipboard "$token"; then + echo "Host clipboard read/write baseline failed" + return 1 + fi + + cat >"$deny_config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-clipboard-deny", + "containment": "seatbelt", + "process": { + "commandLine": "copy_status=0; printf 'clipboard_denied_probe' | /usr/bin/pbcopy >/dev/null 2>&1 || copy_status=$?; read_status=0; /usr/bin/pbpaste >/dev/null 2>&1 || read_status=$?; leaked=0; if [ $copy_status -eq 0 ]; then echo CLIPBOARD_WRITE_LEAK; leaked=1; fi; if [ $read_status -eq 0 ]; then echo CLIPBOARD_READ_LEAK; leaked=1; fi; if [ $leaked -eq 0 ]; then echo CLIPBOARD_DENIED; else exit 1; fi", + "timeout": 10000 + }, + "ui": { + "disable": false, + "clipboard": "none" + } +} +JSON + + local output + output=$("$MXC_EXEC" "$deny_config" 2>&1) || { + echo "$output" + echo "Clipboard deny probe failed" + return 1 + } + if ! grep -q "CLIPBOARD_DENIED" <<<"$output" || + grep -Eq "CLIPBOARD_WRITE_LEAK|CLIPBOARD_READ_LEAK" <<<"$output"; then + return 1 + fi + if ! host_clipboard_matches "$token"; then + echo "Host clipboard changed during denied sandbox probe" + return 1 + fi +} + +test_gui_session() { + local config="$TEST_ROOT/gui.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-gui", + "containment": "seatbelt", + "process": { + "commandLine": "/usr/bin/osascript -e 'tell application \"Finder\" to get name' && echo GUI_SESSION_OK", + "timeout": 10000 + }, + "ui": { + "disable": false, + "clipboard": "all" + }, + "seatbelt": { + "guiAccess": true + } +} +JSON + + local output + output=$("$MXC_EXEC" "$config" 2>&1) || { + echo "$output" + echo "GUI session probe failed" + return 1 + } + grep -q "GUI_SESSION_OK" <<<"$output" +} + +test_public_internet_allow() { + local config="$TEST_ROOT/public-internet.json" + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-public-internet", + "containment": "seatbelt", + "process": { + "commandLine": "curl --fail --silent --show-error --max-time 10 https://example.com >/dev/null && echo PUBLIC_INTERNET_OK", + "timeout": 15000 + }, + "network": { + "defaultPolicy": "allow" + } +} +JSON + + local output + output=$("$MXC_EXEC" "$config" 2>&1) || { + echo "$output" + echo "Public internet probe failed" + return 1 + } + grep -q "PUBLIC_INTERNET_OK" <<<"$output" +} + +test_proxy_traffic_filtering() { + local control_config="$TEST_ROOT/proxy-filtering-control.json" + local config="$TEST_ROOT/proxy-filtering.json" + if ! curl --noproxy '*' --fail --silent --max-time 10 \ + https://example.com >/dev/null; then + echo "Host direct-egress baseline failed" + return 1 + fi + + cat >"$control_config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-proxy-filtering-control", + "containment": "seatbelt", + "process": { + "commandLine": "curl --fail --silent --show-error --max-time 10 https://example.com >/dev/null && echo PROXY_CONTROL_OK", + "timeout": 15000 + }, + "network": { + "defaultPolicy": "allow", + "proxy": { + "builtinTestServer": true + } + } +} +JSON + + local output + output=$("$MXC_EXEC" --experimental --allow-testing-features "$control_config" 2>&1) || { + echo "$output" + echo "Unfiltered proxy control failed" + return 1 + } + if ! grep -q "PROXY_CONTROL_OK" <<<"$output"; then + echo "$output" + echo "Unfiltered proxy control did not reach the negative-test target" + return 1 + fi + + cat >"$config" <<'JSON' +{ + "version": "0.7.0-alpha", + "containerId": "ci-seatbelt-proxy-filtering", + "containment": "seatbelt", + "process": { + "commandLine": "set -e; curl --fail --silent --show-error --max-time 10 https://api.github.com/zen >/dev/null; echo PROXY_ALLOWED; if curl --fail --silent --max-time 5 https://example.com >/dev/null 2>&1; then echo PROXY_FILTER_LEAK; exit 1; else echo PROXY_FILTERED; fi; if curl --noproxy '*' --fail --silent --max-time 5 https://example.com >/dev/null 2>&1; then echo PROXY_DIRECT_BYPASS_OBSERVED; else echo PROXY_DIRECT_BLOCKED_OBSERVED; fi", + "timeout": 30000 + }, + "network": { + "defaultPolicy": "block", + "proxy": { + "builtinTestServer": true + }, + "allowedHosts": [ + "api.github.com" + ] + } +} +JSON + + output=$("$MXC_EXEC" --experimental --allow-testing-features "$config" 2>&1) || { + echo "$output" + echo "Proxy traffic filtering probe failed" + return 1 + } + if grep -q "PROXY_DIRECT_BYPASS_OBSERVED" <<<"$output"; then + echo "INFO: direct traffic bypassed the cooperative proxy, as Seatbelt currently permits" + elif grep -q "PROXY_DIRECT_BLOCKED_OBSERVED" <<<"$output"; then + echo "INFO: direct traffic was blocked independently of the cooperative proxy" + else + echo "$output" + echo "Direct traffic observation was missing" + return 1 + fi + grep -q "PROXY_ALLOWED" <<<"$output" && + grep -q "PROXY_FILTERED" <<<"$output" && + ! grep -q "PROXY_FILTER_LEAK" <<<"$output" +} + +run_test "Seatbelt execution" test_execution +run_test "Seatbelt exit code" test_exit_code +run_test "Seatbelt filesystem policy" test_filesystem_policy +run_test "Seatbelt environment policy" test_environment_policy +run_test "Seatbelt timeout" test_timeout +run_test "Seatbelt network default deny" test_network_default_deny +run_test "Seatbelt builtin proxy startup" test_builtin_proxy_startup + +run_info_test "Seatbelt clipboard allow" test_clipboard_allow +run_info_test "Seatbelt clipboard deny" test_clipboard_deny +run_info_test "Seatbelt GUI session" test_gui_session +run_info_test "Seatbelt public internet allow" test_public_internet_allow +run_info_test "Seatbelt full proxy traffic filtering" test_proxy_traffic_filtering + +echo "================================" +echo "Results: $PASSED passed, $FAILED failed" +echo "Information only: $INFO_PASSED passed, $INFO_FAILED failed" +if [[ $INFO_FAILED -gt 0 ]]; then + echo -e "Informational failures (non-blocking):$INFO_FAILURES" +fi +if [[ $FAILED -gt 0 ]]; then + echo -e "Failures:$FAILURES" + exit 1 +fi From 7fc8290739244a12d34355d4e21372cc99558f5e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:10:21 -0700 Subject: [PATCH 31/93] fix bugs in e2e test for macos --- .github/workflows/Test.Matrix.Job.yml | 8 +++--- ...Tests.yml => Validation.Infra.Testing.yml} | 2 +- tests/scripts/run_seatbelt_all_tests.sh | 26 ++++++++++++------- 3 files changed, 21 insertions(+), 15 deletions(-) rename .github/workflows/{Validation.Infra.PR.Tests.yml => Validation.Infra.Testing.yml} (94%) diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index af606d97c..19213f169 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -14,7 +14,7 @@ permissions: jobs: resolve: - name: Resolve ${{ inputs.plan }} matrix + name: resolve "${{ inputs.plan }}" test matrix runs-on: ubuntu-latest outputs: windows: ${{ steps.matrix.outputs.windows }} @@ -31,7 +31,7 @@ jobs: run: node scripts/ci/resolve-test-matrix.mjs --plan "${{ inputs.plan }}" windows: - name: ${{ matrix.backend || 'Windows tests' }} + name: ${{ matrix.os }} / ${{ matrix.architecture }} / ${{ matrix.backend }} needs: resolve if: needs.resolve.outputs.has_windows == 'true' strategy: @@ -88,7 +88,7 @@ jobs: retention-days: 7 linux: - name: ${{ matrix.backend || 'Linux tests' }} + name: ${{ matrix.os }} / ${{ matrix.architecture }} / ${{ matrix.backend }} needs: resolve if: needs.resolve.outputs.has_linux == 'true' strategy: @@ -151,7 +151,7 @@ jobs: retention-days: 7 macos: - name: ${{ matrix.backend || 'macOS tests' }} + name: ${{ matrix.os }} / ${{ matrix.architecture }} / ${{ matrix.backend }} needs: resolve if: needs.resolve.outputs.has_macos == 'true' strategy: diff --git a/.github/workflows/Validation.Infra.PR.Tests.yml b/.github/workflows/Validation.Infra.Testing.yml similarity index 94% rename from .github/workflows/Validation.Infra.PR.Tests.yml rename to .github/workflows/Validation.Infra.Testing.yml index 923eec762..a28178fe1 100644 --- a/.github/workflows/Validation.Infra.PR.Tests.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -1,4 +1,4 @@ -name: Validation Infrastructure PR Tests +name: Validation Infrastructure Testing on: push: diff --git a/tests/scripts/run_seatbelt_all_tests.sh b/tests/scripts/run_seatbelt_all_tests.sh index 52deced70..e46a9a701 100644 --- a/tests/scripts/run_seatbelt_all_tests.sh +++ b/tests/scripts/run_seatbelt_all_tests.sh @@ -27,6 +27,7 @@ for command_name in curl python3; do done TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/mxc-seatbelt-tests.XXXXXX")" +TEST_ROOT="$(cd "$TEST_ROOT" && pwd -P)" PASSED=0 FAILED=0 FAILURES="" @@ -241,6 +242,7 @@ JSON start_host_server() { local server_script="$TEST_ROOT/server.py" local port_file="$TEST_ROOT/server.port" + rm -f "$port_file" cat >"$server_script" <<'PYTHON' import http.server import socketserver @@ -461,20 +463,24 @@ JSON test_proxy_traffic_filtering() { local control_config="$TEST_ROOT/proxy-filtering-control.json" local config="$TEST_ROOT/proxy-filtering.json" - if ! curl --noproxy '*' --fail --silent --max-time 10 \ - https://example.com >/dev/null; then - echo "Host direct-egress baseline failed" + if [[ -z "${HOST_SERVER_PORT:-}" ]] || + ! kill -0 "$SERVER_PID" 2>/dev/null; then + start_host_server || return 1 + fi + if ! curl --noproxy '*' --fail --silent --max-time 2 \ + "http://127.0.0.1:$HOST_SERVER_PORT" >/dev/null; then + echo "Host local-server baseline failed" return 1 fi - cat >"$control_config" <<'JSON' + cat >"$control_config" </dev/null && echo PROXY_CONTROL_OK", - "timeout": 15000 + "commandLine": "curl --fail --silent --show-error --max-time 5 http://127.0.0.1:$HOST_SERVER_PORT >/dev/null && echo PROXY_CONTROL_OK", + "timeout": 10000 }, "network": { "defaultPolicy": "allow", @@ -497,14 +503,14 @@ JSON return 1 fi - cat >"$config" <<'JSON' + cat >"$config" </dev/null; echo PROXY_ALLOWED; if curl --fail --silent --max-time 5 https://example.com >/dev/null 2>&1; then echo PROXY_FILTER_LEAK; exit 1; else echo PROXY_FILTERED; fi; if curl --noproxy '*' --fail --silent --max-time 5 https://example.com >/dev/null 2>&1; then echo PROXY_DIRECT_BYPASS_OBSERVED; else echo PROXY_DIRECT_BLOCKED_OBSERVED; fi", - "timeout": 30000 + "commandLine": "set -e; curl --fail --silent --show-error --max-time 5 http://127.0.0.1:$HOST_SERVER_PORT >/dev/null; echo PROXY_ALLOWED; if curl --fail --silent --max-time 5 http://localhost:$HOST_SERVER_PORT >/dev/null 2>&1; then echo PROXY_FILTER_LEAK; exit 1; else echo PROXY_FILTERED; fi; if curl --noproxy '*' --fail --silent --max-time 5 http://localhost:$HOST_SERVER_PORT >/dev/null 2>&1; then echo PROXY_DIRECT_BYPASS_OBSERVED; else echo PROXY_DIRECT_BLOCKED_OBSERVED; fi", + "timeout": 20000 }, "network": { "defaultPolicy": "block", @@ -512,7 +518,7 @@ JSON "builtinTestServer": true }, "allowedHosts": [ - "api.github.com" + "127.0.0.1" ] } } From 8d3dfb578dc645c6b818c07ecf144bfe91fc91f2 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:33:57 -0700 Subject: [PATCH 32/93] better naming conventions --- .github/copilot-instructions.md | 9 +++++++ .../{Build.yml => Build.And.Validate.yml} | 24 +++++++------------ .github/workflows/Build.Artifacts.Job.yml | 18 ++++++++++++++ .github/workflows/Scheduled.E2E.yml | 14 ++++------- .github/workflows/Test.Matrix.Job.yml | 8 +++---- .../workflows/Validation.Infra.Testing.yml | 12 +++------- docs/pull-requests.md | 6 ++--- 7 files changed, 49 insertions(+), 42 deletions(-) rename .github/workflows/{Build.yml => Build.And.Validate.yml} (71%) create mode 100644 .github/workflows/Build.Artifacts.Job.yml diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 022ad111d..c703eac9a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -43,6 +43,15 @@ build.bat --with-microvm # Include NanVix micro-VM binaries Requires Xcode Command Line Tools and Rust. Produces an unsigned `mxc-exec-mac` binary (codesigning + notarization happen at release time). Schema `0.7.0-alpha` or later required for macOS/Seatbelt backend. +### GitHub Actions + +`.github/workflows/Build.And.Validate.yml` is the PR/CI entry point. It calls the +workflow-call-only `.github/workflows/Build.Artifacts.Job.yml`, which builds and +uploads the Windows, Linux, and macOS artifacts in parallel. Scheduled E2E and +validation-infrastructure workflows reuse the same artifact workflow before +calling `.github/workflows/Test.Matrix.Job.yml`; keep artifact production and +consumption in the same workflow run. + ### Individual components ``` diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.And.Validate.yml similarity index 71% rename from .github/workflows/Build.yml rename to .github/workflows/Build.And.Validate.yml index 6820133b9..9400ae106 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.And.Validate.yml @@ -1,7 +1,7 @@ # Entry point for GHA PR/CI validation. Mirrors the 1ES `1ES.Build.yml` -# split: this file orchestrates, each stage lives in its own reusable. +# split: this file orchestrates reusable build-and-validation stages. -name: Build +name: Build + Validate on: push: @@ -18,7 +18,7 @@ on: workflow_dispatch: concurrency: - group: build-${{ github.workflow }}-${{ github.ref }} + group: build-validate-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: @@ -38,20 +38,12 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Lint.Job.yml - windows: + build-artifacts: needs: dependency-feed-check - uses: ./.github/workflows/Build.Windows.Job.yml + uses: ./.github/workflows/Build.Artifacts.Job.yml - linux: - needs: dependency-feed-check - uses: ./.github/workflows/Build.Linux.Job.yml - - macos: - needs: dependency-feed-check - uses: ./.github/workflows/Build.MacOS.Job.yml - - pr-e2e-test-matrix: - needs: [windows, linux, macos] + pr-validation-test-matrix: + needs: build-artifacts if: github.event_name == 'pull_request' && github.base_ref == 'main' uses: ./.github/workflows/Test.Matrix.Job.yml with: @@ -61,7 +53,7 @@ jobs: uses: ./.github/workflows/SDK.Unit.Test.Job.yml package-npm-sdk: - needs: [windows, linux, macos] + needs: build-artifacts uses: ./.github/workflows/Package.NpmSdk.Job.yml sdk-integration-tests: diff --git a/.github/workflows/Build.Artifacts.Job.yml b/.github/workflows/Build.Artifacts.Job.yml new file mode 100644 index 000000000..d118cde3a --- /dev/null +++ b/.github/workflows/Build.Artifacts.Job.yml @@ -0,0 +1,18 @@ +name: Build Artifacts + +on: + workflow_call: + +permissions: + actions: read + contents: read + +jobs: + windows: + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + uses: ./.github/workflows/Build.MacOS.Job.yml diff --git a/.github/workflows/Scheduled.E2E.yml b/.github/workflows/Scheduled.E2E.yml index 6e36b2eb0..d24c37199 100644 --- a/.github/workflows/Scheduled.E2E.yml +++ b/.github/workflows/Scheduled.E2E.yml @@ -1,4 +1,4 @@ -name: Scheduled E2E +name: Scheduled E2E Validation Tests on: schedule: @@ -25,17 +25,11 @@ permissions: contents: read jobs: - windows: - uses: ./.github/workflows/Build.Windows.Job.yml - - linux: - uses: ./.github/workflows/Build.Linux.Job.yml - - macos: - uses: ./.github/workflows/Build.MacOS.Job.yml + build-artifacts: + uses: ./.github/workflows/Build.Artifacts.Job.yml test: - needs: [windows, linux, macos] + needs: build-artifacts uses: ./.github/workflows/Test.Matrix.Job.yml with: plan: ${{ github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0' && 'weekly' || github.event_name == 'schedule' && 'nightly' || inputs.plan }} diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Test.Matrix.Job.yml index 19213f169..7f702c84a 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Test.Matrix.Job.yml @@ -1,4 +1,4 @@ -name: Test Matrix +name: E2E Validation Test Matrix on: workflow_call: @@ -31,7 +31,7 @@ jobs: run: node scripts/ci/resolve-test-matrix.mjs --plan "${{ inputs.plan }}" windows: - name: ${{ matrix.os }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + name: ${{ matrix.os }}, ${{ matrix.architecture }}, ${{ matrix.backend }} needs: resolve if: needs.resolve.outputs.has_windows == 'true' strategy: @@ -88,7 +88,7 @@ jobs: retention-days: 7 linux: - name: ${{ matrix.os }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + name: ${{ matrix.os }}, ${{ matrix.architecture }}, ${{ matrix.backend }} needs: resolve if: needs.resolve.outputs.has_linux == 'true' strategy: @@ -151,7 +151,7 @@ jobs: retention-days: 7 macos: - name: ${{ matrix.os }} / ${{ matrix.architecture }} / ${{ matrix.backend }} + name: ${{ matrix.os }}, ${{ matrix.backend }} needs: resolve if: needs.resolve.outputs.has_macos == 'true' strategy: diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml index a28178fe1..8101310d8 100644 --- a/.github/workflows/Validation.Infra.Testing.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -14,17 +14,11 @@ permissions: contents: read jobs: - windows: - uses: ./.github/workflows/Build.Windows.Job.yml - - linux: - uses: ./.github/workflows/Build.Linux.Job.yml - - macos: - uses: ./.github/workflows/Build.MacOS.Job.yml + build-artifacts: + uses: ./.github/workflows/Build.Artifacts.Job.yml test: - needs: [windows, linux, macos] + needs: build-artifacts uses: ./.github/workflows/Test.Matrix.Job.yml with: # Exercise every combination currently rolled out, independent of its diff --git a/docs/pull-requests.md b/docs/pull-requests.md index 617ae9188..fa4d70087 100644 --- a/docs/pull-requests.md +++ b/docs/pull-requests.md @@ -3,9 +3,9 @@ ## GitHub Actions (automatic) Every PR is validated automatically by the GitHub Actions workflows under -`.github/workflows/` (entry point: `Build.yml`). This is the primary PR -signal — it builds and tests on native Windows x64/arm64, Linux x64/arm64, -and macOS arm64 hosts in parallel. +`.github/workflows/` (entry point: `Build.And.Validate.yml`). This is the primary +PR signal — it calls the reusable `Build.Artifacts.Job.yml` workflow, then +tests on native Windows x64/arm64, Linux x64/arm64, and macOS arm64 hosts. ## Azure Pipelines (optional on PRs, required on `main`) From 5ec57a3f76450c9e4421554c7b800a29e2006230 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:54:49 -0700 Subject: [PATCH 33/93] fixed naming since github truncates things --- .github/workflows/Build.And.Validate.yml | 16 ++++++++++++---- .github/workflows/Build.Artifacts.Job.yml | 18 ------------------ .github/workflows/Scheduled.E2E.yml | 16 ++++++++++++++-- .github/workflows/Validation.Infra.Testing.yml | 18 +++++++++++++++--- 4 files changed, 41 insertions(+), 27 deletions(-) delete mode 100644 .github/workflows/Build.Artifacts.Job.yml diff --git a/.github/workflows/Build.And.Validate.yml b/.github/workflows/Build.And.Validate.yml index 9400ae106..60a5e962f 100644 --- a/.github/workflows/Build.And.Validate.yml +++ b/.github/workflows/Build.And.Validate.yml @@ -38,12 +38,20 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Lint.Job.yml - build-artifacts: + windows: needs: dependency-feed-check - uses: ./.github/workflows/Build.Artifacts.Job.yml + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + needs: dependency-feed-check + uses: ./.github/workflows/Build.MacOS.Job.yml pr-validation-test-matrix: - needs: build-artifacts + needs: [windows, linux, macos] if: github.event_name == 'pull_request' && github.base_ref == 'main' uses: ./.github/workflows/Test.Matrix.Job.yml with: @@ -53,7 +61,7 @@ jobs: uses: ./.github/workflows/SDK.Unit.Test.Job.yml package-npm-sdk: - needs: build-artifacts + needs: [windows, linux, macos] uses: ./.github/workflows/Package.NpmSdk.Job.yml sdk-integration-tests: diff --git a/.github/workflows/Build.Artifacts.Job.yml b/.github/workflows/Build.Artifacts.Job.yml deleted file mode 100644 index d118cde3a..000000000 --- a/.github/workflows/Build.Artifacts.Job.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Build Artifacts - -on: - workflow_call: - -permissions: - actions: read - contents: read - -jobs: - windows: - uses: ./.github/workflows/Build.Windows.Job.yml - - linux: - uses: ./.github/workflows/Build.Linux.Job.yml - - macos: - uses: ./.github/workflows/Build.MacOS.Job.yml diff --git a/.github/workflows/Scheduled.E2E.yml b/.github/workflows/Scheduled.E2E.yml index d24c37199..bc35c4c93 100644 --- a/.github/workflows/Scheduled.E2E.yml +++ b/.github/workflows/Scheduled.E2E.yml @@ -25,8 +25,20 @@ permissions: contents: read jobs: - build-artifacts: - uses: ./.github/workflows/Build.Artifacts.Job.yml + dependency-feed-check: + uses: ./.github/workflows/Dependency.Feed.Check.Job.yml + + windows: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + needs: dependency-feed-check + uses: ./.github/workflows/Build.MacOS.Job.yml test: needs: build-artifacts diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml index 8101310d8..aaf3c02ed 100644 --- a/.github/workflows/Validation.Infra.Testing.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -14,11 +14,23 @@ permissions: contents: read jobs: - build-artifacts: - uses: ./.github/workflows/Build.Artifacts.Job.yml + dependency-feed-check: + uses: ./.github/workflows/Dependency.Feed.Check.Job.yml + + windows: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + needs: dependency-feed-check + uses: ./.github/workflows/Build.MacOS.Job.yml test: - needs: build-artifacts + needs: [windows, linux, macos] uses: ./.github/workflows/Test.Matrix.Job.yml with: # Exercise every combination currently rolled out, independent of its From 03f26ffa2b5d9fa9085ca9c8c08aef83fc9d3275 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Wed, 5 Aug 2026 12:11:17 -0700 Subject: [PATCH 34/93] test(wslc): add WSLC e2e suite, guards, and 1ES workflow - e2e_wslc.rs: test_wslc_hello + test_wslc_suite (5-config smoke set: hello-world, exit-code, python-stdlib, network-isolated, large-output), mirroring the microvm/hyperlight suites. - lib.rs: has_wslc_sdk() (wslcsdk.dll next to wxc-exec) and has_wsl_runtime() (wsl --status) skip-guards, so the suite is a clean no-op without the WSLC SDK / WSL2 runtime. - wslc-e2e.yml: executes the suite on the 1ES nested-virt pool 1es-mxc-test-2 (WSL2 needs nested virt, unavailable on GitHub-hosted). Every runtime step is gated to degrade to a skip when prerequisites are absent. - Docs: CI/e2e section in wsl-container-getting-started.md + copilot-instructions. Verified: cargo fmt --check, clippy -D warnings, compiles, and skips cleanly at each guard boundary locally. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 1 + .github/workflows/wslc-e2e.yml | 119 +++++++++ docs/wsl/wsl-container-getting-started.md | 49 ++++ src/testing/wxc_e2e_tests/src/lib.rs | 45 ++++ src/testing/wxc_e2e_tests/tests/e2e_wslc.rs | 263 ++++++++++++++++++++ 5 files changed, 477 insertions(+) create mode 100644 .github/workflows/wslc-e2e.yml create mode 100644 src/testing/wxc_e2e_tests/tests/e2e_wslc.rs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1c6bc338b..b1630814e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -94,6 +94,7 @@ tests\scripts\run_bwrap_all_tests.sh # All Bubblewrap tests (Linux, req # E2E test crate — Rust executor integration tests (from src/) cargo test -p wxc_e2e_tests # Invokes MXC binaries directly cargo test -p wxc_e2e_tests -- --ignored # Include stress tests (run_on_repeat) +cargo test -p wxc_e2e_tests test_wslc_suite # WSLC suite — skips unless wslcsdk.dll (--features wslc) + WSL2 present; only EXECUTES on a nested-virt runner (1ES). CI: .github/workflows/wslc-e2e.yml ``` ## Architecture diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml new file mode 100644 index 000000000..bfd2d4db3 --- /dev/null +++ b/.github/workflows/wslc-e2e.yml @@ -0,0 +1,119 @@ +# ============================================================================ +# WSLC E2E Tests — executes the WSL Container backend on a 1ES nested-virt +# runner. Unlike microvm/hyperlight (which run on GitHub-hosted windows-latest +# and skip when WHP is absent), WSLC needs WSL2 — which requires nested +# virtualization — so it can only EXECUTE on a nested-virt-capable 1ES pool. +# +# Pool: 1es-mxc-test-2 (REPOSITORY-level, runner group Default; confirmed +# working). `shell: powershell` because the raw windows-25h2-pro-x64 CLIENT +# image ships Windows PowerShell 5.1, not pwsh (PS7). Cold-start pickup on the +# idle Stateless pool is ~10-11 min — the long timeout accounts for that plus +# an in-image toolchain/WSL setup on first run. +# +# IMAGE PREREQUISITES (see docs/wsl/wsl-container-getting-started.md): +# The raw client image is a bare desktop. For this workflow to build + run it +# needs: git, the Rust/MSVC toolchain, WSL2 with VirtualMachinePlatform +# enabled, and the WSLC SDK produced by a `--features wslc` build. Enabling +# VirtualMachinePlatform normally needs a reboot, which a Stateless ephemeral +# runner cannot do mid-job — so the robust long-term path is a 1ES Managed +# Image with WSL2 + VirtualMachinePlatform (and ideally the toolchain) baked +# in (the same DISM-at-image-build approach microsoft/ebpf-for-windows uses +# for Hyper-V). Every runtime-dependent step below is guarded so the job +# degrades to a clean SKIP rather than a hard failure when a prerequisite is +# missing. +# ============================================================================ +name: WSLC E2E Tests + +on: + push: + branches: + - user/modanish/1es-github-runner-e2e + workflow_dispatch: + +permissions: + contents: read + +jobs: + wslc-e2e: + name: WXC-Exec WSLC + runs-on: + - self-hosted + - "1ES.Pool=1es-mxc-test-2" + - "JobId=wslc-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + timeout-minutes: 45 + + steps: + - uses: actions/checkout@v4 + + # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root + # only. Surface src/rust-toolchain.toml there so the pinned toolchain is + # honored. Use PowerShell (bare client image may lack bash). + - name: Surface toolchain file at repo root + shell: powershell + run: Copy-Item src\rust-toolchain.toml rust-toolchain.toml -Force + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + target: x86_64-pc-windows-msvc + override: false + rustflags: '' + + - name: Point cargo at the MxcDependencies feed + uses: ./.github/actions/setup-cargo-feed + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: src -> target + key: wslc-e2e-v1 + + - name: Build with WSLC support + working-directory: src + run: cargo build --release --target x86_64-pc-windows-msvc + --no-default-features --features wslc + + - name: Ensure WSL2 runtime + id: wsl + shell: powershell + run: | + $ErrorActionPreference = 'Continue' + Write-Host "=== WSL runtime check ===" + # Best-effort update; do not fail the job if the update path is + # unavailable on this image. + wsl --update 2>&1 | Write-Host + wsl --version 2>&1 | Write-Host + wsl --status 2>&1 | Write-Host + $available = ($LASTEXITCODE -eq 0) + Write-Host "wsl_available = $available" + "wsl_available=$($available.ToString().ToLower())" | + Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Pre-pull WSLC images + if: steps.wsl.outputs.wsl_available == 'true' + shell: powershell + run: | + $exe = "src\target\x86_64-pc-windows-msvc\release\wxc-exec.exe" + ./scripts/setup-wslc.ps1 -Image alpine:latest, python:3.12-alpine -WxcExecPath $exe + + - name: Run WSLC E2E suite + if: steps.wsl.outputs.wsl_available == 'true' + working-directory: src + run: cargo test --release --target x86_64-pc-windows-msvc + -p wxc_e2e_tests test_wslc_suite -- --nocapture + + - name: Note skip when WSL unavailable + if: steps.wsl.outputs.wsl_available != 'true' + shell: powershell + run: | + Write-Host "::warning::WSL2 runtime not available on this runner image — WSLC suite skipped." + Write-Host "The raw client image needs WSL2 + VirtualMachinePlatform baked in (1ES Managed Image)." + + - name: Upload logs on failure + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: wslc-e2e-logs-${{ github.event.pull_request.number || github.run_number }} + retention-days: 7 + path: | + logs/ + **/*.log + wslc-perf-results.json diff --git a/docs/wsl/wsl-container-getting-started.md b/docs/wsl/wsl-container-getting-started.md index 3e5cdb6a9..3b7b6b75d 100644 --- a/docs/wsl/wsl-container-getting-started.md +++ b/docs/wsl/wsl-container-getting-started.md @@ -232,6 +232,55 @@ Paths in `filesystem.readwritePaths` and `filesystem.readonlyPaths` are mounted into the container. Host path `C:\workspace` becomes `/mnt/c/workspace` inside the container. +## Running the E2E test suite + +MXC ships a Rust E2E suite for WSLC in +[`src/testing/wxc_e2e_tests/tests/e2e_wslc.rs`](../../src/testing/wxc_e2e_tests/tests/e2e_wslc.rs), +mirroring the MicroVM and Hyperlight suites. It invokes `wxc-exec.exe` directly +against the core smoke configs (hello-world, exit-code, python-stdlib, +network-isolated, large-output). + +```powershell +# From src/ — build with WSLC support first so wslcsdk.dll is present: +cargo build --release --features wslc --target x86_64-pc-windows-msvc + +# Pre-pull the images the smoke set needs: +.\scripts\setup-wslc.ps1 -Image alpine:latest, python:3.12-alpine + +# Run the suite: +cargo test -p wxc_e2e_tests test_wslc_suite -- --nocapture +``` + +The suite **skips gracefully** (each test passes as a no-op) when any +prerequisite is missing, guarded by three checks: + +| Guard | Skips when | +|---|---| +| `has_wxc_exe()` | `wxc-exec.exe` not built | +| `has_wslc_sdk()` | `wslcsdk.dll` not next to the binary (build without `--features wslc`) | +| `has_wsl_runtime()` | `wsl --status` fails (WSL2 not installed) | + +### CI: why WSLC can only run on a nested-virt runner + +WSLC boots Linux containers inside **WSL2**, which requires **nested +virtualization** on the host. GitHub-hosted `windows-latest` runners do **not** +provide nested virt, so — unlike the MicroVM/Hyperlight suites, which run there +and self-skip on the WHP check — the WSLC suite can only *execute* on a +nested-virt-capable runner such as a **1ES hosted pool**. + +The [`wslc-e2e.yml`](../../.github/workflows/wslc-e2e.yml) workflow targets a +1ES pool for this reason. Beyond nested virt, the runner image must provide: + +- **WSL2 with the `VirtualMachinePlatform` optional feature enabled.** Enabling + it normally requires a reboot, which a Stateless (ephemeral) 1ES runner + cannot do mid-job — so the durable path is a **1ES Managed Image** with WSL2 + + `VirtualMachinePlatform` baked in at image-build time (the same DISM approach + `microsoft/ebpf-for-windows` uses to bake in Hyper-V). +- The **Rust/MSVC toolchain and git** (a raw client image is a bare desktop). + +Every runtime-dependent workflow step is gated so the job degrades to a clean +skip rather than a hard failure when a prerequisite is absent. + ## Troubleshooting | Error | Cause | Fix | diff --git a/src/testing/wxc_e2e_tests/src/lib.rs b/src/testing/wxc_e2e_tests/src/lib.rs index 0de4a0b6a..871ddacf3 100644 --- a/src/testing/wxc_e2e_tests/src/lib.rs +++ b/src/testing/wxc_e2e_tests/src/lib.rs @@ -256,6 +256,51 @@ pub fn has_hyperlight_snapshot() -> bool { } } +/// Return whether the WSLC SDK (`wslcsdk.dll`) is present next to +/// `wxc-exec.exe`. The DLL is copied there by a `--features wslc` build; its +/// absence means the WSLC backend cannot load at runtime, so WSLC E2E tests +/// must skip rather than fail. +pub fn has_wslc_sdk() -> bool { + let Some(exe) = find_binary("wxc-exec.exe") else { + println!("SKIPPED: wxc-exec.exe not found — build with `--features wslc` first"); + return false; + }; + let dll = exe.parent().unwrap_or(Path::new(".")).join("wslcsdk.dll"); + if dll.is_file() { + println!("Using wslcsdk.dll at {}", dll.display()); + true + } else { + println!( + "SKIPPED: wslcsdk.dll not found next to wxc-exec.exe — build with `--features wslc`" + ); + false + } +} + +/// Return whether a usable WSL runtime is available (WSL2 installed and +/// responsive). WSLC boots Linux containers inside WSL2, which requires +/// nested virtualization on the host — unavailable on GitHub-hosted runners, +/// so these tests only execute on a nested-virt-capable (e.g. 1ES) runner. +pub fn has_wsl_runtime() -> bool { + // `wsl.exe --status` exits 0 when the runtime is installed and a default + // distribution/version is configured. On hosts without WSL the command is + // either missing or returns a non-zero status. + let available = Command::new("wsl.exe") + .arg("--status") + .output() + .map(|output| output.status.success()) + .unwrap_or(false); + + if !available { + println!( + "SKIPPED: WSL runtime not available — WSLC requires WSL2 (needs nested virtualization; \ + run `wsl --update`)" + ); + } + + available +} + /// Return whether the Windows Sandbox optional feature is enabled. pub fn has_windows_sandbox_feature() -> bool { let available = Command::new("dism") diff --git a/src/testing/wxc_e2e_tests/tests/e2e_wslc.rs b/src/testing/wxc_e2e_tests/tests/e2e_wslc.rs new file mode 100644 index 000000000..0b71829ec --- /dev/null +++ b/src/testing/wxc_e2e_tests/tests/e2e_wslc.rs @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! WSLC (WSL Container) E2E integration tests. +//! +//! These tests mirror the Windows MicroVM / Hyperlight E2E suites and invoke +//! `wxc-exec.exe` directly with the `wslc` containment backend. WSLC boots +//! Linux containers inside WSL2, which needs nested virtualization on the host +//! (unavailable on GitHub-hosted runners) plus the WSLC SDK (`wslcsdk.dll`, a +//! `--features wslc` build), a WSL2 runtime, and pre-pulled images. Tests skip +//! gracefully when any prerequisite is missing so the suite is a no-op on +//! machines that cannot run it, and only executes on a nested-virt-capable +//! (e.g. 1ES) runner. + +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use wxc_e2e_tests::{ + has_wsl_runtime, has_wslc_sdk, has_wxc_exe, repo_root, run_wxc_config, run_wxc_example, + test_configs_dir, CommandResult, +}; + +static HAS_WXC_EXE: OnceLock = OnceLock::new(); +static HAS_WSLC_SDK: OnceLock = OnceLock::new(); +static HAS_WSL_RUNTIME: OnceLock = OnceLock::new(); + +fn cached_has_wxc_exe() -> bool { + *HAS_WXC_EXE.get_or_init(has_wxc_exe) +} + +fn cached_has_wslc_sdk() -> bool { + *HAS_WSLC_SDK.get_or_init(has_wslc_sdk) +} + +fn cached_has_wsl_runtime() -> bool { + *HAS_WSL_RUNTIME.get_or_init(has_wsl_runtime) +} + +/// Guard: skip test unless the WSLC prerequisites are present. +fn skip_unless_ready() -> bool { + cached_has_wxc_exe() && cached_has_wslc_sdk() && cached_has_wsl_runtime() +} + +// --------------------------------------------------------------------------- +// Individual test (mirrors test_microvm_hello) +// --------------------------------------------------------------------------- + +#[test] +fn test_wslc_hello() { + if !skip_unless_ready() { + return; + } + // The hello-world config lives under tests/examples/, not tests/configs/. + let result = run_wxc_example("wslc_hello_world.json", &["--debug", "--experimental"]); + assert_eq!( + result.code, + Some(0), + "expected exit 0, got {:?}\nstdout: {}\nstderr: {}", + result.code, + result.stdout, + result.stderr + ); + assert!( + result + .combined_output_with_decoded_base64() + .contains("Hello from WSL Container!"), + "output missing greeting\nstdout: {}\nstderr: {}", + result.stdout, + result.stderr + ); +} + +// --------------------------------------------------------------------------- +// Full WSLC smoke suite (mirrors test_microvm_suite) +// --------------------------------------------------------------------------- + +#[derive(Debug)] +struct WslcCase { + config: &'static str, + /// When true the config lives in tests/examples/, otherwise tests/configs/. + from_example: bool, + expected_exit: Option, + description: &'static str, + output_contains: Option<&'static str>, + expect_non_zero: bool, +} + +#[derive(Debug, Serialize)] +struct WslcPerfOutput { + commit: String, + timestamp: String, + results: Vec, +} + +#[derive(Debug, Serialize)] +struct WslcPerfEntry { + test: String, + description: String, + wall_time_ms: u128, + exit_code: Option, + status: String, +} + +#[test] +fn test_wslc_suite() { + if !skip_unless_ready() { + return; + } + wslc_suite(); +} + +fn wslc_suite() { + // Core smoke set. Images required (pre-pull via scripts/setup-wslc.ps1): + // alpine:latest -> hello-world, exit-code, network-isolated, large-output + // python:3.12-alpine -> python-stdlib + let cases = [ + WslcCase { + config: "wslc_hello_world.json", + from_example: true, + expected_exit: Some(0), + description: "Hello world (alpine, uname)", + output_contains: Some("Hello from WSL Container!"), + expect_non_zero: false, + }, + WslcCase { + config: "wslc_exit_code.json", + from_example: false, + expected_exit: Some(42), + description: "Exit code propagation", + output_contains: Some("About to exit with code 42"), + expect_non_zero: false, + }, + WslcCase { + config: "wslc_python_stdlib.json", + from_example: false, + expected_exit: Some(0), + description: "Python stdlib (json, math, hashlib)", + output_contains: Some("pi"), + expect_non_zero: false, + }, + WslcCase { + config: "wslc_network_isolated.json", + from_example: false, + expected_exit: Some(0), + description: "Network isolation (block policy)", + output_contains: Some("Network"), + expect_non_zero: false, + }, + WslcCase { + config: "wslc_large_output.json", + from_example: false, + expected_exit: Some(0), + description: "Large stdout (500 lines)", + output_contains: Some("Large output test complete"), + expect_non_zero: false, + }, + ]; + + let mut perf_entries = Vec::new(); + let mut failures = Vec::new(); + + for case in cases { + let config_path = if case.from_example { + repo_root().join("tests").join("examples").join(case.config) + } else { + test_configs_dir().join(case.config) + }; + if !config_path.exists() { + println!("SKIPPED: config not found: {}", config_path.display()); + continue; + } + + println!("--- {} ({}) ---", case.description, case.config); + let result = if case.from_example { + run_wxc_example(case.config, &["--debug", "--experimental"]) + } else { + run_wxc_config(case.config, &["--debug", "--experimental"]) + }; + + let status = if command_matches(&result, &case) { + "PASS" + } else { + failures.push(format!( + "{} expected {}, got {:?}", + case.config, + expected_exit_description(&case), + result.code + )); + "FAIL" + }; + + perf_entries.push(WslcPerfEntry { + test: case.config.to_string(), + description: case.description.to_string(), + wall_time_ms: result.wall_time_ms, + exit_code: result.code, + status: status.to_string(), + }); + + if status == "FAIL" { + println!( + "--- stdout ---\n{}\n--- stderr ---\n{}", + result.stdout, result.stderr + ); + } else { + println!(" PASS ({} ms)", result.wall_time_ms); + } + } + + write_wslc_perf_results(perf_entries); + + if !failures.is_empty() { + panic!("WSLC E2E failures:\n{}", failures.join("\n")); + } +} + +fn command_matches(result: &CommandResult, case: &WslcCase) -> bool { + if case.expect_non_zero { + if result.code == Some(0) { + return false; + } + } else if result.code != case.expected_exit { + return false; + } + + let Some(expected) = case.output_contains else { + return true; + }; + + result + .combined_output_with_decoded_base64() + .contains(expected) +} + +fn expected_exit_description(case: &WslcCase) -> String { + if case.expect_non_zero { + "non-zero exit".to_string() + } else { + format!("exit {}", case.expected_exit.unwrap_or(0)) + } +} + +// --------------------------------------------------------------------------- +// Perf results output +// --------------------------------------------------------------------------- + +fn write_wslc_perf_results(results: Vec) { + let output = WslcPerfOutput { + commit: std::env::var("GITHUB_SHA").unwrap_or_else(|_| "local".to_string()), + timestamp: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs().to_string()) + .unwrap_or_else(|_| "unknown".to_string()), + results, + }; + let json = + serde_json::to_string_pretty(&output).expect("wslc performance results should serialize"); + let path = repo_root().join("wslc-perf-results.json"); + std::fs::write(&path, json) + .unwrap_or_else(|error| panic!("failed to write {}: {error}", path.display())); + println!("Performance results written to {}", path.display()); +} From f3ecca131d92b1f05aeae906ad4369507520a214 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:19:23 -0700 Subject: [PATCH 35/93] Clearer file naming and debloated matrix generator script. Disabled non-MacOS arm64. --- .github/workflows/Build.And.Validate.yml | 2 +- .../workflows/Validation.Infra.Testing.yml | 2 +- ...ob.yml => Validation.Tests.Matrix.Job.yml} | 4 +- ...E2E.yml => Validation.Tests.Scheduled.yml} | 18 +- .../resolve-validation-test-matrix.test.mjs} | 123 ++++++++----- ...mjs => resolve-validation-test-matrix.mjs} | 161 +++++------------- ...atrix.json => validation-test-matrix.json} | 60 +++---- tests/scripts/README.md | 2 +- 8 files changed, 174 insertions(+), 198 deletions(-) rename .github/workflows/{Test.Matrix.Job.yml => Validation.Tests.Matrix.Job.yml} (98%) rename .github/workflows/{Scheduled.E2E.yml => Validation.Tests.Scheduled.yml} (63%) rename scripts/ci/{tests/resolve-test-matrix.test.mjs => matrix-generator-tests/resolve-validation-test-matrix.test.mjs} (54%) rename scripts/ci/{resolve-test-matrix.mjs => resolve-validation-test-matrix.mjs} (62%) rename scripts/ci/{test-matrix.json => validation-test-matrix.json} (95%) diff --git a/.github/workflows/Build.And.Validate.yml b/.github/workflows/Build.And.Validate.yml index 60a5e962f..225fdeab7 100644 --- a/.github/workflows/Build.And.Validate.yml +++ b/.github/workflows/Build.And.Validate.yml @@ -53,7 +53,7 @@ jobs: pr-validation-test-matrix: needs: [windows, linux, macos] if: github.event_name == 'pull_request' && github.base_ref == 'main' - uses: ./.github/workflows/Test.Matrix.Job.yml + uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml with: plan: pr diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml index aaf3c02ed..85483f07a 100644 --- a/.github/workflows/Validation.Infra.Testing.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -31,7 +31,7 @@ jobs: test: needs: [windows, linux, macos] - uses: ./.github/workflows/Test.Matrix.Job.yml + uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml with: # Exercise every combination currently rolled out, independent of its # normal PR/nightly/weekly schedule. diff --git a/.github/workflows/Test.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml similarity index 98% rename from .github/workflows/Test.Matrix.Job.yml rename to .github/workflows/Validation.Tests.Matrix.Job.yml index 7f702c84a..859b60f2f 100644 --- a/.github/workflows/Test.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -1,4 +1,4 @@ -name: E2E Validation Test Matrix +name: Create Validation Test Matrix on: workflow_call: @@ -28,7 +28,7 @@ jobs: - name: Resolve test combinations id: matrix - run: node scripts/ci/resolve-test-matrix.mjs --plan "${{ inputs.plan }}" + run: node scripts/ci/resolve-validation-test-matrix.mjs --plan "${{ inputs.plan }}" windows: name: ${{ matrix.os }}, ${{ matrix.architecture }}, ${{ matrix.backend }} diff --git a/.github/workflows/Scheduled.E2E.yml b/.github/workflows/Validation.Tests.Scheduled.yml similarity index 63% rename from .github/workflows/Scheduled.E2E.yml rename to .github/workflows/Validation.Tests.Scheduled.yml index bc35c4c93..cca7b82cb 100644 --- a/.github/workflows/Scheduled.E2E.yml +++ b/.github/workflows/Validation.Tests.Scheduled.yml @@ -1,4 +1,4 @@ -name: Scheduled E2E Validation Tests +name: Scheduled Validation Tests on: schedule: @@ -40,8 +40,16 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Build.MacOS.Job.yml - test: - needs: build-artifacts - uses: ./.github/workflows/Test.Matrix.Job.yml + test-nightly: + needs: [windows, linux, macos] + if: github.event_name == 'schedule' || inputs.plan == 'nightly' + uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml with: - plan: ${{ github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0' && 'weekly' || github.event_name == 'schedule' && 'nightly' || inputs.plan }} + plan: nightly + + test-weekly: + needs: [windows, linux, macos] + if: (github.event_name == 'schedule' && github.event.schedule == '0 8 * * 0') || inputs.plan == 'weekly' + uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml + with: + plan: weekly \ No newline at end of file diff --git a/scripts/ci/tests/resolve-test-matrix.test.mjs b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs similarity index 54% rename from scripts/ci/tests/resolve-test-matrix.test.mjs rename to scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs index 6031ebd90..a80d8d7cd 100644 --- a/scripts/ci/tests/resolve-test-matrix.test.mjs +++ b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs @@ -8,12 +8,12 @@ import { readCatalog, resolvePlan, validateCatalog -} from '../resolve-test-matrix.mjs'; +} from '../resolve-validation-test-matrix.mjs'; // Tests load a fresh catalog for each case so negative mutations cannot leak // into later assertions. const testDirectory = path.dirname(fileURLToPath(import.meta.url)); -const catalogPath = path.resolve(testDirectory, '..', 'test-matrix.json'); +const catalogPath = path.resolve(testDirectory, '..', 'validation-test-matrix.json'); function catalog() { return readCatalog(catalogPath); @@ -28,14 +28,26 @@ test('catalog validates and contains all five build targets', () => { assert.doesNotThrow(() => validateCatalog(catalog())); }); -test('current rollout enables three PR jobs and scheduled macOS coverage', () => { +test('trigger categories resolve independently', () => { const pr = resolvePlan(catalog(), 'pr'); const nightly = resolvePlan(catalog(), 'nightly'); const weekly = resolvePlan(catalog(), 'weekly'); - assert.equal(pr.windows.length + pr.linux.length + pr.macos.length, 3); - assert.equal(nightly.windows.length + nightly.linux.length + nightly.macos.length, 1); - assert.equal(weekly.windows.length + weekly.linux.length + weekly.macos.length, 2); + assert.ok( + pr.windows.some(entry => ( + entry.os === 'windows-canary' && entry.backend === 'process-t1' + )) + ); + assert.ok( + nightly.linux.some(entry => ( + entry.os === 'rhel-10.2' && entry.backend === 'bubblewrap' + )) + ); + assert.ok( + weekly.linux.some(entry => ( + entry.os === 'debian-13' && entry.backend === 'lxc' + )) + ); }); test('macOS rollout uses 26 for PR and nightly with 15 added weekly', () => { @@ -53,8 +65,7 @@ test('macOS rollout uses 26 for PR and nightly with 15 added weekly', () => { { plan: 'nightly', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } ]); assert.deepEqual(resolvePlan(catalog(), 'weekly').macos.map(project), [ - { plan: 'weekly', os: 'macos-15', runner: 'macos-15', backend: 'seatbelt' }, - { plan: 'nightly', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } + { plan: 'weekly', os: 'macos-15', runner: 'macos-15', backend: 'seatbelt' } ]); }); @@ -75,21 +86,60 @@ test('enabled plan deduplicates and runs both macOS versions', () => { ); }); -test('weekly includes all enabled nightly combinations', () => { +test('missing enabled trigger does not affect other plans', () => { + const modified = clone(catalog()); + delete modified.triggers.enabled; + + assert.deepEqual(resolvePlan(modified, 'enabled'), { + windows: [], + linux: [], + macos: [] + }); + assert.ok( + resolvePlan(modified, 'pr').windows + .some(entry => entry.os === 'windows-canary' && entry.backend === 'process-t1') + ); +}); + +test('resolved matrices never emit non-macOS arm64 tests', () => { + const modified = clone(catalog()); + const windowsCanary = modified.platforms + .find(platform => platform.id === 'windows-canary'); + windowsCanary.architectures.x64.backends = windowsCanary.architectures.x64.backends + .filter(backend => backend !== 'process-t1'); + modified.triggers.enabled.push({ + os: 'windows-canary', + backends: ['process-t1'] + }); + + for (const plan of ['pr', 'nightly', 'weekly', 'enabled']) { + const resolved = resolvePlan(modified, plan); + assert.ok(resolved.windows.every(entry => entry.architecture === 'x64')); + assert.ok(resolved.linux.every(entry => entry.architecture === 'x64')); + assert.ok(resolved.macos.every(entry => entry.architecture === 'arm64')); + } +}); + +test('weekly does not inherit nightly combinations', () => { const nightly = resolvePlan(catalog(), 'nightly'); const weekly = resolvePlan(catalog(), 'weekly'); for (const family of ['windows', 'linux', 'macos']) { - // The weekly matrix must be a superset, not a separate replacement plan. - const weeklyKeys = new Set( + const nightlyKeys = new Set( + nightly[family].map(entry => `${entry.os}|${entry.architecture}|${entry.backend}`) + ); + const inherited = weekly[family].filter(entry => ( + nightlyKeys.has(`${entry.os}|${entry.architecture}|${entry.backend}`) + )); + assert.deepEqual( + inherited, + [], + `${family} weekly entries unexpectedly overlap nightly` + ); + assert.ok( weekly[family].map(entry => `${entry.os}|${entry.architecture}|${entry.backend}`) + .every(key => !nightlyKeys.has(key)) ); - for (const entry of nightly[family]) { - assert.ok( - weeklyKeys.has(`${entry.os}|${entry.architecture}|${entry.backend}`), - `weekly is missing ${family} nightly entry ${entry.os}/${entry.backend}` - ); - } } }); @@ -114,42 +164,39 @@ test('arm64 never expands Hyperlight or MicroVM', () => { } }); -test('enabled placeholder handlers are rejected', () => { +test('all trigger categories omit placeholder handlers', () => { const modified = clone(catalog()); - modified.enabled.push({ - plan: 'weekly', + modified.triggers.enabled.push({ os: 'ubuntu-24.04', - architecture: 'x64', - backend: 'hyperlight' + backends: ['hyperlight'] }); - assert.throws( - () => validateCatalog(modified), - /enabled entry has no wired handler/ + assert.doesNotThrow(() => validateCatalog(modified)); + assert.ok( + !resolvePlan(modified, 'enabled').linux + .some(entry => entry.backend === 'hyperlight') ); }); -test('enabled handlers must support the selected architecture', () => { - // WSLC remains in the arm64 capability catalog while its current test - // dispatcher is explicitly restricted to x64. +test('enabled trigger respects handler architecture restrictions', () => { const modified = clone(catalog()); - modified.enabled.push({ - plan: 'weekly', + modified.triggers.enabled.push({ os: 'windows-24h2', - architecture: 'arm64', - backend: 'wslc' + backends: ['wslc'] }); - assert.throws( - () => validateCatalog(modified), - /enabled entry handler does not support arm64/ + assert.deepEqual( + resolvePlan(modified, 'enabled').windows + .filter(entry => entry.os === 'windows-24h2' && entry.backend === 'wslc') + .map(entry => entry.architecture), + ['x64'] ); }); -test('duplicate enabled combinations are rejected', () => { +test('duplicate enabled requests are rejected', () => { const modified = clone(catalog()); - modified.enabled.push(clone(modified.enabled[0])); + modified.triggers.enabled.push(clone(modified.triggers.enabled[0])); assert.throws( () => validateCatalog(modified), - /duplicate enabled entry/ + /duplicate enabled request/ ); }); diff --git a/scripts/ci/resolve-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs similarity index 62% rename from scripts/ci/resolve-test-matrix.mjs rename to scripts/ci/resolve-validation-test-matrix.mjs index 75c7a97a0..4f327f6bd 100644 --- a/scripts/ci/resolve-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -10,14 +10,9 @@ import process from 'node:process'; import { fileURLToPath } from 'node:url'; const FAMILIES = ['windows', 'linux', 'macos']; -const PLANS = ['pr', 'nightly', 'weekly']; -const RESOLUTION_PLANS = [...PLANS, 'enabled']; +const PLANS = ['pr', 'nightly', 'weekly', 'enabled']; const ARM64_UNSUPPORTED_BACKENDS = new Set(['hyperlight', 'microvm']); -function combinationKey(plan, os, architecture, backend) { - return `${plan}|${os}|${architecture}|${backend}`; -} - function assertNonEmptyString(value, label) { if (typeof value !== 'string' || value.trim() === '') { throw new Error(`${label} must be a non-empty string`); @@ -124,47 +119,7 @@ export function validateCatalog(catalog) { } } - const enabled = new Set(); - for (const entry of catalog.enabled ?? []) { - if (!PLANS.includes(entry.plan)) { - throw new Error(`enabled entry has unsupported plan ${entry.plan}`); - } - const platform = platforms.get(entry.os); - const architecture = platform?.architectures?.[entry.architecture]; - if (!architecture?.backends?.includes(entry.backend)) { - throw new Error( - `enabled entry is unsupported: ${entry.os}/${entry.architecture}/${entry.backend}` - ); - } - const requested = (catalog.triggers?.[entry.plan] ?? []) - .some(request => request.os === entry.os && request.backends.includes(entry.backend)); - if (!requested) { - throw new Error( - `enabled entry is not present in ${entry.plan}: ${entry.os}/${entry.backend}` - ); - } - const handler = catalog.handlers[platform.family][entry.backend]; - // Capability entries may describe future coverage, but enabled entries - // must have an executable dispatcher command for their architecture. - if (handler.status !== 'wired' || !handler.command) { - throw new Error( - `enabled entry has no wired handler: ${entry.os}/${entry.architecture}/${entry.backend}` - ); - } - if (handler.architectures && !handler.architectures.includes(entry.architecture)) { - throw new Error( - `enabled entry handler does not support ${entry.architecture}: ` - + `${entry.os}/${entry.backend}` - ); - } - const key = combinationKey(entry.plan, entry.os, entry.architecture, entry.backend); - if (enabled.has(key)) { - throw new Error(`duplicate enabled entry: ${key}`); - } - enabled.add(key); - } - - return { platforms, enabled }; + return { platforms }; } export function expandPlan(catalog, plan) { @@ -172,35 +127,35 @@ export function expandPlan(catalog, plan) { throw new Error(`unsupported plan: ${plan}`); } const { platforms } = validateCatalog(catalog); - // Sunday is one run containing the normal nightly set plus weekly additions. - const planNames = plan === 'weekly' ? ['nightly', 'weekly'] : [plan]; const combinations = []; - for (const planName of planNames) { - for (const request of catalog.triggers[planName]) { - const platform = platforms.get(request.os); - // A trigger is architecture-neutral. Expand it only where the platform's - // capability declaration supports the requested backend. - for (const [architecture, details] of Object.entries(platform.architectures)) { - for (const backend of request.backends) { - if (!details.backends.includes(backend)) { - continue; - } - combinations.push({ - plan: planName, - os: platform.id, - os_name: platform.displayName, - family: platform.family, - architecture, - target: details.target, - artifact: details.artifact, - pool: details.pool, - runner: details.runner, - backend, - command: catalog.handlers[platform.family][backend].command, - handler_status: catalog.handlers[platform.family][backend].status - }); + for (const request of catalog.triggers?.[plan] ?? []) { + const platform = platforms.get(request.os); + // A trigger is architecture-neutral. Expand it only where the platform's + // capability declaration supports the requested backend. + for (const [architecture, details] of Object.entries(platform.architectures)) { + for (const backend of request.backends) { + if (!details.backends.includes(backend)) { + continue; + } + const handler = catalog.handlers[platform.family][backend]; + if (handler.architectures && !handler.architectures.includes(architecture)) { + continue; } + combinations.push({ + plan, + os: platform.id, + os_name: platform.displayName, + family: platform.family, + architecture, + target: details.target, + artifact: details.artifact, + pool: details.pool, + runner: details.runner, + backend, + command: handler.command, + handler_status: handler.status + }); } } } @@ -209,54 +164,15 @@ export function expandPlan(catalog, plan) { } export function resolvePlan(catalog, plan) { - if (!RESOLUTION_PLANS.includes(plan)) { + if (!PLANS.includes(plan)) { throw new Error(`unsupported plan: ${plan}`); } - if (plan === 'enabled') { - const { platforms } = validateCatalog(catalog); - const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); - const seen = new Set(); - - for (const entry of catalog.enabled ?? []) { - const key = `${entry.os}|${entry.architecture}|${entry.backend}`; - if (seen.has(key)) { - continue; - } - seen.add(key); - - const platform = platforms.get(entry.os); - const details = platform.architectures[entry.architecture]; - const handler = catalog.handlers[platform.family][entry.backend]; - matrices[platform.family].push({ - plan, - os: platform.id, - os_name: platform.displayName, - architecture: entry.architecture, - target: details.target, - artifact: details.artifact, - pool: details.pool, - runner: details.runner, - backend: entry.backend, - command: handler.command - }); - } - - sortMatrices(matrices); - return matrices; - } - - const { enabled } = validateCatalog(catalog); + validateCatalog(catalog); const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); for (const combination of expandPlan(catalog, plan)) { - const key = combinationKey( - combination.plan, - combination.os, - combination.architecture, - combination.backend - ); - if (enabled.has(key)) { + if (combination.handler_status === 'wired' && combination.command) { // family selects the workflow job and handler_status is validation-only; // neither belongs in the matrix consumed by the runner. const { family, handler_status: _, ...matrixEntry } = combination; @@ -264,10 +180,21 @@ export function resolvePlan(catalog, plan) { } } + suppressNonMacArm64(matrices); sortMatrices(matrices); return matrices; } +// Windows and Linux ARM64 hosted VMs currently lack nested virtualization. +// Keep their catalog entries intact for future enablement, but never emit them +// until suitable test hosts are available. macOS remains ARM64-only. +function suppressNonMacArm64(matrices) { + for (const family of ['windows', 'linux']) { + matrices[family] = matrices[family] + .filter(entry => entry.architecture !== 'arm64'); + } +} + function sortMatrices(matrices) { for (const family of FAMILIES) { // Stable ordering keeps local output and workflow diagnostics reproducible. @@ -316,11 +243,11 @@ const currentFile = fileURLToPath(import.meta.url); if (process.argv[1] && path.resolve(process.argv[1]) === currentFile) { try { const args = parseArguments(process.argv.slice(2)); - const defaultCatalog = path.join(path.dirname(currentFile), 'test-matrix.json'); + const defaultCatalog = path.join(path.dirname(currentFile), 'validation-test-matrix.json'); const catalog = readCatalog(path.resolve(args.catalog ?? defaultCatalog)); writeOutputs(resolvePlan(catalog, args.plan)); } catch (error) { - process.stderr.write(`resolve-test-matrix: ${error.message}\n`); + process.stderr.write(`resolve-validation-test-matrix: ${error.message}\n`); process.exitCode = 1; } } diff --git a/scripts/ci/test-matrix.json b/scripts/ci/validation-test-matrix.json similarity index 95% rename from scripts/ci/test-matrix.json rename to scripts/ci/validation-test-matrix.json index 4e8fad871..f1573a3c7 100644 --- a/scripts/ci/test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -523,38 +523,32 @@ "lxc" ] } + ], + "enabled": [ + { + "os": "windows-25h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "ubuntu-26.04", + "backends": [ + "bubblewrap" + ] + }, + { + "os": "macos-26", + "backends": [ + "seatbelt" + ] + }, + { + "os": "macos-15", + "backends": [ + "seatbelt" + ] + } ] - }, - "enabled": [ - { - "plan": "pr", - "os": "windows-25h2", - "architecture": "x64", - "backend": "process-t3" - }, - { - "plan": "pr", - "os": "ubuntu-26.04", - "architecture": "x64", - "backend": "bubblewrap" - }, - { - "plan": "pr", - "os": "macos-26", - "architecture": "arm64", - "backend": "seatbelt" - }, - { - "plan": "nightly", - "os": "macos-26", - "architecture": "arm64", - "backend": "seatbelt" - }, - { - "plan": "weekly", - "os": "macos-15", - "architecture": "arm64", - "backend": "seatbelt" - } - ] + } } diff --git a/tests/scripts/README.md b/tests/scripts/README.md index a9f93129f..0df2a58af 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -46,7 +46,7 @@ delegate to the existing backend test scripts. The capability map and PR, nightly, and weekly triggers are defined in [`../../scripts/ci/test-matrix.json`](../../scripts/ci/test-matrix.json). -[`../../scripts/ci/resolve-test-matrix.mjs`](../../scripts/ci/resolve-test-matrix.mjs) +[`../../scripts/ci/resolve-validation-test-matrix.mjs`](../../scripts/ci/resolve-validation-test-matrix.mjs) validates that catalog and emits one matrix per OS family. Weekly resolution includes both nightly coverage and weekly additions. From fc95d8f7febf0226a5dfc5362ca5c7959d155e1d Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:27:38 -0700 Subject: [PATCH 36/93] Added new pools for rest of x64 OS versions. --- scripts/ci/validation-test-matrix.json | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index f1573a3c7..432dc0e92 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -10,7 +10,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t1", "isolation-session", @@ -23,7 +23,7 @@ "arm64": { "target": "aarch64-pc-windows-msvc", "artifact": "wxc-binaries-aarch64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t1", "isolation-session", @@ -42,7 +42,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t1", "isolation-session", @@ -55,7 +55,7 @@ "arm64": { "target": "aarch64-pc-windows-msvc", "artifact": "wxc-binaries-aarch64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t1", "isolation-session", @@ -73,7 +73,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t1", "isolation-session" @@ -82,7 +82,7 @@ "arm64": { "target": "aarch64-pc-windows-msvc", "artifact": "wxc-binaries-aarch64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t1", "isolation-session" @@ -110,7 +110,7 @@ "arm64": { "target": "aarch64-pc-windows-msvc", "artifact": "wxc-binaries-aarch64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t3", "wslc", @@ -127,7 +127,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "1es-mxc-e2e-windows-24h2-pro-x64", "backends": [ "process-t3", "wslc", @@ -139,7 +139,7 @@ "arm64": { "target": "aarch64-pc-windows-msvc", "artifact": "wxc-binaries-aarch64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t3", "wslc", @@ -156,7 +156,7 @@ "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "placeholder", + "pool": "1es-mxc-e2e-windows-23h2-enterprise-x64", "backends": [ "process-t3", "wslc", @@ -168,7 +168,7 @@ "arm64": { "target": "aarch64-pc-windows-msvc", "artifact": "wxc-binaries-aarch64-pc-windows-msvc", - "pool": "placeholder", + "pool": "", "backends": [ "process-t3", "wslc", @@ -195,7 +195,7 @@ "arm64": { "target": "aarch64-unknown-linux-gnu", "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "", "backends": [ "bubblewrap", "lxc" @@ -211,7 +211,7 @@ "x64": { "target": "x86_64-unknown-linux-gnu", "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "1es-mxc-e2e-ubuntu-24.04-x64", "backends": [ "bubblewrap", "microvm", @@ -222,7 +222,7 @@ "arm64": { "target": "aarch64-unknown-linux-gnu", "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "", "backends": [ "bubblewrap", "lxc" @@ -238,7 +238,7 @@ "x64": { "target": "x86_64-unknown-linux-gnu", "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "1es-mxc-e2e-rhel-10-x64", "backends": [ "bubblewrap", "hyperlight", @@ -248,7 +248,7 @@ "arm64": { "target": "aarch64-unknown-linux-gnu", "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "", "backends": [ "bubblewrap", "lxc" @@ -264,7 +264,7 @@ "x64": { "target": "x86_64-unknown-linux-gnu", "artifact": "lxc-binaries-x86_64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "1es-mxc-e2e-debian-13-x64", "backends": [ "bubblewrap", "hyperlight", @@ -274,7 +274,7 @@ "arm64": { "target": "aarch64-unknown-linux-gnu", "artifact": "lxc-binaries-aarch64-unknown-linux-gnu", - "pool": "placeholder", + "pool": "", "backends": [ "bubblewrap", "lxc" From 05e2cb80da4c1da9df34f6330580b532b5d99b7e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:51:08 -0700 Subject: [PATCH 37/93] bug fix --- .../resolve-validation-test-matrix.test.mjs | 36 +++++++++++++++---- scripts/ci/resolve-validation-test-matrix.mjs | 7 ++-- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs index a80d8d7cd..e919d5b11 100644 --- a/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs +++ b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs @@ -35,17 +35,17 @@ test('trigger categories resolve independently', () => { assert.ok( pr.windows.some(entry => ( - entry.os === 'windows-canary' && entry.backend === 'process-t1' + entry.os === 'windows-25h2' && entry.backend === 'process-t3' )) ); assert.ok( nightly.linux.some(entry => ( - entry.os === 'rhel-10.2' && entry.backend === 'bubblewrap' + entry.os === 'ubuntu-26.04' && entry.backend === 'bubblewrap' )) ); assert.ok( - weekly.linux.some(entry => ( - entry.os === 'debian-13' && entry.backend === 'lxc' + weekly.macos.some(entry => ( + entry.os === 'macos-15' && entry.backend === 'seatbelt' )) ); }); @@ -97,7 +97,24 @@ test('missing enabled trigger does not affect other plans', () => { }); assert.ok( resolvePlan(modified, 'pr').windows - .some(entry => entry.os === 'windows-canary' && entry.backend === 'process-t1') + .some(entry => entry.os === 'windows-25h2' && entry.backend === 'process-t3') + ); +}); + +test('empty pools skip OS and architecture combinations', () => { + const modified = clone(catalog()); + const windows25h2 = modified.platforms + .find(platform => platform.id === 'windows-25h2'); + windows25h2.architectures.x64.pool = ''; + + assert.doesNotThrow(() => validateCatalog(modified)); + assert.ok( + !expandPlan(modified, 'pr') + .some(entry => entry.os === 'windows-25h2') + ); + assert.ok( + !resolvePlan(modified, 'pr').windows + .some(entry => entry.os === 'windows-25h2') ); }); @@ -107,6 +124,7 @@ test('resolved matrices never emit non-macOS arm64 tests', () => { .find(platform => platform.id === 'windows-canary'); windowsCanary.architectures.x64.backends = windowsCanary.architectures.x64.backends .filter(backend => backend !== 'process-t1'); + windowsCanary.architectures.arm64.pool = 'test-arm64-pool'; modified.triggers.enabled.push({ os: 'windows-canary', backends: ['process-t1'] @@ -143,13 +161,13 @@ test('weekly does not inherit nightly combinations', () => { } }); -test('full plan expands supported backends to both architectures', () => { +test('full plan expands only architectures with available pools', () => { const expanded = expandPlan(catalog(), 'pr'); const ubuntuBubblewrap = expanded .filter(entry => entry.os === 'ubuntu-26.04' && entry.backend === 'bubblewrap'); assert.deepEqual( ubuntuBubblewrap.map(entry => entry.architecture).sort(), - ['arm64', 'x64'] + ['x64'] ); }); @@ -179,6 +197,10 @@ test('all trigger categories omit placeholder handlers', () => { test('enabled trigger respects handler architecture restrictions', () => { const modified = clone(catalog()); + const windows24h2 = modified.platforms + .find(platform => platform.id === 'windows-24h2'); + windows24h2.architectures.x64.pool = 'test-x64-pool'; + windows24h2.architectures.arm64.pool = 'test-arm64-pool'; modified.triggers.enabled.push({ os: 'windows-24h2', backends: ['wslc'] diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index 4f327f6bd..ac252b04e 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -61,8 +61,8 @@ export function validateCatalog(catalog) { if (platform.family === 'macos') { assertNonEmptyString(details.runner, `${platform.id}.${architecture}.runner`); - } else { - assertNonEmptyString(details.pool, `${platform.id}.${architecture}.pool`); + } else if (details.pool != null && typeof details.pool !== 'string') { + throw new Error(`${platform.id}.${architecture}.pool must be a string`); } const backends = new Set(); @@ -134,6 +134,9 @@ export function expandPlan(catalog, plan) { // A trigger is architecture-neutral. Expand it only where the platform's // capability declaration supports the requested backend. for (const [architecture, details] of Object.entries(platform.architectures)) { + if (platform.family !== 'macos' && !details.pool?.trim()) { + continue; + } for (const backend of request.backends) { if (!details.backends.includes(backend)) { continue; From d2320d5670a9fd7a30c44ad43495844bf610475d Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:53:31 -0700 Subject: [PATCH 38/93] Testing additional OS versions --- scripts/ci/validation-test-matrix.json | 38 +++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 432dc0e92..6f55a0763 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -231,8 +231,8 @@ } }, { - "id": "rhel-10.2", - "displayName": "RHEL 10.2", + "id": "rhel-10", + "displayName": "RHEL 10", "family": "linux", "architectures": { "x64": { @@ -463,7 +463,7 @@ ] }, { - "os": "rhel-10.2", + "os": "rhel-10", "backends": [ "bubblewrap" ] @@ -509,7 +509,7 @@ ] }, { - "os": "rhel-10.2", + "os": "rhel-10", "backends": [ "hyperlight", "lxc" @@ -531,12 +531,42 @@ "process-t3" ] }, + { + "os": "windows-24h2", + "backends": [ + "process-t3" + ] + }, + { + "os": "windows-23h2", + "backends": [ + "process-t3" + ] + }, { "os": "ubuntu-26.04", "backends": [ "bubblewrap" ] }, + { + "os": "ubuntu-24.04", + "backends": [ + "bubblewrap" + ] + }, + { + "os": "rhel-10", + "backends": [ + "bubblewrap" + ] + }, + { + "os": "debian-13", + "backends": [ + "bubblewrap" + ] + }, { "os": "macos-26", "backends": [ From 840d36112e3052a888f4755f1123044a4d008f0f Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Wed, 5 Aug 2026 19:10:57 -0700 Subject: [PATCH 39/93] ci(wslc): target new 1es-mxc-e2e-windows-25h2-wsl2 pool (WSL2 baked image) Repoint the WSLC E2E workflow from the scratch 1es-mxc-test-2 pool to the new repository-level pool 1es-mxc-e2e-windows-25h2-wsl2, whose VMs boot from the windows-25h2-wsl2 1ES Managed Image (win11-25h2-pro + a provisioning script that enables Microsoft-Windows-Subsystem-Linux + VirtualMachinePlatform at image-build time, activated by 1ES's mandatory post-provisioning reboot). Nested virt is supported on the pool's Dsv7 SKU. Header comment updated to match; the stale 'no baked image yet' caveat is removed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/wslc-e2e.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml index bfd2d4db3..2def26da6 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/wslc-e2e.yml @@ -4,22 +4,22 @@ # and skip when WHP is absent), WSLC needs WSL2 — which requires nested # virtualization — so it can only EXECUTE on a nested-virt-capable 1ES pool. # -# Pool: 1es-mxc-test-2 (REPOSITORY-level, runner group Default; confirmed -# working). `shell: powershell` because the raw windows-25h2-pro-x64 CLIENT -# image ships Windows PowerShell 5.1, not pwsh (PS7). Cold-start pickup on the -# idle Stateless pool is ~10-11 min — the long timeout accounts for that plus -# an in-image toolchain/WSL setup on first run. +# Pool: 1es-mxc-e2e-windows-25h2-wsl2 (REPOSITORY-level, microsoft/mxc). Its VMs +# boot from the windows-25h2-wsl2 1ES Managed Image, which bakes WSL2 in (see +# below). `shell: powershell` for parity with the sibling client images. Cold- +# start pickup on the idle Stateless pool is ~10-11 min — the long timeout +# accounts for that plus an in-image toolchain/WSL setup on first run. # # IMAGE PREREQUISITES (see docs/wsl/wsl-container-getting-started.md): -# The raw client image is a bare desktop. For this workflow to build + run it -# needs: git, the Rust/MSVC toolchain, WSL2 with VirtualMachinePlatform -# enabled, and the WSLC SDK produced by a `--features wslc` build. Enabling -# VirtualMachinePlatform normally needs a reboot, which a Stateless ephemeral -# runner cannot do mid-job — so the robust long-term path is a 1ES Managed -# Image with WSL2 + VirtualMachinePlatform (and ideally the toolchain) baked -# in (the same DISM-at-image-build approach microsoft/ebpf-for-windows uses -# for Hyper-V). Every runtime-dependent step below is guarded so the job -# degrades to a clean SKIP rather than a hard failure when a prerequisite is +# The pool's windows-25h2-wsl2 Managed Image is win11-25h2-pro with a 1ES +# provisioning script (.azure-pipelines/1es-images/wsl2/Setup.ps1) that enables +# Microsoft-Windows-Subsystem-Linux + VirtualMachinePlatform at image-build +# time; 1ES's mandatory post-provisioning reboot activates them, so runner VMs +# boot WSL2-ready (the DISM-at-image-build approach microsoft/ebpf-for-windows +# uses for Hyper-V). The job still installs git + the Rust/MSVC toolchain and +# finishes WSL2 runtime setup (`wsl --update`), and builds the WSLC SDK via a +# `--features wslc` build. Every runtime-dependent step below is guarded so the +# job degrades to a clean SKIP rather than a hard failure if a prerequisite is # missing. # ============================================================================ name: WSLC E2E Tests @@ -38,7 +38,7 @@ jobs: name: WXC-Exec WSLC runs-on: - self-hosted - - "1ES.Pool=1es-mxc-test-2" + - "1ES.Pool=1es-mxc-e2e-windows-25h2-wsl2" - "JobId=wslc-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" timeout-minutes: 45 From ba633ed0659d083b1cb21b3a583ebe07255f67e9 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Wed, 5 Aug 2026 19:25:57 -0700 Subject: [PATCH 40/93] ci(wslc): make workflow bash-free for the bare 1ES client image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First run on the new WSL2 pool proved the image/pool wiring works (runner picked up the job), but failed at setup-rust-toolchain with 'bash: command not found' — the bare win11-25h2-pro client image ships no bash. - Replace actions-rust-lang/setup-rust-toolchain@v1 (runs internal bash) with a PowerShell rustup install that honors the rust-toolchain.toml channel pin. - setup-cargo-feed: swap its one 'shell: bash' cat step for PowerShell. - Default all job run steps to Windows PowerShell 5.1 (pwsh isn't reliably on PATH on the client image), protecting the unshelled cargo build/test steps. The cargo feed is a sparse registry, so no git is required. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-cargo-feed/action.yml | 4 +- .github/workflows/wslc-e2e.yml | 42 ++++++++++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup-cargo-feed/action.yml b/.github/actions/setup-cargo-feed/action.yml index 00f2973f1..c0de28fb3 100644 --- a/.github/actions/setup-cargo-feed/action.yml +++ b/.github/actions/setup-cargo-feed/action.yml @@ -5,6 +5,6 @@ description: >- runs: using: composite steps: - - shell: bash + - shell: powershell working-directory: ${{ github.workspace }} - run: cat .azure-pipelines/.cargo/config.public.toml >> .cargo/config.toml + run: Get-Content .azure-pipelines\.cargo\config.public.toml | Add-Content -Path .cargo\config.toml diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml index 2def26da6..664a65003 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/wslc-e2e.yml @@ -36,6 +36,13 @@ permissions: jobs: wslc-e2e: name: WXC-Exec WSLC + # Default every `run:` step to Windows PowerShell 5.1 (always present on the + # image). The Windows default shell is pwsh, but the bare client image does + # not reliably ship pwsh on PATH, so steps without an explicit shell (the + # cargo build/test steps) would otherwise fail. + defaults: + run: + shell: powershell runs-on: - self-hosted - "1ES.Pool=1es-mxc-e2e-windows-25h2-wsl2" @@ -52,11 +59,36 @@ jobs: shell: powershell run: Copy-Item src\rust-toolchain.toml rust-toolchain.toml -Force - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - target: x86_64-pc-windows-msvc - override: false - rustflags: '' + # The bare 1ES client image has no rustup/cargo and no bash, so we can't + # use actions-rust-lang/setup-rust-toolchain (it runs internal bash steps + # and fails with "bash: command not found"). Install rustup directly in + # PowerShell; rust-toolchain.toml (surfaced at repo root above) pins + # channel 1.93, which rustup installs and selects automatically. + - name: Install Rust toolchain + shell: powershell + run: | + $ErrorActionPreference = 'Stop' + if (-not (Get-Command rustup -ErrorAction SilentlyContinue)) { + Write-Host "Installing rustup..." + $init = Join-Path $env:RUNNER_TEMP 'rustup-init.exe' + Invoke-WebRequest -Uri 'https://win.rustup.rs/x86_64' -OutFile $init -UseBasicParsing + & $init -y --default-toolchain none --profile minimal + if ($LASTEXITCODE -ne 0) { throw "rustup-init failed ($LASTEXITCODE)" } + $cargoBin = Join-Path $env:USERPROFILE '.cargo\bin' + $cargoBin | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + $env:Path = "$cargoBin;$env:Path" + } + # Install the exact channel pinned by rust-toolchain.toml (repo root), + # reading it from the file so there is no drift with the pin. + $chan = ([regex]::Match((Get-Content rust-toolchain.toml -Raw), 'channel\s*=\s*"([^"]+)"')).Groups[1].Value + if (-not $chan) { throw "could not parse channel from rust-toolchain.toml" } + Write-Host "Pinned Rust channel: $chan" + rustup toolchain install $chan --profile minimal + if ($LASTEXITCODE -ne 0) { throw "rustup toolchain install $chan failed ($LASTEXITCODE)" } + rustup target add --toolchain $chan x86_64-pc-windows-msvc + if ($LASTEXITCODE -ne 0) { throw "rustup target add failed ($LASTEXITCODE)" } + rustc "+$chan" --version + cargo "+$chan" --version - name: Point cargo at the MxcDependencies feed uses: ./.github/actions/setup-cargo-feed From cf660106a90404259e9e127049d5486871c68a40 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Thu, 6 Aug 2026 09:31:58 -0700 Subject: [PATCH 41/93] ci(wslc): probe WSL2 feature state before build (image verification) Add an early best-effort step that reports Microsoft-Windows-Subsystem-Linux + VirtualMachinePlatform feature state and wsl --status right after checkout, so we can confirm the 1ES Managed Image baked WSL2 in even when later build steps fail for unrelated reasons (currently: missing MSVC linker on the bare image). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/wslc-e2e.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml index 664a65003..c5639703d 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/wslc-e2e.yml @@ -52,6 +52,34 @@ jobs: steps: - uses: actions/checkout@v4 + # Image verification: report WSL2 optional-feature state BEFORE the build, + # so we confirm whether the 1ES Managed Image baked WSL2 in even if later + # build/test steps fail for unrelated reasons (e.g. missing MSVC linker). + # Best-effort: never fails the job. + - name: Report WSL2 feature state (image verification) + shell: powershell + run: | + $ErrorActionPreference = 'Continue' + Write-Host "=== Windows optional features (baked into image?) ===" + foreach ($f in 'Microsoft-Windows-Subsystem-Linux','VirtualMachinePlatform') { + try { + $state = (Get-WindowsOptionalFeature -Online -FeatureName $f -ErrorAction Stop).State + } catch { + $state = "query-failed: $($_.Exception.Message)" + } + Write-Host (" {0} = {1}" -f $f, $state) + } + Write-Host "=== wsl.exe presence + status ===" + $wsl = Get-Command wsl.exe -ErrorAction SilentlyContinue + if ($wsl) { + Write-Host "wsl.exe: $($wsl.Source)" + wsl --status 2>&1 | Write-Host + wsl --version 2>&1 | Write-Host + Write-Host "wsl --status exit code: $LASTEXITCODE" + } else { + Write-Host "wsl.exe NOT found on PATH" + } + # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root # only. Surface src/rust-toolchain.toml there so the pinned toolchain is # honored. Use PowerShell (bare client image may lack bash). From cbf42642f45190d642ade8c87318050f51a7837a Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:24:13 -0700 Subject: [PATCH 42/93] added additional package manager support for linux prereq installs --- .../workflows/Validation.Tests.Matrix.Job.yml | 20 +++++++++++++++---- .../resolve-validation-test-matrix.test.mjs | 10 +++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 859b60f2f..7b904d6dc 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -109,12 +109,24 @@ jobs: shell: bash run: | set -euo pipefail + exec > >(tee -a "$RUNNER_TEMP/mxc-ci.log") 2>&1 chmod +x artifacts/bin/lxc-exec case '${{ matrix.command }}' in bubblewrap) if ! command -v bwrap >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y --no-install-recommends bubblewrap + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends bubblewrap + elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y bubblewrap + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y bubblewrap + elif command -v microdnf >/dev/null 2>&1; then + sudo microdnf install -y bubblewrap + else + echo "No supported package manager found to install bubblewrap." >&2 + exit 1 + fi fi command -v bwrap if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then @@ -134,11 +146,11 @@ jobs: if [[ '${{ matrix.command }}' == 'lxc' ]]; then sudo --preserve-env=RUNNER_TEMP bash tests/scripts/run_ci_backend_tests.sh \ '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | - tee "$RUNNER_TEMP/mxc-ci.log" + tee -a "$RUNNER_TEMP/mxc-ci.log" else bash tests/scripts/run_ci_backend_tests.sh \ '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | - tee "$RUNNER_TEMP/mxc-ci.log" + tee -a "$RUNNER_TEMP/mxc-ci.log" fi - name: Upload failure logs diff --git a/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs index e919d5b11..1aeeacb9c 100644 --- a/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs +++ b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs @@ -71,7 +71,15 @@ test('macOS rollout uses 26 for PR and nightly with 15 added weekly', () => { test('enabled plan deduplicates and runs both macOS versions', () => { const enabled = resolvePlan(catalog(), 'enabled'); - assert.equal(enabled.windows.length + enabled.linux.length + enabled.macos.length, 4); + const combinations = Object.values(enabled).flat(); + assert.equal( + new Set( + combinations.map(entry => ( + `${entry.os}|${entry.architecture}|${entry.backend}` + )) + ).size, + combinations.length + ); assert.deepEqual( enabled.macos.map(entry => ({ plan: entry.plan, From 0038997138cf1a264b9c870c28998b88dfd59139 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:25:07 -0700 Subject: [PATCH 43/93] fixing debain package manager issue --- .github/workflows/Validation.Tests.Matrix.Job.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 7b904d6dc..c89f56c2d 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -115,7 +115,9 @@ jobs: bubblewrap) if ! command -v bwrap >/dev/null 2>&1; then if command -v apt-get >/dev/null 2>&1; then - sudo apt-get update + if ! sudo apt-get update; then + echo "WARNING: apt-get update reported repository errors; continuing with available package indexes." >&2 + fi sudo apt-get install -y --no-install-recommends bubblewrap elif command -v dnf >/dev/null 2>&1; then sudo dnf install -y bubblewrap From 8dd44ac53079dbc77be802edc1759729ba966f16 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Thu, 6 Aug 2026 14:44:58 -0700 Subject: [PATCH 44/93] ci(wslc): enable WSL2 on stateful pool VM (enable + reboot if needed) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base image can't carry WSL2 (sysprep resets pending optional features; the WSL2 artifacts aren't in our sub's catalog). Instead, run the pool as 1ES Stateful (agentProfile.kind=Stateful, verified), so the same VM persists across jobs. New 'Prepare WSL2 on stateful VM' step: if the two optional features aren't Enabled, enable via DISM and reboot to finalize (a normal running-VM reboot, no sysprep) — that ends the job; a later job on the same persisted VM sees them Enabled and proceeds. Build/toolchain/WSL steps are gated on prep.ready so a preparing VM does no wasted work. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/wslc-e2e.yml | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml index c5639703d..aa7f28016 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/wslc-e2e.yml @@ -80,10 +80,49 @@ jobs: Write-Host "wsl.exe NOT found on PATH" } + # Stateful-pool WSL2 preparation. This pool is agentProfile.kind=Stateful, so + # the same VM persists across jobs (maxAgentLifetime 7d). WSL2's optional + # features are NOT in the base image, so the FIRST job on a fresh VM enables + # them via DISM and reboots — a normal running-VM reboot finalizes the + # pending features (no sysprep involved, unlike image capture). That reboot + # ends this job (expected). A SUBSEQUENT job on the same persisted VM sees + # the features Enabled and proceeds. This sidesteps the ephemeral-image + # sysprep-reset problem entirely. + - name: Prepare WSL2 on stateful VM (enable + reboot if needed) + id: prep + shell: powershell + run: | + $ErrorActionPreference = 'Stop' + $features = 'Microsoft-Windows-Subsystem-Linux','VirtualMachinePlatform' + $states = @{} + foreach ($f in $features) { $states[$f] = (Get-WindowsOptionalFeature -Online -FeatureName $f).State } + $allEnabled = -not (($features | ForEach-Object { $states[$_] -ne 'Enabled' }) -contains $true) + if ($allEnabled) { + Write-Host "WSL2 features already Enabled on this persisted VM — proceeding." + "ready=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + exit 0 + } + Write-Host "WSL2 features not all enabled: $(($states.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ', ')" + $isAdmin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) + Write-Host "Running elevated: $isAdmin" + foreach ($f in $features) { + if ($states[$f] -ne 'Enabled') { + Write-Host "Enabling $f ..." + $p = Start-Process dism.exe -ArgumentList '/online','/enable-feature',"/featurename:$f",'/all','/norestart' -Wait -PassThru -NoNewWindow + Write-Host " dism exit $($p.ExitCode)" + if ($p.ExitCode -notin @(0,3010)) { throw "dism enable $f failed ($($p.ExitCode))" } + } + } + "ready=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + Write-Host "::warning::WSL2 features enabled on this stateful VM; rebooting to finalize. This job will end — re-run the workflow and the next job on this persisted VM will have WSL2 ready." + shutdown /r /t 15 /c "WSL2 feature finalize reboot (1ES stateful pool)" + Start-Sleep -Seconds 180 + # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root # only. Surface src/rust-toolchain.toml there so the pinned toolchain is # honored. Use PowerShell (bare client image may lack bash). - name: Surface toolchain file at repo root + if: steps.prep.outputs.ready == 'true' shell: powershell run: Copy-Item src\rust-toolchain.toml rust-toolchain.toml -Force @@ -93,6 +132,7 @@ jobs: # PowerShell; rust-toolchain.toml (surfaced at repo root above) pins # channel 1.93, which rustup installs and selects automatically. - name: Install Rust toolchain + if: steps.prep.outputs.ready == 'true' shell: powershell run: | $ErrorActionPreference = 'Stop' @@ -119,20 +159,24 @@ jobs: cargo "+$chan" --version - name: Point cargo at the MxcDependencies feed + if: steps.prep.outputs.ready == 'true' uses: ./.github/actions/setup-cargo-feed - uses: Swatinem/rust-cache@v2 + if: steps.prep.outputs.ready == 'true' with: workspaces: src -> target key: wslc-e2e-v1 - name: Build with WSLC support + if: steps.prep.outputs.ready == 'true' working-directory: src run: cargo build --release --target x86_64-pc-windows-msvc --no-default-features --features wslc - name: Ensure WSL2 runtime id: wsl + if: steps.prep.outputs.ready == 'true' shell: powershell run: | $ErrorActionPreference = 'Continue' From 7cbb52fbc1da8de712f8b9f9208d705f0190e526 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Thu, 6 Aug 2026 14:55:43 -0700 Subject: [PATCH 45/93] ci(wslc): make WSL2 probe non-fatal so prep step runs The diagnostic probe ran 'wsl --status', which exits non-zero when WSL isn't installed; PowerShell propagated that as the step's exit code, failing the step and skipping the downstream 'Prepare WSL2' step (default if: success()). Add 'exit 0' + continue-on-error so the probe never fails the job. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/wslc-e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml index aa7f28016..7ded0e556 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/wslc-e2e.yml @@ -57,6 +57,7 @@ jobs: # build/test steps fail for unrelated reasons (e.g. missing MSVC linker). # Best-effort: never fails the job. - name: Report WSL2 feature state (image verification) + continue-on-error: true shell: powershell run: | $ErrorActionPreference = 'Continue' @@ -79,6 +80,9 @@ jobs: } else { Write-Host "wsl.exe NOT found on PATH" } + # Never fail the job on this diagnostic step (wsl --status exits non-zero + # when WSL isn't installed, which would otherwise fail the PowerShell step). + exit 0 # Stateful-pool WSL2 preparation. This pool is agentProfile.kind=Stateful, so # the same VM persists across jobs (maxAgentLifetime 7d). WSL2's optional From 28ee1d2dcbb09e7732816d4fe8b066f95af0e5d8 Mon Sep 17 00:00:00 2001 From: Huzaifa Danish Date: Thu, 6 Aug 2026 15:01:50 -0700 Subject: [PATCH 46/93] ci(wslc): fix prep step PowerShell parse error (plain ASCII, simpler constructs) The prior prep script failed to parse (em-dashes + nested double-quotes in a \ subexpression). Rewrite with -f format operator, ASCII-only strings, and straightforward loops. Parse-verified locally with PSParser.Tokenize. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/wslc-e2e.yml | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/wslc-e2e.yml index 7ded0e556..73899ea33 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/wslc-e2e.yml @@ -97,29 +97,29 @@ jobs: shell: powershell run: | $ErrorActionPreference = 'Stop' - $features = 'Microsoft-Windows-Subsystem-Linux','VirtualMachinePlatform' - $states = @{} - foreach ($f in $features) { $states[$f] = (Get-WindowsOptionalFeature -Online -FeatureName $f).State } - $allEnabled = -not (($features | ForEach-Object { $states[$_] -ne 'Enabled' }) -contains $true) - if ($allEnabled) { - Write-Host "WSL2 features already Enabled on this persisted VM — proceeding." - "ready=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + $features = @('Microsoft-Windows-Subsystem-Linux','VirtualMachinePlatform') + $disabled = @() + foreach ($f in $features) { + $state = (Get-WindowsOptionalFeature -Online -FeatureName $f).State + Write-Host ("{0} = {1}" -f $f, $state) + if ($state -ne 'Enabled') { $disabled += $f } + } + if ($disabled.Count -eq 0) { + Write-Host 'WSL2 features already Enabled on this persisted VM. Proceeding.' + 'ready=true' | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 exit 0 } - Write-Host "WSL2 features not all enabled: $(($states.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ', ')" $isAdmin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) - Write-Host "Running elevated: $isAdmin" - foreach ($f in $features) { - if ($states[$f] -ne 'Enabled') { - Write-Host "Enabling $f ..." - $p = Start-Process dism.exe -ArgumentList '/online','/enable-feature',"/featurename:$f",'/all','/norestart' -Wait -PassThru -NoNewWindow - Write-Host " dism exit $($p.ExitCode)" - if ($p.ExitCode -notin @(0,3010)) { throw "dism enable $f failed ($($p.ExitCode))" } - } + Write-Host ("Running elevated: {0}" -f $isAdmin) + foreach ($f in $disabled) { + Write-Host ("Enabling {0} ..." -f $f) + $proc = Start-Process dism.exe -ArgumentList '/online','/enable-feature',"/featurename:$f",'/all','/norestart' -Wait -PassThru -NoNewWindow + Write-Host (" dism exit {0}" -f $proc.ExitCode) + if (@(0,3010) -notcontains $proc.ExitCode) { throw ("dism enable {0} failed ({1})" -f $f, $proc.ExitCode) } } - "ready=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - Write-Host "::warning::WSL2 features enabled on this stateful VM; rebooting to finalize. This job will end — re-run the workflow and the next job on this persisted VM will have WSL2 ready." - shutdown /r /t 15 /c "WSL2 feature finalize reboot (1ES stateful pool)" + 'ready=false' | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + Write-Host '::warning::WSL2 features enabled on this stateful VM; rebooting to finalize. This job will end; re-run the workflow and the next job on this persisted VM will have WSL2 ready.' + shutdown /r /t 15 /c "WSL2 feature finalize reboot" Start-Sleep -Seconds 180 # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root From ceef61fbb262baf6b8df81200ed7d343e6a88ecf Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:34:10 -0700 Subject: [PATCH 47/93] force github to run action after downtime --- .github/workflows/Validation.Infra.Testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml index 85483f07a..0c54c089f 100644 --- a/.github/workflows/Validation.Infra.Testing.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -1,5 +1,7 @@ name: Validation Infrastructure Testing +# github was down - forcing this to run now :) + on: push: branches: From d40bf3d5fc665132ab431894f8b9d2a6b9e03368 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:21:27 -0700 Subject: [PATCH 48/93] added support for more microvm, lxc. and hyperlight. they should fail on purpose if no test is configured --- .../workflows/Validation.Tests.Matrix.Job.yml | 39 +-- scripts/ci/install-linux-backend-prereqs.sh | 117 +++++++++ .../resolve-validation-test-matrix.test.mjs | 237 ------------------ scripts/ci/resolve-validation-test-matrix.mjs | 25 +- scripts/ci/validation-test-matrix.json | 77 ++---- ...ests.sh => TEMP_run_seatbelt_all_tests.sh} | 2 +- tests/scripts/run_ci_backend_tests.sh | 6 +- 7 files changed, 165 insertions(+), 338 deletions(-) create mode 100644 scripts/ci/install-linux-backend-prereqs.sh delete mode 100644 scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs rename tests/scripts/{run_seatbelt_all_tests.sh => TEMP_run_seatbelt_all_tests.sh} (99%) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index c89f56c2d..cf7701bb9 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -48,6 +48,11 @@ jobs: name: ${{ matrix.artifact }} path: artifacts/bin + - name: Prepare backend prerequisites + if: matrix.command == 'microvm' + shell: pwsh + run: ./scripts/ci/check-whp.ps1 + - name: Prepare Process Container host if: matrix.backend == 'process-t3' shell: pwsh @@ -109,37 +114,9 @@ jobs: shell: bash run: | set -euo pipefail - exec > >(tee -a "$RUNNER_TEMP/mxc-ci.log") 2>&1 - chmod +x artifacts/bin/lxc-exec - case '${{ matrix.command }}' in - bubblewrap) - if ! command -v bwrap >/dev/null 2>&1; then - if command -v apt-get >/dev/null 2>&1; then - if ! sudo apt-get update; then - echo "WARNING: apt-get update reported repository errors; continuing with available package indexes." >&2 - fi - sudo apt-get install -y --no-install-recommends bubblewrap - elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y bubblewrap - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y bubblewrap - elif command -v microdnf >/dev/null 2>&1; then - sudo microdnf install -y bubblewrap - else - echo "No supported package manager found to install bubblewrap." >&2 - exit 1 - fi - fi - command -v bwrap - if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - fi - ;; - lxc) - command -v lxc-start - sudo -n true - ;; - esac + bash scripts/ci/install-linux-backend-prereqs.sh \ + '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + tee -a "$RUNNER_TEMP/mxc-ci.log" - name: Run backend tests shell: bash diff --git a/scripts/ci/install-linux-backend-prereqs.sh b/scripts/ci/install-linux-backend-prereqs.sh new file mode 100644 index 000000000..3123fa8a7 --- /dev/null +++ b/scripts/ci/install-linux-backend-prereqs.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Installs the host packages a Linux backend needs before its artifact-only +# test suite runs. Distro-aware so the same matrix entry works on Ubuntu, +# Debian, and RHEL images. + +usage() { + echo "Usage: $0 " >&2 +} + +if [[ $# -ne 2 ]]; then + usage + exit 2 +fi + +backend="$1" +binary_directory="$2" + +apt_update() { + # Unrelated third-party repositories on the pool images can fail to + # refresh; the package install below still decides success. + if ! sudo apt-get update; then + echo "WARNING: apt-get update reported repository errors; continuing with available package indexes." >&2 + fi +} + +install_epel() { + local package_manager="$1" + if ! sudo "$package_manager" install -y epel-release; then + sudo "$package_manager" install -y \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm + fi +} + +install_bubblewrap() { + if command -v bwrap >/dev/null 2>&1; then + return + fi + if command -v apt-get >/dev/null 2>&1; then + apt_update + sudo apt-get install -y --no-install-recommends bubblewrap + elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y bubblewrap + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y bubblewrap + elif command -v microdnf >/dev/null 2>&1; then + sudo microdnf install -y bubblewrap + else + echo "No supported package manager found to install bubblewrap." >&2 + exit 1 + fi +} + +install_lxc() { + if command -v lxc-start >/dev/null 2>&1; then + return + fi + if command -v apt-get >/dev/null 2>&1; then + apt_update + # Debian dropped lxc-utils; Ubuntu still ships it. + local packages=(lxc dnsmasq-base iptables bridge-utils) + if apt-cache show lxc-utils >/dev/null 2>&1; then + packages+=(lxc-utils) + fi + sudo apt-get install -y --no-install-recommends "${packages[@]}" + elif command -v dnf >/dev/null 2>&1; then + install_epel dnf + sudo dnf install -y lxc lxc-templates dnsmasq iptables + elif command -v yum >/dev/null 2>&1; then + install_epel yum + sudo yum install -y lxc lxc-templates dnsmasq iptables + elif command -v microdnf >/dev/null 2>&1; then + install_epel microdnf + sudo microdnf install -y lxc lxc-templates dnsmasq iptables + else + echo "No supported package manager found to install LXC." >&2 + exit 1 + fi +} + +chmod +x "$binary_directory/lxc-exec" + +case "$backend" in + bubblewrap) + install_bubblewrap + command -v bwrap + if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + ;; + lxc) + install_lxc + command -v lxc-start + sudo -n true + # Package installs may not activate the AppArmor profile or the + # lxc-net bridge, which lxc-start needs. + if command -v apparmor_parser >/dev/null 2>&1; then + sudo apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null || true + fi + if command -v systemctl >/dev/null 2>&1; then + sudo systemctl start lxc-net 2>/dev/null || true + fi + ;; + microvm) + for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do + test -f "$binary_directory/$file" + done + ;; + hyperlight) + echo "Hyperlight has no artifact-only Linux test prerequisites yet." + ;; + *) + usage + exit 2 + ;; +esac diff --git a/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs b/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs deleted file mode 100644 index 1aeeacb9c..000000000 --- a/scripts/ci/matrix-generator-tests/resolve-validation-test-matrix.test.mjs +++ /dev/null @@ -1,237 +0,0 @@ -import assert from 'node:assert/strict'; -import path from 'node:path'; -import test from 'node:test'; -import { fileURLToPath } from 'node:url'; - -import { - expandPlan, - readCatalog, - resolvePlan, - validateCatalog -} from '../resolve-validation-test-matrix.mjs'; - -// Tests load a fresh catalog for each case so negative mutations cannot leak -// into later assertions. -const testDirectory = path.dirname(fileURLToPath(import.meta.url)); -const catalogPath = path.resolve(testDirectory, '..', 'validation-test-matrix.json'); - -function catalog() { - return readCatalog(catalogPath); -} - -function clone(value) { - // Catalog values are JSON data, so a JSON round-trip is sufficient here. - return JSON.parse(JSON.stringify(value)); -} - -test('catalog validates and contains all five build targets', () => { - assert.doesNotThrow(() => validateCatalog(catalog())); -}); - -test('trigger categories resolve independently', () => { - const pr = resolvePlan(catalog(), 'pr'); - const nightly = resolvePlan(catalog(), 'nightly'); - const weekly = resolvePlan(catalog(), 'weekly'); - - assert.ok( - pr.windows.some(entry => ( - entry.os === 'windows-25h2' && entry.backend === 'process-t3' - )) - ); - assert.ok( - nightly.linux.some(entry => ( - entry.os === 'ubuntu-26.04' && entry.backend === 'bubblewrap' - )) - ); - assert.ok( - weekly.macos.some(entry => ( - entry.os === 'macos-15' && entry.backend === 'seatbelt' - )) - ); -}); - -test('macOS rollout uses 26 for PR and nightly with 15 added weekly', () => { - const project = entry => ({ - plan: entry.plan, - os: entry.os, - runner: entry.runner, - backend: entry.backend - }); - - assert.deepEqual(resolvePlan(catalog(), 'pr').macos.map(project), [ - { plan: 'pr', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } - ]); - assert.deepEqual(resolvePlan(catalog(), 'nightly').macos.map(project), [ - { plan: 'nightly', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } - ]); - assert.deepEqual(resolvePlan(catalog(), 'weekly').macos.map(project), [ - { plan: 'weekly', os: 'macos-15', runner: 'macos-15', backend: 'seatbelt' } - ]); -}); - -test('enabled plan deduplicates and runs both macOS versions', () => { - const enabled = resolvePlan(catalog(), 'enabled'); - const combinations = Object.values(enabled).flat(); - assert.equal( - new Set( - combinations.map(entry => ( - `${entry.os}|${entry.architecture}|${entry.backend}` - )) - ).size, - combinations.length - ); - assert.deepEqual( - enabled.macos.map(entry => ({ - plan: entry.plan, - os: entry.os, - runner: entry.runner, - backend: entry.backend - })), - [ - { plan: 'enabled', os: 'macos-15', runner: 'macos-15', backend: 'seatbelt' }, - { plan: 'enabled', os: 'macos-26', runner: 'macos-26', backend: 'seatbelt' } - ] - ); -}); - -test('missing enabled trigger does not affect other plans', () => { - const modified = clone(catalog()); - delete modified.triggers.enabled; - - assert.deepEqual(resolvePlan(modified, 'enabled'), { - windows: [], - linux: [], - macos: [] - }); - assert.ok( - resolvePlan(modified, 'pr').windows - .some(entry => entry.os === 'windows-25h2' && entry.backend === 'process-t3') - ); -}); - -test('empty pools skip OS and architecture combinations', () => { - const modified = clone(catalog()); - const windows25h2 = modified.platforms - .find(platform => platform.id === 'windows-25h2'); - windows25h2.architectures.x64.pool = ''; - - assert.doesNotThrow(() => validateCatalog(modified)); - assert.ok( - !expandPlan(modified, 'pr') - .some(entry => entry.os === 'windows-25h2') - ); - assert.ok( - !resolvePlan(modified, 'pr').windows - .some(entry => entry.os === 'windows-25h2') - ); -}); - -test('resolved matrices never emit non-macOS arm64 tests', () => { - const modified = clone(catalog()); - const windowsCanary = modified.platforms - .find(platform => platform.id === 'windows-canary'); - windowsCanary.architectures.x64.backends = windowsCanary.architectures.x64.backends - .filter(backend => backend !== 'process-t1'); - windowsCanary.architectures.arm64.pool = 'test-arm64-pool'; - modified.triggers.enabled.push({ - os: 'windows-canary', - backends: ['process-t1'] - }); - - for (const plan of ['pr', 'nightly', 'weekly', 'enabled']) { - const resolved = resolvePlan(modified, plan); - assert.ok(resolved.windows.every(entry => entry.architecture === 'x64')); - assert.ok(resolved.linux.every(entry => entry.architecture === 'x64')); - assert.ok(resolved.macos.every(entry => entry.architecture === 'arm64')); - } -}); - -test('weekly does not inherit nightly combinations', () => { - const nightly = resolvePlan(catalog(), 'nightly'); - const weekly = resolvePlan(catalog(), 'weekly'); - - for (const family of ['windows', 'linux', 'macos']) { - const nightlyKeys = new Set( - nightly[family].map(entry => `${entry.os}|${entry.architecture}|${entry.backend}`) - ); - const inherited = weekly[family].filter(entry => ( - nightlyKeys.has(`${entry.os}|${entry.architecture}|${entry.backend}`) - )); - assert.deepEqual( - inherited, - [], - `${family} weekly entries unexpectedly overlap nightly` - ); - assert.ok( - weekly[family].map(entry => `${entry.os}|${entry.architecture}|${entry.backend}`) - .every(key => !nightlyKeys.has(key)) - ); - } -}); - -test('full plan expands only architectures with available pools', () => { - const expanded = expandPlan(catalog(), 'pr'); - const ubuntuBubblewrap = expanded - .filter(entry => entry.os === 'ubuntu-26.04' && entry.backend === 'bubblewrap'); - assert.deepEqual( - ubuntuBubblewrap.map(entry => entry.architecture).sort(), - ['x64'] - ); -}); - -test('arm64 never expands Hyperlight or MicroVM', () => { - for (const plan of ['pr', 'nightly', 'weekly']) { - const invalid = expandPlan(catalog(), plan) - .filter(entry => ( - entry.architecture === 'arm64' - && ['hyperlight', 'microvm'].includes(entry.backend) - )); - assert.deepEqual(invalid, []); - } -}); - -test('all trigger categories omit placeholder handlers', () => { - const modified = clone(catalog()); - modified.triggers.enabled.push({ - os: 'ubuntu-24.04', - backends: ['hyperlight'] - }); - assert.doesNotThrow(() => validateCatalog(modified)); - assert.ok( - !resolvePlan(modified, 'enabled').linux - .some(entry => entry.backend === 'hyperlight') - ); -}); - -test('enabled trigger respects handler architecture restrictions', () => { - const modified = clone(catalog()); - const windows24h2 = modified.platforms - .find(platform => platform.id === 'windows-24h2'); - windows24h2.architectures.x64.pool = 'test-x64-pool'; - windows24h2.architectures.arm64.pool = 'test-arm64-pool'; - modified.triggers.enabled.push({ - os: 'windows-24h2', - backends: ['wslc'] - }); - assert.deepEqual( - resolvePlan(modified, 'enabled').windows - .filter(entry => entry.os === 'windows-24h2' && entry.backend === 'wslc') - .map(entry => entry.architecture), - ['x64'] - ); -}); - -test('duplicate enabled requests are rejected', () => { - const modified = clone(catalog()); - modified.triggers.enabled.push(clone(modified.triggers.enabled[0])); - assert.throws( - () => validateCatalog(modified), - /duplicate enabled request/ - ); -}); - -test('prerelease platforms use neutral public aliases', () => { - for (const platform of catalog().platforms.filter(entry => entry.prerelease === true)) { - assert.match(platform.id, /^windows-prerelease-[a-z-]+$/); - } -}); diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index ac252b04e..893faa867 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -74,9 +74,10 @@ export function validateCatalog(catalog) { if (architecture === 'arm64' && ARM64_UNSUPPORTED_BACKENDS.has(backend)) { throw new Error(`${backend} cannot be scheduled on arm64 (${platform.id})`); } - if (!catalog.handlers?.[platform.family]?.[backend]) { - throw new Error(`missing ${platform.family} handler entry for ${backend}`); - } + assertNonEmptyString( + catalog.handlers?.[platform.family]?.[backend], + `${platform.family} handler entry for ${backend}` + ); backends.add(backend); } } @@ -141,10 +142,6 @@ export function expandPlan(catalog, plan) { if (!details.backends.includes(backend)) { continue; } - const handler = catalog.handlers[platform.family][backend]; - if (handler.architectures && !handler.architectures.includes(architecture)) { - continue; - } combinations.push({ plan, os: platform.id, @@ -156,8 +153,7 @@ export function expandPlan(catalog, plan) { pool: details.pool, runner: details.runner, backend, - command: handler.command, - handler_status: handler.status + command: catalog.handlers[platform.family][backend] }); } } @@ -175,12 +171,11 @@ export function resolvePlan(catalog, plan) { const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); for (const combination of expandPlan(catalog, plan)) { - if (combination.handler_status === 'wired' && combination.command) { - // family selects the workflow job and handler_status is validation-only; - // neither belongs in the matrix consumed by the runner. - const { family, handler_status: _, ...matrixEntry } = combination; - matrices[family].push(matrixEntry); - } + // A trigger entry means "run this". A backend without a test script fails + // in the dispatcher, which is an actionable result: write the tests or + // remove the backend from the trigger. + const { family, ...matrixEntry } = combination; + matrices[family].push(matrixEntry); } suppressNonMacArm64(matrices); diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 6f55a0763..2d7762d36 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -315,58 +315,22 @@ ], "handlers": { "windows": { - "process-t1": { - "command": "process-container", - "status": "wired" - }, - "process-t3": { - "command": "process-container", - "status": "wired" - }, - "isolation-session": { - "command": "isolation-session", - "status": "wired" - }, - "wslc": { - "command": "wslc", - "status": "wired", - "architectures": [ - "x64" - ] - }, - "windows-sandbox": { - "command": "windows-sandbox", - "status": "wired" - }, - "microvm": { - "command": "microvm", - "status": "wired" - }, - "hyperlight": { - "status": "placeholder" - } + "process-t1": "process-container", + "process-t3": "process-container", + "isolation-session": "isolation-session", + "wslc": "wslc", + "windows-sandbox": "windows-sandbox", + "microvm": "microvm", + "hyperlight": "hyperlight" }, "linux": { - "bubblewrap": { - "command": "bubblewrap", - "status": "wired" - }, - "lxc": { - "command": "lxc", - "status": "wired" - }, - "microvm": { - "status": "placeholder" - }, - "hyperlight": { - "status": "placeholder" - } + "bubblewrap": "bubblewrap", + "lxc": "lxc", + "microvm": "microvm", + "hyperlight": "hyperlight" }, "macos": { - "seatbelt": { - "command": "seatbelt", - "status": "wired" - } + "seatbelt": "seatbelt" } }, "triggers": { @@ -528,7 +492,9 @@ { "os": "windows-25h2", "backends": [ - "process-t3" + "process-t3", + "microvm", + "hyperlight" ] }, { @@ -546,25 +512,30 @@ { "os": "ubuntu-26.04", "backends": [ - "bubblewrap" + "bubblewrap", + "lxc" ] }, { "os": "ubuntu-24.04", "backends": [ - "bubblewrap" + "bubblewrap", + "lxc", + "microvm" ] }, { "os": "rhel-10", "backends": [ - "bubblewrap" + "bubblewrap", + "lxc" ] }, { "os": "debian-13", "backends": [ - "bubblewrap" + "bubblewrap", + "lxc" ] }, { diff --git a/tests/scripts/run_seatbelt_all_tests.sh b/tests/scripts/TEMP_run_seatbelt_all_tests.sh similarity index 99% rename from tests/scripts/run_seatbelt_all_tests.sh rename to tests/scripts/TEMP_run_seatbelt_all_tests.sh index e46a9a701..dfd78c411 100644 --- a/tests/scripts/run_seatbelt_all_tests.sh +++ b/tests/scripts/TEMP_run_seatbelt_all_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Artifact-only macOS Seatbelt executor tests. +# THIS SCRIPT IS FOR TESTING CI INFRASTRUCTURE. THIS IS NOT A STABLE TEST SUITE. set -uo pipefail if [[ $# -ne 1 ]]; then diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index 992154ef5..f50d8a16c 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -22,10 +22,14 @@ release_directory="$repo_root/src/target/release" case "$backend" in microvm) + # Keep unwired commands explicit so accidental activation fails loudly. + # Future test script: run_microvm_tests.sh echo "The MicroVM CI handler is not wired to an artifact-only Linux test entry point yet." >&2 exit 2 ;; hyperlight) + # Keep unwired commands explicit so accidental activation fails loudly. + # Future test script: run_hyperlight_tests.sh echo "The Hyperlight CI handler is not wired to an existing backend test entry point yet." >&2 exit 2 ;; @@ -49,7 +53,7 @@ case "$backend" in seatbelt) test -x "$binary_directory/mxc-exec-mac" test -x "$binary_directory/unix-test-proxy" - bash "$script_root/run_seatbelt_all_tests.sh" "$binary_directory" + bash "$script_root/TEMP_run_seatbelt_all_tests.sh" "$binary_directory" ;; *) usage From 428ac4976f285cc94faf7e550f59a1f349422c0b Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:09:01 -0700 Subject: [PATCH 49/93] testing microvm on 23H2 with additional optional features. also correct ubuntu/debian networking DNS issue --- scripts/ci/install-linux-backend-prereqs.sh | 50 ++++++++++++++++++--- scripts/ci/validation-test-matrix.json | 15 ++++--- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/scripts/ci/install-linux-backend-prereqs.sh b/scripts/ci/install-linux-backend-prereqs.sh index 3123fa8a7..68255fab3 100644 --- a/scripts/ci/install-linux-backend-prereqs.sh +++ b/scripts/ci/install-linux-backend-prereqs.sh @@ -79,8 +79,48 @@ install_lxc() { fi } -chmod +x "$binary_directory/lxc-exec" +# Start the LXC bridge and wait until it can actually serve containers. +# A freshly installed lxc-net needs a moment before lxcbr0 has its IPv4 and +# dnsmasq is answering DHCP/DNS. Without this wait a container can boot into a +# bridge that has no lease to give, which surfaces much later as an unrelated +# name-resolution failure inside the guest. +start_lxc_bridge() { + local bridge="${LXC_BRIDGE:-lxcbr0}" + + if command -v systemctl >/dev/null 2>&1; then + if systemctl list-unit-files lxc-net.service >/dev/null 2>&1 && + systemctl cat lxc-net.service >/dev/null 2>&1; then + sudo systemctl start lxc-net + else + echo "No lxc-net unit on this distribution; skipping bridge startup." + fi + fi + + if ! ip link show "$bridge" >/dev/null 2>&1; then + echo "WARNING: bridge $bridge does not exist; container networking may be unavailable." >&2 + return 0 + fi + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if ip -4 addr show "$bridge" 2>/dev/null | grep -q 'inet '; then + echo "Bridge $bridge is up:" + ip -4 addr show "$bridge" | sed -n 's/^[[:space:]]*\(inet .*\)$/ \1/p' + if pgrep -f "dnsmasq.*$bridge" >/dev/null 2>&1; then + echo " dnsmasq is serving $bridge" + else + echo " WARNING: no dnsmasq bound to $bridge; DHCP and DNS may fail." >&2 + fi + return 0 + fi + sleep 1 + done + + echo "WARNING: $bridge did not receive an IPv4 address within 30s." >&2 + ip addr show "$bridge" || true +} + +chmod +x "$binary_directory/lxc-exec" case "$backend" in bubblewrap) install_bubblewrap @@ -93,14 +133,12 @@ case "$backend" in install_lxc command -v lxc-start sudo -n true - # Package installs may not activate the AppArmor profile or the - # lxc-net bridge, which lxc-start needs. + # Package installs may not activate the AppArmor profile, which + # lxc-start needs. if command -v apparmor_parser >/dev/null 2>&1; then sudo apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null || true fi - if command -v systemctl >/dev/null 2>&1; then - sudo systemctl start lxc-net 2>/dev/null || true - fi + start_lxc_bridge ;; microvm) for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 2d7762d36..7d5a73fed 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -493,35 +493,36 @@ "os": "windows-25h2", "backends": [ "process-t3", - "microvm", - "hyperlight" + "microvm" ] }, { "os": "windows-24h2", "backends": [ - "process-t3" + "process-t3", + "microvm" ] }, { "os": "windows-23h2", "backends": [ - "process-t3" + "process-t3", + "microvm" ] }, { "os": "ubuntu-26.04", "backends": [ "bubblewrap", - "lxc" + "lxc", + "hyperlight" ] }, { "os": "ubuntu-24.04", "backends": [ "bubblewrap", - "lxc", - "microvm" + "lxc" ] }, { From 4ca3db172bcad396cef70d23048dd94f3272a544 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:53:26 -0700 Subject: [PATCH 50/93] Testing process-t1 on unsuitable host. And network test for debian. --- scripts/ci/install-linux-backend-prereqs.sh | 54 +++++++++++++++++++++ scripts/ci/validation-test-matrix.json | 22 +-------- tests/configs/lxc_network_diagnostics.json | 19 ++++++++ tests/scripts/run_ci_backend_tests.ps1 | 29 ++++++----- tests/scripts/run_lxc_network_test.sh | 12 +++++ 5 files changed, 103 insertions(+), 33 deletions(-) create mode 100644 tests/configs/lxc_network_diagnostics.json diff --git a/scripts/ci/install-linux-backend-prereqs.sh b/scripts/ci/install-linux-backend-prereqs.sh index 68255fab3..c16fded05 100644 --- a/scripts/ci/install-linux-backend-prereqs.sh +++ b/scripts/ci/install-linux-backend-prereqs.sh @@ -120,6 +120,59 @@ start_lxc_bridge() { ip addr show "$bridge" || true } +# Report the host-side state that container networking depends on. Purely +# diagnostic: never fails the job, so a networking problem still surfaces as +# the backend test failure rather than as a prerequisite error. +report_lxc_network_diagnostics() { + local bridge="${LXC_BRIDGE:-lxcbr0}" + + echo "--- LXC network diagnostics (host) ---" + + echo "net.ipv4.ip_forward: $(cat /proc/sys/net/ipv4/ip_forward 2>/dev/null || echo unknown)" + + echo "Bridge $bridge:" + ip -4 addr show "$bridge" 2>/dev/null | sed 's/^/ /' || echo " (absent)" + + echo "dnsmasq processes:" + pgrep -af dnsmasq 2>/dev/null | sed 's/^/ /' || echo " (none)" + + echo "NAT rules for the bridge subnet:" + sudo iptables -t nat -S POSTROUTING 2>/dev/null | grep -E '10\.0\.3|MASQUERADE' | + sed 's/^/ /' || echo " (none found)" + + echo "FORWARD policy and bridge rules:" + sudo iptables -S FORWARD 2>/dev/null | grep -E "policy|$bridge" | sed 's/^/ /' || + echo " (none found)" + + echo "Host /etc/resolv.conf nameservers:" + grep '^nameserver' /etc/resolv.conf 2>/dev/null | sed 's/^/ /' || echo " (none)" + + echo "lxc-net configuration:" + grep -E '^(USE_LXC_BRIDGE|LXC_ADDR|LXC_NETMASK|LXC_DHCP_RANGE|LXC_DHCP_CONFILE)' \ + /etc/default/lxc-net 2>/dev/null | sed 's/^/ /' || echo " (no /etc/default/lxc-net)" + + # Prove the host itself can resolve the name the network test uses. If this + # fails, the container was never going to succeed. + if command -v getent >/dev/null 2>&1; then + echo "Host resolution of api.github.com:" + getent ahostsv4 api.github.com 2>/dev/null | head -n 2 | sed 's/^/ /' || + echo " FAILED - the host cannot resolve it either" + fi + + # Ask the bridge's own resolver, which is what a container is handed via + # DHCP. This isolates "dnsmasq is broken" from "the host is fine". + local bridge_ip + bridge_ip="$(ip -4 -o addr show "$bridge" 2>/dev/null | + awk '{print $4}' | cut -d/ -f1 | head -n 1)" + if [[ -n "$bridge_ip" ]] && command -v nslookup >/dev/null 2>&1; then + echo "Resolution via bridge resolver ($bridge_ip):" + nslookup api.github.com "$bridge_ip" 2>&1 | tail -n 4 | sed 's/^/ /' || + echo " FAILED - dnsmasq on $bridge is not answering" + fi + + echo "--- end diagnostics ---" +} + chmod +x "$binary_directory/lxc-exec" case "$backend" in bubblewrap) @@ -139,6 +192,7 @@ case "$backend" in sudo apparmor_parser -rT /etc/apparmor.d/lxc* 2>/dev/null || true fi start_lxc_bridge + report_lxc_network_diagnostics ;; microvm) for file in nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf; do diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 7d5a73fed..2e8da3a8d 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -492,6 +492,7 @@ { "os": "windows-25h2", "backends": [ + "process-t1", "process-t3", "microvm" ] @@ -499,57 +500,38 @@ { "os": "windows-24h2", "backends": [ - "process-t3", "microvm" ] }, { "os": "windows-23h2", "backends": [ - "process-t3", "microvm" ] }, { "os": "ubuntu-26.04", "backends": [ - "bubblewrap", - "lxc", - "hyperlight" + "lxc" ] }, { "os": "ubuntu-24.04", "backends": [ - "bubblewrap", "lxc" ] }, { "os": "rhel-10", "backends": [ - "bubblewrap", "lxc" ] }, { "os": "debian-13", "backends": [ - "bubblewrap", "lxc" ] - }, - { - "os": "macos-26", - "backends": [ - "seatbelt" - ] - }, - { - "os": "macos-15", - "backends": [ - "seatbelt" - ] } ] } diff --git a/tests/configs/lxc_network_diagnostics.json b/tests/configs/lxc_network_diagnostics.json new file mode 100644 index 000000000..a7ab5bcb4 --- /dev/null +++ b/tests/configs/lxc_network_diagnostics.json @@ -0,0 +1,19 @@ +{ + "version": "0.6.0-alpha", + "containerId": "CLI-LXC-Network-Diagnostics", + "containment": "lxc", + "process": { + "commandLine": "echo '--- container interfaces ---'; ip -4 addr show eth0 2>&1 || ifconfig eth0 2>&1; echo '--- container routes ---'; ip route 2>&1; echo '--- container resolv.conf ---'; cat /etc/resolv.conf 2>&1; echo '--- gateway reachability ---'; ping -c 1 -W 3 10.0.3.1 2>&1 || echo 'gateway unreachable'; echo '--- DNS via bridge ---'; nslookup api.github.com 10.0.3.1 2>&1 || echo 'bridge DNS failed'; echo '--- DNS via configured resolver ---'; nslookup api.github.com 2>&1 || echo 'configured resolver failed'; echo '--- direct IP egress (bypasses DNS) ---'; wget -qO- --timeout=5 https://140.82.121.6/zen 2>&1 | head -c 100 || echo 'direct IP fetch failed'", + "timeout": 60000 + }, + "lifecycle": { + "destroyOnExit": true + }, + "lxc": { + "distribution": "alpine", + "release": "3.23" + }, + "network": { + "defaultPolicy": "allow" + } +} diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 index 4b04df3c3..2d82cc408 100644 --- a/tests/scripts/run_ci_backend_tests.ps1 +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -37,7 +37,10 @@ function Assert-File { function Invoke-TestScript { param( [Parameter(Mandatory)][string]$Path, - [string[]]$Arguments = @() + # Splat a hashtable, not an array. Array splatting binds elements + # positionally, so '-BinDir' would be passed as the first positional + # value rather than naming the parameter. + [hashtable]$Arguments = @{} ) # PowerShell scripts do not always replace a previous native exit code. @@ -111,28 +114,28 @@ switch ($Backend) { } } 'isolation-session' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @( - '-WxcExePath', $wxc - ) + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{ + WxcExePath = $wxc + } } 'windows-sandbox' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @( - '-BinDir', $binaryDirectoryPath - ) + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_windows_sandbox_one_shot_tests.ps1') -Arguments @{ + BinDir = $binaryDirectoryPath + } } 'wslc' { # The current WSLC helper hardcodes the x64 target when locating assets. if ($Architecture -ne 'x64') { throw 'The existing WSLC test harness is not architecture-portable yet.' } - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @( - '-WxcExecPath', $wxc - ) + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_wslc_all_tests.ps1') -Arguments @{ + WxcExecPath = $wxc + } } 'microvm' { - Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @( - '-BinDir', $binaryDirectoryPath - ) + Invoke-TestScript -Path (Join-Path $scriptRoot 'run_microvm_tests.ps1') -Arguments @{ + BinDir = $binaryDirectoryPath + } } 'hyperlight' { # Keep unwired commands explicit so accidental activation fails loudly. diff --git a/tests/scripts/run_lxc_network_test.sh b/tests/scripts/run_lxc_network_test.sh index c2d3f93ea..eb8e09f37 100644 --- a/tests/scripts/run_lxc_network_test.sh +++ b/tests/scripts/run_lxc_network_test.sh @@ -16,5 +16,17 @@ if [ ! -f "$LXC_EXEC" ]; then fi echo "Running LXC network test..." + +# When the network test fails it is usually the environment (no DHCP lease, +# dnsmasq not answering, blocked egress) rather than the policy code. Capture +# the container's own view first so the log distinguishes those cases. This is +# diagnostic only and never fails the suite. +if [ "${MXC_LXC_NETWORK_DIAGNOSTICS:-1}" = "1" ]; then + echo "--- LXC network diagnostics (container) ---" + "$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_network_diagnostics.json" 2>&1 || \ + echo "(diagnostic container run failed)" + echo "--- end diagnostics ---" +fi + "$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_network_test.json" echo "LXC network test complete." From 7aa0d46b1cf891b074c801bed5e789987f82ed42 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:23:36 -0700 Subject: [PATCH 51/93] Added additional opt. features to Windows for WSLC and microvm support. --- scripts/ci/validation-test-matrix.json | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 2e8da3a8d..895d0eaa4 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -492,7 +492,6 @@ { "os": "windows-25h2", "backends": [ - "process-t1", "process-t3", "microvm" ] From cb51bddd2e4910bf3de57e16d9896a1070786869 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:47:33 -0700 Subject: [PATCH 52/93] fix microvm install for windows --- tests/scripts/run_microvm_tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/run_microvm_tests.ps1 b/tests/scripts/run_microvm_tests.ps1 index 1b32c94ee..b704189bc 100644 --- a/tests/scripts/run_microvm_tests.ps1 +++ b/tests/scripts/run_microvm_tests.ps1 @@ -88,7 +88,7 @@ $wxcExe = Resolve-Path $WxcExePath # -- Verify MicroVM binaries -------------------------------------------------- -$requiredBinaries = @("nanvixd.exe", "kernel.elf", "python3.12", "nanvix_rootfs.img") +$requiredBinaries = @("nanvixd.exe", "nanvix_rootfs.img", "python3.initrd", "bin\kernel.elf") $binDir = Split-Path $wxcExe $missing = $requiredBinaries | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } From c3fcc0aa898fcf2bca9c71fb129dc41101b8143d Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Sat, 8 Aug 2026 18:21:38 -0700 Subject: [PATCH 53/93] added a ton of diagnosis and redundancy to microvm tests to determine issue --- .../workflows/Validation.Tests.Matrix.Job.yml | 35 ++++++++++++++++++ tests/scripts/run_microvm_tests.ps1 | 36 ++++++++++++++++--- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index cf7701bb9..62485b0e2 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -49,6 +49,40 @@ jobs: path: artifacts/bin - name: Prepare backend prerequisites + if: matrix.command == 'microvm' + shell: pwsh + run: | + $binDir = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin' + + # NanVix boots a VM from these images on every invocation; Defender + # scanning them can push boot past its timeout. + Add-MpPreference -ExclusionPath $binDir + Write-Host "Added Defender exclusion for $binDir" + + # Mirrors the verification in microvm-e2e.yml. + $required = @( + "wxc-exec.exe", + "nanvixd.exe", + "nanvix_rootfs.img", + "python3.initrd", + "bin\kernel.elf", + "snapshots\kernel.vmem", + "snapshots\kernel.whp.cbor" + ) + $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } + if ($missing) { + Write-Host "::error::Missing binaries: $($missing -join ', ')" + exit 1 + } + $leaves = $required | ForEach-Object { Split-Path $_ -Leaf } + Get-ChildItem $binDir -Include $leaves -Recurse | Format-Table FullName, Length + + - name: Diagnose hypervisor environment + if: matrix.command == 'microvm' + shell: pwsh + run: ./scripts/ci/diagnose-whp.ps1 + + - name: Check Windows Hypervisor Platform if: matrix.command == 'microvm' shell: pwsh run: ./scripts/ci/check-whp.ps1 @@ -86,6 +120,7 @@ jobs: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: | ${{ runner.temp }}/mxc-ci.log + ${{ runner.temp }}/mxc-microvm-logs ${{ runner.temp }}/mxc-wpc-tests/logs ${{ runner.temp }}/WinProcessContainer-Tests.results.* ${{ runner.temp }}/mxc_concurrent_oneshot diff --git a/tests/scripts/run_microvm_tests.ps1 b/tests/scripts/run_microvm_tests.ps1 index b704189bc..1c78a67bb 100644 --- a/tests/scripts/run_microvm_tests.ps1 +++ b/tests/scripts/run_microvm_tests.ps1 @@ -30,7 +30,8 @@ param( [switch]$Release, [string]$BinDir, - [string]$ConfigDir + [string]$ConfigDir, + [string]$LogDir ) $ErrorActionPreference = "Stop" @@ -48,6 +49,14 @@ if (-not $ConfigDir) { $ConfigDir = Join-Path $RepoRoot "tests\configs" } +# Default to RUNNER_TEMP so CI's existing log upload picks the files up. +if (-not $LogDir -and $env:RUNNER_TEMP) { + $LogDir = Join-Path $env:RUNNER_TEMP "mxc-microvm-logs" +} +if ($LogDir -and -not (Test-Path $LogDir)) { + New-Item -ItemType Directory -Force -Path $LogDir | Out-Null +} + $WxcExePath = Join-Path $BinDir "wxc-exec.exe" # -- WHP check (local runs only) --------------------------------------------- @@ -88,7 +97,14 @@ $wxcExe = Resolve-Path $WxcExePath # -- Verify MicroVM binaries -------------------------------------------------- -$requiredBinaries = @("nanvixd.exe", "nanvix_rootfs.img", "python3.initrd", "bin\kernel.elf") +$requiredBinaries = @( + "nanvixd.exe", + "nanvix_rootfs.img", + "python3.initrd", + "bin\kernel.elf", + "snapshots\kernel.vmem", + "snapshots\kernel.whp.cbor" +) $binDir = Split-Path $wxcExe $missing = $requiredBinaries | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } @@ -132,8 +148,16 @@ foreach ($test in $tests) { $sw = [System.Diagnostics.Stopwatch]::StartNew() $stdoutFile = [System.IO.Path]::GetTempFileName() $stderrFile = [System.IO.Path]::GetTempFileName() + # The failure tail below only keeps the last few lines, which for a boot + # timeout are the error envelope rather than the boot itself. Keep the full + # --debug log so CI can show where a VM actually stalled. + $logArgs = @() + if ($LogDir) { + $logFile = Join-Path $LogDir "microvm-$([System.IO.Path]::GetFileNameWithoutExtension($test.Config)).log" + $logArgs = @("--log-file", $logFile) + } $process = Start-Process -FilePath $wxcExe ` - -ArgumentList "--debug", "--experimental", $configPath ` + -ArgumentList (@("--debug", "--experimental") + $logArgs + @($configPath)) ` -PassThru -Wait ` -RedirectStandardOutput $stdoutFile ` -RedirectStandardError $stderrFile @@ -153,8 +177,10 @@ foreach ($test in $tests) { $reason = "expected exit=$expectedExit, got exit=$actualExit" } - # Check stdout content if OutputContains is specified - if ($pass -and $test.OutputContains) { + # Check stdout content if OutputContains is specified. Not every test + # defines the key, and StrictMode makes a missing hashtable key throw, so + # test for its presence rather than accessing it directly. + if ($pass -and $test.ContainsKey('OutputContains')) { $combined = "$stdout`n$stderr" if ($combined -notmatch [regex]::Escape($test.OutputContains)) { $pass = $false From daed2e6ad9f4a5df453919c276349a44e733f51a Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Sat, 8 Aug 2026 22:33:26 -0700 Subject: [PATCH 54/93] experimenting with cold start for nanvix --- .../workflows/Validation.Tests.Matrix.Job.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 62485b0e2..cb88e0e58 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -77,6 +77,22 @@ jobs: $leaves = $required | ForEach-Object { Split-Path $_ -Leaf } Get-ChildItem $binDir -Include $leaves -Recurse | Format-Table FullName, Length + # EXPERIMENT: the artifact's WHP snapshots were generated on the build + # machine, and a snapshot is only valid for the host that produced it + # (see nanvix/build_common: prefetched snapshots are untrusted). + # Point NANVIX_HOME at an empty directory so the runner regenerates a + # snapshot on this host via cold boot instead of restoring a foreign + # one. If MicroVM then passes, the imported snapshot is the problem. + $nanvixHome = Join-Path $env:RUNNER_TEMP 'nanvix-home' + New-Item -ItemType Directory -Force -Path $nanvixHome | Out-Null + "NANVIX_HOME=$nanvixHome" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + Write-Host "Set NANVIX_HOME=$nanvixHome (forces host-local snapshot generation)" + + # Capture nanvixd's own stderr/tracing; without this the runner + # silences it, which is why a hung boot produced no diagnostics. + "MXC_NANVIX_TRACE=1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + Write-Host "Set MXC_NANVIX_TRACE=1 (captures nanvixd tracing)" + - name: Diagnose hypervisor environment if: matrix.command == 'microvm' shell: pwsh From 957f95341a3fc41d73c02ee76508040381c7d09f Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:06:43 -0700 Subject: [PATCH 55/93] additional nanvix debugging. last effort --- .../workflows/Validation.Tests.Matrix.Job.yml | 22 +-- scripts/ci/probe-nanvixd.ps1 | 130 ++++++++++++++++++ 2 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 scripts/ci/probe-nanvixd.ps1 diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index cb88e0e58..65c6b46b1 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -77,17 +77,6 @@ jobs: $leaves = $required | ForEach-Object { Split-Path $_ -Leaf } Get-ChildItem $binDir -Include $leaves -Recurse | Format-Table FullName, Length - # EXPERIMENT: the artifact's WHP snapshots were generated on the build - # machine, and a snapshot is only valid for the host that produced it - # (see nanvix/build_common: prefetched snapshots are untrusted). - # Point NANVIX_HOME at an empty directory so the runner regenerates a - # snapshot on this host via cold boot instead of restoring a foreign - # one. If MicroVM then passes, the imported snapshot is the problem. - $nanvixHome = Join-Path $env:RUNNER_TEMP 'nanvix-home' - New-Item -ItemType Directory -Force -Path $nanvixHome | Out-Null - "NANVIX_HOME=$nanvixHome" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - Write-Host "Set NANVIX_HOME=$nanvixHome (forces host-local snapshot generation)" - # Capture nanvixd's own stderr/tracing; without this the runner # silences it, which is why a hung boot produced no diagnostics. "MXC_NANVIX_TRACE=1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 @@ -103,6 +92,14 @@ jobs: shell: pwsh run: ./scripts/ci/check-whp.ps1 + - name: Probe nanvixd directly + if: matrix.command == 'microvm' + timeout-minutes: 5 + shell: pwsh + run: | + ./scripts/ci/probe-nanvixd.ps1 ` + -BinDir (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') + - name: Prepare Process Container host if: matrix.backend == 'process-t3' shell: pwsh @@ -114,6 +111,9 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run backend tests + # A backend that hangs (rather than failing) would otherwise burn the + # full job timeout. Fail fast enough to keep the log useful. + timeout-minutes: 30 shell: pwsh run: | $arguments = @{ diff --git a/scripts/ci/probe-nanvixd.ps1 b/scripts/ci/probe-nanvixd.ps1 new file mode 100644 index 000000000..981c210ef --- /dev/null +++ b/scripts/ci/probe-nanvixd.ps1 @@ -0,0 +1,130 @@ +<# +.SYNOPSIS + Runs nanvixd directly to isolate MicroVM boot failures from MXC. + +.DESCRIPTION + MicroVM tests can hang or time out without revealing whether the fault is + in nanvixd (the VM monitor) or in how wxc-exec invokes it. This probe skips + MXC entirely and cold-boots nanvixd itself, bounded by a timeout so it can + never hang a job. + + Diagnostic only: always exits 0 so it reports findings without failing the + lane. Interpretation: + + - Snapshot files produced -> nanvixd works; suspect MXC's invocation. + - Non-zero exit with stderr -> nanvixd fails; stderr names the reason. + - Timeout, no output -> nanvixd hangs on this host (e.g. WHP + partition creation never returns). + +.PARAMETER BinDir + Directory holding nanvixd.exe and its payload files. + +.PARAMETER TimeoutSeconds + How long to wait before declaring a hang. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string]$BinDir, + + [int]$TimeoutSeconds = 120 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$binPath = (Resolve-Path -LiteralPath $BinDir).Path +$nanvixd = Join-Path $binPath 'nanvixd.exe' + +Write-Host '=== Direct nanvixd probe ===' + +if (-not (Test-Path -LiteralPath $nanvixd -PathType Leaf)) { + Write-Host "nanvixd.exe not found at $nanvixd - skipping probe." + exit 0 +} + +# Generate into a scratch directory so the probe never disturbs the artifact's +# own snapshots, which the real tests rely on. +$probeHome = Join-Path ([System.IO.Path]::GetTempPath()) "nanvixd-probe-$([guid]::NewGuid().ToString('N'))" +New-Item -ItemType Directory -Force -Path $probeHome | Out-Null + +$stdoutFile = Join-Path $probeHome 'stdout.log' +$stderrFile = Join-Path $probeHome 'stderr.log' + +# Mirrors nanvix_common::generate_snapshot: cold boot with -kernel-args snapshot. +$arguments = @( + '-bin-dir', (Join-Path $binPath 'bin'), + '-ramfs', (Join-Path $binPath 'nanvix_rootfs.img'), + '-kernel-args', 'snapshot', + '--', (Join-Path $binPath 'python3.initrd') +) + +Write-Host "Command: nanvixd.exe $($arguments -join ' ')" +Write-Host "Working directory: $probeHome" +Write-Host "Timeout: ${TimeoutSeconds}s" +Write-Host '' + +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +# RUST_LOG surfaces nanvixd's tracing, which the MXC runner suppresses by +# default - that suppression is why earlier hangs produced no diagnostics. +$previousRustLog = $env:RUST_LOG +$env:RUST_LOG = 'debug' +try { + $process = Start-Process -FilePath $nanvixd ` + -ArgumentList $arguments ` + -WorkingDirectory $probeHome ` + -PassThru ` + -RedirectStandardOutput $stdoutFile ` + -RedirectStandardError $stderrFile +} finally { + $env:RUST_LOG = $previousRustLog +} + +if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $stopwatch.Stop() + Write-Host "::warning::nanvixd did not exit within ${TimeoutSeconds}s - it HANGS on this host." + try { $process.Kill($true) } catch { } + $hung = $true +} else { + $stopwatch.Stop() + $hung = $false + Write-Host "nanvixd exited with code $($process.ExitCode) after $($stopwatch.Elapsed.TotalSeconds.ToString('0.0'))s" +} + +foreach ($stream in @(@{ Name = 'stdout'; Path = $stdoutFile }, @{ Name = 'stderr'; Path = $stderrFile })) { + Write-Host '' + Write-Host "--- nanvixd $($stream.Name) ---" + if (Test-Path -LiteralPath $stream.Path) { + $content = Get-Content -LiteralPath $stream.Path -Raw -ErrorAction SilentlyContinue + if ([string]::IsNullOrWhiteSpace($content)) { + Write-Host '(empty)' + } else { + Write-Host $content.TrimEnd() + } + } else { + Write-Host '(not captured)' + } +} + +Write-Host '' +Write-Host '--- Generated snapshot files ---' +$snapshotDir = Join-Path $probeHome 'snapshots' +if (Test-Path -LiteralPath $snapshotDir) { + Get-ChildItem -LiteralPath $snapshotDir | Format-Table Name, Length | Out-String | Write-Host +} else { + Write-Host '(no snapshots directory created)' +} + +Write-Host '' +if ($hung) { + Write-Host 'RESULT: nanvixd hangs on this host - the fault is below MXC.' +} elseif ($process.ExitCode -eq 0) { + Write-Host 'RESULT: nanvixd cold-booted successfully - suspect MXC''s invocation instead.' +} else { + Write-Host 'RESULT: nanvixd failed - see its stderr above for the reason.' +} + +Remove-Item -LiteralPath $probeHome -Recurse -Force -ErrorAction SilentlyContinue + +# Diagnostic only: never fail the job on the probe's outcome. +exit 0 From e758336ad21c5cec17693792d06d764071b43e95 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:37:02 -0700 Subject: [PATCH 56/93] WHP probe --- .../workflows/Validation.Tests.Matrix.Job.yml | 6 + scripts/ci/probe-whp-capabilities.ps1 | 218 ++++++++++++++++++ scripts/ci/validation-test-matrix.json | 24 -- 3 files changed, 224 insertions(+), 24 deletions(-) create mode 100644 scripts/ci/probe-whp-capabilities.ps1 diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 65c6b46b1..5a0b3bc37 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -92,6 +92,12 @@ jobs: shell: pwsh run: ./scripts/ci/check-whp.ps1 + - name: Probe WHP capabilities + if: matrix.command == 'microvm' + timeout-minutes: 5 + shell: pwsh + run: ./scripts/ci/probe-whp-capabilities.ps1 + - name: Probe nanvixd directly if: matrix.command == 'microvm' timeout-minutes: 5 diff --git a/scripts/ci/probe-whp-capabilities.ps1 b/scripts/ci/probe-whp-capabilities.ps1 new file mode 100644 index 000000000..b83dd548d --- /dev/null +++ b/scripts/ci/probe-whp-capabilities.ps1 @@ -0,0 +1,218 @@ +<# +.SYNOPSIS + Reports Windows Hypervisor Platform capabilities via the real WHP API. + +.DESCRIPTION + `Get-WindowsOptionalFeature` only reports whether the HypervisorPlatform + feature is installed, and `HypervisorPresent` only says a hypervisor is + running. Neither proves that WHP can actually create a partition, which is + what a VM monitor such as nanvixd needs. + + This probe P/Invokes WinHvPlatform.dll directly to: + + 1. Query capabilities through WHvGetCapability. + 2. Attempt a real WHvCreatePartition / WHvSetupPartition and delete it. + + Step 2 is the important one: it is the first call that genuinely exercises + the hypervisor, and the most likely place for a host that reports WHP as + "enabled" to fail or block. + + Run this on both a working host and a failing one and compare the output. + + Diagnostic only: always exits 0. +#> +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Write-Host '=== WHP Capability Probe ===' +Write-Host "OS: $([System.Environment]::OSVersion.VersionString)" + +try { + $cs = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue + if ($cs) { + Write-Host "Model: $($cs.Model)" + Write-Host "HypervisorPresent: $($cs.HypervisorPresent)" + } + $cpu = Get-CimInstance -ClassName Win32_Processor -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($cpu) { + Write-Host "CPU: $($cpu.Name)" + Write-Host "VirtualizationFirmwareEnabled: $($cpu.VirtualizationFirmwareEnabled)" + } +} catch { + Write-Host "Could not read system info: $($_.Exception.Message)" +} + +if (-not (Test-Path "$env:SystemRoot\System32\WinHvPlatform.dll")) { + Write-Host '::warning::WinHvPlatform.dll is absent - WHP is not installed.' + exit 0 +} + +$signature = @' +using System; +using System.Runtime.InteropServices; + +public static class Whp +{ + [DllImport("WinHvPlatform.dll")] + public static extern int WHvGetCapability( + uint CapabilityCode, + IntPtr CapabilityBuffer, + uint CapabilityBufferSizeInBytes, + out uint WrittenSizeInBytes); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvCreatePartition(out IntPtr Partition); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvSetupPartition(IntPtr Partition); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvDeletePartition(IntPtr Partition); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvSetPartitionProperty( + IntPtr Partition, + uint PropertyCode, + IntPtr PropertyBuffer, + uint PropertyBufferSizeInBytes); +} +'@ + +try { + Add-Type -TypeDefinition $signature -ErrorAction Stop +} catch { + Write-Host "::warning::Could not bind to WinHvPlatform.dll: $($_.Exception.Message)" + exit 0 +} + +function Get-WhpCapability { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][uint32]$Code + ) + + $buffer = [Runtime.InteropServices.Marshal]::AllocHGlobal(8) + try { + [Runtime.InteropServices.Marshal]::WriteInt64($buffer, 0) + $written = 0 + $hr = [Whp]::WHvGetCapability($Code, $buffer, 8, [ref]$written) + if ($hr -eq 0) { + $value = [Runtime.InteropServices.Marshal]::ReadInt64($buffer) + Write-Host (" {0,-28} 0x{1:X16}" -f $Name, $value) + return $value + } + Write-Host (" {0,-28} FAILED (hr=0x{1:X8})" -f $Name, $hr) + return $null + } finally { + [Runtime.InteropServices.Marshal]::FreeHGlobal($buffer) + } +} + +Write-Host '' +Write-Host '--- WHvGetCapability ---' +$hypervisorPresent = Get-WhpCapability -Name 'HypervisorPresent' -Code 0x00000000 +$features = Get-WhpCapability -Name 'Features' -Code 0x00000001 +Get-WhpCapability -Name 'ExtendedVmExits' -Code 0x00000002 | Out-Null +Get-WhpCapability -Name 'ProcessorVendor' -Code 0x00001000 | Out-Null +Get-WhpCapability -Name 'ProcessorFeatures' -Code 0x00001001 | Out-Null + +# Nested-virtualization capabilities. A host that cannot expose VMX to a guest +# will fail or omit these, which is the difference we are looking for between a +# working runner and one where a VM monitor cannot boot. +Write-Host '' +Write-Host '--- Nested virtualization (VMX) capabilities ---' +Get-WhpCapability -Name 'VmxBasic' -Code 0x00002000 | Out-Null +Get-WhpCapability -Name 'VmxPinbasedCtls' -Code 0x00002001 | Out-Null +Get-WhpCapability -Name 'VmxProcbasedCtls' -Code 0x00002002 | Out-Null +Get-WhpCapability -Name 'VmxEptVpidCap' -Code 0x0000200C | Out-Null + +if ($null -ne $features) { + # Bit layout from WHV_CAPABILITY_FEATURES (x64). + $bits = [ordered]@{ + PartialUnmap = 0 + LocalApicEmulation = 1 + Xsave = 2 + DirtyPageTracking = 3 + SpeculationControl = 4 + ApicRemoteRead = 5 + IdleSuspend = 6 + VirtualPciDeviceSupport = 7 + IommuSupport = 8 + VpHotAddRemove = 9 + DeviceAccessTracking = 10 + } + Write-Host '' + Write-Host '--- Feature bits ---' + foreach ($entry in $bits.GetEnumerator()) { + $set = (($features -shr $entry.Value) -band 1) -eq 1 + Write-Host (" {0,-28} {1}" -f $entry.Key, $set) + } +} + +if ($hypervisorPresent -eq 0) { + Write-Host '' + Write-Host '::warning::WHvCapabilityCodeHypervisorPresent is 0 - WHP reports no usable hypervisor.' +} + +# The decisive test. Feature flags can look correct on a host where partition +# creation still fails or blocks; this is the call a VM monitor makes first. +Write-Host '' +Write-Host '--- WHvCreatePartition (the call that actually exercises the hypervisor) ---' + +$partition = [IntPtr]::Zero +$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() +$hr = [Whp]::WHvCreatePartition([ref]$partition) +$stopwatch.Stop() +Write-Host (" WHvCreatePartition hr=0x{0:X8} ({1} ms)" -f $hr, $stopwatch.ElapsedMilliseconds) + +if ($hr -ne 0) { + Write-Host '::warning::WHvCreatePartition failed - WHP cannot host a VM here even though the feature is enabled.' + Write-Host '' + Write-Host 'RESULT: partition creation FAILED.' + exit 0 +} + +try { + # A partition needs both a processor count and an extended-VM-exit + # configuration before setup will accept it; omitting the latter yields + # WHV_E_INVALID_PARTITION_CONFIG (0x80370304) on an otherwise healthy host. + $propertyBuffer = [Runtime.InteropServices.Marshal]::AllocHGlobal(8) + try { + [Runtime.InteropServices.Marshal]::WriteInt64($propertyBuffer, 0) + [Runtime.InteropServices.Marshal]::WriteInt32($propertyBuffer, 1) + # WHvPartitionPropertyCodeProcessorCount = 0x00001002 + $hrProp = [Whp]::WHvSetPartitionProperty($partition, 0x00001002, $propertyBuffer, 4) + Write-Host (" SetProcessorCount hr=0x{0:X8}" -f $hrProp) + + # WHvPartitionPropertyCodeExtendedVmExits = 0x00000002 + [Runtime.InteropServices.Marshal]::WriteInt64($propertyBuffer, 0) + $hrExits = [Whp]::WHvSetPartitionProperty($partition, 0x00000002, $propertyBuffer, 8) + Write-Host (" SetExtendedVmExits hr=0x{0:X8}" -f $hrExits) + } finally { + [Runtime.InteropServices.Marshal]::FreeHGlobal($propertyBuffer) + } + + $stopwatch.Restart() + $hrSetup = [Whp]::WHvSetupPartition($partition) + $stopwatch.Stop() + Write-Host (" WHvSetupPartition hr=0x{0:X8} ({1} ms)" -f $hrSetup, $stopwatch.ElapsedMilliseconds) + + Write-Host '' + if ($hrSetup -eq 0) { + Write-Host 'RESULT: WHP can create and set up a partition - the hypervisor is usable.' + Write-Host ' A VM monitor hanging here is failing for some other reason.' + } else { + # 0x80370304 here means this probe built an incomplete partition, not + # necessarily that the host is broken; compare against a known-good host. + Write-Host ("RESULT: partition setup returned 0x{0:X8}." -f $hrSetup) + Write-Host ' Compare this value against a host where the VM monitor works.' + } +} finally { + $hrDelete = [Whp]::WHvDeletePartition($partition) + Write-Host (" WHvDeletePartition hr=0x{0:X8}" -f $hrDelete) +} + +exit 0 diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 895d0eaa4..37b18f77a 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -507,30 +507,6 @@ "backends": [ "microvm" ] - }, - { - "os": "ubuntu-26.04", - "backends": [ - "lxc" - ] - }, - { - "os": "ubuntu-24.04", - "backends": [ - "lxc" - ] - }, - { - "os": "rhel-10", - "backends": [ - "lxc" - ] - }, - { - "os": "debian-13", - "backends": [ - "lxc" - ] } ] } From aae46a2bb36bd912ce1d5ab55f25104554e02cdf Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:20:26 -0700 Subject: [PATCH 57/93] last ditch effort to see why nanvix is failing --- .../workflows/Validation.Tests.Matrix.Job.yml | 5 +- scripts/ci/probe-nanvixd.ps1 | 305 ++++++++++++++---- scripts/ci/probe-whp-capabilities.ps1 | 210 ++++++++++-- scripts/ci/validation-test-matrix.json | 1 - src/backends/nanvix/runner/src/lib.rs | 32 +- 5 files changed, 446 insertions(+), 107 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 5a0b3bc37..d95c97a32 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -100,11 +100,12 @@ jobs: - name: Probe nanvixd directly if: matrix.command == 'microvm' - timeout-minutes: 5 + timeout-minutes: 10 shell: pwsh run: | ./scripts/ci/probe-nanvixd.ps1 ` - -BinDir (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') + -BinDir (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') ` + -Mode both - name: Prepare Process Container host if: matrix.backend == 'process-t3' diff --git a/scripts/ci/probe-nanvixd.ps1 b/scripts/ci/probe-nanvixd.ps1 index 981c210ef..8e995be29 100644 --- a/scripts/ci/probe-nanvixd.ps1 +++ b/scripts/ci/probe-nanvixd.ps1 @@ -5,29 +5,58 @@ .DESCRIPTION MicroVM tests can hang or time out without revealing whether the fault is in nanvixd (the VM monitor) or in how wxc-exec invokes it. This probe skips - MXC entirely and cold-boots nanvixd itself, bounded by a timeout so it can - never hang a job. + MXC entirely and boots nanvixd itself, bounded by a timeout so it can never + hang a job. + + Two boot paths are exercised because they fail independently: + + - cold: `-kernel-args snapshot`, the path that builds a snapshot from + scratch. This is what the build machine runs. + - warm: `-snapshot snapshots\kernel.whp.cbor`, the path wxc-exec actually + uses at runtime. MXC ships snapshots produced on the *build* + machine, so a warm start can fail on a test host even when a cold + boot succeeds. + + Getting diagnostics out of nanvixd requires care: by default it writes its + trace to a file under `-log-dir` (default `/logs`), so its stdout and + stderr are empty on a *successful* run as well as a hung one. This probe + passes `-log-to-stdout` and `-console-file` so the boot trace and guest + console are actually captured, keeps everything it collects, and takes a + minidump of a hung process so its thread stacks can be inspected offline. Diagnostic only: always exits 0 so it reports findings without failing the lane. Interpretation: - - Snapshot files produced -> nanvixd works; suspect MXC's invocation. - - Non-zero exit with stderr -> nanvixd fails; stderr names the reason. - - Timeout, no output -> nanvixd hangs on this host (e.g. WHP - partition creation never returns). + - Exit 0 with snapshot files -> that boot path works on this host. + - Non-zero exit -> the captured trace names the reason. + - Timeout -> nanvixd hangs; the trace's last line shows + how far the boot got, and the minidump + shows where the threads are parked. .PARAMETER BinDir Directory holding nanvixd.exe and its payload files. +.PARAMETER Mode + Which boot path(s) to exercise: cold, warm, or both (default). + .PARAMETER TimeoutSeconds How long to wait before declaring a hang. + +.PARAMETER OutputDirectory + Where to keep collected artifacts. Defaults to + `$env:RUNNER_TEMP\mxc-microvm-logs\nanvixd-probe`, which CI already uploads. #> [CmdletBinding()] param( [Parameter(Mandatory)] [string]$BinDir, - [int]$TimeoutSeconds = 120 + [ValidateSet('cold', 'warm', 'both')] + [string]$Mode = 'both', + + [int]$TimeoutSeconds = 120, + + [string]$OutputDirectory ) Set-StrictMode -Version Latest @@ -43,88 +72,224 @@ if (-not (Test-Path -LiteralPath $nanvixd -PathType Leaf)) { exit 0 } -# Generate into a scratch directory so the probe never disturbs the artifact's -# own snapshots, which the real tests rely on. -$probeHome = Join-Path ([System.IO.Path]::GetTempPath()) "nanvixd-probe-$([guid]::NewGuid().ToString('N'))" -New-Item -ItemType Directory -Force -Path $probeHome | Out-Null - -$stdoutFile = Join-Path $probeHome 'stdout.log' -$stderrFile = Join-Path $probeHome 'stderr.log' +if (-not $OutputDirectory) { + $OutputDirectory = if ($env:RUNNER_TEMP) { + Join-Path $env:RUNNER_TEMP 'mxc-microvm-logs\nanvixd-probe' + } else { + Join-Path ([System.IO.Path]::GetTempPath()) "nanvixd-probe-$([guid]::NewGuid().ToString('N'))" + } +} +New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null +Write-Host "Artifacts: $OutputDirectory" -# Mirrors nanvix_common::generate_snapshot: cold boot with -kernel-args snapshot. -$arguments = @( - '-bin-dir', (Join-Path $binPath 'bin'), - '-ramfs', (Join-Path $binPath 'nanvix_rootfs.img'), - '-kernel-args', 'snapshot', - '--', (Join-Path $binPath 'python3.initrd') -) +# MiniDumpWriteDump lets a hung run be diagnosed after the fact: the boot trace +# says how far nanvixd got, the dump says where its threads are parked. +$dumpSignature = @' +using System; +using System.Runtime.InteropServices; -Write-Host "Command: nanvixd.exe $($arguments -join ' ')" -Write-Host "Working directory: $probeHome" -Write-Host "Timeout: ${TimeoutSeconds}s" -Write-Host '' +public static class MiniDump +{ + [DllImport("dbghelp.dll", SetLastError = true)] + public static extern bool MiniDumpWriteDump( + IntPtr hProcess, + uint ProcessId, + IntPtr hFile, + uint DumpType, + IntPtr ExceptionParam, + IntPtr UserStreamParam, + IntPtr CallbackParam); +} +'@ -$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() -# RUST_LOG surfaces nanvixd's tracing, which the MXC runner suppresses by -# default - that suppression is why earlier hangs produced no diagnostics. -$previousRustLog = $env:RUST_LOG -$env:RUST_LOG = 'debug' +$dumpAvailable = $true try { + Add-Type -TypeDefinition $dumpSignature -ErrorAction Stop +} catch { + $dumpAvailable = $false + Write-Host "Minidump capture unavailable: $($_.Exception.Message)" +} + +function Save-HungProcessDump { + param( + [Parameter(Mandatory)][System.Diagnostics.Process]$Process, + [Parameter(Mandatory)][string]$Path + ) + + if (-not $dumpAvailable) { return } + + try { + # MiniDumpNormal (thread stacks) | MiniDumpWithThreadInfo. Deliberately + # not WithFullMemory: nanvixd maps a 256 MB guest and the stacks are + # what identify a hang. + $dumpType = 0x00000000 -bor 0x00001000 + $stream = [System.IO.File]::Create($Path) + try { + $ok = [MiniDump]::MiniDumpWriteDump( + $Process.Handle, + [uint32]$Process.Id, + $stream.SafeFileHandle.DangerousGetHandle(), + $dumpType, + [IntPtr]::Zero, [IntPtr]::Zero, [IntPtr]::Zero) + } finally { + $stream.Dispose() + } + if ($ok) { + $size = (Get-Item -LiteralPath $Path).Length + Write-Host " Minidump written: $Path ($size bytes)" + } else { + $code = [Runtime.InteropServices.Marshal]::GetLastWin32Error() + Write-Host " Minidump failed (win32 error $code)" + Remove-Item -LiteralPath $Path -Force -ErrorAction SilentlyContinue + } + } catch { + Write-Host " Minidump failed: $($_.Exception.Message)" + } +} + +function Show-CapturedFile { + param( + [Parameter(Mandatory)][string]$Label, + [Parameter(Mandatory)][string]$Path + ) + + Write-Host '' + Write-Host "--- $Label ---" + if (-not (Test-Path -LiteralPath $Path)) { + Write-Host '(not captured)' + return + } + $content = Get-Content -LiteralPath $Path -Raw -ErrorAction SilentlyContinue + if ([string]::IsNullOrWhiteSpace($content)) { + Write-Host '(empty)' + return + } + Write-Host $content.TrimEnd() +} + +function Invoke-NanvixdBoot { + param( + [Parameter(Mandatory)][ValidateSet('cold', 'warm')][string]$BootMode + ) + + Write-Host '' + Write-Host "=== $BootMode boot ===" + + $runDir = Join-Path $OutputDirectory $BootMode + New-Item -ItemType Directory -Force -Path $runDir | Out-Null + + $stdoutFile = Join-Path $runDir 'stdout.log' + $stderrFile = Join-Path $runDir 'stderr.log' + $consoleFile = Join-Path $runDir 'guest-console.log' + + # `-log-to-stdout` is essential: without it nanvixd logs to a file under + # `-log-dir` and both stdout and stderr stay empty even on a healthy run, + # which makes an empty capture indistinguishable from an early hang. + $arguments = @( + '-bin-dir', (Join-Path $binPath 'bin'), + '-ramfs', (Join-Path $binPath 'nanvix_rootfs.img'), + '-log-to-stdout', + '-console-file', $consoleFile + ) + + if ($BootMode -eq 'cold') { + # Mirrors nanvix_common::generate_snapshot. Runs in its own directory so + # it never disturbs the artifact's shipped snapshots. + $workingDirectory = Join-Path $runDir 'home' + New-Item -ItemType Directory -Force -Path $workingDirectory | Out-Null + $arguments += @('-kernel-args', 'snapshot') + } else { + # Mirrors nanvix_runner::spawn_nanvixd: cwd is the snapshot home and the + # snapshot path is relative to it. + $snapshot = Join-Path $binPath 'snapshots\kernel.whp.cbor' + if (-not (Test-Path -LiteralPath $snapshot)) { + Write-Host "No shipped snapshot at $snapshot - skipping warm boot." + return + } + $workingDirectory = $binPath + + # The guest runs the bootstrap script from the mounted staging dir, so + # stage one exactly as wxc_common::microvm_staging does. Without it + # CPython would wait on stdin and look like a hang. + $stagingDir = Join-Path $runDir 'staging' + New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null + @( + 'import sys' + "sys.argv = ['/mnt/bootstrap.py']" + "print('nanvixd-probe-ok')" + ) -join "`n" | Set-Content -LiteralPath (Join-Path $stagingDir 'bootstrap.py') -Encoding ascii + + $arguments += @('-snapshot', 'snapshots\kernel.whp.cbor', '-mount', $stagingDir) + } + + $arguments += @('--', (Join-Path $binPath 'python3.initrd')) + + Write-Host "Command: nanvixd.exe $($arguments -join ' ')" + Write-Host "Working directory: $workingDirectory" + Write-Host "Timeout: ${TimeoutSeconds}s" + + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $process = Start-Process -FilePath $nanvixd ` -ArgumentList $arguments ` - -WorkingDirectory $probeHome ` + -WorkingDirectory $workingDirectory ` -PassThru ` -RedirectStandardOutput $stdoutFile ` -RedirectStandardError $stderrFile -} finally { - $env:RUST_LOG = $previousRustLog -} -if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { - $stopwatch.Stop() - Write-Host "::warning::nanvixd did not exit within ${TimeoutSeconds}s - it HANGS on this host." - try { $process.Kill($true) } catch { } - $hung = $true -} else { + $hung = -not $process.WaitForExit($TimeoutSeconds * 1000) $stopwatch.Stop() - $hung = $false - Write-Host "nanvixd exited with code $($process.ExitCode) after $($stopwatch.Elapsed.TotalSeconds.ToString('0.0'))s" -} -foreach ($stream in @(@{ Name = 'stdout'; Path = $stdoutFile }, @{ Name = 'stderr'; Path = $stderrFile })) { - Write-Host '' - Write-Host "--- nanvixd $($stream.Name) ---" - if (Test-Path -LiteralPath $stream.Path) { - $content = Get-Content -LiteralPath $stream.Path -Raw -ErrorAction SilentlyContinue - if ([string]::IsNullOrWhiteSpace($content)) { - Write-Host '(empty)' - } else { - Write-Host $content.TrimEnd() + if ($hung) { + Write-Host "::warning::nanvixd did not exit within ${TimeoutSeconds}s - it HANGS on this host ($BootMode boot)." + Save-HungProcessDump -Process $process -Path (Join-Path $runDir 'nanvixd-hang.dmp') + try { $process.Kill($true) } catch { } + } else { + Write-Host "nanvixd exited with code $($process.ExitCode) after $($stopwatch.Elapsed.TotalSeconds.ToString('0.0'))s" + } + + Show-CapturedFile -Label 'nanvixd stdout' -Path $stdoutFile + Show-CapturedFile -Label 'nanvixd stderr (boot trace)' -Path $stderrFile + Show-CapturedFile -Label 'guest console' -Path $consoleFile + + # Fallback for a nanvixd build that ignores -log-to-stdout. + $logDir = Join-Path $workingDirectory 'logs' + if (Test-Path -LiteralPath $logDir) { + Write-Host '' + Write-Host '--- nanvixd log directory ---' + foreach ($file in Get-ChildItem -LiteralPath $logDir -File) { + Write-Host " $($file.Name) ($($file.Length) bytes)" + Copy-Item -LiteralPath $file.FullName -Destination $runDir -Force -ErrorAction SilentlyContinue } + } + + Write-Host '' + Write-Host '--- Generated snapshot files ---' + $snapshotDir = Join-Path $workingDirectory 'snapshots' + if (Test-Path -LiteralPath $snapshotDir) { + Get-ChildItem -LiteralPath $snapshotDir | Format-Table Name, Length | Out-String | Write-Host } else { - Write-Host '(not captured)' + Write-Host '(no snapshots directory created)' } -} -Write-Host '' -Write-Host '--- Generated snapshot files ---' -$snapshotDir = Join-Path $probeHome 'snapshots' -if (Test-Path -LiteralPath $snapshotDir) { - Get-ChildItem -LiteralPath $snapshotDir | Format-Table Name, Length | Out-String | Write-Host -} else { - Write-Host '(no snapshots directory created)' + Write-Host '' + if ($hung) { + Write-Host "RESULT ($BootMode): nanvixd HANGS on this host - the fault is below MXC." + Write-Host ' The last line of the boot trace above shows how far it got.' + } elseif ($process.ExitCode -eq 0) { + Write-Host "RESULT ($BootMode): nanvixd booted successfully." + } else { + Write-Host "RESULT ($BootMode): nanvixd failed - see the boot trace above for the reason." + } } -Write-Host '' -if ($hung) { - Write-Host 'RESULT: nanvixd hangs on this host - the fault is below MXC.' -} elseif ($process.ExitCode -eq 0) { - Write-Host 'RESULT: nanvixd cold-booted successfully - suspect MXC''s invocation instead.' -} else { - Write-Host 'RESULT: nanvixd failed - see its stderr above for the reason.' +$modes = if ($Mode -eq 'both') { @('cold', 'warm') } else { @($Mode) } +foreach ($bootMode in $modes) { + try { + Invoke-NanvixdBoot -BootMode $bootMode + } catch { + Write-Host "::warning::$bootMode probe raised: $($_.Exception.Message)" + } } -Remove-Item -LiteralPath $probeHome -Recurse -Force -ErrorAction SilentlyContinue - # Diagnostic only: never fail the job on the probe's outcome. exit 0 diff --git a/scripts/ci/probe-whp-capabilities.ps1 b/scripts/ci/probe-whp-capabilities.ps1 index b83dd548d..0dec08db9 100644 --- a/scripts/ci/probe-whp-capabilities.ps1 +++ b/scripts/ci/probe-whp-capabilities.ps1 @@ -11,11 +11,16 @@ This probe P/Invokes WinHvPlatform.dll directly to: 1. Query capabilities through WHvGetCapability. - 2. Attempt a real WHvCreatePartition / WHvSetupPartition and delete it. + 2. Build a complete partition (WHvCreatePartition / WHvSetupPartition / + WHvCreateVirtualProcessor), map a guest page, and actually execute + guest code on it. - Step 2 is the important one: it is the first call that genuinely exercises - the hypervisor, and the most likely place for a host that reports WHP as - "enabled" to fail or block. + Step 2 is the decisive one. Capability flags can look correct on a host + where a partition cannot be set up, and a partition can be set up on a host + where the hypervisor never actually executes guest instructions -- which is + what a VM monitor such as nanvixd needs. The guest here is a single `hlt` + placed at the x86 reset vector, so a healthy host reports exit reason + `WHvRunVpExitReasonX64Halt` and no register or paging setup is required. Run this on both a working host and a failing one and compare the output. @@ -78,6 +83,42 @@ public static class Whp uint PropertyCode, IntPtr PropertyBuffer, uint PropertyBufferSizeInBytes); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvCreateVirtualProcessor( + IntPtr Partition, + uint VpIndex, + uint Flags); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvDeleteVirtualProcessor( + IntPtr Partition, + uint VpIndex); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvMapGpaRange( + IntPtr Partition, + IntPtr SourceAddress, + ulong GuestAddress, + ulong SizeInBytes, + uint Flags); + + [DllImport("WinHvPlatform.dll")] + public static extern int WHvRunVirtualProcessor( + IntPtr Partition, + uint VpIndex, + IntPtr ExitContext, + uint ExitContextSizeInBytes); + + [DllImport("kernel32.dll")] + public static extern IntPtr VirtualAlloc( + IntPtr lpAddress, + UIntPtr dwSize, + uint flAllocationType, + uint flProtect); + + [DllImport("kernel32.dll")] + public static extern bool VirtualFree(IntPtr lpAddress, UIntPtr dwSize, uint dwFreeType); } '@ @@ -158,39 +199,60 @@ if ($hypervisorPresent -eq 0) { } # The decisive test. Feature flags can look correct on a host where partition -# creation still fails or blocks; this is the call a VM monitor makes first. +# setup fails, and setup can succeed on a host that never executes guest code. Write-Host '' -Write-Host '--- WHvCreatePartition (the call that actually exercises the hypervisor) ---' +Write-Host '--- Partition setup and guest execution (what a VM monitor actually needs) ---' + +# WHV_PARTITION_PROPERTY_CODE values from WinHvPlatformDefs.h. These are easy to +# get wrong: ProcessorCount is 0x00001fff, NOT 0x00001002 (ProcessorClFlushSize), +# and ExtendedVmExits is 0x00000001, NOT 0x00000002 (ExceptionExitBitmap). +# Setting the wrong code makes WHvSetupPartition fail with +# WHV_E_INVALID_PARTITION_CONFIG (0x80370304) on a perfectly healthy host. +$PropertyCodeExtendedVmExits = 0x00000001 +$PropertyCodeProcessorCount = 0x00001fff + +# WHV_RUN_VP_EXIT_REASON values worth naming in the output. +$exitReasons = @{ + 0x00000000 = 'None' + 0x00000001 = 'MemoryAccess' + 0x00000002 = 'X64IoPortAccess' + 0x00000004 = 'UnrecoverableException' + 0x00000005 = 'InvalidVpRegisterValue' + 0x00000006 = 'UnsupportedFeature' + 0x00000007 = 'X64InterruptWindow' + 0x00000008 = 'X64Halt' + 0x00001000 = 'X64MsrAccess' + 0x00001001 = 'X64Cpuid' + 0x00001002 = 'Exception' + 0x00002001 = 'Canceled' +} $partition = [IntPtr]::Zero $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() $hr = [Whp]::WHvCreatePartition([ref]$partition) $stopwatch.Stop() -Write-Host (" WHvCreatePartition hr=0x{0:X8} ({1} ms)" -f $hr, $stopwatch.ElapsedMilliseconds) +Write-Host (" WHvCreatePartition hr=0x{0:X8} ({1} ms)" -f $hr, $stopwatch.ElapsedMilliseconds) if ($hr -ne 0) { Write-Host '::warning::WHvCreatePartition failed - WHP cannot host a VM here even though the feature is enabled.' Write-Host '' - Write-Host 'RESULT: partition creation FAILED.' + Write-Host 'RESULT: partition creation FAILED - the hypervisor is unusable on this host.' exit 0 } +$guestMemory = @() +$vpCreated = $false try { - # A partition needs both a processor count and an extended-VM-exit - # configuration before setup will accept it; omitting the latter yields - # WHV_E_INVALID_PARTITION_CONFIG (0x80370304) on an otherwise healthy host. $propertyBuffer = [Runtime.InteropServices.Marshal]::AllocHGlobal(8) try { [Runtime.InteropServices.Marshal]::WriteInt64($propertyBuffer, 0) [Runtime.InteropServices.Marshal]::WriteInt32($propertyBuffer, 1) - # WHvPartitionPropertyCodeProcessorCount = 0x00001002 - $hrProp = [Whp]::WHvSetPartitionProperty($partition, 0x00001002, $propertyBuffer, 4) - Write-Host (" SetProcessorCount hr=0x{0:X8}" -f $hrProp) + $hrProp = [Whp]::WHvSetPartitionProperty($partition, $PropertyCodeProcessorCount, $propertyBuffer, 4) + Write-Host (" SetProcessorCount(1) hr=0x{0:X8}" -f $hrProp) - # WHvPartitionPropertyCodeExtendedVmExits = 0x00000002 [Runtime.InteropServices.Marshal]::WriteInt64($propertyBuffer, 0) - $hrExits = [Whp]::WHvSetPartitionProperty($partition, 0x00000002, $propertyBuffer, 8) - Write-Host (" SetExtendedVmExits hr=0x{0:X8}" -f $hrExits) + $hrExits = [Whp]::WHvSetPartitionProperty($partition, $PropertyCodeExtendedVmExits, $propertyBuffer, 8) + Write-Host (" SetExtendedVmExits(0) hr=0x{0:X8}" -f $hrExits) } finally { [Runtime.InteropServices.Marshal]::FreeHGlobal($propertyBuffer) } @@ -198,21 +260,113 @@ try { $stopwatch.Restart() $hrSetup = [Whp]::WHvSetupPartition($partition) $stopwatch.Stop() - Write-Host (" WHvSetupPartition hr=0x{0:X8} ({1} ms)" -f $hrSetup, $stopwatch.ElapsedMilliseconds) + Write-Host (" WHvSetupPartition hr=0x{0:X8} ({1} ms)" -f $hrSetup, $stopwatch.ElapsedMilliseconds) + if ($hrSetup -ne 0) { + Write-Host '' + Write-Host ('::warning::WHvSetupPartition failed with 0x{0:X8}.' -f $hrSetup) + Write-Host 'RESULT: partition setup FAILED - a VM monitor cannot start here.' + exit 0 + } - Write-Host '' - if ($hrSetup -eq 0) { - Write-Host 'RESULT: WHP can create and set up a partition - the hypervisor is usable.' - Write-Host ' A VM monitor hanging here is failing for some other reason.' - } else { - # 0x80370304 here means this probe built an incomplete partition, not - # necessarily that the host is broken; compare against a known-good host. - Write-Host ("RESULT: partition setup returned 0x{0:X8}." -f $hrSetup) - Write-Host ' Compare this value against a host where the VM monitor works.' + $stopwatch.Restart() + $hrVp = [Whp]::WHvCreateVirtualProcessor($partition, 0, 0) + $stopwatch.Stop() + Write-Host (" WHvCreateVirtualProcessor hr=0x{0:X8} ({1} ms)" -f $hrVp, $stopwatch.ElapsedMilliseconds) + if ($hrVp -ne 0) { + Write-Host '' + Write-Host '::warning::WHvCreateVirtualProcessor failed - no vCPU can be created on this host.' + Write-Host 'RESULT: vCPU creation FAILED.' + exit 0 + } + $vpCreated = $true + + # Map HLT-filled pages at the addresses a fresh vCPU can start from and let + # the guest run. WHP resets a vCPU to real mode at CS.Base 0xF0000 / + # RIP 0xFFF0 (linear 0xFFFF0, page 0xFF000); the zero page is mapped too so + # the probe stays valid if that reset state ever changes. Either way the + # first instruction fetched is HLT (0xF4). + $pageSize = 4096 + $MEM_COMMIT_RESERVE = 0x3000 + $PAGE_EXECUTE_READWRITE = 0x40 + # WHvMapGpaRangeFlagRead | Write | Execute + $mapFlags = 0x00000007 + $guestPages = @( + @{ Name = 'reset vector'; Gpa = [uint64]0xFF000 }, + @{ Name = 'zero page'; Gpa = [uint64]0 } + ) + + foreach ($page in $guestPages) { + $host_address = [Whp]::VirtualAlloc([IntPtr]::Zero, [UIntPtr]::new($pageSize), $MEM_COMMIT_RESERVE, $PAGE_EXECUTE_READWRITE) + if ($host_address -eq [IntPtr]::Zero) { + Write-Host '::warning::VirtualAlloc for guest memory failed - cannot complete the execution test.' + exit 0 + } + $guestMemory += $host_address + for ($offset = 0; $offset -lt $pageSize; $offset++) { + [Runtime.InteropServices.Marshal]::WriteByte($host_address, $offset, 0xF4) + } + + $hrMap = [Whp]::WHvMapGpaRange($partition, $host_address, $page.Gpa, $pageSize, $mapFlags) + Write-Host (" WHvMapGpaRange {0,-11} hr=0x{1:X8}" -f $page.Name, $hrMap) + if ($hrMap -ne 0) { + Write-Host '' + Write-Host '::warning::WHvMapGpaRange failed - guest memory cannot be mapped on this host.' + Write-Host 'RESULT: guest memory mapping FAILED.' + exit 0 + } + } + + # WHV_RUN_VP_EXIT_CONTEXT is 224 bytes (x64, SDK 10.0.26100). + $exitContextSize = 224 + $exitContext = [Runtime.InteropServices.Marshal]::AllocHGlobal($exitContextSize) + try { + # If the hypervisor never actually dispatches guest code this call is + # where a host blocks; the CI step timeout bounds it. + $stopwatch.Restart() + $hrRun = [Whp]::WHvRunVirtualProcessor($partition, 0, $exitContext, $exitContextSize) + $stopwatch.Stop() + $exitReason = [Runtime.InteropServices.Marshal]::ReadInt32($exitContext) + $reasonName = if ($exitReasons.ContainsKey($exitReason)) { $exitReasons[$exitReason] } else { 'Unknown' } + # WHV_RUN_VP_EXIT_CONTEXT layout: ExitReason(0), Reserved(4), + # VpContext(8) { ExecutionState, InstructionLength/Cr8, Reserved, + # Reserved2, Cs @16, Rip @32, Rflags @40 }, union @48. For a + # MemoryAccess exit the union holds WHV_MEMORY_ACCESS_CONTEXT with + # Gpa at offset 72. + $csBase = [Runtime.InteropServices.Marshal]::ReadInt64($exitContext, 16) + $rip = [Runtime.InteropServices.Marshal]::ReadInt64($exitContext, 32) + Write-Host (" WHvRunVirtualProcessor hr=0x{0:X8} ({1} ms)" -f $hrRun, $stopwatch.ElapsedMilliseconds) + Write-Host (" Guest exit reason 0x{0:X8} ({1})" -f $exitReason, $reasonName) + Write-Host (" Guest CS.Base/RIP 0x{0:X16} / 0x{1:X16}" -f $csBase, $rip) + if ($exitReason -eq 0x00000001) { + $gpa = [Runtime.InteropServices.Marshal]::ReadInt64($exitContext, 72) + Write-Host (" Faulting GPA 0x{0:X16}" -f $gpa) + } + + Write-Host '' + if ($hrRun -eq 0 -and $exitReason -eq 0x00000008) { + Write-Host 'RESULT: WHP executed guest code and halted as expected.' + Write-Host ' The hypervisor is fully usable here - a VM monitor that hangs' + Write-Host ' or fails is doing so for a reason above WHP.' + } elseif ($hrRun -ne 0) { + Write-Host ('::warning::WHvRunVirtualProcessor failed with 0x{0:X8}.' -f $hrRun) + Write-Host 'RESULT: the hypervisor refused to run guest code on this host.' + } else { + Write-Host ("::warning::Guest exited for {0} instead of X64Halt." -f $reasonName) + Write-Host 'RESULT: guest code ran but did not reach HLT - compare against a working host.' + } + } finally { + [Runtime.InteropServices.Marshal]::FreeHGlobal($exitContext) } } finally { + if ($vpCreated) { + $hrDeleteVp = [Whp]::WHvDeleteVirtualProcessor($partition, 0) + Write-Host (" WHvDeleteVirtualProcessor hr=0x{0:X8}" -f $hrDeleteVp) + } $hrDelete = [Whp]::WHvDeletePartition($partition) - Write-Host (" WHvDeletePartition hr=0x{0:X8}" -f $hrDelete) + Write-Host (" WHvDeletePartition hr=0x{0:X8}" -f $hrDelete) + foreach ($address in $guestMemory) { + [void][Whp]::VirtualFree($address, [UIntPtr]::Zero, 0x8000) + } } exit 0 diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 37b18f77a..2c71420e5 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -492,7 +492,6 @@ { "os": "windows-25h2", "backends": [ - "process-t3", "microvm" ] }, diff --git a/src/backends/nanvix/runner/src/lib.rs b/src/backends/nanvix/runner/src/lib.rs index fc0c2070b..aa2762664 100644 --- a/src/backends/nanvix/runner/src/lib.rs +++ b/src/backends/nanvix/runner/src/lib.rs @@ -23,11 +23,15 @@ //! ## Diagnostics //! //! By default the runner sets `RUST_LOG=off` in nanvixd's environment, which -//! suppresses the per-run `%LOCALAPPDATA%\nanvix\logs\nanvixd_*.log` trace -//! file and noticeably reduces warm-start latency. Set `MXC_NANVIX_TRACE=1` -//! (or `true`/`yes`, case-insensitive) before invoking wxc-exec to let -//! nanvixd use its own `RUST_LOG` default and to capture nanvixd's stderr -//! for inclusion in the wxc-exec log. +//! suppresses its trace output and noticeably reduces warm-start latency. Set +//! `MXC_NANVIX_TRACE=1` (or `true`/`yes`, case-insensitive) before invoking +//! wxc-exec to let nanvixd use its own `RUST_LOG` default and to capture +//! nanvixd's stderr for inclusion in the wxc-exec log. +//! +//! Trace mode also passes `-log-to-stdout` to nanvixd. Without it nanvixd +//! writes its trace to a file under `-log-dir` (default `/logs`, i.e. the +//! snapshot home) and the captured stderr is empty — which is exactly the +//! wrong behavior when diagnosing a VM that never finished booting. //! //! ## Exit codes //! @@ -676,6 +680,15 @@ impl NanVixScriptRunner { let mut cmd = Command::new(&paths.nanvixd); + if trace { + // Route nanvixd's trace to its standard streams. Its default is a + // log file under `-log-dir` (i.e. `/logs`), which + // would leave the stderr we just piped empty and give a stalled + // boot no visible diagnostics at all. Must precede the `--` + // separator, so it is added before the per-platform arguments. + cmd.arg("-log-to-stdout"); + } + // Host networking is opt-in. When enabled, attach the host network // backend; nanvixd parses this flag regardless of argument order, so // it is added up front for both the Windows (snapshot) and Linux @@ -735,7 +748,7 @@ impl NanVixScriptRunner { .stdout(Stdio::inherit()) .stderr(stderr); if !trace { - // Suppress nanvixd's env_logger output and per-run log file. + // Suppress nanvixd's trace output and per-run log file. cmd.env("RUST_LOG", "off"); } cmd.spawn().map_err(|e| { @@ -893,6 +906,13 @@ impl NanVixScriptRunner { if timed_out.load(Ordering::SeqCst) { let _ = child.kill(); + // Emit the captured trace before returning. A timeout is the one + // failure mode where nanvixd never reports a reason itself, so + // dropping its stderr here would leave the log with nothing but + // "timed out" — the exact case these diagnostics exist for. + if !stderr_output.is_empty() { + let _ = writeln!(logger, "NanVix stderr:\n{}", stderr_output); + } let err = NanVixError::Timeout { script_timeout_ms: script_timeout, total_ms: timeout_ms, From 223fd43e332242ef547d2ed5b0ded1499695045b Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:36:54 -0700 Subject: [PATCH 58/93] adding limited WSLC support, changing how windows hosts are setup --- .github/workflows/1es-smoke.yml | 61 --- .../workflows/Validation.Tests.Matrix.Job.yml | 74 +--- ...wslc-e2e.yml => Validation.Tests.WSLC.yml} | 111 +++--- ...ckend-prereqs.sh => prepare-linux-host.sh} | 6 +- scripts/ci/prepare-windows-host.ps1 | 179 +++++++++ scripts/ci/probe-nanvixd.ps1 | 295 -------------- scripts/ci/probe-whp-capabilities.ps1 | 372 ------------------ src/backends/nanvix/runner/src/lib.rs | 32 +- tests/scripts/run_microvm_tests.ps1 | 21 +- 9 files changed, 252 insertions(+), 899 deletions(-) delete mode 100644 .github/workflows/1es-smoke.yml rename .github/workflows/{wslc-e2e.yml => Validation.Tests.WSLC.yml} (72%) rename scripts/ci/{install-linux-backend-prereqs.sh => prepare-linux-host.sh} (97%) create mode 100644 scripts/ci/prepare-windows-host.ps1 delete mode 100644 scripts/ci/probe-nanvixd.ps1 delete mode 100644 scripts/ci/probe-whp-capabilities.ps1 diff --git a/.github/workflows/1es-smoke.yml b/.github/workflows/1es-smoke.yml deleted file mode 100644 index 55f5ef668..000000000 --- a/.github/workflows/1es-smoke.yml +++ /dev/null @@ -1,61 +0,0 @@ -# ============================================================================ -# 1ES connectivity smoke test — proves microsoft/mxc can dispatch a job to the -# 1ES Hosted GitHub Runner pool. -# -# THROWAWAY / PROOF workflow. Not part of the real CI matrix (Build.yml). -# -# Pool: 1es-mxc-test-2 — REPOSITORY-level pool bound directly to microsoft/mxc -# (runner group 1 / Default). Confirmed working: run 30931840357 registered -# runner 21 (bf347e45c000000) after a ~10.5-min COLD start on the idle pool. -# -# NOTE: uses `shell: powershell` (Windows PowerShell 5.1) because the raw -# windows-25h2-pro-x64 CLIENT image ships powershell.exe, NOT pwsh.exe (PS7). -# Cold-start pickup on an idle Stateless pool is ~10-11 min — allow a full 15. -# ============================================================================ -name: 1ES Connectivity Smoke - -on: - push: - branches: - - user/modanish/1es-github-runner-e2e - -permissions: - contents: read - -jobs: - smoke: - name: 1ES pickup - runs-on: - - self-hosted - - "1ES.Pool=1es-mxc-test-2" - - "JobId=smoke-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" - timeout-minutes: 15 - - steps: - - name: Hello from the 1ES runner - shell: powershell - run: | - Write-Host "=== Job picked up on the 1ES pool ===" - Write-Host "Hostname : $(hostname)" - Write-Host "User : $(whoami)" - Write-Host "OS : $([System.Environment]::OSVersion.VersionString)" - Write-Host "PSVersion : $($PSVersionTable.PSVersion)" - Write-Host "RUNNER_NAME: $env:RUNNER_NAME" - Write-Host "Cores : $env:NUMBER_OF_PROCESSORS" - - - name: Confirm this is the expected Azure pool VM (IMDS) - shell: powershell - run: | - try { - $c = (Invoke-RestMethod -Headers @{Metadata="true"} ` - -Uri "http://169.254.169.254/metadata/instance?api-version=2021-02-01").compute - Write-Host "vmSize : $($c.vmSize)" - Write-Host "location : $($c.location)" - Write-Host "vmId : $($c.vmId)" - if ($env:GITHUB_STEP_SUMMARY) { - "## 1ES runner picked up`n- **vmSize**: $($c.vmSize)`n- **location**: $($c.location)`n- **host**: $(hostname)" | - Add-Content -Path $env:GITHUB_STEP_SUMMARY - } - } catch { - Write-Host "::warning::IMDS unreachable: $($_.Exception.Message)" - } diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index d95c97a32..f9bb7a448 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -49,73 +49,12 @@ jobs: path: artifacts/bin - name: Prepare backend prerequisites - if: matrix.command == 'microvm' shell: pwsh run: | - $binDir = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin' - - # NanVix boots a VM from these images on every invocation; Defender - # scanning them can push boot past its timeout. - Add-MpPreference -ExclusionPath $binDir - Write-Host "Added Defender exclusion for $binDir" - - # Mirrors the verification in microvm-e2e.yml. - $required = @( - "wxc-exec.exe", - "nanvixd.exe", - "nanvix_rootfs.img", - "python3.initrd", - "bin\kernel.elf", - "snapshots\kernel.vmem", - "snapshots\kernel.whp.cbor" - ) - $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } - if ($missing) { - Write-Host "::error::Missing binaries: $($missing -join ', ')" - exit 1 - } - $leaves = $required | ForEach-Object { Split-Path $_ -Leaf } - Get-ChildItem $binDir -Include $leaves -Recurse | Format-Table FullName, Length - - # Capture nanvixd's own stderr/tracing; without this the runner - # silences it, which is why a hung boot produced no diagnostics. - "MXC_NANVIX_TRACE=1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - Write-Host "Set MXC_NANVIX_TRACE=1 (captures nanvixd tracing)" - - - name: Diagnose hypervisor environment - if: matrix.command == 'microvm' - shell: pwsh - run: ./scripts/ci/diagnose-whp.ps1 - - - name: Check Windows Hypervisor Platform - if: matrix.command == 'microvm' - shell: pwsh - run: ./scripts/ci/check-whp.ps1 - - - name: Probe WHP capabilities - if: matrix.command == 'microvm' - timeout-minutes: 5 - shell: pwsh - run: ./scripts/ci/probe-whp-capabilities.ps1 - - - name: Probe nanvixd directly - if: matrix.command == 'microvm' - timeout-minutes: 10 - shell: pwsh - run: | - ./scripts/ci/probe-nanvixd.ps1 ` - -BinDir (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') ` - -Mode both - - - name: Prepare Process Container host - if: matrix.backend == 'process-t3' - shell: pwsh - run: | - $hostPrep = Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin\wxc-host-prep.exe' - & $hostPrep prepare-system-drive - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - & $hostPrep prepare-null-device --no-sacl - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & ./scripts/ci/prepare-windows-host.ps1 ` + -Backend '${{ matrix.backend }}' ` + -BinaryDirectory (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') *>&1 | + Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append - name: Run backend tests # A backend that hangs (rather than failing) would otherwise burn the @@ -134,7 +73,7 @@ jobs: $arguments.ExpectedTier = 'T3' } & ./tests/scripts/run_ci_backend_tests.ps1 @arguments *>&1 | - Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') + Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append - name: Upload failure logs if: failure() || cancelled() @@ -143,7 +82,6 @@ jobs: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: | ${{ runner.temp }}/mxc-ci.log - ${{ runner.temp }}/mxc-microvm-logs ${{ runner.temp }}/mxc-wpc-tests/logs ${{ runner.temp }}/WinProcessContainer-Tests.results.* ${{ runner.temp }}/mxc_concurrent_oneshot @@ -172,7 +110,7 @@ jobs: shell: bash run: | set -euo pipefail - bash scripts/ci/install-linux-backend-prereqs.sh \ + bash scripts/ci/prepare-linux-host.sh \ '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee -a "$RUNNER_TEMP/mxc-ci.log" diff --git a/.github/workflows/wslc-e2e.yml b/.github/workflows/Validation.Tests.WSLC.yml similarity index 72% rename from .github/workflows/wslc-e2e.yml rename to .github/workflows/Validation.Tests.WSLC.yml index 73899ea33..e9494fb21 100644 --- a/.github/workflows/wslc-e2e.yml +++ b/.github/workflows/Validation.Tests.WSLC.yml @@ -28,6 +28,7 @@ on: push: branches: - user/modanish/1es-github-runner-e2e + - user/emichlin/validation-infra-lite workflow_dispatch: permissions: @@ -122,61 +123,61 @@ jobs: shutdown /r /t 15 /c "WSL2 feature finalize reboot" Start-Sleep -Seconds 180 - # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root - # only. Surface src/rust-toolchain.toml there so the pinned toolchain is - # honored. Use PowerShell (bare client image may lack bash). - - name: Surface toolchain file at repo root - if: steps.prep.outputs.ready == 'true' - shell: powershell - run: Copy-Item src\rust-toolchain.toml rust-toolchain.toml -Force - - # The bare 1ES client image has no rustup/cargo and no bash, so we can't - # use actions-rust-lang/setup-rust-toolchain (it runs internal bash steps - # and fails with "bash: command not found"). Install rustup directly in - # PowerShell; rust-toolchain.toml (surfaced at repo root above) pins - # channel 1.93, which rustup installs and selects automatically. - - name: Install Rust toolchain - if: steps.prep.outputs.ready == 'true' - shell: powershell - run: | - $ErrorActionPreference = 'Stop' - if (-not (Get-Command rustup -ErrorAction SilentlyContinue)) { - Write-Host "Installing rustup..." - $init = Join-Path $env:RUNNER_TEMP 'rustup-init.exe' - Invoke-WebRequest -Uri 'https://win.rustup.rs/x86_64' -OutFile $init -UseBasicParsing - & $init -y --default-toolchain none --profile minimal - if ($LASTEXITCODE -ne 0) { throw "rustup-init failed ($LASTEXITCODE)" } - $cargoBin = Join-Path $env:USERPROFILE '.cargo\bin' - $cargoBin | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 - $env:Path = "$cargoBin;$env:Path" - } - # Install the exact channel pinned by rust-toolchain.toml (repo root), - # reading it from the file so there is no drift with the pin. - $chan = ([regex]::Match((Get-Content rust-toolchain.toml -Raw), 'channel\s*=\s*"([^"]+)"')).Groups[1].Value - if (-not $chan) { throw "could not parse channel from rust-toolchain.toml" } - Write-Host "Pinned Rust channel: $chan" - rustup toolchain install $chan --profile minimal - if ($LASTEXITCODE -ne 0) { throw "rustup toolchain install $chan failed ($LASTEXITCODE)" } - rustup target add --toolchain $chan x86_64-pc-windows-msvc - if ($LASTEXITCODE -ne 0) { throw "rustup target add failed ($LASTEXITCODE)" } - rustc "+$chan" --version - cargo "+$chan" --version - - - name: Point cargo at the MxcDependencies feed - if: steps.prep.outputs.ready == 'true' - uses: ./.github/actions/setup-cargo-feed - - - uses: Swatinem/rust-cache@v2 - if: steps.prep.outputs.ready == 'true' - with: - workspaces: src -> target - key: wslc-e2e-v1 - - - name: Build with WSLC support - if: steps.prep.outputs.ready == 'true' - working-directory: src - run: cargo build --release --target x86_64-pc-windows-msvc - --no-default-features --features wslc + # # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root + # # only. Surface src/rust-toolchain.toml there so the pinned toolchain is + # # honored. Use PowerShell (bare client image may lack bash). + # - name: Surface toolchain file at repo root + # if: steps.prep.outputs.ready == 'true' + # shell: powershell + # run: Copy-Item src\rust-toolchain.toml rust-toolchain.toml -Force + + # # The bare 1ES client image has no rustup/cargo and no bash, so we can't + # # use actions-rust-lang/setup-rust-toolchain (it runs internal bash steps + # # and fails with "bash: command not found"). Install rustup directly in + # # PowerShell; rust-toolchain.toml (surfaced at repo root above) pins + # # channel 1.93, which rustup installs and selects automatically. + # - name: Install Rust toolchain + # if: steps.prep.outputs.ready == 'true' + # shell: powershell + # run: | + # $ErrorActionPreference = 'Stop' + # if (-not (Get-Command rustup -ErrorAction SilentlyContinue)) { + # Write-Host "Installing rustup..." + # $init = Join-Path $env:RUNNER_TEMP 'rustup-init.exe' + # Invoke-WebRequest -Uri 'https://win.rustup.rs/x86_64' -OutFile $init -UseBasicParsing + # & $init -y --default-toolchain none --profile minimal + # if ($LASTEXITCODE -ne 0) { throw "rustup-init failed ($LASTEXITCODE)" } + # $cargoBin = Join-Path $env:USERPROFILE '.cargo\bin' + # $cargoBin | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 + # $env:Path = "$cargoBin;$env:Path" + # } + # # Install the exact channel pinned by rust-toolchain.toml (repo root), + # # reading it from the file so there is no drift with the pin. + # $chan = ([regex]::Match((Get-Content rust-toolchain.toml -Raw), 'channel\s*=\s*"([^"]+)"')).Groups[1].Value + # if (-not $chan) { throw "could not parse channel from rust-toolchain.toml" } + # Write-Host "Pinned Rust channel: $chan" + # rustup toolchain install $chan --profile minimal + # if ($LASTEXITCODE -ne 0) { throw "rustup toolchain install $chan failed ($LASTEXITCODE)" } + # rustup target add --toolchain $chan x86_64-pc-windows-msvc + # if ($LASTEXITCODE -ne 0) { throw "rustup target add failed ($LASTEXITCODE)" } + # rustc "+$chan" --version + # cargo "+$chan" --version + + # - name: Point cargo at the MxcDependencies feed + # if: steps.prep.outputs.ready == 'true' + # uses: ./.github/actions/setup-cargo-feed + + # - uses: Swatinem/rust-cache@v2 + # if: steps.prep.outputs.ready == 'true' + # with: + # workspaces: src -> target + # key: wslc-e2e-v1 + + # - name: Build with WSLC support + # if: steps.prep.outputs.ready == 'true' + # working-directory: src + # run: cargo build --release --target x86_64-pc-windows-msvc + # --no-default-features --features wslc - name: Ensure WSL2 runtime id: wsl diff --git a/scripts/ci/install-linux-backend-prereqs.sh b/scripts/ci/prepare-linux-host.sh similarity index 97% rename from scripts/ci/install-linux-backend-prereqs.sh rename to scripts/ci/prepare-linux-host.sh index c16fded05..7cf28b946 100644 --- a/scripts/ci/install-linux-backend-prereqs.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -# Installs the host packages a Linux backend needs before its artifact-only -# test suite runs. Distro-aware so the same matrix entry works on Ubuntu, -# Debian, and RHEL images. +# Prepares a Linux host for a backend's artifact-only test suite by installing +# the packages and starting the services it needs. Distro-aware so the same +# matrix entry works on Ubuntu, Debian, and RHEL images. usage() { echo "Usage: $0 " >&2 diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 new file mode 100644 index 000000000..a932f5d7c --- /dev/null +++ b/scripts/ci/prepare-windows-host.ps1 @@ -0,0 +1,179 @@ +#Requires -Version 7.0 + +<# +.SYNOPSIS + Prepares a Windows host for a backend's artifact-only test suite. + +.PARAMETER Backend + Matrix backend id (not the handler command), because process-t1 and + process-t3 share a handler but differ in host preparation. + +.PARAMETER BinaryDirectory + Directory holding the downloaded build artifact. + +.EXAMPLE + ./scripts/ci/prepare-windows-host.ps1 -Backend process-t3 -BinaryDirectory artifacts/bin +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateSet( + 'process-t1', + 'process-t3', + 'isolation-session', + 'wslc', + 'windows-sandbox', + 'microvm', + 'hyperlight' + )] + [string]$Backend, + + [Parameter(Mandatory)] + [string]$BinaryDirectory +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Exit-WithError { + param([Parameter(Mandatory)][string]$Message) + + Write-Host "::error::$Message" + exit 1 +} + +function Assert-RequiredFile { + param( + [Parameter(Mandatory)][string[]]$RelativePath + ) + + $missing = $RelativePath | Where-Object { -not (Test-Path (Join-Path $BinaryDirectory $_)) } + if ($missing) { + Exit-WithError "Missing binaries: $($missing -join ', ')" + } + + $leaves = $RelativePath | ForEach-Object { Split-Path $_ -Leaf } + Get-ChildItem $BinaryDirectory -Include $leaves -Recurse | Format-Table FullName, Length +} + +# Report whether the artifact carries a complete WHP warm-start snapshot set. +# A partial set is worth calling out: the runner treats a complete set as the +# signal to warm-boot, so a missing half silently costs a cold boot per run. +function Write-SnapshotAvailability { + $snapshots = @('snapshots\kernel.vmem', 'snapshots\kernel.whp.cbor') + $present = @($snapshots | Where-Object { Test-Path (Join-Path $BinaryDirectory $_) }) + + if ($present.Count -eq $snapshots.Count) { + Write-Host 'WHP warm-start snapshots are present; the runner will warm-boot.' + } elseif ($present.Count -eq 0) { + Write-Host 'WHP warm-start snapshots are absent; the runner will cold-boot (slower, expected on cross-arch or offline builds).' + } else { + Write-Host "WARNING: incomplete WHP snapshot set (found: $($present -join ', ')); the runner will cold-boot." + } +} + +# Report the hypervisor state a VM-backed backend depends on. Purely +# diagnostic: never fails, so a hypervisor problem surfaces as the explicit +# check below rather than as an unexplained collection error. +function Write-HypervisorDiagnostic { + Write-Host '=== Hypervisor Diagnostics ===' + try { + Write-Host "OS: $([System.Environment]::OSVersion)" + + $computerSystem = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue + $hypervisorPresent = if ($null -eq $computerSystem) { 'unknown' } else { $computerSystem.HypervisorPresent } + Write-Host "HypervisorPresent: $hypervisorPresent" + Write-Host "WinHvPlatform.dll exists: $(Test-Path "$env:SystemRoot\System32\WinHvPlatform.dll")" + + $feature = Get-WindowsOptionalFeature -Online -FeatureName HypervisorPlatform -ErrorAction SilentlyContinue + $featureState = if ($null -eq $feature) { 'unknown' } else { $feature.State } + Write-Host "HypervisorPlatform feature state: $featureState" + } catch { + Write-Host "WARNING: hypervisor diagnostics could not be collected: $($_.Exception.Message)" + } + Write-Host '=== end diagnostics ===' +} + +# The feature can be enabled while the hypervisor is not actually running (for +# example when a host reboot is still pending), so both are required. +function Assert-HypervisorPlatform { + try { + $feature = Get-WindowsOptionalFeature -Online -FeatureName HypervisorPlatform -ErrorAction SilentlyContinue + } catch { + # Querying optional features needs elevation; a host that cannot answer + # cannot be certified as WHP-capable. + Exit-WithError "Unable to query the HypervisorPlatform feature: $($_.Exception.Message)" + } + + if ($null -eq $feature -or $feature.State -ne 'Enabled') { + Exit-WithError 'Windows Hypervisor Platform is not enabled. This backend requires WHP.' + } + + $computerSystem = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue + if ($null -eq $computerSystem -or -not $computerSystem.HypervisorPresent) { + Exit-WithError 'HypervisorPresent is false - WHP feature is enabled but hypervisor is not running.' + } + + Write-Host 'WHP is enabled and hypervisor is present.' +} + +function Initialize-ProcessContainerHost { + $hostPrep = Join-Path $BinaryDirectory 'wxc-host-prep.exe' + if (-not (Test-Path $hostPrep)) { + Exit-WithError "wxc-host-prep.exe not found in $BinaryDirectory" + } + + # The AppContainer tier needs the system-drive ACEs and the \Device\Null + # security descriptor. --no-sacl keeps the descriptor within what a CI host + # can grant without SeSecurityPrivilege. + & $hostPrep prepare-system-drive + if ($LASTEXITCODE -ne 0) { + Exit-WithError "wxc-host-prep prepare-system-drive failed with exit code $LASTEXITCODE" + } + + & $hostPrep prepare-null-device --no-sacl + if ($LASTEXITCODE -ne 0) { + Exit-WithError "wxc-host-prep prepare-null-device failed with exit code $LASTEXITCODE" + } +} + +function Initialize-MicroVmHost { + # NanVix binaries are pinned GitHub release assets fetched and checksum- + # verified at build time, so the build either staged all of them or failed. + # Their absence here means a broken artifact, not a host problem. + Assert-RequiredFile @( + 'wxc-exec.exe', + 'nanvixd.exe', + 'nanvix_rootfs.img', + 'python3.initrd', + 'bin\kernel.elf' + ) + + # The WHP snapshots are a warm-start cache generated by running nanvixd on + # an x86_64 build host, not a release asset. The build legitimately skips + # them (cross-arch or offline NANVIX_BIN builds) and the runner cold-boots + # when they are absent, so report the boot path instead of failing. + Write-SnapshotAvailability + + # NanVix boots a VM from these images on every invocation; Defender + # scanning them can push boot past its timeout. + Add-MpPreference -ExclusionPath $BinaryDirectory + Write-Host "Added Defender exclusion for $BinaryDirectory" + + Write-HypervisorDiagnostic + Assert-HypervisorPlatform +} + +if (-not (Test-Path $BinaryDirectory)) { + Exit-WithError "Binary directory not found: $BinaryDirectory" +} +$BinaryDirectory = (Resolve-Path $BinaryDirectory).Path + +Write-Host "Preparing Windows host for backend '$Backend' using $BinaryDirectory" + +switch ($Backend) { + 'process-t3' { Initialize-ProcessContainerHost } + 'microvm' { Initialize-MicroVmHost } + default { Write-Host "$Backend has no artifact-only Windows test prerequisites yet." } +} diff --git a/scripts/ci/probe-nanvixd.ps1 b/scripts/ci/probe-nanvixd.ps1 deleted file mode 100644 index 8e995be29..000000000 --- a/scripts/ci/probe-nanvixd.ps1 +++ /dev/null @@ -1,295 +0,0 @@ -<# -.SYNOPSIS - Runs nanvixd directly to isolate MicroVM boot failures from MXC. - -.DESCRIPTION - MicroVM tests can hang or time out without revealing whether the fault is - in nanvixd (the VM monitor) or in how wxc-exec invokes it. This probe skips - MXC entirely and boots nanvixd itself, bounded by a timeout so it can never - hang a job. - - Two boot paths are exercised because they fail independently: - - - cold: `-kernel-args snapshot`, the path that builds a snapshot from - scratch. This is what the build machine runs. - - warm: `-snapshot snapshots\kernel.whp.cbor`, the path wxc-exec actually - uses at runtime. MXC ships snapshots produced on the *build* - machine, so a warm start can fail on a test host even when a cold - boot succeeds. - - Getting diagnostics out of nanvixd requires care: by default it writes its - trace to a file under `-log-dir` (default `/logs`), so its stdout and - stderr are empty on a *successful* run as well as a hung one. This probe - passes `-log-to-stdout` and `-console-file` so the boot trace and guest - console are actually captured, keeps everything it collects, and takes a - minidump of a hung process so its thread stacks can be inspected offline. - - Diagnostic only: always exits 0 so it reports findings without failing the - lane. Interpretation: - - - Exit 0 with snapshot files -> that boot path works on this host. - - Non-zero exit -> the captured trace names the reason. - - Timeout -> nanvixd hangs; the trace's last line shows - how far the boot got, and the minidump - shows where the threads are parked. - -.PARAMETER BinDir - Directory holding nanvixd.exe and its payload files. - -.PARAMETER Mode - Which boot path(s) to exercise: cold, warm, or both (default). - -.PARAMETER TimeoutSeconds - How long to wait before declaring a hang. - -.PARAMETER OutputDirectory - Where to keep collected artifacts. Defaults to - `$env:RUNNER_TEMP\mxc-microvm-logs\nanvixd-probe`, which CI already uploads. -#> -[CmdletBinding()] -param( - [Parameter(Mandatory)] - [string]$BinDir, - - [ValidateSet('cold', 'warm', 'both')] - [string]$Mode = 'both', - - [int]$TimeoutSeconds = 120, - - [string]$OutputDirectory -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -$binPath = (Resolve-Path -LiteralPath $BinDir).Path -$nanvixd = Join-Path $binPath 'nanvixd.exe' - -Write-Host '=== Direct nanvixd probe ===' - -if (-not (Test-Path -LiteralPath $nanvixd -PathType Leaf)) { - Write-Host "nanvixd.exe not found at $nanvixd - skipping probe." - exit 0 -} - -if (-not $OutputDirectory) { - $OutputDirectory = if ($env:RUNNER_TEMP) { - Join-Path $env:RUNNER_TEMP 'mxc-microvm-logs\nanvixd-probe' - } else { - Join-Path ([System.IO.Path]::GetTempPath()) "nanvixd-probe-$([guid]::NewGuid().ToString('N'))" - } -} -New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null -Write-Host "Artifacts: $OutputDirectory" - -# MiniDumpWriteDump lets a hung run be diagnosed after the fact: the boot trace -# says how far nanvixd got, the dump says where its threads are parked. -$dumpSignature = @' -using System; -using System.Runtime.InteropServices; - -public static class MiniDump -{ - [DllImport("dbghelp.dll", SetLastError = true)] - public static extern bool MiniDumpWriteDump( - IntPtr hProcess, - uint ProcessId, - IntPtr hFile, - uint DumpType, - IntPtr ExceptionParam, - IntPtr UserStreamParam, - IntPtr CallbackParam); -} -'@ - -$dumpAvailable = $true -try { - Add-Type -TypeDefinition $dumpSignature -ErrorAction Stop -} catch { - $dumpAvailable = $false - Write-Host "Minidump capture unavailable: $($_.Exception.Message)" -} - -function Save-HungProcessDump { - param( - [Parameter(Mandatory)][System.Diagnostics.Process]$Process, - [Parameter(Mandatory)][string]$Path - ) - - if (-not $dumpAvailable) { return } - - try { - # MiniDumpNormal (thread stacks) | MiniDumpWithThreadInfo. Deliberately - # not WithFullMemory: nanvixd maps a 256 MB guest and the stacks are - # what identify a hang. - $dumpType = 0x00000000 -bor 0x00001000 - $stream = [System.IO.File]::Create($Path) - try { - $ok = [MiniDump]::MiniDumpWriteDump( - $Process.Handle, - [uint32]$Process.Id, - $stream.SafeFileHandle.DangerousGetHandle(), - $dumpType, - [IntPtr]::Zero, [IntPtr]::Zero, [IntPtr]::Zero) - } finally { - $stream.Dispose() - } - if ($ok) { - $size = (Get-Item -LiteralPath $Path).Length - Write-Host " Minidump written: $Path ($size bytes)" - } else { - $code = [Runtime.InteropServices.Marshal]::GetLastWin32Error() - Write-Host " Minidump failed (win32 error $code)" - Remove-Item -LiteralPath $Path -Force -ErrorAction SilentlyContinue - } - } catch { - Write-Host " Minidump failed: $($_.Exception.Message)" - } -} - -function Show-CapturedFile { - param( - [Parameter(Mandatory)][string]$Label, - [Parameter(Mandatory)][string]$Path - ) - - Write-Host '' - Write-Host "--- $Label ---" - if (-not (Test-Path -LiteralPath $Path)) { - Write-Host '(not captured)' - return - } - $content = Get-Content -LiteralPath $Path -Raw -ErrorAction SilentlyContinue - if ([string]::IsNullOrWhiteSpace($content)) { - Write-Host '(empty)' - return - } - Write-Host $content.TrimEnd() -} - -function Invoke-NanvixdBoot { - param( - [Parameter(Mandatory)][ValidateSet('cold', 'warm')][string]$BootMode - ) - - Write-Host '' - Write-Host "=== $BootMode boot ===" - - $runDir = Join-Path $OutputDirectory $BootMode - New-Item -ItemType Directory -Force -Path $runDir | Out-Null - - $stdoutFile = Join-Path $runDir 'stdout.log' - $stderrFile = Join-Path $runDir 'stderr.log' - $consoleFile = Join-Path $runDir 'guest-console.log' - - # `-log-to-stdout` is essential: without it nanvixd logs to a file under - # `-log-dir` and both stdout and stderr stay empty even on a healthy run, - # which makes an empty capture indistinguishable from an early hang. - $arguments = @( - '-bin-dir', (Join-Path $binPath 'bin'), - '-ramfs', (Join-Path $binPath 'nanvix_rootfs.img'), - '-log-to-stdout', - '-console-file', $consoleFile - ) - - if ($BootMode -eq 'cold') { - # Mirrors nanvix_common::generate_snapshot. Runs in its own directory so - # it never disturbs the artifact's shipped snapshots. - $workingDirectory = Join-Path $runDir 'home' - New-Item -ItemType Directory -Force -Path $workingDirectory | Out-Null - $arguments += @('-kernel-args', 'snapshot') - } else { - # Mirrors nanvix_runner::spawn_nanvixd: cwd is the snapshot home and the - # snapshot path is relative to it. - $snapshot = Join-Path $binPath 'snapshots\kernel.whp.cbor' - if (-not (Test-Path -LiteralPath $snapshot)) { - Write-Host "No shipped snapshot at $snapshot - skipping warm boot." - return - } - $workingDirectory = $binPath - - # The guest runs the bootstrap script from the mounted staging dir, so - # stage one exactly as wxc_common::microvm_staging does. Without it - # CPython would wait on stdin and look like a hang. - $stagingDir = Join-Path $runDir 'staging' - New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null - @( - 'import sys' - "sys.argv = ['/mnt/bootstrap.py']" - "print('nanvixd-probe-ok')" - ) -join "`n" | Set-Content -LiteralPath (Join-Path $stagingDir 'bootstrap.py') -Encoding ascii - - $arguments += @('-snapshot', 'snapshots\kernel.whp.cbor', '-mount', $stagingDir) - } - - $arguments += @('--', (Join-Path $binPath 'python3.initrd')) - - Write-Host "Command: nanvixd.exe $($arguments -join ' ')" - Write-Host "Working directory: $workingDirectory" - Write-Host "Timeout: ${TimeoutSeconds}s" - - $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() - $process = Start-Process -FilePath $nanvixd ` - -ArgumentList $arguments ` - -WorkingDirectory $workingDirectory ` - -PassThru ` - -RedirectStandardOutput $stdoutFile ` - -RedirectStandardError $stderrFile - - $hung = -not $process.WaitForExit($TimeoutSeconds * 1000) - $stopwatch.Stop() - - if ($hung) { - Write-Host "::warning::nanvixd did not exit within ${TimeoutSeconds}s - it HANGS on this host ($BootMode boot)." - Save-HungProcessDump -Process $process -Path (Join-Path $runDir 'nanvixd-hang.dmp') - try { $process.Kill($true) } catch { } - } else { - Write-Host "nanvixd exited with code $($process.ExitCode) after $($stopwatch.Elapsed.TotalSeconds.ToString('0.0'))s" - } - - Show-CapturedFile -Label 'nanvixd stdout' -Path $stdoutFile - Show-CapturedFile -Label 'nanvixd stderr (boot trace)' -Path $stderrFile - Show-CapturedFile -Label 'guest console' -Path $consoleFile - - # Fallback for a nanvixd build that ignores -log-to-stdout. - $logDir = Join-Path $workingDirectory 'logs' - if (Test-Path -LiteralPath $logDir) { - Write-Host '' - Write-Host '--- nanvixd log directory ---' - foreach ($file in Get-ChildItem -LiteralPath $logDir -File) { - Write-Host " $($file.Name) ($($file.Length) bytes)" - Copy-Item -LiteralPath $file.FullName -Destination $runDir -Force -ErrorAction SilentlyContinue - } - } - - Write-Host '' - Write-Host '--- Generated snapshot files ---' - $snapshotDir = Join-Path $workingDirectory 'snapshots' - if (Test-Path -LiteralPath $snapshotDir) { - Get-ChildItem -LiteralPath $snapshotDir | Format-Table Name, Length | Out-String | Write-Host - } else { - Write-Host '(no snapshots directory created)' - } - - Write-Host '' - if ($hung) { - Write-Host "RESULT ($BootMode): nanvixd HANGS on this host - the fault is below MXC." - Write-Host ' The last line of the boot trace above shows how far it got.' - } elseif ($process.ExitCode -eq 0) { - Write-Host "RESULT ($BootMode): nanvixd booted successfully." - } else { - Write-Host "RESULT ($BootMode): nanvixd failed - see the boot trace above for the reason." - } -} - -$modes = if ($Mode -eq 'both') { @('cold', 'warm') } else { @($Mode) } -foreach ($bootMode in $modes) { - try { - Invoke-NanvixdBoot -BootMode $bootMode - } catch { - Write-Host "::warning::$bootMode probe raised: $($_.Exception.Message)" - } -} - -# Diagnostic only: never fail the job on the probe's outcome. -exit 0 diff --git a/scripts/ci/probe-whp-capabilities.ps1 b/scripts/ci/probe-whp-capabilities.ps1 deleted file mode 100644 index 0dec08db9..000000000 --- a/scripts/ci/probe-whp-capabilities.ps1 +++ /dev/null @@ -1,372 +0,0 @@ -<# -.SYNOPSIS - Reports Windows Hypervisor Platform capabilities via the real WHP API. - -.DESCRIPTION - `Get-WindowsOptionalFeature` only reports whether the HypervisorPlatform - feature is installed, and `HypervisorPresent` only says a hypervisor is - running. Neither proves that WHP can actually create a partition, which is - what a VM monitor such as nanvixd needs. - - This probe P/Invokes WinHvPlatform.dll directly to: - - 1. Query capabilities through WHvGetCapability. - 2. Build a complete partition (WHvCreatePartition / WHvSetupPartition / - WHvCreateVirtualProcessor), map a guest page, and actually execute - guest code on it. - - Step 2 is the decisive one. Capability flags can look correct on a host - where a partition cannot be set up, and a partition can be set up on a host - where the hypervisor never actually executes guest instructions -- which is - what a VM monitor such as nanvixd needs. The guest here is a single `hlt` - placed at the x86 reset vector, so a healthy host reports exit reason - `WHvRunVpExitReasonX64Halt` and no register or paging setup is required. - - Run this on both a working host and a failing one and compare the output. - - Diagnostic only: always exits 0. -#> -[CmdletBinding()] -param() - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -Write-Host '=== WHP Capability Probe ===' -Write-Host "OS: $([System.Environment]::OSVersion.VersionString)" - -try { - $cs = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue - if ($cs) { - Write-Host "Model: $($cs.Model)" - Write-Host "HypervisorPresent: $($cs.HypervisorPresent)" - } - $cpu = Get-CimInstance -ClassName Win32_Processor -ErrorAction SilentlyContinue | Select-Object -First 1 - if ($cpu) { - Write-Host "CPU: $($cpu.Name)" - Write-Host "VirtualizationFirmwareEnabled: $($cpu.VirtualizationFirmwareEnabled)" - } -} catch { - Write-Host "Could not read system info: $($_.Exception.Message)" -} - -if (-not (Test-Path "$env:SystemRoot\System32\WinHvPlatform.dll")) { - Write-Host '::warning::WinHvPlatform.dll is absent - WHP is not installed.' - exit 0 -} - -$signature = @' -using System; -using System.Runtime.InteropServices; - -public static class Whp -{ - [DllImport("WinHvPlatform.dll")] - public static extern int WHvGetCapability( - uint CapabilityCode, - IntPtr CapabilityBuffer, - uint CapabilityBufferSizeInBytes, - out uint WrittenSizeInBytes); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvCreatePartition(out IntPtr Partition); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvSetupPartition(IntPtr Partition); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvDeletePartition(IntPtr Partition); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvSetPartitionProperty( - IntPtr Partition, - uint PropertyCode, - IntPtr PropertyBuffer, - uint PropertyBufferSizeInBytes); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvCreateVirtualProcessor( - IntPtr Partition, - uint VpIndex, - uint Flags); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvDeleteVirtualProcessor( - IntPtr Partition, - uint VpIndex); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvMapGpaRange( - IntPtr Partition, - IntPtr SourceAddress, - ulong GuestAddress, - ulong SizeInBytes, - uint Flags); - - [DllImport("WinHvPlatform.dll")] - public static extern int WHvRunVirtualProcessor( - IntPtr Partition, - uint VpIndex, - IntPtr ExitContext, - uint ExitContextSizeInBytes); - - [DllImport("kernel32.dll")] - public static extern IntPtr VirtualAlloc( - IntPtr lpAddress, - UIntPtr dwSize, - uint flAllocationType, - uint flProtect); - - [DllImport("kernel32.dll")] - public static extern bool VirtualFree(IntPtr lpAddress, UIntPtr dwSize, uint dwFreeType); -} -'@ - -try { - Add-Type -TypeDefinition $signature -ErrorAction Stop -} catch { - Write-Host "::warning::Could not bind to WinHvPlatform.dll: $($_.Exception.Message)" - exit 0 -} - -function Get-WhpCapability { - param( - [Parameter(Mandatory)][string]$Name, - [Parameter(Mandatory)][uint32]$Code - ) - - $buffer = [Runtime.InteropServices.Marshal]::AllocHGlobal(8) - try { - [Runtime.InteropServices.Marshal]::WriteInt64($buffer, 0) - $written = 0 - $hr = [Whp]::WHvGetCapability($Code, $buffer, 8, [ref]$written) - if ($hr -eq 0) { - $value = [Runtime.InteropServices.Marshal]::ReadInt64($buffer) - Write-Host (" {0,-28} 0x{1:X16}" -f $Name, $value) - return $value - } - Write-Host (" {0,-28} FAILED (hr=0x{1:X8})" -f $Name, $hr) - return $null - } finally { - [Runtime.InteropServices.Marshal]::FreeHGlobal($buffer) - } -} - -Write-Host '' -Write-Host '--- WHvGetCapability ---' -$hypervisorPresent = Get-WhpCapability -Name 'HypervisorPresent' -Code 0x00000000 -$features = Get-WhpCapability -Name 'Features' -Code 0x00000001 -Get-WhpCapability -Name 'ExtendedVmExits' -Code 0x00000002 | Out-Null -Get-WhpCapability -Name 'ProcessorVendor' -Code 0x00001000 | Out-Null -Get-WhpCapability -Name 'ProcessorFeatures' -Code 0x00001001 | Out-Null - -# Nested-virtualization capabilities. A host that cannot expose VMX to a guest -# will fail or omit these, which is the difference we are looking for between a -# working runner and one where a VM monitor cannot boot. -Write-Host '' -Write-Host '--- Nested virtualization (VMX) capabilities ---' -Get-WhpCapability -Name 'VmxBasic' -Code 0x00002000 | Out-Null -Get-WhpCapability -Name 'VmxPinbasedCtls' -Code 0x00002001 | Out-Null -Get-WhpCapability -Name 'VmxProcbasedCtls' -Code 0x00002002 | Out-Null -Get-WhpCapability -Name 'VmxEptVpidCap' -Code 0x0000200C | Out-Null - -if ($null -ne $features) { - # Bit layout from WHV_CAPABILITY_FEATURES (x64). - $bits = [ordered]@{ - PartialUnmap = 0 - LocalApicEmulation = 1 - Xsave = 2 - DirtyPageTracking = 3 - SpeculationControl = 4 - ApicRemoteRead = 5 - IdleSuspend = 6 - VirtualPciDeviceSupport = 7 - IommuSupport = 8 - VpHotAddRemove = 9 - DeviceAccessTracking = 10 - } - Write-Host '' - Write-Host '--- Feature bits ---' - foreach ($entry in $bits.GetEnumerator()) { - $set = (($features -shr $entry.Value) -band 1) -eq 1 - Write-Host (" {0,-28} {1}" -f $entry.Key, $set) - } -} - -if ($hypervisorPresent -eq 0) { - Write-Host '' - Write-Host '::warning::WHvCapabilityCodeHypervisorPresent is 0 - WHP reports no usable hypervisor.' -} - -# The decisive test. Feature flags can look correct on a host where partition -# setup fails, and setup can succeed on a host that never executes guest code. -Write-Host '' -Write-Host '--- Partition setup and guest execution (what a VM monitor actually needs) ---' - -# WHV_PARTITION_PROPERTY_CODE values from WinHvPlatformDefs.h. These are easy to -# get wrong: ProcessorCount is 0x00001fff, NOT 0x00001002 (ProcessorClFlushSize), -# and ExtendedVmExits is 0x00000001, NOT 0x00000002 (ExceptionExitBitmap). -# Setting the wrong code makes WHvSetupPartition fail with -# WHV_E_INVALID_PARTITION_CONFIG (0x80370304) on a perfectly healthy host. -$PropertyCodeExtendedVmExits = 0x00000001 -$PropertyCodeProcessorCount = 0x00001fff - -# WHV_RUN_VP_EXIT_REASON values worth naming in the output. -$exitReasons = @{ - 0x00000000 = 'None' - 0x00000001 = 'MemoryAccess' - 0x00000002 = 'X64IoPortAccess' - 0x00000004 = 'UnrecoverableException' - 0x00000005 = 'InvalidVpRegisterValue' - 0x00000006 = 'UnsupportedFeature' - 0x00000007 = 'X64InterruptWindow' - 0x00000008 = 'X64Halt' - 0x00001000 = 'X64MsrAccess' - 0x00001001 = 'X64Cpuid' - 0x00001002 = 'Exception' - 0x00002001 = 'Canceled' -} - -$partition = [IntPtr]::Zero -$stopwatch = [System.Diagnostics.Stopwatch]::StartNew() -$hr = [Whp]::WHvCreatePartition([ref]$partition) -$stopwatch.Stop() -Write-Host (" WHvCreatePartition hr=0x{0:X8} ({1} ms)" -f $hr, $stopwatch.ElapsedMilliseconds) - -if ($hr -ne 0) { - Write-Host '::warning::WHvCreatePartition failed - WHP cannot host a VM here even though the feature is enabled.' - Write-Host '' - Write-Host 'RESULT: partition creation FAILED - the hypervisor is unusable on this host.' - exit 0 -} - -$guestMemory = @() -$vpCreated = $false -try { - $propertyBuffer = [Runtime.InteropServices.Marshal]::AllocHGlobal(8) - try { - [Runtime.InteropServices.Marshal]::WriteInt64($propertyBuffer, 0) - [Runtime.InteropServices.Marshal]::WriteInt32($propertyBuffer, 1) - $hrProp = [Whp]::WHvSetPartitionProperty($partition, $PropertyCodeProcessorCount, $propertyBuffer, 4) - Write-Host (" SetProcessorCount(1) hr=0x{0:X8}" -f $hrProp) - - [Runtime.InteropServices.Marshal]::WriteInt64($propertyBuffer, 0) - $hrExits = [Whp]::WHvSetPartitionProperty($partition, $PropertyCodeExtendedVmExits, $propertyBuffer, 8) - Write-Host (" SetExtendedVmExits(0) hr=0x{0:X8}" -f $hrExits) - } finally { - [Runtime.InteropServices.Marshal]::FreeHGlobal($propertyBuffer) - } - - $stopwatch.Restart() - $hrSetup = [Whp]::WHvSetupPartition($partition) - $stopwatch.Stop() - Write-Host (" WHvSetupPartition hr=0x{0:X8} ({1} ms)" -f $hrSetup, $stopwatch.ElapsedMilliseconds) - if ($hrSetup -ne 0) { - Write-Host '' - Write-Host ('::warning::WHvSetupPartition failed with 0x{0:X8}.' -f $hrSetup) - Write-Host 'RESULT: partition setup FAILED - a VM monitor cannot start here.' - exit 0 - } - - $stopwatch.Restart() - $hrVp = [Whp]::WHvCreateVirtualProcessor($partition, 0, 0) - $stopwatch.Stop() - Write-Host (" WHvCreateVirtualProcessor hr=0x{0:X8} ({1} ms)" -f $hrVp, $stopwatch.ElapsedMilliseconds) - if ($hrVp -ne 0) { - Write-Host '' - Write-Host '::warning::WHvCreateVirtualProcessor failed - no vCPU can be created on this host.' - Write-Host 'RESULT: vCPU creation FAILED.' - exit 0 - } - $vpCreated = $true - - # Map HLT-filled pages at the addresses a fresh vCPU can start from and let - # the guest run. WHP resets a vCPU to real mode at CS.Base 0xF0000 / - # RIP 0xFFF0 (linear 0xFFFF0, page 0xFF000); the zero page is mapped too so - # the probe stays valid if that reset state ever changes. Either way the - # first instruction fetched is HLT (0xF4). - $pageSize = 4096 - $MEM_COMMIT_RESERVE = 0x3000 - $PAGE_EXECUTE_READWRITE = 0x40 - # WHvMapGpaRangeFlagRead | Write | Execute - $mapFlags = 0x00000007 - $guestPages = @( - @{ Name = 'reset vector'; Gpa = [uint64]0xFF000 }, - @{ Name = 'zero page'; Gpa = [uint64]0 } - ) - - foreach ($page in $guestPages) { - $host_address = [Whp]::VirtualAlloc([IntPtr]::Zero, [UIntPtr]::new($pageSize), $MEM_COMMIT_RESERVE, $PAGE_EXECUTE_READWRITE) - if ($host_address -eq [IntPtr]::Zero) { - Write-Host '::warning::VirtualAlloc for guest memory failed - cannot complete the execution test.' - exit 0 - } - $guestMemory += $host_address - for ($offset = 0; $offset -lt $pageSize; $offset++) { - [Runtime.InteropServices.Marshal]::WriteByte($host_address, $offset, 0xF4) - } - - $hrMap = [Whp]::WHvMapGpaRange($partition, $host_address, $page.Gpa, $pageSize, $mapFlags) - Write-Host (" WHvMapGpaRange {0,-11} hr=0x{1:X8}" -f $page.Name, $hrMap) - if ($hrMap -ne 0) { - Write-Host '' - Write-Host '::warning::WHvMapGpaRange failed - guest memory cannot be mapped on this host.' - Write-Host 'RESULT: guest memory mapping FAILED.' - exit 0 - } - } - - # WHV_RUN_VP_EXIT_CONTEXT is 224 bytes (x64, SDK 10.0.26100). - $exitContextSize = 224 - $exitContext = [Runtime.InteropServices.Marshal]::AllocHGlobal($exitContextSize) - try { - # If the hypervisor never actually dispatches guest code this call is - # where a host blocks; the CI step timeout bounds it. - $stopwatch.Restart() - $hrRun = [Whp]::WHvRunVirtualProcessor($partition, 0, $exitContext, $exitContextSize) - $stopwatch.Stop() - $exitReason = [Runtime.InteropServices.Marshal]::ReadInt32($exitContext) - $reasonName = if ($exitReasons.ContainsKey($exitReason)) { $exitReasons[$exitReason] } else { 'Unknown' } - # WHV_RUN_VP_EXIT_CONTEXT layout: ExitReason(0), Reserved(4), - # VpContext(8) { ExecutionState, InstructionLength/Cr8, Reserved, - # Reserved2, Cs @16, Rip @32, Rflags @40 }, union @48. For a - # MemoryAccess exit the union holds WHV_MEMORY_ACCESS_CONTEXT with - # Gpa at offset 72. - $csBase = [Runtime.InteropServices.Marshal]::ReadInt64($exitContext, 16) - $rip = [Runtime.InteropServices.Marshal]::ReadInt64($exitContext, 32) - Write-Host (" WHvRunVirtualProcessor hr=0x{0:X8} ({1} ms)" -f $hrRun, $stopwatch.ElapsedMilliseconds) - Write-Host (" Guest exit reason 0x{0:X8} ({1})" -f $exitReason, $reasonName) - Write-Host (" Guest CS.Base/RIP 0x{0:X16} / 0x{1:X16}" -f $csBase, $rip) - if ($exitReason -eq 0x00000001) { - $gpa = [Runtime.InteropServices.Marshal]::ReadInt64($exitContext, 72) - Write-Host (" Faulting GPA 0x{0:X16}" -f $gpa) - } - - Write-Host '' - if ($hrRun -eq 0 -and $exitReason -eq 0x00000008) { - Write-Host 'RESULT: WHP executed guest code and halted as expected.' - Write-Host ' The hypervisor is fully usable here - a VM monitor that hangs' - Write-Host ' or fails is doing so for a reason above WHP.' - } elseif ($hrRun -ne 0) { - Write-Host ('::warning::WHvRunVirtualProcessor failed with 0x{0:X8}.' -f $hrRun) - Write-Host 'RESULT: the hypervisor refused to run guest code on this host.' - } else { - Write-Host ("::warning::Guest exited for {0} instead of X64Halt." -f $reasonName) - Write-Host 'RESULT: guest code ran but did not reach HLT - compare against a working host.' - } - } finally { - [Runtime.InteropServices.Marshal]::FreeHGlobal($exitContext) - } -} finally { - if ($vpCreated) { - $hrDeleteVp = [Whp]::WHvDeleteVirtualProcessor($partition, 0) - Write-Host (" WHvDeleteVirtualProcessor hr=0x{0:X8}" -f $hrDeleteVp) - } - $hrDelete = [Whp]::WHvDeletePartition($partition) - Write-Host (" WHvDeletePartition hr=0x{0:X8}" -f $hrDelete) - foreach ($address in $guestMemory) { - [void][Whp]::VirtualFree($address, [UIntPtr]::Zero, 0x8000) - } -} - -exit 0 diff --git a/src/backends/nanvix/runner/src/lib.rs b/src/backends/nanvix/runner/src/lib.rs index aa2762664..fc0c2070b 100644 --- a/src/backends/nanvix/runner/src/lib.rs +++ b/src/backends/nanvix/runner/src/lib.rs @@ -23,15 +23,11 @@ //! ## Diagnostics //! //! By default the runner sets `RUST_LOG=off` in nanvixd's environment, which -//! suppresses its trace output and noticeably reduces warm-start latency. Set -//! `MXC_NANVIX_TRACE=1` (or `true`/`yes`, case-insensitive) before invoking -//! wxc-exec to let nanvixd use its own `RUST_LOG` default and to capture -//! nanvixd's stderr for inclusion in the wxc-exec log. -//! -//! Trace mode also passes `-log-to-stdout` to nanvixd. Without it nanvixd -//! writes its trace to a file under `-log-dir` (default `/logs`, i.e. the -//! snapshot home) and the captured stderr is empty — which is exactly the -//! wrong behavior when diagnosing a VM that never finished booting. +//! suppresses the per-run `%LOCALAPPDATA%\nanvix\logs\nanvixd_*.log` trace +//! file and noticeably reduces warm-start latency. Set `MXC_NANVIX_TRACE=1` +//! (or `true`/`yes`, case-insensitive) before invoking wxc-exec to let +//! nanvixd use its own `RUST_LOG` default and to capture nanvixd's stderr +//! for inclusion in the wxc-exec log. //! //! ## Exit codes //! @@ -680,15 +676,6 @@ impl NanVixScriptRunner { let mut cmd = Command::new(&paths.nanvixd); - if trace { - // Route nanvixd's trace to its standard streams. Its default is a - // log file under `-log-dir` (i.e. `/logs`), which - // would leave the stderr we just piped empty and give a stalled - // boot no visible diagnostics at all. Must precede the `--` - // separator, so it is added before the per-platform arguments. - cmd.arg("-log-to-stdout"); - } - // Host networking is opt-in. When enabled, attach the host network // backend; nanvixd parses this flag regardless of argument order, so // it is added up front for both the Windows (snapshot) and Linux @@ -748,7 +735,7 @@ impl NanVixScriptRunner { .stdout(Stdio::inherit()) .stderr(stderr); if !trace { - // Suppress nanvixd's trace output and per-run log file. + // Suppress nanvixd's env_logger output and per-run log file. cmd.env("RUST_LOG", "off"); } cmd.spawn().map_err(|e| { @@ -906,13 +893,6 @@ impl NanVixScriptRunner { if timed_out.load(Ordering::SeqCst) { let _ = child.kill(); - // Emit the captured trace before returning. A timeout is the one - // failure mode where nanvixd never reports a reason itself, so - // dropping its stderr here would leave the log with nothing but - // "timed out" — the exact case these diagnostics exist for. - if !stderr_output.is_empty() { - let _ = writeln!(logger, "NanVix stderr:\n{}", stderr_output); - } let err = NanVixError::Timeout { script_timeout_ms: script_timeout, total_ms: timeout_ms, diff --git a/tests/scripts/run_microvm_tests.ps1 b/tests/scripts/run_microvm_tests.ps1 index 1c78a67bb..dcf6002b5 100644 --- a/tests/scripts/run_microvm_tests.ps1 +++ b/tests/scripts/run_microvm_tests.ps1 @@ -30,8 +30,7 @@ param( [switch]$Release, [string]$BinDir, - [string]$ConfigDir, - [string]$LogDir + [string]$ConfigDir ) $ErrorActionPreference = "Stop" @@ -49,14 +48,6 @@ if (-not $ConfigDir) { $ConfigDir = Join-Path $RepoRoot "tests\configs" } -# Default to RUNNER_TEMP so CI's existing log upload picks the files up. -if (-not $LogDir -and $env:RUNNER_TEMP) { - $LogDir = Join-Path $env:RUNNER_TEMP "mxc-microvm-logs" -} -if ($LogDir -and -not (Test-Path $LogDir)) { - New-Item -ItemType Directory -Force -Path $LogDir | Out-Null -} - $WxcExePath = Join-Path $BinDir "wxc-exec.exe" # -- WHP check (local runs only) --------------------------------------------- @@ -148,16 +139,8 @@ foreach ($test in $tests) { $sw = [System.Diagnostics.Stopwatch]::StartNew() $stdoutFile = [System.IO.Path]::GetTempFileName() $stderrFile = [System.IO.Path]::GetTempFileName() - # The failure tail below only keeps the last few lines, which for a boot - # timeout are the error envelope rather than the boot itself. Keep the full - # --debug log so CI can show where a VM actually stalled. - $logArgs = @() - if ($LogDir) { - $logFile = Join-Path $LogDir "microvm-$([System.IO.Path]::GetFileNameWithoutExtension($test.Config)).log" - $logArgs = @("--log-file", $logFile) - } $process = Start-Process -FilePath $wxcExe ` - -ArgumentList (@("--debug", "--experimental") + $logArgs + @($configPath)) ` + -ArgumentList "--debug", "--experimental", $configPath ` -PassThru -Wait ` -RedirectStandardOutput $stdoutFile ` -RedirectStandardError $stderrFile From c30e47b1d9f6619d36e93b049973b8cea237546e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:42:04 -0700 Subject: [PATCH 59/93] cold start for microvm on windows, and adding wslc test suite --- .../workflows/Validation.Tests.Matrix.Job.yml | 2 + .github/workflows/Validation.Tests.WSLC.yml | 2 +- scripts/ci/prepare-windows-host.ps1 | 247 ++++++++++++++---- scripts/ci/validation-test-matrix.json | 9 +- tests/scripts/run_microvm_tests.ps1 | 98 ++++++- 5 files changed, 289 insertions(+), 69 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index f9bb7a448..74ef5ac39 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -50,6 +50,8 @@ jobs: - name: Prepare backend prerequisites shell: pwsh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | & ./scripts/ci/prepare-windows-host.ps1 ` -Backend '${{ matrix.backend }}' ` diff --git a/.github/workflows/Validation.Tests.WSLC.yml b/.github/workflows/Validation.Tests.WSLC.yml index e9494fb21..5b2530128 100644 --- a/.github/workflows/Validation.Tests.WSLC.yml +++ b/.github/workflows/Validation.Tests.WSLC.yml @@ -28,7 +28,7 @@ on: push: branches: - user/modanish/1es-github-runner-e2e - - user/emichlin/validation-infra-lite + # - user/emichlin/validation-infra-lite workflow_dispatch: permissions: diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index a932f5d7c..f254597c0 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -57,19 +57,149 @@ function Assert-RequiredFile { Get-ChildItem $BinaryDirectory -Include $leaves -Recurse | Format-Table FullName, Length } -# Report whether the artifact carries a complete WHP warm-start snapshot set. -# A partial set is worth calling out: the runner treats a complete set as the -# signal to warm-boot, so a missing half silently costs a cold boot per run. -function Write-SnapshotAvailability { - $snapshots = @('snapshots\kernel.vmem', 'snapshots\kernel.whp.cbor') - $present = @($snapshots | Where-Object { Test-Path (Join-Path $BinaryDirectory $_) }) - - if ($present.Count -eq $snapshots.Count) { - Write-Host 'WHP warm-start snapshots are present; the runner will warm-boot.' - } elseif ($present.Count -eq 0) { - Write-Host 'WHP warm-start snapshots are absent; the runner will cold-boot (slower, expected on cross-arch or offline builds).' - } else { - Write-Host "WARNING: incomplete WHP snapshot set (found: $($present -join ', ')); the runner will cold-boot." +# Fetch the pinned NanVix release onto the test host, mirroring the offline-build +# contract in docs/nanvix-microvm/nanvix.md: flat binaries plus bin/, verified +# against checksums.json, and no snapshots (the runtime cold-boots and +# regenerates a verified one on first use). +function Install-NanvixBinaries { + $configDir = Join-Path $PSScriptRoot '..\..\src\backends\nanvix\binaries' + $versionsPath = Join-Path $configDir 'versions.json' + $checksumsPath = Join-Path $configDir 'checksums.json' + foreach ($path in $versionsPath, $checksumsPath) { + if (-not (Test-Path $path)) { + Exit-WithError "NanVix pin file not found: $path" + } + } + + $release = (Get-Content $versionsPath -Raw | ConvertFrom-Json).nanvix_python + $checksums = (Get-Content $checksumsPath -Raw | ConvertFrom-Json).windows + $prefix = [System.IO.Path]::GetFileNameWithoutExtension($release.asset) + $binSubdir = Join-Path $BinaryDirectory 'bin' + + # The build VM's copies are discarded: its snapshots are a host-specific WHP + # memory image, and a partial set would silently cost a cold boot per run. + Write-Host 'Discarding build-staged NanVix binaries in favor of the pinned release.' + Remove-Item (Join-Path $BinaryDirectory 'snapshots') -Recurse -Force -ErrorAction SilentlyContinue + foreach ($name in $release.binaries) { + Remove-Item (Join-Path $BinaryDirectory $name) -Force -ErrorAction SilentlyContinue + } + Remove-Item (Join-Path $binSubdir 'kernel.elf') -Force -ErrorAction SilentlyContinue + + $url = "https://github.com/nanvix/nanvix-python/releases/download/$($release.tag)/$($release.asset)" + $archive = Join-Path ([System.IO.Path]::GetTempPath()) $release.asset + Write-Host "Downloading nanvix/nanvix-python $($release.tag)..." + + try { + $curlArgs = @( + '--silent', '--show-error', '--fail', '--location', + '--retry', '5', '--retry-delay', '5', '--retry-all-errors', + '--output', $archive + ) + $token = if ($env:GITHUB_TOKEN) { $env:GITHUB_TOKEN } else { $env:GH_TOKEN } + if ($token) { + $curlArgs += @('--header', "Authorization: Bearer $token") + } + $curlArgs += $url + + & curl.exe @curlArgs + if ($LASTEXITCODE -ne 0) { + Exit-WithError "curl failed for $url (exit code $LASTEXITCODE)" + } + + New-Item -ItemType Directory -Force -Path $binSubdir | Out-Null + + # nanvixd.exe and kernel.elf live under /bin/; the rest at /. + Expand-NanvixEntry -Archive $archive -Entry "$prefix/bin/nanvixd.exe" -StripComponents 2 -Destination $BinaryDirectory + Expand-NanvixEntry -Archive $archive -Entry "$prefix/bin/kernel.elf" -StripComponents 2 -Destination $binSubdir + foreach ($name in $release.binaries | Where-Object { $_ -ne 'nanvixd.exe' }) { + Expand-NanvixEntry -Archive $archive -Entry "$prefix/$name" -StripComponents 1 -Destination $BinaryDirectory + } + } finally { + Remove-Item $archive -Force -ErrorAction SilentlyContinue + } + + Assert-NanvixChecksum -Path (Join-Path $binSubdir 'kernel.elf') -Expected $checksums.'kernel.elf' + foreach ($name in $release.binaries) { + Assert-NanvixChecksum -Path (Join-Path $BinaryDirectory $name) -Expected $checksums.$name + } + + Write-Host "NanVix $($release.tag) staged and verified; the runner will cold-boot and regenerate its snapshot." +} + +function Expand-NanvixEntry { + param( + [Parameter(Mandatory)][string]$Archive, + [Parameter(Mandatory)][string]$Entry, + [Parameter(Mandatory)][int]$StripComponents, + [Parameter(Mandatory)][string]$Destination + ) + + & tar.exe -xf $Archive -C $Destination --strip-components $StripComponents $Entry + if ($LASTEXITCODE -ne 0) { + Exit-WithError "tar failed to extract $Entry (exit code $LASTEXITCODE)" + } +} + +function Assert-NanvixChecksum { + param( + [Parameter(Mandatory)][string]$Path, + [string]$Expected + ) + + if (-not (Test-Path $Path)) { + Exit-WithError "NanVix binary missing after extraction: $Path" + } + if (-not $Expected) { + Exit-WithError "No pinned checksum for $(Split-Path $Path -Leaf)" + } + + $actual = (Get-FileHash -Path $Path -Algorithm SHA256).Hash + if ($actual -ne $Expected) { + Exit-WithError "Checksum mismatch for $(Split-Path $Path -Leaf): expected $($Expected.ToLowerInvariant()), got $($actual.ToLowerInvariant())" + } + Write-Host " $(Split-Path $Path -Leaf) verified" +} + +# Read a Windows optional feature's state without throwing, so both the +# diagnostic and assertion paths can share one query. A host that cannot answer +# (querying needs elevation) reports the reason as its state rather than +# aborting, which keeps the failure message actionable. +function Get-OptionalFeatureState { + param([Parameter(Mandatory)][string]$Name) + + try { + $feature = Get-WindowsOptionalFeature -Online -FeatureName $Name -ErrorAction Stop + } catch { + return "query-failed: $($_.Exception.Message.Trim())" + } + + if ($null -eq $feature) { + return 'unknown' + } + return [string]$feature.State +} + +# Require every named optional feature to be Enabled. Enabling one needs a +# reboot the runner cannot take mid-job, so this verifies rather than installs: +# a mis-imaged pool fails here with a pointed message instead of surfacing +# later as an opaque backend error. $Remedy names the image-level fix. +function Assert-RequiredFeature { + param( + [Parameter(Mandatory)][string[]]$Name, + [Parameter(Mandatory)][string]$Remedy + ) + + $notEnabled = @() + foreach ($feature in $Name) { + $state = Get-OptionalFeatureState -Name $feature + Write-Host " $feature = $state" + if ($state -ne 'Enabled') { + $notEnabled += "$feature ($state)" + } + } + + if ($notEnabled) { + Exit-WithError "Required Windows optional feature(s) not enabled: $($notEnabled -join '; '). $Remedy" } } @@ -78,37 +208,20 @@ function Write-SnapshotAvailability { # check below rather than as an unexplained collection error. function Write-HypervisorDiagnostic { Write-Host '=== Hypervisor Diagnostics ===' - try { - Write-Host "OS: $([System.Environment]::OSVersion)" + Write-Host "OS: $([System.Environment]::OSVersion)" - $computerSystem = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue - $hypervisorPresent = if ($null -eq $computerSystem) { 'unknown' } else { $computerSystem.HypervisorPresent } - Write-Host "HypervisorPresent: $hypervisorPresent" - Write-Host "WinHvPlatform.dll exists: $(Test-Path "$env:SystemRoot\System32\WinHvPlatform.dll")" - - $feature = Get-WindowsOptionalFeature -Online -FeatureName HypervisorPlatform -ErrorAction SilentlyContinue - $featureState = if ($null -eq $feature) { 'unknown' } else { $feature.State } - Write-Host "HypervisorPlatform feature state: $featureState" - } catch { - Write-Host "WARNING: hypervisor diagnostics could not be collected: $($_.Exception.Message)" - } + $computerSystem = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue + $hypervisorPresent = if ($null -eq $computerSystem) { 'unknown' } else { $computerSystem.HypervisorPresent } + Write-Host "HypervisorPresent: $hypervisorPresent" + Write-Host "WinHvPlatform.dll exists: $(Test-Path "$env:SystemRoot\System32\WinHvPlatform.dll")" Write-Host '=== end diagnostics ===' } # The feature can be enabled while the hypervisor is not actually running (for # example when a host reboot is still pending), so both are required. function Assert-HypervisorPlatform { - try { - $feature = Get-WindowsOptionalFeature -Online -FeatureName HypervisorPlatform -ErrorAction SilentlyContinue - } catch { - # Querying optional features needs elevation; a host that cannot answer - # cannot be certified as WHP-capable. - Exit-WithError "Unable to query the HypervisorPlatform feature: $($_.Exception.Message)" - } - - if ($null -eq $feature -or $feature.State -ne 'Enabled') { - Exit-WithError 'Windows Hypervisor Platform is not enabled. This backend requires WHP.' - } + Assert-RequiredFeature -Name 'HypervisorPlatform' ` + -Remedy 'This backend requires Windows Hypervisor Platform on the runner image.' $computerSystem = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue if ($null -eq $computerSystem -or -not $computerSystem.HypervisorPresent) { @@ -139,25 +252,11 @@ function Initialize-ProcessContainerHost { } function Initialize-MicroVmHost { - # NanVix binaries are pinned GitHub release assets fetched and checksum- - # verified at build time, so the build either staged all of them or failed. - # Their absence here means a broken artifact, not a host problem. - Assert-RequiredFile @( - 'wxc-exec.exe', - 'nanvixd.exe', - 'nanvix_rootfs.img', - 'python3.initrd', - 'bin\kernel.elf' - ) - - # The WHP snapshots are a warm-start cache generated by running nanvixd on - # an x86_64 build host, not a release asset. The build legitimately skips - # them (cross-arch or offline NANVIX_BIN builds) and the runner cold-boots - # when they are absent, so report the boot path instead of failing. - Write-SnapshotAvailability + Assert-RequiredFile @('wxc-exec.exe') + Install-NanvixBinaries - # NanVix boots a VM from these images on every invocation; Defender - # scanning them can push boot past its timeout. + # NanVix boots a VM from these images on every invocation; Defender scanning + # them can push boot past its timeout. Add-MpPreference -ExclusionPath $BinaryDirectory Write-Host "Added Defender exclusion for $BinaryDirectory" @@ -165,6 +264,43 @@ function Initialize-MicroVmHost { Assert-HypervisorPlatform } +# WSL2 is baked into the pool image, so this verifies rather than installs: +# enabling a feature needs a reboot the runner cannot take mid-job. Container +# images are pulled by the suite itself (tests/scripts/run_wslc_all_tests.ps1). +function Initialize-WslcHost { + # wslcsdk.dll ships beside wxc-exec.exe only in a --features wslc build. + Assert-RequiredFile @('wxc-exec.exe', 'wslcsdk.dll') + + Assert-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform' ` + -Remedy 'WSL2 must be baked into the runner image; enabling these features requires a host reboot this job cannot take.' + + if ((Invoke-Wsl @('--version')) -ne 0) { + Exit-WithError 'wsl --version failed; the WSL2 runtime is not usable on this runner.' + } + + # Diagnostic only: --status exits non-zero with no distribution installed, + # which is expected since WSLC creates its own containers via the SDK. + Invoke-Wsl @('--status') | Out-Null +} + +# wsl.exe emits UTF-16LE, which the default console encoding renders as +# null-separated garbage. Returns the exit code. +function Invoke-Wsl { + param([Parameter(Mandatory)][string[]]$Arguments) + + $previousEncoding = [Console]::OutputEncoding + try { + [Console]::OutputEncoding = [System.Text.Encoding]::Unicode + & wsl.exe @Arguments 2>&1 | Write-Host + return $LASTEXITCODE + } catch { + Write-Host "WARNING: wsl.exe $($Arguments -join ' ') could not be run: $($_.Exception.Message)" + return 1 + } finally { + [Console]::OutputEncoding = $previousEncoding + } +} + if (-not (Test-Path $BinaryDirectory)) { Exit-WithError "Binary directory not found: $BinaryDirectory" } @@ -175,5 +311,6 @@ Write-Host "Preparing Windows host for backend '$Backend' using $BinaryDirectory switch ($Backend) { 'process-t3' { Initialize-ProcessContainerHost } 'microvm' { Initialize-MicroVmHost } + 'wslc' { Initialize-WslcHost } default { Write-Host "$Backend has no artifact-only Windows test prerequisites yet." } } diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 2c71420e5..d473eec1d 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -492,19 +492,22 @@ { "os": "windows-25h2", "backends": [ - "microvm" + "microvm", + "wslc" ] }, { "os": "windows-24h2", "backends": [ - "microvm" + "microvm", + "wslc" ] }, { "os": "windows-23h2", "backends": [ - "microvm" + "microvm", + "wslc" ] } ] diff --git a/tests/scripts/run_microvm_tests.ps1 b/tests/scripts/run_microvm_tests.ps1 index dcf6002b5..ffe4c37f7 100644 --- a/tests/scripts/run_microvm_tests.ps1 +++ b/tests/scripts/run_microvm_tests.ps1 @@ -21,16 +21,30 @@ .PARAMETER ConfigDir Path to test configs directory. Defaults to \tests\configs +.PARAMETER NanvixBin + Offline mode. Directory of pre-fetched NanVix binaries to stage next to + wxc-exec.exe instead of relying on a --features microvm build. Contents are + checksum-verified; snapshots inside it are ignored (see + docs/nanvix-microvm/nanvix.md). Off unless explicitly passed. + +.PARAMETER ColdStart + Give each test a fresh NANVIX_HOME so the VM cold-boots every run instead of + reusing a warm-start snapshot. Off by default. + .EXAMPLE .\run_microvm_tests.ps1 .\run_microvm_tests.ps1 -Release .\run_microvm_tests.ps1 -BinDir C:\build\output + .\run_microvm_tests.ps1 -NanvixBin $env:NANVIX_BIN + .\run_microvm_tests.ps1 -ColdStart #> param( [switch]$Release, [string]$BinDir, - [string]$ConfigDir + [string]$ConfigDir, + [string]$NanvixBin, + [switch]$ColdStart ) $ErrorActionPreference = "Stop" @@ -86,15 +100,57 @@ if (-not (Test-Path $WxcExePath)) { $wxcExe = Resolve-Path $WxcExePath +# -- Offline mode ------------------------------------------------------------- + +# Stage pre-fetched binaries next to wxc-exec.exe. The runner resolves them from +# its own directory only, so NANVIX_BIN cannot be used in place. +if ($NanvixBin) { + if (-not (Test-Path $NanvixBin)) { + Write-Host "ERROR: NANVIX_BIN directory not found: $NanvixBin" -ForegroundColor Red + exit 1 + } + + $checksumsPath = Join-Path $RepoRoot "src\backends\nanvix\binaries\checksums.json" + $checksums = (Get-Content $checksumsPath -Raw | ConvertFrom-Json).windows + Write-Host "Offline mode: staging NanVix binaries from $NanvixBin" -ForegroundColor Cyan + + foreach ($rel in @("nanvixd.exe", "nanvix_rootfs.img", "python3.initrd", "bin\kernel.elf")) { + $source = Join-Path $NanvixBin $rel + if (-not (Test-Path $source)) { + Write-Host "ERROR: $rel missing from $NanvixBin" -ForegroundColor Red + exit 1 + } + + $expected = $checksums.($rel | Split-Path -Leaf) + $actual = (Get-FileHash -Path $source -Algorithm SHA256).Hash + if ($actual -ne $expected) { + Write-Host "ERROR: checksum mismatch for ${rel}: expected $expected, got $actual" -ForegroundColor Red + exit 1 + } + + $destination = Join-Path $BinDir $rel + New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null + Copy-Item $source $destination -Force + Write-Host " $rel staged and verified" + } + + # Snapshots in NANVIX_BIN are not covered by checksums.json, so they are + # never staged; drop any stale ones so no unverified image is warm-booted. + Remove-Item (Join-Path $BinDir "snapshots") -Recurse -Force -ErrorAction SilentlyContinue +} + # -- Verify MicroVM binaries -------------------------------------------------- +# Local runs get these from `cargo build --features microvm`, which stages them +# next to wxc-exec.exe. CI gets them from scripts/ci/prepare-windows-host.ps1, +# which downloads and checksum-verifies the pinned release, so this check is a +# guard for the local path. Snapshots are excluded: they are a warm-start cache +# the runner generates on demand, not a shipped artifact. $requiredBinaries = @( "nanvixd.exe", "nanvix_rootfs.img", "python3.initrd", - "bin\kernel.elf", - "snapshots\kernel.vmem", - "snapshots\kernel.whp.cbor" + "bin\kernel.elf" ) $binDir = Split-Path $wxcExe $missing = $requiredBinaries | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } @@ -136,15 +192,37 @@ foreach ($test in $tests) { Write-Host "`n--- $($test.Description) ($($test.Config)) ---" -ForegroundColor White + # A fresh NANVIX_HOME per test means no snapshot is carried over, so the VM + # cold-boots every run. The runner's 60s boot grace covers the extra time. + $nanvixHome = $null + $previousNanvixHome = $env:NANVIX_HOME + if ($ColdStart) { + $nanvixHome = Join-Path ([System.IO.Path]::GetTempPath()) "mxc-nanvix-$([guid]::NewGuid().ToString('N'))" + New-Item -ItemType Directory -Force -Path $nanvixHome | Out-Null + $env:NANVIX_HOME = $nanvixHome + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() $stdoutFile = [System.IO.Path]::GetTempFileName() $stderrFile = [System.IO.Path]::GetTempFileName() - $process = Start-Process -FilePath $wxcExe ` - -ArgumentList "--debug", "--experimental", $configPath ` - -PassThru -Wait ` - -RedirectStandardOutput $stdoutFile ` - -RedirectStandardError $stderrFile - $sw.Stop() + try { + $process = Start-Process -FilePath $wxcExe ` + -ArgumentList "--debug", "--experimental", $configPath ` + -PassThru -Wait ` + -RedirectStandardOutput $stdoutFile ` + -RedirectStandardError $stderrFile + } finally { + $sw.Stop() + if ($nanvixHome) { + if ($null -eq $previousNanvixHome) { + Remove-Item Env:\NANVIX_HOME -ErrorAction SilentlyContinue + } else { + $env:NANVIX_HOME = $previousNanvixHome + } + # Snapshots are large; do not let them accumulate across tests. + Remove-Item $nanvixHome -Recurse -Force -ErrorAction SilentlyContinue + } + } $actualExit = $process.ExitCode $expectedExit = $test.ExpectedExit From c8bdb8a92ec94adfa3eea8d452e4f593588b12f9 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:47:37 -0700 Subject: [PATCH 60/93] fix incorrect pwsh ver call --- .github/actions/setup-cargo-feed/action.yml | 2 +- .github/workflows/Validation.Tests.WSLC.yml | 228 -------------------- 2 files changed, 1 insertion(+), 229 deletions(-) delete mode 100644 .github/workflows/Validation.Tests.WSLC.yml diff --git a/.github/actions/setup-cargo-feed/action.yml b/.github/actions/setup-cargo-feed/action.yml index c0de28fb3..eb6d73459 100644 --- a/.github/actions/setup-cargo-feed/action.yml +++ b/.github/actions/setup-cargo-feed/action.yml @@ -5,6 +5,6 @@ description: >- runs: using: composite steps: - - shell: powershell + - shell: pwsh working-directory: ${{ github.workspace }} run: Get-Content .azure-pipelines\.cargo\config.public.toml | Add-Content -Path .cargo\config.toml diff --git a/.github/workflows/Validation.Tests.WSLC.yml b/.github/workflows/Validation.Tests.WSLC.yml deleted file mode 100644 index 5b2530128..000000000 --- a/.github/workflows/Validation.Tests.WSLC.yml +++ /dev/null @@ -1,228 +0,0 @@ -# ============================================================================ -# WSLC E2E Tests — executes the WSL Container backend on a 1ES nested-virt -# runner. Unlike microvm/hyperlight (which run on GitHub-hosted windows-latest -# and skip when WHP is absent), WSLC needs WSL2 — which requires nested -# virtualization — so it can only EXECUTE on a nested-virt-capable 1ES pool. -# -# Pool: 1es-mxc-e2e-windows-25h2-wsl2 (REPOSITORY-level, microsoft/mxc). Its VMs -# boot from the windows-25h2-wsl2 1ES Managed Image, which bakes WSL2 in (see -# below). `shell: powershell` for parity with the sibling client images. Cold- -# start pickup on the idle Stateless pool is ~10-11 min — the long timeout -# accounts for that plus an in-image toolchain/WSL setup on first run. -# -# IMAGE PREREQUISITES (see docs/wsl/wsl-container-getting-started.md): -# The pool's windows-25h2-wsl2 Managed Image is win11-25h2-pro with a 1ES -# provisioning script (.azure-pipelines/1es-images/wsl2/Setup.ps1) that enables -# Microsoft-Windows-Subsystem-Linux + VirtualMachinePlatform at image-build -# time; 1ES's mandatory post-provisioning reboot activates them, so runner VMs -# boot WSL2-ready (the DISM-at-image-build approach microsoft/ebpf-for-windows -# uses for Hyper-V). The job still installs git + the Rust/MSVC toolchain and -# finishes WSL2 runtime setup (`wsl --update`), and builds the WSLC SDK via a -# `--features wslc` build. Every runtime-dependent step below is guarded so the -# job degrades to a clean SKIP rather than a hard failure if a prerequisite is -# missing. -# ============================================================================ -name: WSLC E2E Tests - -on: - push: - branches: - - user/modanish/1es-github-runner-e2e - # - user/emichlin/validation-infra-lite - workflow_dispatch: - -permissions: - contents: read - -jobs: - wslc-e2e: - name: WXC-Exec WSLC - # Default every `run:` step to Windows PowerShell 5.1 (always present on the - # image). The Windows default shell is pwsh, but the bare client image does - # not reliably ship pwsh on PATH, so steps without an explicit shell (the - # cargo build/test steps) would otherwise fail. - defaults: - run: - shell: powershell - runs-on: - - self-hosted - - "1ES.Pool=1es-mxc-e2e-windows-25h2-wsl2" - - "JobId=wslc-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" - timeout-minutes: 45 - - steps: - - uses: actions/checkout@v4 - - # Image verification: report WSL2 optional-feature state BEFORE the build, - # so we confirm whether the 1ES Managed Image baked WSL2 in even if later - # build/test steps fail for unrelated reasons (e.g. missing MSVC linker). - # Best-effort: never fails the job. - - name: Report WSL2 feature state (image verification) - continue-on-error: true - shell: powershell - run: | - $ErrorActionPreference = 'Continue' - Write-Host "=== Windows optional features (baked into image?) ===" - foreach ($f in 'Microsoft-Windows-Subsystem-Linux','VirtualMachinePlatform') { - try { - $state = (Get-WindowsOptionalFeature -Online -FeatureName $f -ErrorAction Stop).State - } catch { - $state = "query-failed: $($_.Exception.Message)" - } - Write-Host (" {0} = {1}" -f $f, $state) - } - Write-Host "=== wsl.exe presence + status ===" - $wsl = Get-Command wsl.exe -ErrorAction SilentlyContinue - if ($wsl) { - Write-Host "wsl.exe: $($wsl.Source)" - wsl --status 2>&1 | Write-Host - wsl --version 2>&1 | Write-Host - Write-Host "wsl --status exit code: $LASTEXITCODE" - } else { - Write-Host "wsl.exe NOT found on PATH" - } - # Never fail the job on this diagnostic step (wsl --status exits non-zero - # when WSL isn't installed, which would otherwise fail the PowerShell step). - exit 0 - - # Stateful-pool WSL2 preparation. This pool is agentProfile.kind=Stateful, so - # the same VM persists across jobs (maxAgentLifetime 7d). WSL2's optional - # features are NOT in the base image, so the FIRST job on a fresh VM enables - # them via DISM and reboots — a normal running-VM reboot finalizes the - # pending features (no sysprep involved, unlike image capture). That reboot - # ends this job (expected). A SUBSEQUENT job on the same persisted VM sees - # the features Enabled and proceeds. This sidesteps the ephemeral-image - # sysprep-reset problem entirely. - - name: Prepare WSL2 on stateful VM (enable + reboot if needed) - id: prep - shell: powershell - run: | - $ErrorActionPreference = 'Stop' - $features = @('Microsoft-Windows-Subsystem-Linux','VirtualMachinePlatform') - $disabled = @() - foreach ($f in $features) { - $state = (Get-WindowsOptionalFeature -Online -FeatureName $f).State - Write-Host ("{0} = {1}" -f $f, $state) - if ($state -ne 'Enabled') { $disabled += $f } - } - if ($disabled.Count -eq 0) { - Write-Host 'WSL2 features already Enabled on this persisted VM. Proceeding.' - 'ready=true' | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - exit 0 - } - $isAdmin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) - Write-Host ("Running elevated: {0}" -f $isAdmin) - foreach ($f in $disabled) { - Write-Host ("Enabling {0} ..." -f $f) - $proc = Start-Process dism.exe -ArgumentList '/online','/enable-feature',"/featurename:$f",'/all','/norestart' -Wait -PassThru -NoNewWindow - Write-Host (" dism exit {0}" -f $proc.ExitCode) - if (@(0,3010) -notcontains $proc.ExitCode) { throw ("dism enable {0} failed ({1})" -f $f, $proc.ExitCode) } - } - 'ready=false' | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - Write-Host '::warning::WSL2 features enabled on this stateful VM; rebooting to finalize. This job will end; re-run the workflow and the next job on this persisted VM will have WSL2 ready.' - shutdown /r /t 15 /c "WSL2 feature finalize reboot" - Start-Sleep -Seconds 180 - - # # setup-rust-toolchain auto-reads rust-toolchain.toml from the repo root - # # only. Surface src/rust-toolchain.toml there so the pinned toolchain is - # # honored. Use PowerShell (bare client image may lack bash). - # - name: Surface toolchain file at repo root - # if: steps.prep.outputs.ready == 'true' - # shell: powershell - # run: Copy-Item src\rust-toolchain.toml rust-toolchain.toml -Force - - # # The bare 1ES client image has no rustup/cargo and no bash, so we can't - # # use actions-rust-lang/setup-rust-toolchain (it runs internal bash steps - # # and fails with "bash: command not found"). Install rustup directly in - # # PowerShell; rust-toolchain.toml (surfaced at repo root above) pins - # # channel 1.93, which rustup installs and selects automatically. - # - name: Install Rust toolchain - # if: steps.prep.outputs.ready == 'true' - # shell: powershell - # run: | - # $ErrorActionPreference = 'Stop' - # if (-not (Get-Command rustup -ErrorAction SilentlyContinue)) { - # Write-Host "Installing rustup..." - # $init = Join-Path $env:RUNNER_TEMP 'rustup-init.exe' - # Invoke-WebRequest -Uri 'https://win.rustup.rs/x86_64' -OutFile $init -UseBasicParsing - # & $init -y --default-toolchain none --profile minimal - # if ($LASTEXITCODE -ne 0) { throw "rustup-init failed ($LASTEXITCODE)" } - # $cargoBin = Join-Path $env:USERPROFILE '.cargo\bin' - # $cargoBin | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8 - # $env:Path = "$cargoBin;$env:Path" - # } - # # Install the exact channel pinned by rust-toolchain.toml (repo root), - # # reading it from the file so there is no drift with the pin. - # $chan = ([regex]::Match((Get-Content rust-toolchain.toml -Raw), 'channel\s*=\s*"([^"]+)"')).Groups[1].Value - # if (-not $chan) { throw "could not parse channel from rust-toolchain.toml" } - # Write-Host "Pinned Rust channel: $chan" - # rustup toolchain install $chan --profile minimal - # if ($LASTEXITCODE -ne 0) { throw "rustup toolchain install $chan failed ($LASTEXITCODE)" } - # rustup target add --toolchain $chan x86_64-pc-windows-msvc - # if ($LASTEXITCODE -ne 0) { throw "rustup target add failed ($LASTEXITCODE)" } - # rustc "+$chan" --version - # cargo "+$chan" --version - - # - name: Point cargo at the MxcDependencies feed - # if: steps.prep.outputs.ready == 'true' - # uses: ./.github/actions/setup-cargo-feed - - # - uses: Swatinem/rust-cache@v2 - # if: steps.prep.outputs.ready == 'true' - # with: - # workspaces: src -> target - # key: wslc-e2e-v1 - - # - name: Build with WSLC support - # if: steps.prep.outputs.ready == 'true' - # working-directory: src - # run: cargo build --release --target x86_64-pc-windows-msvc - # --no-default-features --features wslc - - - name: Ensure WSL2 runtime - id: wsl - if: steps.prep.outputs.ready == 'true' - shell: powershell - run: | - $ErrorActionPreference = 'Continue' - Write-Host "=== WSL runtime check ===" - # Best-effort update; do not fail the job if the update path is - # unavailable on this image. - wsl --update 2>&1 | Write-Host - wsl --version 2>&1 | Write-Host - wsl --status 2>&1 | Write-Host - $available = ($LASTEXITCODE -eq 0) - Write-Host "wsl_available = $available" - "wsl_available=$($available.ToString().ToLower())" | - Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - - - name: Pre-pull WSLC images - if: steps.wsl.outputs.wsl_available == 'true' - shell: powershell - run: | - $exe = "src\target\x86_64-pc-windows-msvc\release\wxc-exec.exe" - ./scripts/setup-wslc.ps1 -Image alpine:latest, python:3.12-alpine -WxcExecPath $exe - - - name: Run WSLC E2E suite - if: steps.wsl.outputs.wsl_available == 'true' - working-directory: src - run: cargo test --release --target x86_64-pc-windows-msvc - -p wxc_e2e_tests test_wslc_suite -- --nocapture - - - name: Note skip when WSL unavailable - if: steps.wsl.outputs.wsl_available != 'true' - shell: powershell - run: | - Write-Host "::warning::WSL2 runtime not available on this runner image — WSLC suite skipped." - Write-Host "The raw client image needs WSL2 + VirtualMachinePlatform baked in (1ES Managed Image)." - - - name: Upload logs on failure - if: failure() || cancelled() - uses: actions/upload-artifact@v4 - with: - name: wslc-e2e-logs-${{ github.event.pull_request.number || github.run_number }} - retention-days: 7 - path: | - logs/ - **/*.log - wslc-perf-results.json From de534cf7ec8b2b3c7a26fc43ed9d2a80e822b3e6 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:45:58 -0700 Subject: [PATCH 61/93] removed wslc cargo tests --- .github/copilot-instructions.md | 4 +- docs/wsl/wsl-container-getting-started.md | 49 ---- src/testing/wxc_e2e_tests/src/lib.rs | 45 ---- src/testing/wxc_e2e_tests/tests/e2e_wslc.rs | 263 -------------------- 4 files changed, 3 insertions(+), 358 deletions(-) delete mode 100644 src/testing/wxc_e2e_tests/tests/e2e_wslc.rs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 27119dcb7..0c44345ac 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -105,7 +105,9 @@ tests\scripts\run_bwrap_all_tests.sh # All Bubblewrap tests (Linux, req # E2E test crate — Rust executor integration tests (from src/) cargo test -p wxc_e2e_tests # Invokes MXC binaries directly cargo test -p wxc_e2e_tests -- --ignored # Include stress tests (run_on_repeat) -cargo test -p wxc_e2e_tests test_wslc_suite # WSLC suite — skips unless wslcsdk.dll (--features wslc) + WSL2 present; only EXECUTES on a nested-virt runner (1ES). CI: .github/workflows/wslc-e2e.yml + +# WSLC has no cargo E2E suite — it is covered by tests\scripts\run_wslc_all_tests.ps1, +# which the validation matrix runs via tests\scripts\run_ci_backend_tests.ps1. ``` ## Architecture diff --git a/docs/wsl/wsl-container-getting-started.md b/docs/wsl/wsl-container-getting-started.md index d22153785..cbdf9e6a8 100644 --- a/docs/wsl/wsl-container-getting-started.md +++ b/docs/wsl/wsl-container-getting-started.md @@ -368,55 +368,6 @@ Paths in `filesystem.readwritePaths` and `filesystem.readonlyPaths` are mounted into the container. Host path `C:\workspace` becomes `/mnt/c/workspace` inside the container. -## Running the E2E test suite - -MXC ships a Rust E2E suite for WSLC in -[`src/testing/wxc_e2e_tests/tests/e2e_wslc.rs`](../../src/testing/wxc_e2e_tests/tests/e2e_wslc.rs), -mirroring the MicroVM and Hyperlight suites. It invokes `wxc-exec.exe` directly -against the core smoke configs (hello-world, exit-code, python-stdlib, -network-isolated, large-output). - -```powershell -# From src/ — build with WSLC support first so wslcsdk.dll is present: -cargo build --release --features wslc --target x86_64-pc-windows-msvc - -# Pre-pull the images the smoke set needs: -.\scripts\setup-wslc.ps1 -Image alpine:latest, python:3.12-alpine - -# Run the suite: -cargo test -p wxc_e2e_tests test_wslc_suite -- --nocapture -``` - -The suite **skips gracefully** (each test passes as a no-op) when any -prerequisite is missing, guarded by three checks: - -| Guard | Skips when | -|---|---| -| `has_wxc_exe()` | `wxc-exec.exe` not built | -| `has_wslc_sdk()` | `wslcsdk.dll` not next to the binary (build without `--features wslc`) | -| `has_wsl_runtime()` | `wsl --status` fails (WSL2 not installed) | - -### CI: why WSLC can only run on a nested-virt runner - -WSLC boots Linux containers inside **WSL2**, which requires **nested -virtualization** on the host. GitHub-hosted `windows-latest` runners do **not** -provide nested virt, so — unlike the MicroVM/Hyperlight suites, which run there -and self-skip on the WHP check — the WSLC suite can only *execute* on a -nested-virt-capable runner such as a **1ES hosted pool**. - -The [`wslc-e2e.yml`](../../.github/workflows/wslc-e2e.yml) workflow targets a -1ES pool for this reason. Beyond nested virt, the runner image must provide: - -- **WSL2 with the `VirtualMachinePlatform` optional feature enabled.** Enabling - it normally requires a reboot, which a Stateless (ephemeral) 1ES runner - cannot do mid-job — so the durable path is a **1ES Managed Image** with WSL2 + - `VirtualMachinePlatform` baked in at image-build time (the same DISM approach - `microsoft/ebpf-for-windows` uses to bake in Hyper-V). -- The **Rust/MSVC toolchain and git** (a raw client image is a bare desktop). - -Every runtime-dependent workflow step is gated so the job degrades to a clean -skip rather than a hard failure when a prerequisite is absent. - ## Troubleshooting | Error | Cause | Fix | diff --git a/src/testing/wxc_e2e_tests/src/lib.rs b/src/testing/wxc_e2e_tests/src/lib.rs index e2d1f39d9..cea6f4fb3 100644 --- a/src/testing/wxc_e2e_tests/src/lib.rs +++ b/src/testing/wxc_e2e_tests/src/lib.rs @@ -256,51 +256,6 @@ pub fn has_hyperlight_snapshot() -> bool { } } -/// Return whether the WSLC SDK (`wslcsdk.dll`) is present next to -/// `wxc-exec.exe`. The DLL is copied there by a `--features wslc` build; its -/// absence means the WSLC backend cannot load at runtime, so WSLC E2E tests -/// must skip rather than fail. -pub fn has_wslc_sdk() -> bool { - let Some(exe) = find_binary("wxc-exec.exe") else { - println!("SKIPPED: wxc-exec.exe not found — build with `--features wslc` first"); - return false; - }; - let dll = exe.parent().unwrap_or(Path::new(".")).join("wslcsdk.dll"); - if dll.is_file() { - println!("Using wslcsdk.dll at {}", dll.display()); - true - } else { - println!( - "SKIPPED: wslcsdk.dll not found next to wxc-exec.exe — build with `--features wslc`" - ); - false - } -} - -/// Return whether a usable WSL runtime is available (WSL2 installed and -/// responsive). WSLC boots Linux containers inside WSL2, which requires -/// nested virtualization on the host — unavailable on GitHub-hosted runners, -/// so these tests only execute on a nested-virt-capable (e.g. 1ES) runner. -pub fn has_wsl_runtime() -> bool { - // `wsl.exe --status` exits 0 when the runtime is installed and a default - // distribution/version is configured. On hosts without WSL the command is - // either missing or returns a non-zero status. - let available = Command::new("wsl.exe") - .arg("--status") - .output() - .map(|output| output.status.success()) - .unwrap_or(false); - - if !available { - println!( - "SKIPPED: WSL runtime not available — WSLC requires WSL2 (needs nested virtualization; \ - run `wsl --update`)" - ); - } - - available -} - /// Return whether the Windows Sandbox optional feature is enabled. pub fn has_windows_sandbox_feature() -> bool { let available = Command::new("dism") diff --git a/src/testing/wxc_e2e_tests/tests/e2e_wslc.rs b/src/testing/wxc_e2e_tests/tests/e2e_wslc.rs deleted file mode 100644 index 0b71829ec..000000000 --- a/src/testing/wxc_e2e_tests/tests/e2e_wslc.rs +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -//! WSLC (WSL Container) E2E integration tests. -//! -//! These tests mirror the Windows MicroVM / Hyperlight E2E suites and invoke -//! `wxc-exec.exe` directly with the `wslc` containment backend. WSLC boots -//! Linux containers inside WSL2, which needs nested virtualization on the host -//! (unavailable on GitHub-hosted runners) plus the WSLC SDK (`wslcsdk.dll`, a -//! `--features wslc` build), a WSL2 runtime, and pre-pulled images. Tests skip -//! gracefully when any prerequisite is missing so the suite is a no-op on -//! machines that cannot run it, and only executes on a nested-virt-capable -//! (e.g. 1ES) runner. - -use std::sync::OnceLock; -use std::time::{SystemTime, UNIX_EPOCH}; - -use serde::Serialize; -use wxc_e2e_tests::{ - has_wsl_runtime, has_wslc_sdk, has_wxc_exe, repo_root, run_wxc_config, run_wxc_example, - test_configs_dir, CommandResult, -}; - -static HAS_WXC_EXE: OnceLock = OnceLock::new(); -static HAS_WSLC_SDK: OnceLock = OnceLock::new(); -static HAS_WSL_RUNTIME: OnceLock = OnceLock::new(); - -fn cached_has_wxc_exe() -> bool { - *HAS_WXC_EXE.get_or_init(has_wxc_exe) -} - -fn cached_has_wslc_sdk() -> bool { - *HAS_WSLC_SDK.get_or_init(has_wslc_sdk) -} - -fn cached_has_wsl_runtime() -> bool { - *HAS_WSL_RUNTIME.get_or_init(has_wsl_runtime) -} - -/// Guard: skip test unless the WSLC prerequisites are present. -fn skip_unless_ready() -> bool { - cached_has_wxc_exe() && cached_has_wslc_sdk() && cached_has_wsl_runtime() -} - -// --------------------------------------------------------------------------- -// Individual test (mirrors test_microvm_hello) -// --------------------------------------------------------------------------- - -#[test] -fn test_wslc_hello() { - if !skip_unless_ready() { - return; - } - // The hello-world config lives under tests/examples/, not tests/configs/. - let result = run_wxc_example("wslc_hello_world.json", &["--debug", "--experimental"]); - assert_eq!( - result.code, - Some(0), - "expected exit 0, got {:?}\nstdout: {}\nstderr: {}", - result.code, - result.stdout, - result.stderr - ); - assert!( - result - .combined_output_with_decoded_base64() - .contains("Hello from WSL Container!"), - "output missing greeting\nstdout: {}\nstderr: {}", - result.stdout, - result.stderr - ); -} - -// --------------------------------------------------------------------------- -// Full WSLC smoke suite (mirrors test_microvm_suite) -// --------------------------------------------------------------------------- - -#[derive(Debug)] -struct WslcCase { - config: &'static str, - /// When true the config lives in tests/examples/, otherwise tests/configs/. - from_example: bool, - expected_exit: Option, - description: &'static str, - output_contains: Option<&'static str>, - expect_non_zero: bool, -} - -#[derive(Debug, Serialize)] -struct WslcPerfOutput { - commit: String, - timestamp: String, - results: Vec, -} - -#[derive(Debug, Serialize)] -struct WslcPerfEntry { - test: String, - description: String, - wall_time_ms: u128, - exit_code: Option, - status: String, -} - -#[test] -fn test_wslc_suite() { - if !skip_unless_ready() { - return; - } - wslc_suite(); -} - -fn wslc_suite() { - // Core smoke set. Images required (pre-pull via scripts/setup-wslc.ps1): - // alpine:latest -> hello-world, exit-code, network-isolated, large-output - // python:3.12-alpine -> python-stdlib - let cases = [ - WslcCase { - config: "wslc_hello_world.json", - from_example: true, - expected_exit: Some(0), - description: "Hello world (alpine, uname)", - output_contains: Some("Hello from WSL Container!"), - expect_non_zero: false, - }, - WslcCase { - config: "wslc_exit_code.json", - from_example: false, - expected_exit: Some(42), - description: "Exit code propagation", - output_contains: Some("About to exit with code 42"), - expect_non_zero: false, - }, - WslcCase { - config: "wslc_python_stdlib.json", - from_example: false, - expected_exit: Some(0), - description: "Python stdlib (json, math, hashlib)", - output_contains: Some("pi"), - expect_non_zero: false, - }, - WslcCase { - config: "wslc_network_isolated.json", - from_example: false, - expected_exit: Some(0), - description: "Network isolation (block policy)", - output_contains: Some("Network"), - expect_non_zero: false, - }, - WslcCase { - config: "wslc_large_output.json", - from_example: false, - expected_exit: Some(0), - description: "Large stdout (500 lines)", - output_contains: Some("Large output test complete"), - expect_non_zero: false, - }, - ]; - - let mut perf_entries = Vec::new(); - let mut failures = Vec::new(); - - for case in cases { - let config_path = if case.from_example { - repo_root().join("tests").join("examples").join(case.config) - } else { - test_configs_dir().join(case.config) - }; - if !config_path.exists() { - println!("SKIPPED: config not found: {}", config_path.display()); - continue; - } - - println!("--- {} ({}) ---", case.description, case.config); - let result = if case.from_example { - run_wxc_example(case.config, &["--debug", "--experimental"]) - } else { - run_wxc_config(case.config, &["--debug", "--experimental"]) - }; - - let status = if command_matches(&result, &case) { - "PASS" - } else { - failures.push(format!( - "{} expected {}, got {:?}", - case.config, - expected_exit_description(&case), - result.code - )); - "FAIL" - }; - - perf_entries.push(WslcPerfEntry { - test: case.config.to_string(), - description: case.description.to_string(), - wall_time_ms: result.wall_time_ms, - exit_code: result.code, - status: status.to_string(), - }); - - if status == "FAIL" { - println!( - "--- stdout ---\n{}\n--- stderr ---\n{}", - result.stdout, result.stderr - ); - } else { - println!(" PASS ({} ms)", result.wall_time_ms); - } - } - - write_wslc_perf_results(perf_entries); - - if !failures.is_empty() { - panic!("WSLC E2E failures:\n{}", failures.join("\n")); - } -} - -fn command_matches(result: &CommandResult, case: &WslcCase) -> bool { - if case.expect_non_zero { - if result.code == Some(0) { - return false; - } - } else if result.code != case.expected_exit { - return false; - } - - let Some(expected) = case.output_contains else { - return true; - }; - - result - .combined_output_with_decoded_base64() - .contains(expected) -} - -fn expected_exit_description(case: &WslcCase) -> String { - if case.expect_non_zero { - "non-zero exit".to_string() - } else { - format!("exit {}", case.expected_exit.unwrap_or(0)) - } -} - -// --------------------------------------------------------------------------- -// Perf results output -// --------------------------------------------------------------------------- - -fn write_wslc_perf_results(results: Vec) { - let output = WslcPerfOutput { - commit: std::env::var("GITHUB_SHA").unwrap_or_else(|_| "local".to_string()), - timestamp: SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_secs().to_string()) - .unwrap_or_else(|_| "unknown".to_string()), - results, - }; - let json = - serde_json::to_string_pretty(&output).expect("wslc performance results should serialize"); - let path = repo_root().join("wslc-perf-results.json"); - std::fs::write(&path, json) - .unwrap_or_else(|error| panic!("failed to write {}: {error}", path.display())); - println!("Performance results written to {}", path.display()); -} From af7134b4c71c90d6354960df4bc5c9ebf2f9837e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:15:39 -0700 Subject: [PATCH 62/93] installing wsl --- scripts/ci/prepare-windows-host.ps1 | 18 ++++++++++++++---- scripts/ci/validation-test-matrix.json | 12 +++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index f254597c0..a7a7fa89e 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -264,9 +264,10 @@ function Initialize-MicroVmHost { Assert-HypervisorPlatform } -# WSL2 is baked into the pool image, so this verifies rather than installs: -# enabling a feature needs a reboot the runner cannot take mid-job. Container -# images are pulled by the suite itself (tests/scripts/run_wslc_all_tests.ps1). +# The optional features must be baked into the pool image (enabling one needs a +# reboot this job cannot take), but the WSL runtime package is installed here if +# missing. Container images are pulled by the suite itself +# (tests/scripts/run_wslc_all_tests.ps1). function Initialize-WslcHost { # wslcsdk.dll ships beside wxc-exec.exe only in a --features wslc build. Assert-RequiredFile @('wxc-exec.exe', 'wslcsdk.dll') @@ -274,8 +275,17 @@ function Initialize-WslcHost { Assert-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform' ` -Remedy 'WSL2 must be baked into the runner image; enabling these features requires a host reboot this job cannot take.' + # The optional features can be enabled while the WSL runtime package itself + # is absent. Installing that package needs no reboot once the features are + # on, so it is safe to do mid-job. if ((Invoke-Wsl @('--version')) -ne 0) { - Exit-WithError 'wsl --version failed; the WSL2 runtime is not usable on this runner.' + Write-Host 'WSL runtime not installed; installing (features are already enabled, so no reboot is needed)...' + if ((Invoke-Wsl @('--install', '--no-distribution')) -ne 0) { + Exit-WithError 'wsl --install failed; the WSL2 runtime could not be installed on this runner.' + } + if ((Invoke-Wsl @('--version')) -ne 0) { + Exit-WithError 'wsl --version still fails after install; the WSL2 runtime is not usable on this runner.' + } } # Diagnostic only: --status exits non-zero with no distribution installed, diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index d473eec1d..c957b0a3e 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -2,15 +2,15 @@ "schemaVersion": 1, "platforms": [ { - "id": "windows-prerelease-a", - "displayName": "Windows Pre-release A", + "id": "windows-prerelease-t1-process-container", + "displayName": "Windows Prerelease T1 Process Container", "family": "windows", "prerelease": true, "architectures": { "x64": { "target": "x86_64-pc-windows-msvc", "artifact": "wxc-binaries-x86_64-pc-windows-msvc", - "pool": "", + "pool": "1es-mxc-windows-prerelease-t1-x64", "backends": [ "process-t1", "isolation-session", @@ -489,6 +489,12 @@ } ], "enabled": [ + { + "os": "windows-prerelease-t1-process-container", + "backends": [ + "process-t3" + ] + }, { "os": "windows-25h2", "backends": [ From 5430ce525555092ad2bdf2626e9ded798f383bb0 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:17:38 -0700 Subject: [PATCH 63/93] disabling support for microvm, for now --- scripts/ci/validation-test-matrix.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index c957b0a3e..9b3912915 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -498,21 +498,18 @@ { "os": "windows-25h2", "backends": [ - "microvm", "wslc" ] }, { "os": "windows-24h2", "backends": [ - "microvm", "wslc" ] }, { "os": "windows-23h2", "backends": [ - "microvm", "wslc" ] } From 3686d6ba38f0c4725c28695ba7873f5a2dc8524f Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:25:20 -0700 Subject: [PATCH 64/93] prerelease windows tests --- scripts/ci/validation-test-matrix.json | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 9b3912915..537be2285 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "platforms": [ { - "id": "windows-prerelease-t1-process-container", + "id": "windows-prerelease-process-container", "displayName": "Windows Prerelease T1 Process Container", "family": "windows", "prerelease": true, @@ -13,6 +13,7 @@ "pool": "1es-mxc-windows-prerelease-t1-x64", "backends": [ "process-t1", + "process-t3", "isolation-session", "wslc", "windows-sandbox", @@ -26,6 +27,7 @@ "pool": "", "backends": [ "process-t1", + "process-t3", "isolation-session", "wslc", "windows-sandbox" @@ -45,6 +47,7 @@ "pool": "", "backends": [ "process-t1", + "process-t3", "isolation-session", "wslc", "windows-sandbox", @@ -58,6 +61,7 @@ "pool": "", "backends": [ "process-t1", + "process-t3", "isolation-session", "wslc", "windows-sandbox" @@ -76,7 +80,12 @@ "pool": "", "backends": [ "process-t1", - "isolation-session" + "process-t3", + "isolation-session", + "wslc", + "windows-sandbox", + "microvm", + "hyperlight" ] }, "arm64": { @@ -85,7 +94,10 @@ "pool": "", "backends": [ "process-t1", - "isolation-session" + "process-t3", + "isolation-session", + "wslc", + "windows-sandbox" ] } } @@ -368,7 +380,7 @@ ] }, { - "os": "windows-prerelease-a", + "os": "windows-prerelease-process-container", "backends": [ "process-t1", "isolation-session", @@ -490,9 +502,11 @@ ], "enabled": [ { - "os": "windows-prerelease-t1-process-container", + "os": "windows-prerelease-process-container", "backends": [ - "process-t3" + "process-t1", + "process-t3", + "isolation-session" ] }, { From 57b2f42aae1298264ed92cb54fdfb80a724500ca Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:03:48 -0700 Subject: [PATCH 65/93] added wsl install, modified isolation session test script --- .../workflows/Validation.Tests.Matrix.Job.yml | 2 - scripts/ci/prepare-windows-host.ps1 | 115 ++---------------- tests/scripts/run_microvm_tests.ps1 | 98 ++------------- 3 files changed, 20 insertions(+), 195 deletions(-) diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 74ef5ac39..f9bb7a448 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -50,8 +50,6 @@ jobs: - name: Prepare backend prerequisites shell: pwsh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | & ./scripts/ci/prepare-windows-host.ps1 ` -Backend '${{ matrix.backend }}' ` diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index a7a7fa89e..74354a3be 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -57,109 +57,6 @@ function Assert-RequiredFile { Get-ChildItem $BinaryDirectory -Include $leaves -Recurse | Format-Table FullName, Length } -# Fetch the pinned NanVix release onto the test host, mirroring the offline-build -# contract in docs/nanvix-microvm/nanvix.md: flat binaries plus bin/, verified -# against checksums.json, and no snapshots (the runtime cold-boots and -# regenerates a verified one on first use). -function Install-NanvixBinaries { - $configDir = Join-Path $PSScriptRoot '..\..\src\backends\nanvix\binaries' - $versionsPath = Join-Path $configDir 'versions.json' - $checksumsPath = Join-Path $configDir 'checksums.json' - foreach ($path in $versionsPath, $checksumsPath) { - if (-not (Test-Path $path)) { - Exit-WithError "NanVix pin file not found: $path" - } - } - - $release = (Get-Content $versionsPath -Raw | ConvertFrom-Json).nanvix_python - $checksums = (Get-Content $checksumsPath -Raw | ConvertFrom-Json).windows - $prefix = [System.IO.Path]::GetFileNameWithoutExtension($release.asset) - $binSubdir = Join-Path $BinaryDirectory 'bin' - - # The build VM's copies are discarded: its snapshots are a host-specific WHP - # memory image, and a partial set would silently cost a cold boot per run. - Write-Host 'Discarding build-staged NanVix binaries in favor of the pinned release.' - Remove-Item (Join-Path $BinaryDirectory 'snapshots') -Recurse -Force -ErrorAction SilentlyContinue - foreach ($name in $release.binaries) { - Remove-Item (Join-Path $BinaryDirectory $name) -Force -ErrorAction SilentlyContinue - } - Remove-Item (Join-Path $binSubdir 'kernel.elf') -Force -ErrorAction SilentlyContinue - - $url = "https://github.com/nanvix/nanvix-python/releases/download/$($release.tag)/$($release.asset)" - $archive = Join-Path ([System.IO.Path]::GetTempPath()) $release.asset - Write-Host "Downloading nanvix/nanvix-python $($release.tag)..." - - try { - $curlArgs = @( - '--silent', '--show-error', '--fail', '--location', - '--retry', '5', '--retry-delay', '5', '--retry-all-errors', - '--output', $archive - ) - $token = if ($env:GITHUB_TOKEN) { $env:GITHUB_TOKEN } else { $env:GH_TOKEN } - if ($token) { - $curlArgs += @('--header', "Authorization: Bearer $token") - } - $curlArgs += $url - - & curl.exe @curlArgs - if ($LASTEXITCODE -ne 0) { - Exit-WithError "curl failed for $url (exit code $LASTEXITCODE)" - } - - New-Item -ItemType Directory -Force -Path $binSubdir | Out-Null - - # nanvixd.exe and kernel.elf live under /bin/; the rest at /. - Expand-NanvixEntry -Archive $archive -Entry "$prefix/bin/nanvixd.exe" -StripComponents 2 -Destination $BinaryDirectory - Expand-NanvixEntry -Archive $archive -Entry "$prefix/bin/kernel.elf" -StripComponents 2 -Destination $binSubdir - foreach ($name in $release.binaries | Where-Object { $_ -ne 'nanvixd.exe' }) { - Expand-NanvixEntry -Archive $archive -Entry "$prefix/$name" -StripComponents 1 -Destination $BinaryDirectory - } - } finally { - Remove-Item $archive -Force -ErrorAction SilentlyContinue - } - - Assert-NanvixChecksum -Path (Join-Path $binSubdir 'kernel.elf') -Expected $checksums.'kernel.elf' - foreach ($name in $release.binaries) { - Assert-NanvixChecksum -Path (Join-Path $BinaryDirectory $name) -Expected $checksums.$name - } - - Write-Host "NanVix $($release.tag) staged and verified; the runner will cold-boot and regenerate its snapshot." -} - -function Expand-NanvixEntry { - param( - [Parameter(Mandatory)][string]$Archive, - [Parameter(Mandatory)][string]$Entry, - [Parameter(Mandatory)][int]$StripComponents, - [Parameter(Mandatory)][string]$Destination - ) - - & tar.exe -xf $Archive -C $Destination --strip-components $StripComponents $Entry - if ($LASTEXITCODE -ne 0) { - Exit-WithError "tar failed to extract $Entry (exit code $LASTEXITCODE)" - } -} - -function Assert-NanvixChecksum { - param( - [Parameter(Mandatory)][string]$Path, - [string]$Expected - ) - - if (-not (Test-Path $Path)) { - Exit-WithError "NanVix binary missing after extraction: $Path" - } - if (-not $Expected) { - Exit-WithError "No pinned checksum for $(Split-Path $Path -Leaf)" - } - - $actual = (Get-FileHash -Path $Path -Algorithm SHA256).Hash - if ($actual -ne $Expected) { - Exit-WithError "Checksum mismatch for $(Split-Path $Path -Leaf): expected $($Expected.ToLowerInvariant()), got $($actual.ToLowerInvariant())" - } - Write-Host " $(Split-Path $Path -Leaf) verified" -} - # Read a Windows optional feature's state without throwing, so both the # diagnostic and assertion paths can share one query. A host that cannot answer # (querying needs elevation) reports the reason as its state rather than @@ -252,8 +149,16 @@ function Initialize-ProcessContainerHost { } function Initialize-MicroVmHost { - Assert-RequiredFile @('wxc-exec.exe') - Install-NanvixBinaries + # Staged next to wxc-exec.exe by the --features microvm build, so their + # absence means a broken artifact rather than a host problem. Snapshots are + # excluded: they are a warm-start cache the runner regenerates on demand. + Assert-RequiredFile @( + 'wxc-exec.exe', + 'nanvixd.exe', + 'nanvix_rootfs.img', + 'python3.initrd', + 'bin\kernel.elf' + ) # NanVix boots a VM from these images on every invocation; Defender scanning # them can push boot past its timeout. diff --git a/tests/scripts/run_microvm_tests.ps1 b/tests/scripts/run_microvm_tests.ps1 index ffe4c37f7..dcf6002b5 100644 --- a/tests/scripts/run_microvm_tests.ps1 +++ b/tests/scripts/run_microvm_tests.ps1 @@ -21,30 +21,16 @@ .PARAMETER ConfigDir Path to test configs directory. Defaults to \tests\configs -.PARAMETER NanvixBin - Offline mode. Directory of pre-fetched NanVix binaries to stage next to - wxc-exec.exe instead of relying on a --features microvm build. Contents are - checksum-verified; snapshots inside it are ignored (see - docs/nanvix-microvm/nanvix.md). Off unless explicitly passed. - -.PARAMETER ColdStart - Give each test a fresh NANVIX_HOME so the VM cold-boots every run instead of - reusing a warm-start snapshot. Off by default. - .EXAMPLE .\run_microvm_tests.ps1 .\run_microvm_tests.ps1 -Release .\run_microvm_tests.ps1 -BinDir C:\build\output - .\run_microvm_tests.ps1 -NanvixBin $env:NANVIX_BIN - .\run_microvm_tests.ps1 -ColdStart #> param( [switch]$Release, [string]$BinDir, - [string]$ConfigDir, - [string]$NanvixBin, - [switch]$ColdStart + [string]$ConfigDir ) $ErrorActionPreference = "Stop" @@ -100,57 +86,15 @@ if (-not (Test-Path $WxcExePath)) { $wxcExe = Resolve-Path $WxcExePath -# -- Offline mode ------------------------------------------------------------- - -# Stage pre-fetched binaries next to wxc-exec.exe. The runner resolves them from -# its own directory only, so NANVIX_BIN cannot be used in place. -if ($NanvixBin) { - if (-not (Test-Path $NanvixBin)) { - Write-Host "ERROR: NANVIX_BIN directory not found: $NanvixBin" -ForegroundColor Red - exit 1 - } - - $checksumsPath = Join-Path $RepoRoot "src\backends\nanvix\binaries\checksums.json" - $checksums = (Get-Content $checksumsPath -Raw | ConvertFrom-Json).windows - Write-Host "Offline mode: staging NanVix binaries from $NanvixBin" -ForegroundColor Cyan - - foreach ($rel in @("nanvixd.exe", "nanvix_rootfs.img", "python3.initrd", "bin\kernel.elf")) { - $source = Join-Path $NanvixBin $rel - if (-not (Test-Path $source)) { - Write-Host "ERROR: $rel missing from $NanvixBin" -ForegroundColor Red - exit 1 - } - - $expected = $checksums.($rel | Split-Path -Leaf) - $actual = (Get-FileHash -Path $source -Algorithm SHA256).Hash - if ($actual -ne $expected) { - Write-Host "ERROR: checksum mismatch for ${rel}: expected $expected, got $actual" -ForegroundColor Red - exit 1 - } - - $destination = Join-Path $BinDir $rel - New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null - Copy-Item $source $destination -Force - Write-Host " $rel staged and verified" - } - - # Snapshots in NANVIX_BIN are not covered by checksums.json, so they are - # never staged; drop any stale ones so no unverified image is warm-booted. - Remove-Item (Join-Path $BinDir "snapshots") -Recurse -Force -ErrorAction SilentlyContinue -} - # -- Verify MicroVM binaries -------------------------------------------------- -# Local runs get these from `cargo build --features microvm`, which stages them -# next to wxc-exec.exe. CI gets them from scripts/ci/prepare-windows-host.ps1, -# which downloads and checksum-verifies the pinned release, so this check is a -# guard for the local path. Snapshots are excluded: they are a warm-start cache -# the runner generates on demand, not a shipped artifact. $requiredBinaries = @( "nanvixd.exe", "nanvix_rootfs.img", "python3.initrd", - "bin\kernel.elf" + "bin\kernel.elf", + "snapshots\kernel.vmem", + "snapshots\kernel.whp.cbor" ) $binDir = Split-Path $wxcExe $missing = $requiredBinaries | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } @@ -192,37 +136,15 @@ foreach ($test in $tests) { Write-Host "`n--- $($test.Description) ($($test.Config)) ---" -ForegroundColor White - # A fresh NANVIX_HOME per test means no snapshot is carried over, so the VM - # cold-boots every run. The runner's 60s boot grace covers the extra time. - $nanvixHome = $null - $previousNanvixHome = $env:NANVIX_HOME - if ($ColdStart) { - $nanvixHome = Join-Path ([System.IO.Path]::GetTempPath()) "mxc-nanvix-$([guid]::NewGuid().ToString('N'))" - New-Item -ItemType Directory -Force -Path $nanvixHome | Out-Null - $env:NANVIX_HOME = $nanvixHome - } - $sw = [System.Diagnostics.Stopwatch]::StartNew() $stdoutFile = [System.IO.Path]::GetTempFileName() $stderrFile = [System.IO.Path]::GetTempFileName() - try { - $process = Start-Process -FilePath $wxcExe ` - -ArgumentList "--debug", "--experimental", $configPath ` - -PassThru -Wait ` - -RedirectStandardOutput $stdoutFile ` - -RedirectStandardError $stderrFile - } finally { - $sw.Stop() - if ($nanvixHome) { - if ($null -eq $previousNanvixHome) { - Remove-Item Env:\NANVIX_HOME -ErrorAction SilentlyContinue - } else { - $env:NANVIX_HOME = $previousNanvixHome - } - # Snapshots are large; do not let them accumulate across tests. - Remove-Item $nanvixHome -Recurse -Force -ErrorAction SilentlyContinue - } - } + $process = Start-Process -FilePath $wxcExe ` + -ArgumentList "--debug", "--experimental", $configPath ` + -PassThru -Wait ` + -RedirectStandardOutput $stdoutFile ` + -RedirectStandardError $stderrFile + $sw.Stop() $actualExit = $process.ExitCode $expectedExit = $test.ExpectedExit From 23ed9e6962b29c19434f1277b074af5767965426 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:04:04 -0700 Subject: [PATCH 66/93] prev. commit, continued --- scripts/ci/prepare-windows-host.ps1 | 40 +++++++++++++------ tests/scripts/run_isolation_session_tests.ps1 | 36 ++++++++++++++--- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index 74354a3be..e11faee83 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -180,16 +180,25 @@ function Initialize-WslcHost { Assert-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform' ` -Remedy 'WSL2 must be baked into the runner image; enabling these features requires a host reboot this job cannot take.' - # The optional features can be enabled while the WSL runtime package itself - # is absent. Installing that package needs no reboot once the features are - # on, so it is safe to do mid-job. - if ((Invoke-Wsl @('--version')) -ne 0) { - Write-Host 'WSL runtime not installed; installing (features are already enabled, so no reboot is needed)...' - if ((Invoke-Wsl @('--install', '--no-distribution')) -ne 0) { - Exit-WithError 'wsl --install failed; the WSL2 runtime could not be installed on this runner.' + # The optional features can be enabled while the modern (Store/MSIX) WSL + # runtime is absent — the image may still carry only the legacy inbox + # wsl.exe. `wsl --version` exists only on the modern runtime, so it doubles + # as the presence probe. Installing needs no reboot once the features are on. + # + # Probe quietly: the legacy wsl.exe answers an unknown switch with its full + # usage text, which would bury the real progress messages below. + if ((Invoke-Wsl @('--version') -Quiet) -ne 0) { + Write-Host 'Modern WSL runtime absent; installing via wsl --update (features are already enabled, so no reboot is needed)...' + + # --web-download avoids the Microsoft Store, which CI images generally + # cannot use. Older wsl.exe builds reject the flag, so retry without it. + if ((Invoke-Wsl @('--update', '--web-download') -Quiet) -ne 0 -and + (Invoke-Wsl @('--update')) -ne 0) { + Exit-WithError 'wsl --update failed; the WSL2 runtime could not be installed on this runner.' } + if ((Invoke-Wsl @('--version')) -ne 0) { - Exit-WithError 'wsl --version still fails after install; the WSL2 runtime is not usable on this runner.' + Exit-WithError 'wsl --version still fails after wsl --update; the WSL2 runtime is not usable on this runner.' } } @@ -199,15 +208,22 @@ function Initialize-WslcHost { } # wsl.exe emits UTF-16LE, which the default console encoding renders as -# null-separated garbage. Returns the exit code. +# null-separated garbage. Returns the exit code. -Quiet suppresses output for +# probes, where the legacy wsl.exe dumps its whole usage text on an unknown +# switch. function Invoke-Wsl { - param([Parameter(Mandatory)][string[]]$Arguments) + param( + [Parameter(Mandatory)][string[]]$Arguments, + [switch]$Quiet + ) $previousEncoding = [Console]::OutputEncoding try { [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - & wsl.exe @Arguments 2>&1 | Write-Host - return $LASTEXITCODE + $output = & wsl.exe @Arguments 2>&1 + $exitCode = $LASTEXITCODE + if (-not $Quiet) { $output | Write-Host } + return $exitCode } catch { Write-Host "WARNING: wsl.exe $($Arguments -join ' ') could not be run: $($_.Exception.Message)" return 1 diff --git a/tests/scripts/run_isolation_session_tests.ps1 b/tests/scripts/run_isolation_session_tests.ps1 index 957edff3c..5d9903f03 100644 --- a/tests/scripts/run_isolation_session_tests.ps1 +++ b/tests/scripts/run_isolation_session_tests.ps1 @@ -137,18 +137,44 @@ function Invoke-StateAwareProbe { @{ Stdout = $out; ExitCode = $LASTEXITCODE } } +# Helper: StrictMode-safe property read; returns $null when absent. This suite +# inherits Set-StrictMode -Version Latest from run_ci_backend_tests.ps1, under +# which touching a missing property is a terminating error — and an error +# envelope has no 'result' (and vice versa). +function Get-EnvelopeProperty { + param($Object, [Parameter(Mandatory)][string]$Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + $probe = Invoke-StateAwareProbe -Request @{ phase = 'provision'; containment = 'isolation_session' } $probeEnv = $null try { $probeEnv = $probe.Stdout | ConvertFrom-Json } catch { } -if ($null -ne $probeEnv -and $probeEnv.error.code -eq 'backend_unavailable') { - Write-Host "SKIPPED: wxc-exec reports backend_unavailable (likely built without --features isolation_session)" -ForegroundColor Yellow + +$probeError = Get-EnvelopeProperty $probeEnv 'error' +if ($null -ne $probeError) { + $code = [string](Get-EnvelopeProperty $probeError 'code') + $message = [string](Get-EnvelopeProperty $probeError 'message') + if ($code -eq 'backend_unavailable') { + Write-Host "SKIPPED: wxc-exec reports backend_unavailable (likely built without --features isolation_session)" -ForegroundColor Yellow + } else { + # Not a known prerequisite signal, so annotate the run: the job stays + # green (per the skip policy above) but the cause stays visible. + Write-Host "::warning::IsolationSession backend probe failed (code=$code): $message" + Write-Host "SKIPPED: IsolationSession provision probe failed (code=$code)" -ForegroundColor Yellow + } exit 0 } + # On a healthy build the probe successfully provisions an agent user -- # deprovision it immediately so the probe doesn't leak. -if ($null -ne $probeEnv -and $null -ne $probeEnv.result -and $null -ne $probeEnv.result.sandboxId) { - $probeSandboxId = [string]$probeEnv.result.sandboxId - $probeAgent = if ($probeEnv.result.metadata) { [string]$probeEnv.result.metadata.agentUserName } else { '' } +$probeResult = Get-EnvelopeProperty $probeEnv 'result' +$probeSandboxId = [string](Get-EnvelopeProperty $probeResult 'sandboxId') +if ($probeSandboxId) { + $probeMetadata = Get-EnvelopeProperty $probeResult 'metadata' + $probeAgent = if ($probeMetadata) { [string](Get-EnvelopeProperty $probeMetadata 'agentUserName') } else { '' } Write-Host "Backend probe: provisioned $probeSandboxId (agentUserName=$probeAgent), deprovisioning ..." -ForegroundColor DarkGray $deprov = Invoke-StateAwareProbe -Request @{ phase = 'deprovision'; sandboxId = $probeSandboxId } if ($deprov.ExitCode -ne 0) { From c3930fae323d7bff8227579daccc477f922a3382 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:30:07 -0700 Subject: [PATCH 67/93] Adjusting WSLC install --- scripts/ci/prepare-windows-host.ps1 | 104 ++++++++++++++++++------ scripts/ci/validation-test-matrix.json | 105 +++---------------------- tests/scripts/run_wslc_all_tests.ps1 | 20 ++++- 3 files changed, 107 insertions(+), 122 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index e11faee83..cf53a2a6d 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -180,58 +180,114 @@ function Initialize-WslcHost { Assert-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform' ` -Remedy 'WSL2 must be baked into the runner image; enabling these features requires a host reboot this job cannot take.' - # The optional features can be enabled while the modern (Store/MSIX) WSL - # runtime is absent — the image may still carry only the legacy inbox - # wsl.exe. `wsl --version` exists only on the modern runtime, so it doubles - # as the presence probe. Installing needs no reboot once the features are on. + # The features can be enabled while the modern (Store/MSIX) WSL runtime is + # absent — the image may still carry only the legacy inbox wsl.exe, which + # has no --version. Installing needs no reboot once the features are on. # # Probe quietly: the legacy wsl.exe answers an unknown switch with its full - # usage text, which would bury the real progress messages below. - if ((Invoke-Wsl @('--version') -Quiet) -ne 0) { - Write-Host 'Modern WSL runtime absent; installing via wsl --update (features are already enabled, so no reboot is needed)...' - + # usage text, which would bury the real progress messages. + if ($null -eq (Get-InstalledWslVersion)) { + Write-Host 'Modern WSL runtime absent; installing via wsl --update...' # --web-download avoids the Microsoft Store, which CI images generally # cannot use. Older wsl.exe builds reject the flag, so retry without it. if ((Invoke-Wsl @('--update', '--web-download') -Quiet) -ne 0 -and (Invoke-Wsl @('--update')) -ne 0) { Exit-WithError 'wsl --update failed; the WSL2 runtime could not be installed on this runner.' } + } - if ((Invoke-Wsl @('--version')) -ne 0) { - Exit-WithError 'wsl --version still fails after wsl --update; the WSL2 runtime is not usable on this runner.' + # WSLC needs a runtime at least as new as the pinned WSLC SDK, and those + # builds ship only on the pre-release ring — the stable ring lands well + # behind it. Without this the SDK fails at run time with + # "WSLC runtime unavailable. Missing components: WslPackage". + $required = Get-RequiredWslVersion + $installed = Get-InstalledWslVersion + if ($null -ne $required -and ($null -eq $installed -or $installed -lt $required)) { + Write-Host "WSL $installed is older than the $required WSLC requires; updating to pre-release..." + if ((Invoke-Wsl @('--update', '--pre-release', '--web-download') -Quiet) -ne 0 -and + (Invoke-Wsl @('--update', '--pre-release')) -ne 0) { + Exit-WithError "wsl --update --pre-release failed; WSLC requires WSL $required or newer." } + $installed = Get-InstalledWslVersion + } + + if ($null -eq $installed) { + Exit-WithError 'wsl --version failed after updating; the WSL2 runtime is not usable on this runner.' } + if ($null -ne $required -and $installed -lt $required) { + Exit-WithError "WSL $installed is installed, but WSLC requires $required or newer." + } + Write-Host "WSL runtime $installed is ready (WSLC requires $required or newer)." # Diagnostic only: --status exits non-zero with no distribution installed, # which is expected since WSLC creates its own containers via the SDK. Invoke-Wsl @('--status') | Out-Null } +# Minimum WSL runtime for WSLC, read from the pinned SDK version so the two +# cannot drift. The SDK's own runtime error names this same version. +function Get-RequiredWslVersion { + $buildScript = Join-Path $PSScriptRoot '..\..\src\backends\wslc\common\build.rs' + if (-not (Test-Path $buildScript)) { + Write-Host "WARNING: $buildScript not found; skipping the WSL version gate." + return $null + } + + $match = [regex]::Match((Get-Content $buildScript -Raw), 'WSLC_SDK_VERSION:\s*&str\s*=\s*"([0-9]+(?:\.[0-9]+)+)"') + if (-not $match.Success) { + Write-Host 'WARNING: could not parse WSLC_SDK_VERSION; skipping the WSL version gate.' + return $null + } + return [version]$match.Groups[1].Value +} + +# Installed modern-runtime version, or $null when wsl.exe is the legacy inbox +# build (no --version) or otherwise unusable. +function Get-InstalledWslVersion { + $result = Invoke-WslCapture -Arguments @('--version') + if ($result.ExitCode -ne 0) { + return $null + } + + $match = [regex]::Match($result.Output, '(?im)^\s*WSL version:\s*([0-9]+(?:\.[0-9]+)+)') + if (-not $match.Success) { + return $null + } + return [version]$match.Groups[1].Value +} + # wsl.exe emits UTF-16LE, which the default console encoding renders as -# null-separated garbage. Returns the exit code. -Quiet suppresses output for -# probes, where the legacy wsl.exe dumps its whole usage text on an unknown -# switch. -function Invoke-Wsl { - param( - [Parameter(Mandatory)][string[]]$Arguments, - [switch]$Quiet - ) +# null-separated garbage. Returns @{ ExitCode; Output } with the output decoded. +function Invoke-WslCapture { + param([Parameter(Mandatory)][string[]]$Arguments) $previousEncoding = [Console]::OutputEncoding try { [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - $output = & wsl.exe @Arguments 2>&1 - $exitCode = $LASTEXITCODE - if (-not $Quiet) { $output | Write-Host } - return $exitCode + $output = & wsl.exe @Arguments 2>&1 | Out-String + return @{ ExitCode = $LASTEXITCODE; Output = $output } } catch { - Write-Host "WARNING: wsl.exe $($Arguments -join ' ') could not be run: $($_.Exception.Message)" - return 1 + return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" } } finally { [Console]::OutputEncoding = $previousEncoding } } +# Run wsl.exe and return its exit code. -Quiet suppresses output for probes, +# where the legacy wsl.exe dumps its whole usage text on an unknown switch. +function Invoke-Wsl { + param( + [Parameter(Mandatory)][string[]]$Arguments, + [switch]$Quiet + ) + + $result = Invoke-WslCapture -Arguments $Arguments + if (-not $Quiet -and $result.Output.Trim()) { + Write-Host $result.Output.Trim() + } + return $result.ExitCode +} + if (-not (Test-Path $BinaryDirectory)) { Exit-WithError "Binary directory not found: $BinaryDirectory" } diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 537be2285..abca901b1 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -346,64 +346,19 @@ } }, "triggers": { - "pr": [ - { - "os": "windows-canary", - "backends": [ - "process-t1" - ] - }, - { - "os": "windows-25h2", - "backends": [ - "process-t3" - ] - }, - { - "os": "macos-26", - "backends": [ - "seatbelt" - ] - }, - { - "os": "ubuntu-26.04", - "backends": [ - "bubblewrap" - ] - } - ], + "pr": [], "nightly": [ - { - "os": "windows-prerelease-isolation-session", - "backends": [ - "isolation-session" - ] - }, { "os": "windows-prerelease-process-container", "backends": [ "process-t1", - "isolation-session", - "wslc", - "windows-sandbox", - "microvm", - "hyperlight" - ] - }, - { - "os": "windows-canary", - "backends": [ - "process-t1" + "isolation-session" ] }, { "os": "windows-25h2", "backends": [ - "process-t3", - "wslc", - "windows-sandbox", - "microvm", - "hyperlight" + "process-t3" ] }, { @@ -425,69 +380,29 @@ ] }, { - "os": "ubuntu-26.04", - "backends": [ - "bubblewrap", - "hyperlight", - "lxc" - ] - }, - { - "os": "ubuntu-24.04", - "backends": [ - "microvm" - ] - }, - { - "os": "rhel-10", - "backends": [ - "bubblewrap" - ] - } - ], - "weekly": [ - { - "os": "windows-canary", - "backends": [ - "isolation-session" - ] - }, - { - "os": "windows-24h2", - "backends": [ - "wslc", - "windows-sandbox", - "microvm", - "hyperlight" - ] - }, - { - "os": "windows-23h2", + "os": "macos-15", "backends": [ - "wslc", - "windows-sandbox", - "microvm", - "hyperlight" + "seatbelt" ] }, { - "os": "macos-15", + "os": "ubuntu-26.04", "backends": [ - "seatbelt" + "bubblewrap", + "lxc" ] }, { "os": "ubuntu-24.04", "backends": [ "bubblewrap", - "hyperlight", "lxc" ] }, { "os": "rhel-10", "backends": [ - "hyperlight", + "bubblewrap", "lxc" ] }, @@ -495,11 +410,11 @@ "os": "debian-13", "backends": [ "bubblewrap", - "hyperlight", "lxc" ] } ], + "weekly": [], "enabled": [ { "os": "windows-prerelease-process-container", diff --git a/tests/scripts/run_wslc_all_tests.ps1 b/tests/scripts/run_wslc_all_tests.ps1 index 4f5a11332..e7a62b6ad 100644 --- a/tests/scripts/run_wslc_all_tests.ps1 +++ b/tests/scripts/run_wslc_all_tests.ps1 @@ -92,6 +92,17 @@ if (-not $SkipSetup) { } } +# Helper: StrictMode-safe property read; returns $null when the property (or the +# object) is absent. Lets the optional-config-field reads below work under the +# Set-StrictMode -Version Latest that run_ci_backend_tests.ps1 imposes. +function Get-JsonProperty { + param($Object, [Parameter(Mandatory)][string]$Name) + if ($null -eq $Object) { return $null } + $prop = $Object.PSObject.Properties[$Name] + if ($null -eq $prop) { return $null } + return $prop.Value +} + # Helper: run a single WSLC test config function Run-WslcTest { param( @@ -109,9 +120,12 @@ function Run-WslcTest { return @{ Name = $ConfigFile; Pass = $true; Skipped = $true; Reason = "File not found" } } - # Skip if the config references a tar file that doesn't exist locally + # Skip if the config references a tar file that doesn't exist locally. + # Read the chain defensively: this suite inherits Set-StrictMode -Version + # Latest from run_ci_backend_tests.ps1, under which touching a missing + # property is a terminating error, and most configs have no wslc.imageTarPath. $configJson = Get-Content $configPath -Raw | ConvertFrom-Json - $tarPath = $configJson.experimental.wslc.imageTarPath + $tarPath = Get-JsonProperty (Get-JsonProperty (Get-JsonProperty $configJson 'experimental') 'wslc') 'imageTarPath' if ($tarPath -and -not (Test-Path $tarPath)) { Write-Host " $ConfigFile ... " -NoNewline Write-Host "SKIP (tar not found: $tarPath)" -ForegroundColor Yellow @@ -164,7 +178,7 @@ function Run-WslcTest { # PostExitCheck runs after exit/output gates pass. Receives ($id, $output) # and must return truthy. Use for externally-observable state assertions. if ($pass -and $PostExitCheck) { - $containerId = $configJson.containerId + $containerId = Get-JsonProperty $configJson 'containerId' try { $checkResult = & $PostExitCheck $containerId $output if (-not $checkResult) { From 68a96e19b5f13fd1841b0532392380c85e8f10eb Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:51:31 -0700 Subject: [PATCH 68/93] made changes to images for wslc and isolation session --- scripts/ci/prepare-windows-host.ps1 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index cf53a2a6d..0ace3d4f1 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -180,12 +180,7 @@ function Initialize-WslcHost { Assert-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform' ` -Remedy 'WSL2 must be baked into the runner image; enabling these features requires a host reboot this job cannot take.' - # The features can be enabled while the modern (Store/MSIX) WSL runtime is - # absent — the image may still carry only the legacy inbox wsl.exe, which - # has no --version. Installing needs no reboot once the features are on. - # - # Probe quietly: the legacy wsl.exe answers an unknown switch with its full - # usage text, which would bury the real progress messages. + # check for WSL installed and up to date, and update if needed. if ($null -eq (Get-InstalledWslVersion)) { Write-Host 'Modern WSL runtime absent; installing via wsl --update...' # --web-download avoids the Microsoft Store, which CI images generally From bb27b47182c688ce2f2977beb236a95a0651d68a Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:05:25 -0700 Subject: [PATCH 69/93] force new test --- scripts/ci/check-whp.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/check-whp.ps1 b/scripts/ci/check-whp.ps1 index b943a457e..85232dda8 100644 --- a/scripts/ci/check-whp.ps1 +++ b/scripts/ci/check-whp.ps1 @@ -14,3 +14,4 @@ Write-Host "WHP is enabled and hypervisor is present." if ($env:GITHUB_OUTPUT) { Add-Content -Path $env:GITHUB_OUTPUT -Value "whp_available=true" } + From ccd5bbd01c80e3194303676a213ab0e6d206cc23 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:50:03 -0700 Subject: [PATCH 70/93] force again --- scripts/ci/check-whp.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ci/check-whp.ps1 b/scripts/ci/check-whp.ps1 index 85232dda8..c251e0281 100644 --- a/scripts/ci/check-whp.ps1 +++ b/scripts/ci/check-whp.ps1 @@ -13,5 +13,4 @@ if ($null -eq $cs -or -not $cs.HypervisorPresent) { Write-Host "WHP is enabled and hypervisor is present." if ($env:GITHUB_OUTPUT) { Add-Content -Path $env:GITHUB_OUTPUT -Value "whp_available=true" -} - +} \ No newline at end of file From 9c95106d8badf9a98095f04991f4d56c08af6ed0 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:19:01 -0700 Subject: [PATCH 71/93] force attempt 3 --- scripts/ci/check-whp.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/check-whp.ps1 b/scripts/ci/check-whp.ps1 index c251e0281..b943a457e 100644 --- a/scripts/ci/check-whp.ps1 +++ b/scripts/ci/check-whp.ps1 @@ -13,4 +13,4 @@ if ($null -eq $cs -or -not $cs.HypervisorPresent) { Write-Host "WHP is enabled and hypervisor is present." if ($env:GITHUB_OUTPUT) { Add-Content -Path $env:GITHUB_OUTPUT -Value "whp_available=true" -} \ No newline at end of file +} From 3d002a1af5bef94be0f4b6aa1a9d2eb36ca66239 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:40:28 -0700 Subject: [PATCH 72/93] pausing isolation session before getting feedback, and last effort to install wsl --- scripts/ci/prepare-windows-host.ps1 | 143 ++++++++++------------------ 1 file changed, 52 insertions(+), 91 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index 0ace3d4f1..e841cd326 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -180,108 +180,69 @@ function Initialize-WslcHost { Assert-RequiredFeature -Name 'Microsoft-Windows-Subsystem-Linux', 'VirtualMachinePlatform' ` -Remedy 'WSL2 must be baked into the runner image; enabling these features requires a host reboot this job cannot take.' - # check for WSL installed and up to date, and update if needed. - if ($null -eq (Get-InstalledWslVersion)) { - Write-Host 'Modern WSL runtime absent; installing via wsl --update...' - # --web-download avoids the Microsoft Store, which CI images generally - # cannot use. Older wsl.exe builds reject the flag, so retry without it. - if ((Invoke-Wsl @('--update', '--web-download') -Quiet) -ne 0 -and - (Invoke-Wsl @('--update')) -ne 0) { - Exit-WithError 'wsl --update failed; the WSL2 runtime could not be installed on this runner.' - } + Write-Host "=== wsl.exe presence + status ===" + $wsl = Get-Command wsl.exe -ErrorAction SilentlyContinue + if ($wsl) { + Write-Host "wsl.exe: $($wsl.Source)" + wsl --status 2>&1 | Write-Host + wsl --version 2>&1 | Write-Host + Write-Host "wsl --status exit code: $LASTEXITCODE" + } else { + Write-Host "wsl.exe NOT found on PATH" + Exit-WithError 'WSL2 is not installed on this runner. The runner image must include WSL2 for this backend.' } - # WSLC needs a runtime at least as new as the pinned WSLC SDK, and those - # builds ship only on the pre-release ring — the stable ring lands well - # behind it. Without this the SDK fails at run time with - # "WSLC runtime unavailable. Missing components: WslPackage". - $required = Get-RequiredWslVersion - $installed = Get-InstalledWslVersion - if ($null -ne $required -and ($null -eq $installed -or $installed -lt $required)) { - Write-Host "WSL $installed is older than the $required WSLC requires; updating to pre-release..." - if ((Invoke-Wsl @('--update', '--pre-release', '--web-download') -Quiet) -ne 0 -and - (Invoke-Wsl @('--update', '--pre-release')) -ne 0) { - Exit-WithError "wsl --update --pre-release failed; WSLC requires WSL $required or newer." - } - $installed = Get-InstalledWslVersion - } + Write-Host "=== installing WSL ===" - if ($null -eq $installed) { - Exit-WithError 'wsl --version failed after updating; the WSL2 runtime is not usable on this runner.' - } - if ($null -ne $required -and $installed -lt $required) { - Exit-WithError "WSL $installed is installed, but WSLC requires $required or newer." - } - Write-Host "WSL runtime $installed is ready (WSLC requires $required or newer)." + wsl --install 2>&1 | Write-Host + Write-Host "=== post-install status check ===" + wsl --status 2>&1 | Write-Host + Write-Host "=== post-install version check ===" + wsl --version 2>&1 | Write-Host - # Diagnostic only: --status exits non-zero with no distribution installed, - # which is expected since WSLC creates its own containers via the SDK. - Invoke-Wsl @('--status') | Out-Null -} + Write-Host "=== updating WSL to pre-release ===" -# Minimum WSL runtime for WSLC, read from the pinned SDK version so the two -# cannot drift. The SDK's own runtime error names this same version. -function Get-RequiredWslVersion { - $buildScript = Join-Path $PSScriptRoot '..\..\src\backends\wslc\common\build.rs' - if (-not (Test-Path $buildScript)) { - Write-Host "WARNING: $buildScript not found; skipping the WSL version gate." - return $null - } + wsl --update --prerelease 2>&1 | Write-Host + Write-Host "=== post-update status check ===" + wsl --status 2>&1 | Write-Host + Write-Host "=== post-update version check ===" + wsl --version 2>&1 | Write-Host - $match = [regex]::Match((Get-Content $buildScript -Raw), 'WSLC_SDK_VERSION:\s*&str\s*=\s*"([0-9]+(?:\.[0-9]+)+)"') - if (-not $match.Success) { - Write-Host 'WARNING: could not parse WSLC_SDK_VERSION; skipping the WSL version gate.' - return $null - } - return [version]$match.Groups[1].Value + Write-Host "=== done. ===" } -# Installed modern-runtime version, or $null when wsl.exe is the legacy inbox -# build (no --version) or otherwise unusable. -function Get-InstalledWslVersion { - $result = Invoke-WslCapture -Arguments @('--version') - if ($result.ExitCode -ne 0) { - return $null - } - - $match = [regex]::Match($result.Output, '(?im)^\s*WSL version:\s*([0-9]+(?:\.[0-9]+)+)') - if (-not $match.Success) { - return $null - } - return [version]$match.Groups[1].Value -} # wsl.exe emits UTF-16LE, which the default console encoding renders as # null-separated garbage. Returns @{ ExitCode; Output } with the output decoded. -function Invoke-WslCapture { - param([Parameter(Mandatory)][string[]]$Arguments) - - $previousEncoding = [Console]::OutputEncoding - try { - [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - $output = & wsl.exe @Arguments 2>&1 | Out-String - return @{ ExitCode = $LASTEXITCODE; Output = $output } - } catch { - return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" } - } finally { - [Console]::OutputEncoding = $previousEncoding - } -} - -# Run wsl.exe and return its exit code. -Quiet suppresses output for probes, -# where the legacy wsl.exe dumps its whole usage text on an unknown switch. -function Invoke-Wsl { - param( - [Parameter(Mandatory)][string[]]$Arguments, - [switch]$Quiet - ) - - $result = Invoke-WslCapture -Arguments $Arguments - if (-not $Quiet -and $result.Output.Trim()) { - Write-Host $result.Output.Trim() - } - return $result.ExitCode -} +# function Invoke-WslCapture { +# param([Parameter(Mandatory)][string[]]$Arguments) + +# $previousEncoding = [Console]::OutputEncoding +# try { +# [Console]::OutputEncoding = [System.Text.Encoding]::Unicode +# $output = & wsl.exe @Arguments 2>&1 | Out-String +# return @{ ExitCode = $LASTEXITCODE; Output = $output } +# } catch { +# return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" } +# } finally { +# [Console]::OutputEncoding = $previousEncoding +# } +# } + +# # Run wsl.exe and return its exit code. -Quiet suppresses output for probes, +# # where the legacy wsl.exe dumps its whole usage text on an unknown switch. +# function Invoke-Wsl { +# param( +# [Parameter(Mandatory)][string[]]$Arguments, +# [switch]$Quiet +# ) + +# $result = Invoke-WslCapture -Arguments $Arguments +# if (-not $Quiet -and $result.Output.Trim()) { +# Write-Host $result.Output.Trim() +# } +# return $result.ExitCode +# } if (-not (Test-Path $BinaryDirectory)) { Exit-WithError "Binary directory not found: $BinaryDirectory" From fdbf531ebd17bad37467804992bf4189ffee4aaa Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:24:29 -0700 Subject: [PATCH 73/93] correcting order of wsl commands --- scripts/ci/prepare-windows-host.ps1 | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index e841cd326..9271ad7ef 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -184,29 +184,35 @@ function Initialize-WslcHost { $wsl = Get-Command wsl.exe -ErrorAction SilentlyContinue if ($wsl) { Write-Host "wsl.exe: $($wsl.Source)" - wsl --status 2>&1 | Write-Host - wsl --version 2>&1 | Write-Host - Write-Host "wsl --status exit code: $LASTEXITCODE" } else { Write-Host "wsl.exe NOT found on PATH" Exit-WithError 'WSL2 is not installed on this runner. The runner image must include WSL2 for this backend.' } - Write-Host "=== installing WSL ===" + $previousEncoding = [Console]::OutputEncoding + [Console]::OutputEncoding = [System.Text.Encoding]::Unicode + $output = wsl --status 2>&1 | Out-String + Write-Host $output + [Console]::OutputEncoding = $previousEncoding - wsl --install 2>&1 | Write-Host - Write-Host "=== post-install status check ===" - wsl --status 2>&1 | Write-Host - Write-Host "=== post-install version check ===" - wsl --version 2>&1 | Write-Host + # if unicode output mentions wsl.exe --install, skip version check for now + if ($output -match 'wsl.exe --install') { + Write-Host "=== installing WSL ===" + + [Console]::OutputEncoding = [System.Text.Encoding]::Unicode + wsl --install 2>&1 | Write-Host + wsl --status 2>&1 | Write-Host + wsl --version 2>&1 | Write-Host + [Console]::OutputEncoding = $previousEncoding + } Write-Host "=== updating WSL to pre-release ===" + [Console]::OutputEncoding = [System.Text.Encoding]::Unicode wsl --update --prerelease 2>&1 | Write-Host - Write-Host "=== post-update status check ===" wsl --status 2>&1 | Write-Host - Write-Host "=== post-update version check ===" wsl --version 2>&1 | Write-Host + [Console]::OutputEncoding = $previousEncoding Write-Host "=== done. ===" } From 49ba4e4019593c897128dfad4ab9bfda3f8a4c19 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:52:56 -0700 Subject: [PATCH 74/93] trying with wsl.exe --- scripts/ci/prepare-windows-host.ps1 | 17 ++++++++++------- scripts/ci/validation-test-matrix.json | 8 -------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index 9271ad7ef..4059f4570 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -191,7 +191,7 @@ function Initialize-WslcHost { $previousEncoding = [Console]::OutputEncoding [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - $output = wsl --status 2>&1 | Out-String + $output = wsl.exe --status 2>&1 | Out-String Write-Host $output [Console]::OutputEncoding = $previousEncoding @@ -200,18 +200,21 @@ function Initialize-WslcHost { Write-Host "=== installing WSL ===" [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - wsl --install 2>&1 | Write-Host - wsl --status 2>&1 | Write-Host - wsl --version 2>&1 | Write-Host + wsl.exe --install --web-download --no-distribution 2>&1 | Write-Host + if ($LASTEXITCODE -ne 0) { + throw "WSL installation failed: $LASTEXITCODE" + } + wsl.exe --status 2>&1 | Write-Host + wsl.exe --version 2>&1 | Write-Host [Console]::OutputEncoding = $previousEncoding } Write-Host "=== updating WSL to pre-release ===" [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - wsl --update --prerelease 2>&1 | Write-Host - wsl --status 2>&1 | Write-Host - wsl --version 2>&1 | Write-Host + wsl.exe --update --pre-release --web-download 2>&1 | Write-Host + wsl.exe --status 2>&1 | Write-Host + wsl.exe --version 2>&1 | Write-Host [Console]::OutputEncoding = $previousEncoding Write-Host "=== done. ===" diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index abca901b1..42221ccc5 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -416,14 +416,6 @@ ], "weekly": [], "enabled": [ - { - "os": "windows-prerelease-process-container", - "backends": [ - "process-t1", - "process-t3", - "isolation-session" - ] - }, { "os": "windows-25h2", "backends": [ From a8ab3c10f4bb6a2e6936c9e62e5c5ec308910630 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:32:35 -0700 Subject: [PATCH 75/93] reverted some WSL changes. --- scripts/ci/prepare-windows-host.ps1 | 154 ++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 44 deletions(-) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index 4059f4570..8b18b82f6 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -195,63 +195,129 @@ function Initialize-WslcHost { Write-Host $output [Console]::OutputEncoding = $previousEncoding - # if unicode output mentions wsl.exe --install, skip version check for now + # # if unicode output mentions wsl.exe --install, skip version check for now if ($output -match 'wsl.exe --install') { - Write-Host "=== installing WSL ===" + Exit-WithError 'WSL2 is not installed on this runner. The runner image must include WSL2 for this backend.' + # Write-Host "=== installing WSL ===" + + # [Console]::OutputEncoding = [System.Text.Encoding]::Unicode + # wsl.exe --install --web-download --no-distribution 2>&1 | Write-Host + # if ($LASTEXITCODE -ne 0) { + # [Console]::OutputEncoding = $previousEncoding + # throw "WSL installation failed: $LASTEXITCODE" + # } + # wsl.exe --status 2>&1 | Write-Host + # wsl.exe --version 2>&1 | Write-Host + # [Console]::OutputEncoding = $previousEncoding + } - [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - wsl.exe --install --web-download --no-distribution 2>&1 | Write-Host - if ($LASTEXITCODE -ne 0) { - throw "WSL installation failed: $LASTEXITCODE" + if ((Invoke-Wsl @('--version') -Quiet) -ne 0) { + Write-Host 'version command failed, so WSL2 is installed but not updated.' + Write-Host "=== updating inbox WSL to modern version ===" + + if ((Invoke-Wsl @('--update', '--web-download') -Quiet) -ne 0 -and + (Invoke-Wsl @('--update')) -ne 0) { + Exit-WithError 'wsl --update failed; the WSL2 runtime could not be installed on this runner.' } - wsl.exe --status 2>&1 | Write-Host - wsl.exe --version 2>&1 | Write-Host - [Console]::OutputEncoding = $previousEncoding + + if ((Invoke-Wsl @('--version') -Quiet) -ne 0) { + Exit-WithError 'wsl --version failed after updating; the WSL2 runtime is not usable on this runner.' + } + + Write-Host 'WSL2 is installed and updated (not prerelease, yet)' + } - Write-Host "=== updating WSL to pre-release ===" + # WSLC needs a runtime at least as new as the pinned WSLC SDK, and those + # builds ship only on the pre-release ring — the stable ring lands well + # behind it. Without this the SDK fails at run time with + # "WSLC runtime unavailable. Missing components: WslPackage". + $required = Get-RequiredWslVersion + $installed = Get-InstalledWslVersion + if ($null -ne $required -and ($null -eq $installed -or $installed -lt $required)) { + Write-Host "WSL $installed is older than the $required WSLC requires; updating to pre-release..." + if ((Invoke-Wsl @('--update', '--pre-release', '--web-download') -Quiet) -ne 0 -and + (Invoke-Wsl @('--update', '--pre-release')) -ne 0) { + Exit-WithError "wsl --update --pre-release failed; WSLC requires WSL $required or newer." + } + $installed = Get-InstalledWslVersion + } - [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - wsl.exe --update --pre-release --web-download 2>&1 | Write-Host - wsl.exe --status 2>&1 | Write-Host - wsl.exe --version 2>&1 | Write-Host - [Console]::OutputEncoding = $previousEncoding + if ($null -eq $installed) { + Exit-WithError 'wsl --version failed after updating; the WSL2 runtime is not usable on this runner.' + } + if ($null -ne $required -and $installed -lt $required) { + Exit-WithError "WSL $installed is installed, but WSLC requires $required or newer." + } + Write-Host "WSL runtime $installed is ready (WSLC requires $required or newer)." Write-Host "=== done. ===" } +# Minimum WSL runtime for WSLC, read from the pinned SDK version so the two +# cannot drift. The SDK's own runtime error names this same version. +function Get-RequiredWslVersion { + $buildScript = Join-Path $PSScriptRoot '..\..\src\backends\wslc\common\build.rs' + if (-not (Test-Path $buildScript)) { + Write-Host "WARNING: $buildScript not found; skipping the WSL version gate." + return $null + } + + $match = [regex]::Match((Get-Content $buildScript -Raw), 'WSLC_SDK_VERSION:\s*&str\s*=\s*"([0-9]+(?:\.[0-9]+)+)"') + if (-not $match.Success) { + Write-Host 'WARNING: could not parse WSLC_SDK_VERSION; skipping the WSL version gate.' + return $null + } + return [version]$match.Groups[1].Value +} + +# Installed modern-runtime version, or $null when wsl.exe is the legacy inbox +# build (no --version) or otherwise unusable. +function Get-InstalledWslVersion { + $result = Invoke-WslCapture -Arguments @('--version') + if ($result.ExitCode -ne 0) { + return $null + } + + $match = [regex]::Match($result.Output, '(?im)^\s*WSL version:\s*([0-9]+(?:\.[0-9]+)+)') + if (-not $match.Success) { + return $null + } + return [version]$match.Groups[1].Value +} + # wsl.exe emits UTF-16LE, which the default console encoding renders as # null-separated garbage. Returns @{ ExitCode; Output } with the output decoded. -# function Invoke-WslCapture { -# param([Parameter(Mandatory)][string[]]$Arguments) - -# $previousEncoding = [Console]::OutputEncoding -# try { -# [Console]::OutputEncoding = [System.Text.Encoding]::Unicode -# $output = & wsl.exe @Arguments 2>&1 | Out-String -# return @{ ExitCode = $LASTEXITCODE; Output = $output } -# } catch { -# return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" } -# } finally { -# [Console]::OutputEncoding = $previousEncoding -# } -# } - -# # Run wsl.exe and return its exit code. -Quiet suppresses output for probes, -# # where the legacy wsl.exe dumps its whole usage text on an unknown switch. -# function Invoke-Wsl { -# param( -# [Parameter(Mandatory)][string[]]$Arguments, -# [switch]$Quiet -# ) - -# $result = Invoke-WslCapture -Arguments $Arguments -# if (-not $Quiet -and $result.Output.Trim()) { -# Write-Host $result.Output.Trim() -# } -# return $result.ExitCode -# } +function Invoke-WslCapture { + param([Parameter(Mandatory)][string[]]$Arguments) + + $previousEncoding = [Console]::OutputEncoding + try { + [Console]::OutputEncoding = [System.Text.Encoding]::Unicode + $output = & wsl.exe @Arguments 2>&1 | Out-String + return @{ ExitCode = $LASTEXITCODE; Output = $output } + } catch { + return @{ ExitCode = 1; Output = "wsl.exe could not be run: $($_.Exception.Message)" } + } finally { + [Console]::OutputEncoding = $previousEncoding + } +} + +# Run wsl.exe and return its exit code. -Quiet suppresses output for probes, +# where the legacy wsl.exe dumps its whole usage text on an unknown switch. +function Invoke-Wsl { + param( + [Parameter(Mandatory)][string[]]$Arguments, + [switch]$Quiet + ) + + $result = Invoke-WslCapture -Arguments $Arguments + if (-not $Quiet -and $result.Output.Trim()) { + Write-Host $result.Output.Trim() + } + return $result.ExitCode +} if (-not (Test-Path $BinaryDirectory)) { Exit-WithError "Binary directory not found: $BinaryDirectory" From 6d71a804ebf2b64fb46d69920a2306a8bbafb1c2 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:25:43 -0700 Subject: [PATCH 76/93] quick redattempt with a few options to install WSL --- scripts/ci/prepare-windows-host.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index 8b18b82f6..bdc1f19fc 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -209,6 +209,7 @@ function Initialize-WslcHost { # wsl.exe --status 2>&1 | Write-Host # wsl.exe --version 2>&1 | Write-Host # [Console]::OutputEncoding = $previousEncoding + # } if ((Invoke-Wsl @('--version') -Quiet) -ne 0) { From 2f708e871a9cbfca06115d1012b0fc4255a034a2 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:27:25 -0700 Subject: [PATCH 77/93] testing pre-release build for WSLC --- scripts/ci/validation-test-matrix.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 42221ccc5..27fcbecc9 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -416,6 +416,12 @@ ], "weekly": [], "enabled": [ + { + "os": "windows-prerelease-process-container", + "backends": [ + "wslc" + ] + }, { "os": "windows-25h2", "backends": [ From 2d8f16e7ff15fb87693e5dfa51371327bd11fd1e Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:04:28 -0700 Subject: [PATCH 78/93] getting ready for PR by double-checking lxc and bubblewrap --- scripts/ci/validation-test-matrix.json | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 27fcbecc9..2847961c5 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -417,27 +417,31 @@ "weekly": [], "enabled": [ { - "os": "windows-prerelease-process-container", + "os": "ubuntu-24.04", "backends": [ - "wslc" + "lxc", + "bubblewrap" ] }, { - "os": "windows-25h2", + "os": "ubuntu-26.04", "backends": [ - "wslc" + "lxc", + "bubblewrap" ] }, { - "os": "windows-24h2", + "os": "rhel-10", "backends": [ - "wslc" + "lxc", + "bubblewrap" ] }, { - "os": "windows-23h2", + "os": "debian-13", "backends": [ - "wslc" + "lxc", + "bubblewrap" ] } ] From 8de15cbfc4e5afd628458ac912759e9b125ff8c9 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:06:09 -0700 Subject: [PATCH 79/93] Removing PR trigger --- .../workflows/{Build.And.Validate.yml => Build.yml} | 12 +----------- .github/workflows/Validation.Infra.Testing.yml | 2 -- scripts/ci/prepare-windows-host.ps1 | 12 ------------ 3 files changed, 1 insertion(+), 25 deletions(-) rename .github/workflows/{Build.And.Validate.yml => Build.yml} (77%) diff --git a/.github/workflows/Build.And.Validate.yml b/.github/workflows/Build.yml similarity index 77% rename from .github/workflows/Build.And.Validate.yml rename to .github/workflows/Build.yml index 225fdeab7..4f41d91af 100644 --- a/.github/workflows/Build.And.Validate.yml +++ b/.github/workflows/Build.yml @@ -1,7 +1,4 @@ -# Entry point for GHA PR/CI validation. Mirrors the 1ES `1ES.Build.yml` -# split: this file orchestrates reusable build-and-validation stages. - -name: Build + Validate +name: Build on: push: @@ -50,13 +47,6 @@ jobs: needs: dependency-feed-check uses: ./.github/workflows/Build.MacOS.Job.yml - pr-validation-test-matrix: - needs: [windows, linux, macos] - if: github.event_name == 'pull_request' && github.base_ref == 'main' - uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml - with: - plan: pr - sdk-unit-tests: uses: ./.github/workflows/SDK.Unit.Test.Job.yml diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml index 0c54c089f..85483f07a 100644 --- a/.github/workflows/Validation.Infra.Testing.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -1,7 +1,5 @@ name: Validation Infrastructure Testing -# github was down - forcing this to run now :) - on: push: branches: diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index bdc1f19fc..d260bd069 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -198,18 +198,6 @@ function Initialize-WslcHost { # # if unicode output mentions wsl.exe --install, skip version check for now if ($output -match 'wsl.exe --install') { Exit-WithError 'WSL2 is not installed on this runner. The runner image must include WSL2 for this backend.' - # Write-Host "=== installing WSL ===" - - # [Console]::OutputEncoding = [System.Text.Encoding]::Unicode - # wsl.exe --install --web-download --no-distribution 2>&1 | Write-Host - # if ($LASTEXITCODE -ne 0) { - # [Console]::OutputEncoding = $previousEncoding - # throw "WSL installation failed: $LASTEXITCODE" - # } - # wsl.exe --status 2>&1 | Write-Host - # wsl.exe --version 2>&1 | Write-Host - # [Console]::OutputEncoding = $previousEncoding - # } if ((Invoke-Wsl @('--version') -Quiet) -ne 0) { From adec4f10a288538203e6b4b8c8126bd763b9bd9c Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:32:31 -0700 Subject: [PATCH 80/93] Removed temp macOS testing suite --- scripts/ci/validation-test-matrix.json | 12 - tests/scripts/TEMP_run_seatbelt_all_tests.sh | 569 ------------------- tests/scripts/run_ci_backend_tests.sh | 3 +- 3 files changed, 2 insertions(+), 582 deletions(-) delete mode 100644 tests/scripts/TEMP_run_seatbelt_all_tests.sh diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 2847961c5..37dc30126 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -373,18 +373,6 @@ "process-t3" ] }, - { - "os": "macos-26", - "backends": [ - "seatbelt" - ] - }, - { - "os": "macos-15", - "backends": [ - "seatbelt" - ] - }, { "os": "ubuntu-26.04", "backends": [ diff --git a/tests/scripts/TEMP_run_seatbelt_all_tests.sh b/tests/scripts/TEMP_run_seatbelt_all_tests.sh deleted file mode 100644 index dfd78c411..000000000 --- a/tests/scripts/TEMP_run_seatbelt_all_tests.sh +++ /dev/null @@ -1,569 +0,0 @@ -#!/bin/bash -# THIS SCRIPT IS FOR TESTING CI INFRASTRUCTURE. THIS IS NOT A STABLE TEST SUITE. -set -uo pipefail - -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 - exit 2 -fi - -BINARY_DIRECTORY="$(cd "$1" && pwd)" -MXC_EXEC="$BINARY_DIRECTORY/mxc-exec-mac" -UNIX_TEST_PROXY="$BINARY_DIRECTORY/unix-test-proxy" - -if [[ ! -x "$MXC_EXEC" ]]; then - echo "Error: executable mxc-exec-mac not found in $BINARY_DIRECTORY" >&2 - exit 1 -fi -if [[ ! -x "$UNIX_TEST_PROXY" ]]; then - echo "Error: executable unix-test-proxy not found in $BINARY_DIRECTORY" >&2 - exit 1 -fi -for command_name in curl python3; do - if ! command -v "$command_name" >/dev/null; then - echo "Error: required command not found: $command_name" >&2 - exit 1 - fi -done - -TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/mxc-seatbelt-tests.XXXXXX")" -TEST_ROOT="$(cd "$TEST_ROOT" && pwd -P)" -PASSED=0 -FAILED=0 -FAILURES="" -INFO_PASSED=0 -INFO_FAILED=0 -INFO_FAILURES="" -SERVER_PID="" - -cleanup() { - if [[ -n "$SERVER_PID" ]]; then - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - fi - rm -rf "$TEST_ROOT" -} -trap cleanup EXIT - -run_test() { - local name="$1" - local function_name="$2" - echo "=== $name ===" - if "$function_name"; then - echo "PASS: $name" - PASSED=$((PASSED + 1)) - else - echo "FAIL: $name" - FAILED=$((FAILED + 1)) - FAILURES="${FAILURES}\n - ${name}" - fi - echo "" -} - -run_info_test() { - local name="$1" - local function_name="$2" - echo "=== $name (INFORMATION ONLY) ===" - if "$function_name"; then - echo "INFO-PASS: $name" - INFO_PASSED=$((INFO_PASSED + 1)) - else - echo "INFO-FAIL: $name" - INFO_FAILED=$((INFO_FAILED + 1)) - INFO_FAILURES="${INFO_FAILURES}\n - ${name}" - fi - echo "" -} - -seed_host_clipboard() { - python3 - "$1" <<'PYTHON' -import subprocess -import sys - -token = sys.argv[1].encode() -subprocess.run(["/usr/bin/pbcopy"], input=token, timeout=5, check=True) -result = subprocess.run( - ["/usr/bin/pbpaste"], - capture_output=True, - timeout=5, - check=True, -) -sys.exit(0 if result.stdout == token else 1) -PYTHON -} - -host_clipboard_matches() { - python3 - "$1" <<'PYTHON' -import subprocess -import sys - -result = subprocess.run( - ["/usr/bin/pbpaste"], - capture_output=True, - timeout=5, - check=True, -) -sys.exit(0 if result.stdout == sys.argv[1].encode() else 1) -PYTHON -} - -test_execution() { - local config="$TEST_ROOT/execution.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-execution", - "containment": "seatbelt", - "process": { "commandLine": "printf 'SEATBELT_EXEC_OK\\n'" } -} -JSON - - local output - output=$("$MXC_EXEC" "$config" 2>&1) || { - echo "$output" - return 1 - } - grep -q "SEATBELT_EXEC_OK" <<<"$output" -} - -test_exit_code() { - local config="$TEST_ROOT/exit-code.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-exit-code", - "containment": "seatbelt", - "process": { "commandLine": "exit 7" } -} -JSON - - local output status - if output=$("$MXC_EXEC" "$config" 2>&1); then - echo "Expected exit code 7, got 0" - return 1 - else - status=$? - fi - if [[ $status -ne 7 ]]; then - echo "$output" - echo "Expected exit code 7, got $status" - return 1 - fi -} - -test_filesystem_policy() { - local allowed="$TEST_ROOT/allowed" - local denied="$TEST_ROOT/denied" - local config="$TEST_ROOT/filesystem.json" - mkdir -p "$allowed" "$denied" - printf 'host secret\n' >"$denied/secret.txt" - - cat >"$config" < '$allowed/output.txt'; if cat '$denied/secret.txt' >/dev/null 2>&1; then echo DENIED_PATH_LEAK; exit 1; fi; echo FILESYSTEM_OK" - }, - "filesystem": { - "readwritePaths": ["$allowed"], - "deniedPaths": ["$denied"] - } -} -JSON - - local output - output=$("$MXC_EXEC" "$config" 2>&1) || { - echo "$output" - return 1 - } - grep -q "FILESYSTEM_OK" <<<"$output" && - ! grep -q "DENIED_PATH_LEAK" <<<"$output" && - grep -q "sandbox write" "$allowed/output.txt" -} - -test_environment_policy() { - local config="$TEST_ROOT/environment.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-environment", - "containment": "seatbelt", - "process": { - "commandLine": "printf 'HOST=[%s] CONFIG=[%s]\\n' \"$MXC_HOST_MARKER\" \"$MXC_CONFIG_MARKER\"", - "env": ["MXC_CONFIG_MARKER=from_config"] - } -} -JSON - - local output - output=$(MXC_HOST_MARKER=must_not_leak "$MXC_EXEC" "$config" 2>&1) || { - echo "$output" - return 1 - } - grep -q "HOST=\[\] CONFIG=\[from_config\]" <<<"$output" && - ! grep -q "must_not_leak" <<<"$output" -} - -test_timeout() { - local config="$TEST_ROOT/timeout.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-timeout", - "containment": "seatbelt", - "process": { - "commandLine": "echo TIMEOUT_STARTED; /bin/sleep 10; echo TIMEOUT_LEAK", - "timeout": 1000 - } -} -JSON - - local output status start elapsed - start=$SECONDS - if output=$("$MXC_EXEC" "$config" 2>&1); then - echo "$output" - echo "Expected timed-out execution to fail" - return 1 - else - status=$? - fi - elapsed=$((SECONDS - start)) - if [[ $status -eq 0 || $elapsed -ge 8 ]]; then - echo "$output" - echo "Timeout did not terminate promptly (status=$status elapsed=${elapsed}s)" - return 1 - fi - grep -q "TIMEOUT_STARTED" <<<"$output" && - ! grep -q "TIMEOUT_LEAK" <<<"$output" -} - -start_host_server() { - local server_script="$TEST_ROOT/server.py" - local port_file="$TEST_ROOT/server.port" - rm -f "$port_file" - cat >"$server_script" <<'PYTHON' -import http.server -import socketserver -import sys - -class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self): - body = b"HOST_SERVER_OK\n" - self.send_response(200) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, _format, *_args): - pass - -with socketserver.TCPServer(("127.0.0.1", 0), Handler) as server: - with open(sys.argv[1], "w", encoding="utf-8") as port_file: - port_file.write(str(server.server_address[1])) - server.serve_forever() -PYTHON - - python3 "$server_script" "$port_file" >"$TEST_ROOT/server.log" 2>&1 & - SERVER_PID=$! - for _ in {1..50}; do - [[ -s "$port_file" ]] && break - sleep 0.1 - done - [[ -s "$port_file" ]] || { - cat "$TEST_ROOT/server.log" - return 1 - } - HOST_SERVER_PORT="$(cat "$port_file")" - curl --fail --silent --max-time 2 "http://127.0.0.1:$HOST_SERVER_PORT" | - grep -q "HOST_SERVER_OK" -} - -test_network_default_deny() { - start_host_server || return 1 - - local config="$TEST_ROOT/network-deny.json" - cat >"$config" </dev/null 2>&1; then echo NETWORK_LEAK; exit 1; else echo NETWORK_BLOCKED; fi" - }, - "network": { "defaultPolicy": "block" } -} -JSON - - local output - output=$("$MXC_EXEC" "$config" 2>&1) || { - echo "$output" - return 1 - } - grep -q "NETWORK_BLOCKED" <<<"$output" && - ! grep -q "NETWORK_LEAK" <<<"$output" -} - -test_builtin_proxy_startup() { - local config="$TEST_ROOT/proxy.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-proxy", - "containment": "seatbelt", - "process": { - "commandLine": "printf 'HTTP_PROXY=%s HTTPS_PROXY=%s\\n' \"$HTTP_PROXY\" \"$HTTPS_PROXY\"" - }, - "network": { - "defaultPolicy": "block", - "proxy": { "builtinTestServer": true } - } -} -JSON - - local output - output=$("$MXC_EXEC" --experimental --allow-testing-features "$config" 2>&1) || { - echo "$output" - return 1 - } - grep -Eq "HTTP_PROXY=http://127\.0\.0\.1:[0-9]+ HTTPS_PROXY=http://127\.0\.0\.1:[0-9]+" \ - <<<"$output" -} - -test_clipboard_allow() { - local token="mxc_seatbelt_clipboard_$$_${RANDOM}" - local allow_config="$TEST_ROOT/clipboard-allow.json" - - cat >"$allow_config" <&1) || { - echo "$output" - echo "Clipboard allow probe failed" - return 1 - } - if ! grep -q "$token" <<<"$output"; then - echo "$output" - echo "Clipboard allow probe did not return its token" - return 1 - fi -} - -test_clipboard_deny() { - local token="mxc_seatbelt_clipboard_deny_$$_${RANDOM}" - local deny_config="$TEST_ROOT/clipboard-deny.json" - if ! seed_host_clipboard "$token"; then - echo "Host clipboard read/write baseline failed" - return 1 - fi - - cat >"$deny_config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-clipboard-deny", - "containment": "seatbelt", - "process": { - "commandLine": "copy_status=0; printf 'clipboard_denied_probe' | /usr/bin/pbcopy >/dev/null 2>&1 || copy_status=$?; read_status=0; /usr/bin/pbpaste >/dev/null 2>&1 || read_status=$?; leaked=0; if [ $copy_status -eq 0 ]; then echo CLIPBOARD_WRITE_LEAK; leaked=1; fi; if [ $read_status -eq 0 ]; then echo CLIPBOARD_READ_LEAK; leaked=1; fi; if [ $leaked -eq 0 ]; then echo CLIPBOARD_DENIED; else exit 1; fi", - "timeout": 10000 - }, - "ui": { - "disable": false, - "clipboard": "none" - } -} -JSON - - local output - output=$("$MXC_EXEC" "$deny_config" 2>&1) || { - echo "$output" - echo "Clipboard deny probe failed" - return 1 - } - if ! grep -q "CLIPBOARD_DENIED" <<<"$output" || - grep -Eq "CLIPBOARD_WRITE_LEAK|CLIPBOARD_READ_LEAK" <<<"$output"; then - return 1 - fi - if ! host_clipboard_matches "$token"; then - echo "Host clipboard changed during denied sandbox probe" - return 1 - fi -} - -test_gui_session() { - local config="$TEST_ROOT/gui.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-gui", - "containment": "seatbelt", - "process": { - "commandLine": "/usr/bin/osascript -e 'tell application \"Finder\" to get name' && echo GUI_SESSION_OK", - "timeout": 10000 - }, - "ui": { - "disable": false, - "clipboard": "all" - }, - "seatbelt": { - "guiAccess": true - } -} -JSON - - local output - output=$("$MXC_EXEC" "$config" 2>&1) || { - echo "$output" - echo "GUI session probe failed" - return 1 - } - grep -q "GUI_SESSION_OK" <<<"$output" -} - -test_public_internet_allow() { - local config="$TEST_ROOT/public-internet.json" - cat >"$config" <<'JSON' -{ - "version": "0.7.0-alpha", - "containerId": "ci-seatbelt-public-internet", - "containment": "seatbelt", - "process": { - "commandLine": "curl --fail --silent --show-error --max-time 10 https://example.com >/dev/null && echo PUBLIC_INTERNET_OK", - "timeout": 15000 - }, - "network": { - "defaultPolicy": "allow" - } -} -JSON - - local output - output=$("$MXC_EXEC" "$config" 2>&1) || { - echo "$output" - echo "Public internet probe failed" - return 1 - } - grep -q "PUBLIC_INTERNET_OK" <<<"$output" -} - -test_proxy_traffic_filtering() { - local control_config="$TEST_ROOT/proxy-filtering-control.json" - local config="$TEST_ROOT/proxy-filtering.json" - if [[ -z "${HOST_SERVER_PORT:-}" ]] || - ! kill -0 "$SERVER_PID" 2>/dev/null; then - start_host_server || return 1 - fi - if ! curl --noproxy '*' --fail --silent --max-time 2 \ - "http://127.0.0.1:$HOST_SERVER_PORT" >/dev/null; then - echo "Host local-server baseline failed" - return 1 - fi - - cat >"$control_config" </dev/null && echo PROXY_CONTROL_OK", - "timeout": 10000 - }, - "network": { - "defaultPolicy": "allow", - "proxy": { - "builtinTestServer": true - } - } -} -JSON - - local output - output=$("$MXC_EXEC" --experimental --allow-testing-features "$control_config" 2>&1) || { - echo "$output" - echo "Unfiltered proxy control failed" - return 1 - } - if ! grep -q "PROXY_CONTROL_OK" <<<"$output"; then - echo "$output" - echo "Unfiltered proxy control did not reach the negative-test target" - return 1 - fi - - cat >"$config" </dev/null; echo PROXY_ALLOWED; if curl --fail --silent --max-time 5 http://localhost:$HOST_SERVER_PORT >/dev/null 2>&1; then echo PROXY_FILTER_LEAK; exit 1; else echo PROXY_FILTERED; fi; if curl --noproxy '*' --fail --silent --max-time 5 http://localhost:$HOST_SERVER_PORT >/dev/null 2>&1; then echo PROXY_DIRECT_BYPASS_OBSERVED; else echo PROXY_DIRECT_BLOCKED_OBSERVED; fi", - "timeout": 20000 - }, - "network": { - "defaultPolicy": "block", - "proxy": { - "builtinTestServer": true - }, - "allowedHosts": [ - "127.0.0.1" - ] - } -} -JSON - - output=$("$MXC_EXEC" --experimental --allow-testing-features "$config" 2>&1) || { - echo "$output" - echo "Proxy traffic filtering probe failed" - return 1 - } - if grep -q "PROXY_DIRECT_BYPASS_OBSERVED" <<<"$output"; then - echo "INFO: direct traffic bypassed the cooperative proxy, as Seatbelt currently permits" - elif grep -q "PROXY_DIRECT_BLOCKED_OBSERVED" <<<"$output"; then - echo "INFO: direct traffic was blocked independently of the cooperative proxy" - else - echo "$output" - echo "Direct traffic observation was missing" - return 1 - fi - grep -q "PROXY_ALLOWED" <<<"$output" && - grep -q "PROXY_FILTERED" <<<"$output" && - ! grep -q "PROXY_FILTER_LEAK" <<<"$output" -} - -run_test "Seatbelt execution" test_execution -run_test "Seatbelt exit code" test_exit_code -run_test "Seatbelt filesystem policy" test_filesystem_policy -run_test "Seatbelt environment policy" test_environment_policy -run_test "Seatbelt timeout" test_timeout -run_test "Seatbelt network default deny" test_network_default_deny -run_test "Seatbelt builtin proxy startup" test_builtin_proxy_startup - -run_info_test "Seatbelt clipboard allow" test_clipboard_allow -run_info_test "Seatbelt clipboard deny" test_clipboard_deny -run_info_test "Seatbelt GUI session" test_gui_session -run_info_test "Seatbelt public internet allow" test_public_internet_allow -run_info_test "Seatbelt full proxy traffic filtering" test_proxy_traffic_filtering - -echo "================================" -echo "Results: $PASSED passed, $FAILED failed" -echo "Information only: $INFO_PASSED passed, $INFO_FAILED failed" -if [[ $INFO_FAILED -gt 0 ]]; then - echo -e "Informational failures (non-blocking):$INFO_FAILURES" -fi -if [[ $FAILED -gt 0 ]]; then - echo -e "Failures:$FAILURES" - exit 1 -fi diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index f50d8a16c..c8a7c3ef3 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -53,7 +53,8 @@ case "$backend" in seatbelt) test -x "$binary_directory/mxc-exec-mac" test -x "$binary_directory/unix-test-proxy" - bash "$script_root/TEMP_run_seatbelt_all_tests.sh" "$binary_directory" + echo "The Seatbelt CI handler is not wired to an existing backend test entry point yet." >&2 + exit 2 ;; *) usage From 990357cf19125ab438e0b3f4f7e492ba23733d3f Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:56:57 -0700 Subject: [PATCH 81/93] Cleanup unnecessary changes + address feedback --- .github/actions/setup-cargo-feed/action.yml | 4 +-- .github/workflows/Build.Linux.Job.yml | 8 ++--- .../workflows/Validation.Tests.Scheduled.yml | 1 - scripts/ci/validation-test-matrix.json | 33 ++++++++++--------- tests/configs/lxc_network_diagnostics.json | 19 ----------- tests/scripts/run_lxc_network_test.sh | 13 -------- 6 files changed, 23 insertions(+), 55 deletions(-) delete mode 100644 tests/configs/lxc_network_diagnostics.json diff --git a/.github/actions/setup-cargo-feed/action.yml b/.github/actions/setup-cargo-feed/action.yml index eb6d73459..00f2973f1 100644 --- a/.github/actions/setup-cargo-feed/action.yml +++ b/.github/actions/setup-cargo-feed/action.yml @@ -5,6 +5,6 @@ description: >- runs: using: composite steps: - - shell: pwsh + - shell: bash working-directory: ${{ github.workspace }} - run: Get-Content .azure-pipelines\.cargo\config.public.toml | Add-Content -Path .cargo\config.toml + run: cat .azure-pipelines/.cargo/config.public.toml >> .cargo/config.toml diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index 295a8ccd0..bf69045d6 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -97,10 +97,10 @@ jobs: # Runs the Bubblewrap executor characterization tests. lxc-exec was built # into src/target//release above, where find_binary() locates it. - # - name: Test executor characterization (wxc_e2e_tests) - # working-directory: src - # run: cargo test --locked --release --target ${{ matrix.target }} - # -p wxc_e2e_tests + - name: Test executor characterization (wxc_e2e_tests) + working-directory: src + run: cargo test --locked --release --target ${{ matrix.target }} + -p wxc_e2e_tests ############### #################### diff --git a/.github/workflows/Validation.Tests.Scheduled.yml b/.github/workflows/Validation.Tests.Scheduled.yml index cca7b82cb..9bc3bf9ad 100644 --- a/.github/workflows/Validation.Tests.Scheduled.yml +++ b/.github/workflows/Validation.Tests.Scheduled.yml @@ -12,7 +12,6 @@ on: default: nightly type: choice options: - - pr - nightly - weekly diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 37dc30126..682937da4 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -352,25 +352,30 @@ "os": "windows-prerelease-process-container", "backends": [ "process-t1", - "isolation-session" + "process-t3", + "isolation-session", + "wslc" ] }, { "os": "windows-25h2", "backends": [ - "process-t3" + "process-t3", + "wslc" ] }, { "os": "windows-24h2", "backends": [ - "process-t3" + "process-t3", + "wslc" ] }, { "os": "windows-23h2", "backends": [ - "process-t3" + "process-t3", + "wslc" ] }, { @@ -405,31 +410,27 @@ "weekly": [], "enabled": [ { - "os": "ubuntu-24.04", + "os": "windows-prerelease-process-container", "backends": [ - "lxc", - "bubblewrap" + "wslc" ] }, { - "os": "ubuntu-26.04", + "os": "windows-25h2", "backends": [ - "lxc", - "bubblewrap" + "wslc" ] }, { - "os": "rhel-10", + "os": "windows-24h2", "backends": [ - "lxc", - "bubblewrap" + "wslc" ] }, { - "os": "debian-13", + "os": "windows-23h2", "backends": [ - "lxc", - "bubblewrap" + "wslc" ] } ] diff --git a/tests/configs/lxc_network_diagnostics.json b/tests/configs/lxc_network_diagnostics.json deleted file mode 100644 index a7ab5bcb4..000000000 --- a/tests/configs/lxc_network_diagnostics.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "version": "0.6.0-alpha", - "containerId": "CLI-LXC-Network-Diagnostics", - "containment": "lxc", - "process": { - "commandLine": "echo '--- container interfaces ---'; ip -4 addr show eth0 2>&1 || ifconfig eth0 2>&1; echo '--- container routes ---'; ip route 2>&1; echo '--- container resolv.conf ---'; cat /etc/resolv.conf 2>&1; echo '--- gateway reachability ---'; ping -c 1 -W 3 10.0.3.1 2>&1 || echo 'gateway unreachable'; echo '--- DNS via bridge ---'; nslookup api.github.com 10.0.3.1 2>&1 || echo 'bridge DNS failed'; echo '--- DNS via configured resolver ---'; nslookup api.github.com 2>&1 || echo 'configured resolver failed'; echo '--- direct IP egress (bypasses DNS) ---'; wget -qO- --timeout=5 https://140.82.121.6/zen 2>&1 | head -c 100 || echo 'direct IP fetch failed'", - "timeout": 60000 - }, - "lifecycle": { - "destroyOnExit": true - }, - "lxc": { - "distribution": "alpine", - "release": "3.23" - }, - "network": { - "defaultPolicy": "allow" - } -} diff --git a/tests/scripts/run_lxc_network_test.sh b/tests/scripts/run_lxc_network_test.sh index eb8e09f37..751b07964 100644 --- a/tests/scripts/run_lxc_network_test.sh +++ b/tests/scripts/run_lxc_network_test.sh @@ -15,18 +15,5 @@ if [ ! -f "$LXC_EXEC" ]; then exit 1 fi -echo "Running LXC network test..." - -# When the network test fails it is usually the environment (no DHCP lease, -# dnsmasq not answering, blocked egress) rather than the policy code. Capture -# the container's own view first so the log distinguishes those cases. This is -# diagnostic only and never fails the suite. -if [ "${MXC_LXC_NETWORK_DIAGNOSTICS:-1}" = "1" ]; then - echo "--- LXC network diagnostics (container) ---" - "$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_network_diagnostics.json" 2>&1 || \ - echo "(diagnostic container run failed)" - echo "--- end diagnostics ---" -fi - "$LXC_EXEC" "$REPO_DIR/tests/configs/lxc_network_test.json" echo "LXC network test complete." From 7f0629b611bb00c442c36f9d010ac4cfa9eb7357 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:59:28 -0700 Subject: [PATCH 82/93] update copilot documentation + fix microvm still causing issues --- .github/copilot-instructions.md | 61 ++++++++++++++++++++++++--- .github/workflows/Build.Linux.Job.yml | 2 +- docs/pull-requests.md | 8 ++-- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b8c7b1e3a..c1964d683 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -45,12 +45,51 @@ Requires Xcode Command Line Tools and Rust. Produces an unsigned `mxc-exec-mac` ### GitHub Actions -`.github/workflows/Build.And.Validate.yml` is the PR/CI entry point. It calls the -workflow-call-only `.github/workflows/Build.Artifacts.Job.yml`, which builds and -uploads the Windows, Linux, and macOS artifacts in parallel. Scheduled E2E and -validation-infrastructure workflows reuse the same artifact workflow before -calling `.github/workflows/Test.Matrix.Job.yml`; keep artifact production and -consumption in the same workflow run. +`.github/workflows/Build.yml` is the PR/CI entry point. It fans out to the +workflow-call-only `Build.Windows.Job.yml`, `Build.Linux.Job.yml`, and +`Build.MacOS.Job.yml`, which build and upload the per-target artifacts in +parallel, then to the lint / versioning / SDK jobs. + +**Validation (E2E) test infrastructure.** Backend E2E tests run from those same +build artifacts — never from a fresh build — so artifact production and +consumption stay in one workflow run: + +- `.github/workflows/Validation.Tests.Scheduled.yml` — scheduled entry point. + The `nightly` plan runs Mon–Sat; Sunday runs `nightly` *and* `weekly`. + `workflow_dispatch` takes a `plan` input to run one on demand. +- `.github/workflows/Validation.Infra.Testing.yml` — push-triggered entry point + for iterating on the validation infrastructure itself, on a dedicated branch. +- `.github/workflows/Validation.Tests.Matrix.Job.yml` — workflow-call-only, + takes a `plan` input. Its `resolve` job expands the plan into per-family + matrices, then the `windows` / `linux` / `macos` jobs each download the + artifact, prepare the host, and run the backend suite. + +An entry point must build the artifacts (call the three `Build.*.Job.yml` +workflows) before calling the matrix job. + +**The matrix is declarative:** + +- `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each + with per-architecture target/artifact/1ES pool and the backends that platform + supports), `handlers` (backend id → test-dispatcher command), and `triggers` + (which OS/backend pairs each plan runs). +- `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and + expands a plan (`pr`, `nightly`, `weekly`, `enabled`) into GitHub Actions + matrices. It rejects an invalid catalog before any specialized test runner is + allocated, so add a backend to a trigger only where the platform declares it. +- A non-macOS platform architecture with an empty `pool` is never scheduled, + which is how a catalog entry stays declared but dormant. macOS entries use a + GitHub-hosted `runner` instead of a 1ES `pool`. + +**Host preparation** happens in the matrix job before the tests, keyed by the +matrix `backend` id: `scripts/ci/prepare-windows-host.ps1` and +`scripts/ci/prepare-linux-host.sh`. A backend with no prerequisites is an +explicit no-op, so the step runs unconditionally for every entry. + +**Test dispatch** goes through `tests/scripts/run_ci_backend_tests.ps1` +(Windows) and `tests/scripts/run_ci_backend_tests.sh` (Linux/macOS), which map +the handler command to the repository's existing backend suite. A handler with +no wired suite fails loudly rather than reporting a false success. ### Individual components @@ -108,6 +147,16 @@ cargo test -p wxc_e2e_tests -- --ignored # Include stress tests (run_on_repea # WSLC has no cargo E2E suite — it is covered by tests\scripts\run_wslc_all_tests.ps1, # which the validation matrix runs via tests\scripts\run_ci_backend_tests.ps1. + +# CI validation entry points — run a backend suite against a downloaded artifact +# the way the validation matrix does. Take the handler command (see the +# `handlers` map in scripts/ci/validation-test-matrix.json), not the matrix +# backend id, except where a tier must be pinned. +tests\scripts\run_ci_backend_tests.ps1 -Backend process-container -BinaryDirectory -Architecture x64 -ExpectedTier T1 +tests\scripts\run_ci_backend_tests.sh + +# Resolve a plan locally to see exactly what CI would schedule +node scripts/ci/resolve-validation-test-matrix.mjs --plan nightly ``` ## Architecture diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index 4eb47ec5b..8b013231a 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -13,7 +13,7 @@ jobs: - arch: x64 runner: ubuntu-latest target: x86_64-unknown-linux-gnu - features: hyperlight microvm + features: hyperlight - arch: arm64 runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu diff --git a/docs/pull-requests.md b/docs/pull-requests.md index fa4d70087..ac1407e40 100644 --- a/docs/pull-requests.md +++ b/docs/pull-requests.md @@ -3,9 +3,11 @@ ## GitHub Actions (automatic) Every PR is validated automatically by the GitHub Actions workflows under -`.github/workflows/` (entry point: `Build.And.Validate.yml`). This is the primary -PR signal — it calls the reusable `Build.Artifacts.Job.yml` workflow, then -tests on native Windows x64/arm64, Linux x64/arm64, and macOS arm64 hosts. +`.github/workflows/` (entry point: `Build.yml`). This is the primary PR signal — +it fans out to the reusable `Build.Windows.Job.yml`, `Build.Linux.Job.yml`, and +`Build.MacOS.Job.yml` workflows, which build and test on native Windows +x64/arm64, Linux x64/arm64, and macOS arm64 hosts, then runs the lint, +versioning, and SDK jobs. ## Azure Pipelines (optional on PRs, required on `main`) From f6f59368cb2a3ecca48269aeab11ae4603cdc783 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:07:19 -0700 Subject: [PATCH 83/93] adding cargo e2e tests back, for now --- .github/workflows/Build.Linux.Job.yml | 64 ++++++++++++++------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index 8b013231a..414366d1a 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -85,15 +85,15 @@ jobs: # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec # always includes the Bubblewrap backend (bwrap_common is a non-optional # dependency), so the binary built above can drive it. - # - name: Install Bubblewrap - # working-directory: ${{ github.workspace }} - # run: | - # sudo apt-get update - # sudo apt-get install -y bubblewrap - # # Ubuntu 24.04 runners restrict unprivileged user namespaces via - # # AppArmor, which blocks `bwrap --unshare-user`. Relax it so the - # # sandbox can start (no-op on kernels without this knob). - # sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + - name: Install Bubblewrap + working-directory: ${{ github.workspace }} + run: | + sudo apt-get update + sudo apt-get install -y bubblewrap + # Ubuntu 24.04 runners restrict unprivileged user namespaces via + # AppArmor, which blocks `bwrap --unshare-user`. Relax it so the + # sandbox can start (no-op on kernels without this knob). + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true # Runs the Bubblewrap executor characterization tests. lxc-exec was built # into src/target//release above, where find_binary() locates it. @@ -126,24 +126,25 @@ jobs: run: cargo build --locked --release --target ${{ matrix.target }} -p unix_test_proxy - - name: Verify artifact payload - shell: bash - working-directory: ${{ github.workspace }} - run: | - set -euo pipefail - bin_dir="src/target/${{ matrix.target }}/release" - required=(lxc-exec unix-test-proxy) - if [[ "${{ matrix.arch }}" == "x64" ]]; then - required+=(nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf) - fi - missing=() - for file in "${required[@]}"; do - [[ -f "$bin_dir/$file" ]] || missing+=("$file") - done - if (( ${#missing[@]} > 0 )); then - printf 'Missing artifact files: %s\n' "${missing[*]}" >&2 - exit 1 - fi + # MICROVM NOT WORKING CURRENTLY + # - name: Verify artifact payload + # shell: bash + # working-directory: ${{ github.workspace }} + # run: | + # set -euo pipefail + # bin_dir="src/target/${{ matrix.target }}/release" + # required=(lxc-exec unix-test-proxy) + # if [[ "${{ matrix.arch }}" == "x64" ]]; then + # required+=(nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf) + # fi + # missing=() + # for file in "${required[@]}"; do + # [[ -f "$bin_dir/$file" ]] || missing+=("$file") + # done + # if (( ${#missing[@]} > 0 )); then + # printf 'Missing artifact files: %s\n' "${missing[*]}" >&2 + # exit 1 + # fi - name: Upload binaries uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -152,9 +153,10 @@ jobs: path: | src/target/${{ matrix.target }}/release/lxc-exec src/target/${{ matrix.target }}/release/unix-test-proxy - src/target/${{ matrix.target }}/release/nanvixd.elf - src/target/${{ matrix.target }}/release/nanvix_rootfs.img - src/target/${{ matrix.target }}/release/python3.initrd - src/target/${{ matrix.target }}/release/bin/kernel.elf if-no-files-found: error retention-days: 1 + + # src/target/${{ matrix.target }}/release/nanvixd.elf + # src/target/${{ matrix.target }}/release/nanvix_rootfs.img + # src/target/${{ matrix.target }}/release/python3.initrd + # src/target/${{ matrix.target }}/release/bin/kernel.elf From e9229de33ed8795d2d6f2f01217eb1eccb190dde Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:35:56 -0700 Subject: [PATCH 84/93] Running full nightly matrix --- .github/workflows/Build.Windows.Job.yml | 74 ++++++++++--------- .github/workflows/Build.yml | 2 +- .../workflows/Validation.Infra.Testing.yml | 2 +- .../workflows/Validation.Tests.Matrix.Job.yml | 20 ++--- 4 files changed, 50 insertions(+), 48 deletions(-) diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index bf9af3304..81f195d41 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -71,34 +71,35 @@ jobs: --no-default-features --features "${{ matrix.features }}" - - name: Verify artifact payload - shell: pwsh - run: | - $binDir = "target\${{ matrix.target }}\release" - $required = @( - "wxc-exec.exe", - "wxc-host-prep.exe", - "winhttp-proxy-shim.exe", - "wxc-windows-sandbox-daemon.exe", - "wxc-windows-sandbox-guest.exe", - "wxc-test-proxy.exe", - "wxc-test-driver.exe", - "wxc-ui-probe.exe", - "mxc-diagnostic-console.exe", - "wslcsdk.dll" - ) - if ("${{ matrix.arch }}" -eq "x64") { - $required += @( - "nanvixd.exe", - "nanvix_rootfs.img", - "python3.initrd", - "bin\kernel.elf" - ) - } - $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } - if ($missing) { - throw "Missing artifact files: $($missing -join ', ')" - } + # MICROVM NOT WORKING CURRENTLY + # - name: Verify artifact payload + # shell: pwsh + # run: | + # $binDir = "target\${{ matrix.target }}\release" + # $required = @( + # "wxc-exec.exe", + # "wxc-host-prep.exe", + # "winhttp-proxy-shim.exe", + # "wxc-windows-sandbox-daemon.exe", + # "wxc-windows-sandbox-guest.exe", + # "wxc-test-proxy.exe", + # "wxc-test-driver.exe", + # "wxc-ui-probe.exe", + # "mxc-diagnostic-console.exe", + # "wslcsdk.dll" + # ) + # if ("${{ matrix.arch }}" -eq "x64") { + # $required += @( + # "nanvixd.exe", + # "nanvix_rootfs.img", + # "python3.initrd", + # "bin\kernel.elf" + # ) + # } + # $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } + # if ($missing) { + # throw "Missing artifact files: $($missing -join ', ')" + # } - name: Upload binaries uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -112,15 +113,16 @@ jobs: src/target/${{ matrix.target }}/release/wxc-windows-sandbox-guest.exe src/target/${{ matrix.target }}/release/wxc-wslc-daemon.exe src/target/${{ matrix.target }}/release/wxc-test-proxy.exe - src/target/${{ matrix.target }}/release/wxc-test-driver.exe - src/target/${{ matrix.target }}/release/wxc-ui-probe.exe src/target/${{ matrix.target }}/release/mxc-diagnostic-console.exe src/target/${{ matrix.target }}/release/wslcsdk.dll - src/target/${{ matrix.target }}/release/nanvixd.exe - src/target/${{ matrix.target }}/release/nanvix_rootfs.img - src/target/${{ matrix.target }}/release/python3.initrd - src/target/${{ matrix.target }}/release/bin/kernel.elf - src/target/${{ matrix.target }}/release/snapshots/kernel.vmem - src/target/${{ matrix.target }}/release/snapshots/kernel.whp.cbor if-no-files-found: error retention-days: 1 + + # src/target/${{ matrix.target }}/release/nanvixd.exe + # src/target/${{ matrix.target }}/release/nanvix_rootfs.img + # src/target/${{ matrix.target }}/release/python3.initrd + # src/target/${{ matrix.target }}/release/bin/kernel.elf + # src/target/${{ matrix.target }}/release/snapshots/kernel.vmem + # src/target/${{ matrix.target }}/release/snapshots/kernel.whp.cbor + # src/target/${{ matrix.target }}/release/wxc-test-driver.exe + # src/target/${{ matrix.target }}/release/wxc-ui-probe.exe diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 4f41d91af..31848a082 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -15,7 +15,7 @@ on: workflow_dispatch: concurrency: - group: build-validate-${{ github.workflow }}-${{ github.ref }} + group: build-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml index 85483f07a..ab8c4ffec 100644 --- a/.github/workflows/Validation.Infra.Testing.yml +++ b/.github/workflows/Validation.Infra.Testing.yml @@ -35,4 +35,4 @@ jobs: with: # Exercise every combination currently rolled out, independent of its # normal PR/nightly/weekly schedule. - plan: enabled + plan: nightly diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index f9bb7a448..a1cd3bbb0 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -24,7 +24,7 @@ jobs: has_linux: ${{ steps.matrix.outputs.has_linux }} has_macos: ${{ steps.matrix.outputs.has_macos }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Resolve test combinations id: matrix @@ -40,10 +40,10 @@ jobs: runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Download ${{ matrix.target }} artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3d3c42e5aac5ba805825da76410c181273ba90b1 with: name: ${{ matrix.artifact }} path: artifacts/bin @@ -77,7 +77,7 @@ jobs: - name: Upload failure logs if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3d3c42e5aac5ba805825da76410c181273ba90b1 with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: | @@ -98,10 +98,10 @@ jobs: runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Download ${{ matrix.target }} artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3d3c42e5aac5ba805825da76410c181273ba90b1 with: name: ${{ matrix.artifact }} path: artifacts/bin @@ -130,7 +130,7 @@ jobs: - name: Upload failure logs if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@3d3c42e5aac5ba805825da76410c181273ba90b1 with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: ${{ runner.temp }}/mxc-ci.log @@ -147,10 +147,10 @@ jobs: runs-on: ${{ matrix.runner }} timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Download ${{ matrix.target }} artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3d3c42e5aac5ba805825da76410c181273ba90b1 with: name: ${{ matrix.artifact }} path: artifacts/bin @@ -166,7 +166,7 @@ jobs: - name: Upload failure logs if: failure() || cancelled() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@3d3c42e5aac5ba805825da76410c181273ba90b1 with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: ${{ runner.temp }}/mxc-ci.log From 8a37be42711f26ded78656988b5333aced6c1a86 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:55:07 -0700 Subject: [PATCH 85/93] Fixing action SHAs --- .github/workflows/Build.Windows.Job.yml | 4 ++-- .../workflows/Validation.Tests.Matrix.Job.yml | 20 +++++++++---------- sdk/node/tests/integration/test-helpers.ts | 2 ++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index 81f195d41..b61ad79d5 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -115,6 +115,8 @@ jobs: src/target/${{ matrix.target }}/release/wxc-test-proxy.exe src/target/${{ matrix.target }}/release/mxc-diagnostic-console.exe src/target/${{ matrix.target }}/release/wslcsdk.dll + src/target/${{ matrix.target }}/release/wxc-ui-probe.exe + src/target/${{ matrix.target }}/release/wxc-test-driver.exe if-no-files-found: error retention-days: 1 @@ -124,5 +126,3 @@ jobs: # src/target/${{ matrix.target }}/release/bin/kernel.elf # src/target/${{ matrix.target }}/release/snapshots/kernel.vmem # src/target/${{ matrix.target }}/release/snapshots/kernel.whp.cbor - # src/target/${{ matrix.target }}/release/wxc-test-driver.exe - # src/target/${{ matrix.target }}/release/wxc-ui-probe.exe diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index a1cd3bbb0..05b423c9f 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -24,7 +24,7 @@ jobs: has_linux: ${{ steps.matrix.outputs.has_linux }} has_macos: ${{ steps.matrix.outputs.has_macos }} steps: - - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Resolve test combinations id: matrix @@ -40,10 +40,10 @@ jobs: runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 90 steps: - - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Download ${{ matrix.target }} artifact - uses: actions/download-artifact@v3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: ${{ matrix.artifact }} path: artifacts/bin @@ -77,7 +77,7 @@ jobs: - name: Upload failure logs if: failure() || cancelled() - uses: actions/upload-artifact@v3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: | @@ -98,10 +98,10 @@ jobs: runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] timeout-minutes: 90 steps: - - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Download ${{ matrix.target }} artifact - uses: actions/download-artifact@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: ${{ matrix.artifact }} path: artifacts/bin @@ -130,7 +130,7 @@ jobs: - name: Upload failure logs if: failure() || cancelled() - uses: actions/upload-artifact@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: ${{ runner.temp }}/mxc-ci.log @@ -147,10 +147,10 @@ jobs: runs-on: ${{ matrix.runner }} timeout-minutes: 90 steps: - - uses: actions/checkout@v3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Download ${{ matrix.target }} artifact - uses: actions/download-artifact@v3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: ${{ matrix.artifact }} path: artifacts/bin @@ -166,7 +166,7 @@ jobs: - name: Upload failure logs if: failure() || cancelled() - uses: actions/upload-artifact@3d3c42e5aac5ba805825da76410c181273ba90b1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: logs-${{ inputs.plan }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.backend }}-${{ github.run_attempt }} path: ${{ runner.temp }}/mxc-ci.log diff --git a/sdk/node/tests/integration/test-helpers.ts b/sdk/node/tests/integration/test-helpers.ts index 77da91bd0..1ef6d902d 100644 --- a/sdk/node/tests/integration/test-helpers.ts +++ b/sdk/node/tests/integration/test-helpers.ts @@ -48,6 +48,8 @@ export function getSdkBinDir(): string { export const EXPECTED_WINDOWS_BINARIES = [ 'wxc-exec.exe', 'wxc-host-prep.exe', + 'wxc-ui-probe.exe', + 'wxc-test-driver.exe', 'winhttp-proxy-shim.exe', 'wxc-test-proxy.exe', 'wxc-windows-sandbox-daemon.exe', From 3cd8a621be14ff6f31d5a711ba4fb55735b39ec7 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:18:29 -0700 Subject: [PATCH 86/93] Added additional documentation --- .github/workflows/Build.Linux.Job.yml | 2 +- docs/pull-requests.md | 2 + tests/scripts/README.md | 86 ++++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index 414366d1a..dd41eb912 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -78,7 +78,7 @@ jobs: --no-default-features "${features[@]}" #################### - # E2E Tests are being moved to the Test.Matrix.Job.yml workflow, which runs cross-platform validation + # E2E Tests are being moved to the Validation.Tests.Matrix.Job.yml workflow, which runs cross-platform validation ############### # Bubblewrap is required to run the executor characterization tests in diff --git a/docs/pull-requests.md b/docs/pull-requests.md index ac1407e40..12789789e 100644 --- a/docs/pull-requests.md +++ b/docs/pull-requests.md @@ -9,6 +9,8 @@ it fans out to the reusable `Build.Windows.Job.yml`, `Build.Linux.Job.yml`, and x64/arm64, Linux x64/arm64, and macOS arm64 hosts, then runs the lint, versioning, and SDK jobs. +A change that touches only `docs/**` does not trigger `Build.yml`. + ## Azure Pipelines (optional on PRs, required on `main`) The ADO pipeline (`MXC-PR-Build`) is the Azure version of the PR pipeline. The official diff --git a/tests/scripts/README.md b/tests/scripts/README.md index 60b2554d0..279579521 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -9,11 +9,21 @@ All scripts accept a `-Release` switch to use the release build (default: debug) ## Prerequisites -- Windows 11 +Shared: + - Rust toolchain installed (`rustup`, `cargo`) - Built binaries (`cargo build` from `src/`) + +Windows (`.ps1`): + +- Windows 11 - PowerShell 7+ (`pwsh`) +Linux / macOS (`.sh`): + +- Bash, plus the per-backend prerequisites listed in the backend's doc (for + example `bwrap` for Bubblewrap, the LXC stack for LXC) + ## Scripts | Script | Description | Extra prerequisites | @@ -33,9 +43,69 @@ All scripts accept a `-Release` switch to use the release build (default: debug) | `run_processcontainer_proxy_tests.ps1` | Process container proxy tests | `wxc-exec.exe` | | `run_on_repeat.ps1` | Stress test (loops core tests) | `wxc-exec.exe` | -These scripts are local helpers. Not every script is run by CI because several -depend on local OS features such as Windows Sandbox, WHP, proxy setup, or stress -test duration. +### Linux suites + +| Script | Description | Extra prerequisites | +|--------|-------------|---------------------| +| `run_bwrap_all_tests.sh` | All Bubblewrap tests | `lxc-exec`, `bwrap` | +| `run_lxc_all_tests.sh` | All LXC tests | `lxc-exec`, LXC stack, root | + +Individual `run_bwrap_*.sh` / `run_lxc_*.sh` scripts run one case each; the +aggregate scripts above are what CI dispatches to. + +Not every script runs in CI: several depend on local OS features such as +Windows Sandbox, WHP, proxy setup, or stress-test duration. The ones CI does +run are reached through the dispatchers below rather than being invoked +directly. + +### CI dispatch + +The validation matrix (see `scripts/ci/validation-test-matrix.json` and +`.github/workflows/Validation.Tests.Matrix.Job.yml`) never builds from source. +It downloads a build artifact, prepares the host, and then hands off to one of +these dispatchers, which map a backend handler to the suites above: + +| Dispatcher | Platforms | Handlers | +|------------|-----------|----------| +| `run_ci_backend_tests.ps1` | Windows | `process-container` (needs `-ExpectedTier`), `isolation-session`, `windows-sandbox`, `wslc`, `microvm`, `hyperlight` | +| `run_ci_backend_tests.sh` | Linux, macOS | `bubblewrap`, `lxc`, `seatbelt`, `microvm`, `hyperlight` | + +Pass the **handler** name from the catalog's `handlers` map, not the matrix +backend id (they differ where one handler serves several entries — `process-t1` +and `process-t3` both dispatch to `process-container` and are distinguished by +`-ExpectedTier`): + +```powershell +tests\scripts\run_ci_backend_tests.ps1 -Backend process-container ` + -BinaryDirectory -Architecture x64 -ExpectedTier T1 +``` + +```bash +tests/scripts/run_ci_backend_tests.sh bubblewrap +``` + +A handler with no wired suite exits non-zero on purpose, so accidentally +enabling it in a trigger fails loudly instead of reporting a false success. + +To see exactly what a plan would schedule without pushing: + +```bash +node scripts/ci/resolve-validation-test-matrix.mjs --plan nightly +``` + +**Skip semantics.** Several suites degrade gracefully on an unsupported host: +the IsolationSession suites decide availability from a single `wxc-exec --probe` +read of `probes.isolationSessionAvailable` (covering both a host that cannot +activate the API and a binary built without `--features isolation_session`), +print `SKIPPED`, and exit 0. + +Because a skip exits 0 and the dispatchers propagate only the exit code, **a +green CI job does not by itself prove the suite ran.** Anything treating these +suites as validation evidence must check the `SKIPPED` line or the executed +count, not just the exit status — the matrix entry says the host is expected to +support the backend, so a silent skip there is a gap in coverage rather than a +graceful degradation. Independently, a run that reaches the summary having +executed zero tests always fails, since it substantiates nothing. ### Manual smoke tests @@ -67,9 +137,11 @@ itself and takes a `-ComputerName` / `-VMName` plus `-Credential`. | `push_batch_and_config_files_to_vm.ps1` | `tests\configs\`, `examples\`, runner batch files, helper scripts | TShell (active `Open-Device` session) | | `push_sdk_integration_tests_to_vm.ps1` | SDK integration test artifacts (`sdk\bin\x64`, compiled tests, `node_modules`, `package.json`, `run-tests.js`) | PowerShell Remoting (`-ComputerName`/`-VMName` + `-Credential`) | -CI currently runs the MicroVM Rust E2E suite when WHP is available. Other -executor E2E tests are local/prerequisite-gated and should be run on machines -with the required Windows features and binaries. +Backend E2E coverage runs on a schedule (not on PRs) through the validation +matrix described under [CI dispatch](#ci-dispatch), against binaries downloaded +from the build artifacts. Suites whose backend is not yet wired into a trigger — +and any test needing a Windows feature or hardware the pool images lack — remain +local/prerequisite-gated and should be run on a machine that has them. ## Test ownership From f18c79149b33199985618a86636bbc91f13baba8 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:12:56 -0700 Subject: [PATCH 87/93] Addressed PR feedback, removed handler-backend distinction --- .github/copilot-instructions.md | 34 +- .github/workflows/Build.Linux.Job.yml | 9 +- .github/workflows/Build.MacOS.Job.yml | 2 +- .github/workflows/Build.yml | 7 +- .../workflows/Validation.Tests.Matrix.Job.yml | 37 +- docs/ci-validation-infrastructure.md | 319 ++++++++++++++++++ docs/pull-requests.md | 2 - scripts/ci/keys/RPM-GPG-KEY-EPEL-10 | 29 ++ scripts/ci/prepare-linux-host.sh | 54 ++- scripts/ci/prepare-windows-host.ps1 | 3 +- scripts/ci/resolve-validation-test-matrix.mjs | 44 +-- scripts/ci/validation-test-matrix.json | 48 +-- tests/scripts/README.md | 26 +- tests/scripts/run_ci_backend_tests.ps1 | 125 ++++--- tests/scripts/run_ci_backend_tests.sh | 10 +- 15 files changed, 548 insertions(+), 201 deletions(-) create mode 100644 docs/ci-validation-infrastructure.md create mode 100644 scripts/ci/keys/RPM-GPG-KEY-EPEL-10 diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c1964d683..e09c726bc 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -50,8 +50,11 @@ workflow-call-only `Build.Windows.Job.yml`, `Build.Linux.Job.yml`, and `Build.MacOS.Job.yml`, which build and upload the per-target artifacts in parallel, then to the lint / versioning / SDK jobs. -**Validation (E2E) test infrastructure.** Backend E2E tests run from those same -build artifacts — never from a fresh build — so artifact production and +**Validation (E2E) test infrastructure.** Fully documented in +[`docs/ci-validation-infrastructure.md`](../docs/ci-validation-infrastructure.md) +(matrix contents, job names, per-backend coverage and status, and the runbook +for adding/removing an OS, backend, or plan). Backend E2E tests run from those +same build artifacts — never from a fresh build — so artifact production and consumption stay in one workflow run: - `.github/workflows/Validation.Tests.Scheduled.yml` — scheduled entry point. @@ -71,12 +74,14 @@ workflows) before calling the matrix job. - `scripts/ci/validation-test-matrix.json` is the catalog: `platforms` (each with per-architecture target/artifact/1ES pool and the backends that platform - supports), `handlers` (backend id → test-dispatcher command), and `triggers` - (which OS/backend pairs each plan runs). + supports) and `triggers` (which OS/backend pairs each plan runs). The + `triggers` keys *are* the plan list — the resolver reads them at run time, so + adding a plan needs no script change. - `scripts/ci/resolve-validation-test-matrix.mjs` validates that catalog and - expands a plan (`pr`, `nightly`, `weekly`, `enabled`) into GitHub Actions - matrices. It rejects an invalid catalog before any specialized test runner is - allocated, so add a backend to a trigger only where the platform declares it. + expands a plan (currently `pr`, `nightly`, `weekly`, `enabled`) into GitHub + Actions matrices. It rejects an invalid catalog before any specialized test + runner is allocated, so add a backend to a trigger only where the platform + declares it. - A non-macOS platform architecture with an empty `pool` is never scheduled, which is how a catalog entry stays declared but dormant. macOS entries use a GitHub-hosted `runner` instead of a 1ES `pool`. @@ -88,8 +93,11 @@ explicit no-op, so the step runs unconditionally for every entry. **Test dispatch** goes through `tests/scripts/run_ci_backend_tests.ps1` (Windows) and `tests/scripts/run_ci_backend_tests.sh` (Linux/macOS), which map -the handler command to the repository's existing backend suite. A handler with -no wired suite fails loudly rather than reporting a false success. +the matrix `backend` id to the repository's existing backend suite. Ids that +share a suite get their own case (`process-t1` and `process-t3` both run +`WinProcessContainer-Tests.ps1`, which derives the tier it expects from the +host's own `--probe`). A backend with no wired suite fails loudly rather than +reporting a false success. ### Individual components @@ -149,10 +157,9 @@ cargo test -p wxc_e2e_tests -- --ignored # Include stress tests (run_on_repea # which the validation matrix runs via tests\scripts\run_ci_backend_tests.ps1. # CI validation entry points — run a backend suite against a downloaded artifact -# the way the validation matrix does. Take the handler command (see the -# `handlers` map in scripts/ci/validation-test-matrix.json), not the matrix -# backend id, except where a tier must be pinned. -tests\scripts\run_ci_backend_tests.ps1 -Backend process-container -BinaryDirectory -Architecture x64 -ExpectedTier T1 +# the way the validation matrix does. Take the matrix backend id exactly as it +# appears in scripts/ci/validation-test-matrix.json. +tests\scripts\run_ci_backend_tests.ps1 -Backend process-t1 -BinaryDirectory -Architecture x64 tests\scripts\run_ci_backend_tests.sh # Resolve a plan locally to see exactly what CI would schedule @@ -212,6 +219,7 @@ Core references: - `docs/authoring-a-new-feature.md` — step-by-step guide for adding experimental features (which files to touch, in what order) - `docs/examples.md` — annotated configuration examples (see also `tests/examples/` and `tests/configs/`) - `docs/diagnostics.md` — diagnostic logging knobs (env vars, log file format) +- `docs/ci-validation-infrastructure.md` — validation (E2E) test matrix: workflows and job names, catalog format, per-backend coverage and status, and the runbook for adding/removing an OS, backend, or plan - `docs/host-prep.md` — `wxc-host-prep.exe` host setup binary (`prepare-system-drive` / `unprepare-system-drive` for the AppContainer ACEs on the system-drive root, plus `prepare-null-device` / `verify-null-device` / `dump-null-device` for the `\Device\Null` security descriptor that AppContainer-based backends require). Owns elevation via embedded `requireAdministrator` manifest — `wxc-exec.exe` no longer self-elevates. - `docs/sandbox-policy/v1/policy.md` — sandbox policy v1 specification diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index dd41eb912..9c9abc7e6 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -77,10 +77,6 @@ jobs: cargo test --locked --release --target "${{ matrix.target }}" \ --no-default-features "${features[@]}" - #################### - # E2E Tests are being moved to the Validation.Tests.Matrix.Job.yml workflow, which runs cross-platform validation - ############### - # Bubblewrap is required to run the executor characterization tests in # wxc_e2e_tests (they skip via has_bwrap() when it is absent). lxc-exec # always includes the Bubblewrap backend (bwrap_common is a non-optional @@ -101,10 +97,7 @@ jobs: working-directory: src run: cargo test --locked --release --target ${{ matrix.target }} -p wxc_e2e_tests - - ############### - #################### - + # PLM (Permissive Learning Mode) is functionally Windows-only, but the # crate builds cross-platform: the lib's helper modules compile on every # target, and the binary has a no-op stub `fn main()` for non-Windows so diff --git a/.github/workflows/Build.MacOS.Job.yml b/.github/workflows/Build.MacOS.Job.yml index a45b63d65..ce97d3596 100644 --- a/.github/workflows/Build.MacOS.Job.yml +++ b/.github/workflows/Build.MacOS.Job.yml @@ -6,7 +6,7 @@ on: jobs: build: name: arm64 - runs-on: macos-26 + runs-on: macos-15 defaults: run: working-directory: src diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 31848a082..6b3a8cc52 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -1,14 +1,13 @@ +# Entry point for GHA PR/CI validation. Mirrors the 1ES `1ES.Build.yml` +# split: this file orchestrates, each stage lives in its own reusable. + name: Build on: push: - paths-ignore: - - 'docs/**' branches: [main] pull_request: - paths-ignore: - - 'docs/**' branches: [main, 'feature/*', 'user/*'] workflow_call: diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index 05b423c9f..b1bede25f 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.windows) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - timeout-minutes: 90 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -49,6 +49,7 @@ jobs: path: artifacts/bin - name: Prepare backend prerequisites + timeout-minutes: 15 shell: pwsh run: | & ./scripts/ci/prepare-windows-host.ps1 ` @@ -59,20 +60,13 @@ jobs: - name: Run backend tests # A backend that hangs (rather than failing) would otherwise burn the # full job timeout. Fail fast enough to keep the log useful. - timeout-minutes: 30 + timeout-minutes: 45 shell: pwsh run: | - $arguments = @{ - Backend = '${{ matrix.command }}' - BinaryDirectory = (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') - Architecture = '${{ matrix.architecture }}' - } - if ('${{ matrix.backend }}' -eq 'process-t1') { - $arguments.ExpectedTier = 'T1' - } elseif ('${{ matrix.backend }}' -eq 'process-t3') { - $arguments.ExpectedTier = 'T3' - } - & ./tests/scripts/run_ci_backend_tests.ps1 @arguments *>&1 | + & ./tests/scripts/run_ci_backend_tests.ps1 ` + -Backend '${{ matrix.backend }}' ` + -BinaryDirectory (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') ` + -Architecture '${{ matrix.architecture }}' *>&1 | Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append - name: Upload failure logs @@ -96,7 +90,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.linux) }} runs-on: [ self-hosted, "1ES.Pool=${{ matrix.pool }}", "JobId=mxc-e2e-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" ] - timeout-minutes: 90 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -107,24 +101,26 @@ jobs: path: artifacts/bin - name: Prepare backend prerequisites + timeout-minutes: 15 shell: bash run: | set -euo pipefail bash scripts/ci/prepare-linux-host.sh \ - '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee -a "$RUNNER_TEMP/mxc-ci.log" - name: Run backend tests + timeout-minutes: 45 shell: bash run: | set -euo pipefail - if [[ '${{ matrix.command }}' == 'lxc' ]]; then + if [[ '${{ matrix.backend }}' == 'lxc' ]]; then sudo --preserve-env=RUNNER_TEMP bash tests/scripts/run_ci_backend_tests.sh \ - '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee -a "$RUNNER_TEMP/mxc-ci.log" else bash tests/scripts/run_ci_backend_tests.sh \ - '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee -a "$RUNNER_TEMP/mxc-ci.log" fi @@ -145,7 +141,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.resolve.outputs.macos) }} runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -156,12 +152,13 @@ jobs: path: artifacts/bin - name: Run backend tests + timeout-minutes: 60 shell: bash run: | set -euo pipefail chmod +x artifacts/bin/mxc-exec-mac artifacts/bin/unix-test-proxy bash tests/scripts/run_ci_backend_tests.sh \ - '${{ matrix.command }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | + '${{ matrix.backend }}' "$GITHUB_WORKSPACE/artifacts/bin" 2>&1 | tee "$RUNNER_TEMP/mxc-ci.log" - name: Upload failure logs diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md new file mode 100644 index 000000000..0c34af3f6 --- /dev/null +++ b/docs/ci-validation-infrastructure.md @@ -0,0 +1,319 @@ +# Validation (E2E) test infrastructure + +How MXC runs its backend end-to-end suites across real operating systems, what +each job covers today, and what to change when you need to add, remove, or +retire something. + +This document describes the GitHub Actions validation matrix only. PR-time +build/lint/SDK validation is covered by [`pull-requests.md`](pull-requests.md); +the individual local test scripts are documented in +[`tests/scripts/README.md`](../tests/scripts/README.md). + +## At a glance + +- Validation tests **never build from source**. They download the artifacts + produced by `Build.Windows.Job.yml` / `Build.Linux.Job.yml` / + `Build.MacOS.Job.yml` in the same workflow run, so what gets tested is exactly + what got built. +- The matrix is **declarative**. `scripts/ci/validation-test-matrix.json` is the + only file you edit to change *what runs where*; + `scripts/ci/resolve-validation-test-matrix.mjs` validates it and expands a + plan into GitHub Actions matrices. +- Validation runs **on a schedule, not on PRs**. + +## Moving parts + +| File | Role | +|------|------| +| `.github/workflows/Validation.Tests.Scheduled.yml` | Scheduled entry point. Builds artifacts, then calls the matrix job. | +| `.github/workflows/Validation.Tests.Matrix.Job.yml` | `workflow_call`-only. Resolves the plan and runs the per-family test jobs. | +| `scripts/ci/validation-test-matrix.json` | The matrix: OS versions, backends, triggers. | +| `scripts/ci/resolve-validation-test-matrix.mjs` | Matrix validator + plan expander. Emits the GitHub Actions matrices. | +| `scripts/ci/prepare-windows-host.ps1` | Per-backend Windows host preparation / prerequisite assertions. | +| `scripts/ci/prepare-linux-host.sh` | Per-backend Linux package install and service startup (distro-aware). | +| `tests/scripts/run_ci_backend_tests.ps1` | Windows dispatcher: backend id → existing backend suite. | +| `tests/scripts/run_ci_backend_tests.sh` | Linux/macOS dispatcher: backend id → existing backend suite. | + +### Flow + +``` +Validation.Tests.Scheduled.yml (or Validation.Infra.Testing.yml) + └─ dependency-feed-check + ├─ windows / linux / macos → Build.*.Job.yml (upload artifacts) + └─ test-nightly / test-weekly → Validation.Tests.Matrix.Job.yml + └─ resolve → resolve-validation-test-matrix.mjs --plan + ├─ windows job (matrix) → download artifact → prepare-windows-host.ps1 → run_ci_backend_tests.ps1 + ├─ linux job (matrix) → download artifact → prepare-linux-host.sh → run_ci_backend_tests.sh + └─ macos job (matrix) → download artifact → run_ci_backend_tests.sh +``` + +An entry point **must** build the artifacts before calling the matrix job — the +test jobs only ever `download-artifact`. + +## Jobs + +### `Validation.Tests.Scheduled.yml` — "Scheduled Validation Tests" + +| Job | What it does | +|-----|--------------| +| `dependency-feed-check` | Resolves the locked crate graph through the public `MxcDependencies` feed. Gates the builds. | +| `windows` | `Build.Windows.Job.yml` — x64 + arm64 release build, unit tests, uploads `wxc-binaries-`. | +| `linux` | `Build.Linux.Job.yml` — x64 + arm64 release build, unit tests, `wxc_e2e_tests`, uploads `lxc-binaries-`. | +| `macos` | `Build.MacOS.Job.yml` — arm64 release build, unit + `wxc_e2e_tests`, uploads `mxc-binaries-aarch64-apple-darwin`. | +| `test-nightly` | Calls the matrix job with `plan: nightly`. Runs on every schedule tick and on a `nightly` dispatch. | +| `test-weekly` | Calls the matrix job with `plan: weekly`. Runs only on the Sunday cron and on a `weekly` dispatch. | + +Build artifacts are kept for 1 day — they exist only to feed these jobs. + +### `Validation.Tests.Matrix.Job.yml` — "Create Validation Test Matrix" + +| Job | Runner | What it does | +|-----|--------|--------------| +| `resolve` | `ubuntu-latest` | Runs the resolver, emits one matrix per OS family plus `has_` flags so an empty family is skipped rather than failing on an empty matrix. | +| `windows` | `[self-hosted, 1ES.Pool=, JobId=mxc-e2e-…]` | Download artifact → `prepare-windows-host.ps1 -Backend ` → `run_ci_backend_tests.ps1 -Backend `. | +| `linux` | `[self-hosted, 1ES.Pool=, JobId=mxc-e2e-…]` | Download artifact → `prepare-linux-host.sh ` → `run_ci_backend_tests.sh ` (under `sudo` for LXC). | +| `macos` | GitHub-hosted `${{ matrix.runner }}` | Download artifact → `chmod +x` → `run_ci_backend_tests.sh `. No host-prep step. | + +Per-job display name: `, , ` (macOS omits +the architecture). Job timeout 60 min; host prep 15 min; the test step 45 min +(60 on macOS), so a hung backend fails while the log is still useful. On failure +or cancellation the job uploads `mxc-ci.log` plus the Process Container log +directories as `logs-----`, kept 7 days. + +## The catalog + +`scripts/ci/validation-test-matrix.json` has two sections. + +### `platforms` + +Declares an OS image and, per architecture, the build it consumes, the host pool +it runs on, and **which backends that platform is capable of running**. This is +a capability declaration, not a schedule. + +| Field | Meaning | +|-------|---------| +| `id` | Stable key referenced by `triggers`. Also the value shown in job names. | +| `displayName` | Human label (emitted as `os_name`). | +| `family` | `windows` \| `linux` \| `macos` — selects the matrix job and the dispatcher. | +| `prerelease` | `true` marks an unreleased Windows image. Its `id` must be a neutral alias matching `windows-prerelease-`, because the id is public in job names. | +| `architectures..target` | Rust target triple. | +| `architectures.<…>.artifact` | Build artifact name to download. | +| `architectures.<…>.pool` | 1ES pool name (Windows/Linux). **An empty string means "declared but never scheduled"** — the entry stays documented but dormant. | +| `architectures.<…>.runner` | GitHub-hosted runner label (macOS only; required there). | +| `architectures.<…>.backends` | Backend ids this platform/arch can run. | + +Current platforms: + +| Platform id | Family | x64 pool | arm64 pool | Declared backends (x64) | +|-------------|--------|----------|------------|--------------------------| +| `windows-prerelease-process-container` | windows | `1es-mxc-windows-prerelease-t1-x64` | *(dormant)* | process-t1, process-t3, isolation-session, wslc, windows-sandbox, microvm, hyperlight | +| `windows-prerelease-isolation-session` | windows | *(dormant)* | *(dormant)* | same as above | +| `windows-canary` | windows | *(dormant)* | *(dormant)* | same as above | +| `windows-25h2` | windows | `1es-mxc-e2e-windows-25h2-pro-x64` | *(dormant)* | process-t3, wslc, windows-sandbox, microvm, hyperlight | +| `windows-24h2` | windows | `1es-mxc-e2e-windows-24h2-pro-x64` | *(dormant)* | process-t3, wslc, windows-sandbox, microvm, hyperlight | +| `windows-23h2` | windows | `1es-mxc-e2e-windows-23h2-enterprise-x64` | *(dormant)* | process-t3, wslc, windows-sandbox, microvm, hyperlight | +| `ubuntu-26.04` | linux | `1es-mxc-e2e-ubuntu-26.04-x64` | *(dormant)* | bubblewrap, hyperlight, lxc | +| `ubuntu-24.04` | linux | `1es-mxc-e2e-ubuntu-24.04-x64` | *(dormant)* | bubblewrap, microvm, hyperlight, lxc | +| `rhel-10` | linux | `1es-mxc-e2e-rhel-10-x64` | *(dormant)* | bubblewrap, hyperlight, lxc | +| `debian-13` | linux | `1es-mxc-e2e-debian-13-x64` | *(dormant)* | bubblewrap, hyperlight, lxc | +| `macos-26` | macos | — | runner `macos-26` | seatbelt | +| `macos-15` | macos | — | runner `macos-15` | seatbelt | + +ARM64 is declared throughout but never emitted: no Azure VM SKU offers nested +virtualization on ARM CPUs yet, so the resolver filters Windows/Linux ARM64 out +after expansion (`suppressNonMacArm64`). macOS is ARM64-only. + +### Backend ids + +A backend id is passed straight through: the matrix job hands it to the host-prep +script and then to the dispatcher, which has one `switch`/`case` per id. Ids that +share a suite each keep their own case so they can diverge later without a +mapping table — `process-t1` and `process-t3` both run +`WinProcessContainer-Tests.ps1` today. Teaching the Process Container test suite to +accept an explicit tier (so a T1 host can also be exercised +at the T3 fallback) is a worthwhile future improvement; see +[Possible future improvements](#possible-future-improvements). + +An unwired backend fails loudly on purpose: adding it to a trigger produces a +red job ("write the tests or remove it"), never a green no-op. The dispatchers' +accepted-id lists (`ValidateSet` on Windows, the `case` arms on Unix) are what +catch a typo'd id in the catalog. + +### `triggers` + +Names the OS/backend pairs a plan runs. Entries are architecture-neutral: +expansion emits a job for every architecture of that platform that declares the +backend **and** has a non-empty pool. + +| Plan | Wired to | Contents today | +|------|----------|----------------| +| `nightly` | scheduled Mon–Sun, plus `Validation.Infra.Testing.yml` | 4 Windows platforms, 4 Linux platforms | +| `weekly` | scheduled Sunday | empty | +| `pr` | *(nothing — `Build.yml` does not call the matrix job)* | empty; reserved for a potential future PR-time subset | +| `enabled` | *(nothing — resolvable locally only)* | reserved for testing this infrastructure and rapid iteration | + +Resolved `nightly` today = **17 jobs**: 9 Windows (prerelease × process-t1, +isolation-session, wslc; 25H2/24H2/23H2 × process-t3 + wslc) and +8 Linux (each of the four distros × bubblewrap + lxc). macOS resolves empty +because Seatbelt has no wired suite. + +## Backend status + +Snapshot of what the matrix actually proves today. Update this table as backends +get fixed or wired. + +| Backend | Status | Notes | +|---------|--------|-------| +| Process T1 | ✅ Good | Prerelease Windows only. Remaining failures are genuine MXC bugs or harness limitations. | +| Process T3 | ✅ Good | Non-prerelease Windows builds only, until the testing suite is updated. | +| Bubblewrap | ✅ Good | | +| LXC | ⚠️ Mostly good | Some networking tests fail on distros other than Ubuntu 24.04; host-vs-MXC cause not yet isolated. | +| WSLC | ⚠️ Mostly good | Can hit a download rate limit while updating WSL / pulling container images. Planned fix: split into several jobs spaced ~15 min apart. | +| IsolationSession | ⚠️ Blocked | `Feature_AgentSessionsBaseSupport` is not enabled on the pool image yet. | +| Windows Sandbox | ⛔ Not scheduled | Dispatcher case is wired; no trigger entry yet. | +| MicroVM | ⛔ Not working | Windows cold and warm starts hang; no Linux suite. The artifact payload is currently commented out in the build jobs. | +| Hyperlight | ⛔ Not implemented | No suite on any platform. | +| Seatbelt | ⛔ Not implemented | The backend itself is healthy; there is no official E2E suite to dispatch to. | + +## Host preparation + +Preparation runs before the tests, keyed by the matrix backend id. A backend +with no prerequisites is an explicit no-op, so the step runs unconditionally for +every entry. + +`prepare-windows-host.ps1`: + +- `process-t3` — runs `wxc-host-prep.exe prepare-system-drive` and + `prepare-null-device --no-sacl`. +- `microvm` — asserts the NanVix payload is in the artifact, adds a Defender + exclusion for the binary directory, and requires the Windows Hypervisor + Platform feature *and* a running hypervisor. +- `wslc` — asserts `wslcsdk.dll` shipped, requires the WSL and + VirtualMachinePlatform optional features to be baked into the image, then + installs/updates the WSL runtime (including the pre-release ring) up to the + minimum version parsed from `WSLC_SDK_VERSION` in + `src/backends/wslc/common/build.rs`. +- everything else — prints a "no prerequisites yet" line. + +Windows optional features are **verified, never enabled**: turning one on needs a +reboot the job cannot take, so a mis-imaged pool fails here with a pointed +message instead of surfacing later as an opaque backend error. + +`prepare-linux-host.sh`: + +- `bubblewrap` — installs `bwrap` (apt/dnf/yum/microdnf) and relaxes + `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only). +- `lxc` — installs the LXC stack (plus EPEL on RHEL-likes, verified against the + committed key under `scripts/ci/keys/`), reloads the AppArmor profile, starts + and waits for `lxcbr0`, and prints network diagnostics. +- `microvm` — asserts the NanVix payload exists. +- `hyperlight` — no-op. + +macOS has no preparation step. + +## Runbook + +Always finish with a local resolve, which runs the full catalog validation: + +```bash +node scripts/ci/resolve-validation-test-matrix.mjs --plan nightly +``` + +An invalid catalog fails here and in the `resolve` job — before any specialized +test runner is allocated. + +### Schedule an existing backend on an existing OS + +1. Add the backend id to that platform/arch's `backends` list in + `validation-test-matrix.json` if it isn't already declared. +2. Add it to the platform's entry under the plan you want in `triggers`, + creating the `{ "os": …, "backends": [] }` entry if the platform isn't listed. +3. Confirm the platform/arch has a non-empty `pool` (or `runner` on macOS) — + otherwise it silently resolves to nothing. +4. Resolve locally and check the new combination appears. + +### Stop running something + +- **Temporarily, one backend:** remove it from the `triggers` entry. The + platform keeps declaring the capability. +- **Temporarily, a whole platform/arch:** blank its `pool` (`""`). It stays + documented but is never scheduled. +- **Permanently:** remove the trigger entry, then the `backends` entries, then + the platform. If that leaves a backend id declared nowhere, decide whether to + keep its dispatcher and host-prep branches (harmless) or delete them too. + +### Add a new backend + +1. **Catalog:** add the id to the `backends` list of every platform/arch that + can run it. There is no separate registration step — the id *is* the + dispatcher argument. +2. **Dispatcher:** add a case to `run_ci_backend_tests.ps1` (`ValidateSet` + + `switch`) or `run_ci_backend_tests.sh` (`usage` + `case`), pointing at the + suite. Until a suite exists, leave the explicit throw / `exit 2` so + accidental activation fails loudly. +3. **Host prep:** add a branch to `prepare-windows-host.ps1` (`ValidateSet` + + `switch`) or `prepare-linux-host.sh` (`usage` + `case`). Skip only if there + is genuinely nothing to install or assert. +4. **Artifact:** make sure everything the suite needs is in the + `Upload binaries` list of the relevant `Build.*.Job.yml`, and that the build + enables the backend's cargo feature. +5. **Trigger:** add the OS/backend pair to a plan. + +If two ids should run the same suite, give each its own `case` and have both call +the shared function — that is how `process-t1` and `process-t3` are wired. Keep +that split in the dispatcher, not in the workflow YAML, so a case can start +passing a distinguishing argument later without touching the matrix. + +### Add a new OS image + +1. Stand up the 1ES pool (Windows/Linux) with the required optional features + already baked into the image — the jobs verify but never enable them. +2. Add a `platforms` entry: `id`, `displayName`, `family`, and per-architecture + `target`, `artifact`, `pool`/`runner`, and `backends`. +3. For a Windows prerelease image set `"prerelease": true` and use a neutral + `windows-prerelease-` id — the id appears in public job names. +4. For a new Linux distro, check that `prepare-linux-host.sh` handles its + package manager and service layout. +5. Add it to a plan's `triggers`, then resolve locally. + +### Wire an unwired backend to a suite + +Replace the explicit failure in the dispatcher with the suite invocation, add +any host prerequisites, then add the OS/backend pair to a trigger. Verify via +`Validation.Infra.Testing.yml` on your own branch before touching the schedule. + +### Change the schedule + +Everything schedule-related lives in `Validation.Tests.Scheduled.yml`: the two +`cron` entries, the `if:` conditions on `test-nightly` / `test-weekly`, and the +`workflow_dispatch` `plan` choices. Keep the three in sync — a new plan needs a +cron *and* a job condition *and* a dispatch choice. + +### Add a new plan + +1. Add the key to `triggers` in the catalog. That is what defines the plan — + `resolve-validation-test-matrix.mjs` derives its plan list from these keys, + so it needs no edit. +2. Add a job that calls `Validation.Tests.Matrix.Job.yml` with that plan, plus a + `workflow_dispatch` choice if it should be runnable on demand. + +### Enable ARM64 + +Set the ARM64 `pool` for the platform *and* remove or narrow +`suppressNonMacArm64` in the resolver. Note that the resolver rejects +`hyperlight` and `microvm` on ARM64 outright (x64-only runtimes), and the WSLC +dispatcher still refuses non-x64. + +## Important to Note + +- **A green job does not prove a suite ran.** Several suites (notably + IsolationSession) print `SKIPPED` and exit 0 on an unsupported host, and the + dispatchers propagate only the exit code. A matrix entry asserts the host + *should* support the backend, so a silent skip there is a coverage gap — check + the `SKIPPED` line or the executed count in the log, not just the exit status. +- **Empty pool = invisible.** A trigger entry pointing at a platform whose pool + is blank resolves to zero jobs and reports nothing. Resolve locally after any + catalog edit. +- **All OS build jobs must pass** before validation testing happens. +- **Artifacts live one day.** Re-running a test job long after the build has + expired fails at download; re-run the whole workflow instead. diff --git a/docs/pull-requests.md b/docs/pull-requests.md index 12789789e..ac1407e40 100644 --- a/docs/pull-requests.md +++ b/docs/pull-requests.md @@ -9,8 +9,6 @@ it fans out to the reusable `Build.Windows.Job.yml`, `Build.Linux.Job.yml`, and x64/arm64, Linux x64/arm64, and macOS arm64 hosts, then runs the lint, versioning, and SDK jobs. -A change that touches only `docs/**` does not trigger `Build.yml`. - ## Azure Pipelines (optional on PRs, required on `main`) The ADO pipeline (`MXC-PR-Build`) is the Azure version of the PR pipeline. The official diff --git a/scripts/ci/keys/RPM-GPG-KEY-EPEL-10 b/scripts/ci/keys/RPM-GPG-KEY-EPEL-10 new file mode 100644 index 000000000..e8b251ef8 --- /dev/null +++ b/scripts/ci/keys/RPM-GPG-KEY-EPEL-10 @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA +Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK +Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L +URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf +mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2 +XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI +uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60 +GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix +fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z +BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77 +HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB +tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB +CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC +BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p +l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp +yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg +Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u +scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs +7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK +F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg +sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB +0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY +1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt +TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k +p9Dfd2pKuxK9Mys= +=mhQZ +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 7cf28b946..ab59b9c42 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -25,12 +25,54 @@ apt_update() { fi } +# Repo-committed copy of Fedora's EPEL 10 signing key (see +# https://docs.fedoraproject.org/en-US/security/cryptography/signatures/), +# used to verify the fallback RPM download +epel_gpg_key() { + echo "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/keys/RPM-GPG-KEY-EPEL-10" +} + install_epel() { local package_manager="$1" - if ! sudo "$package_manager" install -y epel-release; then - sudo "$package_manager" install -y \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm + # Prefer the package manager's own repos first; only reach out to + # Fedora directly as a fallback, and verify the download's GPG + # signature against our committed key before trusting it. + if sudo "$package_manager" install -y epel-release; then + return + fi + + echo "epel-release is not available from $package_manager's configured repos; falling back to a signature-verified direct download from Fedora." >&2 + + local gpg_key + gpg_key="$(epel_gpg_key)" + if [[ ! -f "$gpg_key" ]]; then + echo "ERROR: missing committed EPEL GPG key at $gpg_key; refusing to install an unverifiable package." >&2 + exit 1 + fi + + local tmp_dir + tmp_dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$tmp_dir'" RETURN + + local rpm_path="$tmp_dir/epel-release-latest-10.noarch.rpm" + curl -fsSL -o "$rpm_path" \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm + + sudo rpm --import "$gpg_key" + local checksig_output + if ! checksig_output="$(sudo rpm --checksig "$rpm_path" 2>&1)"; then + echo "$checksig_output" >&2 + echo "ERROR: GPG signature verification failed for $rpm_path; aborting install." >&2 + exit 1 fi + echo "$checksig_output" + if echo "$checksig_output" | grep -qiE 'NOT OK|MISSING KEYS|NOKEY'; then + echo "ERROR: GPG signature verification reported a problem for $rpm_path; aborting install." >&2 + exit 1 + fi + + sudo "$package_manager" install -y "$rpm_path" } install_bubblewrap() { @@ -90,7 +132,9 @@ start_lxc_bridge() { if command -v systemctl >/dev/null 2>&1; then if systemctl list-unit-files lxc-net.service >/dev/null 2>&1 && systemctl cat lxc-net.service >/dev/null 2>&1; then - sudo systemctl start lxc-net + if ! sudo systemctl start lxc-net; then + echo "WARNING: failed to start lxc-net; container networking may be unavailable." >&2 + fi else echo "No lxc-net unit on this distribution; skipping bridge startup." fi @@ -178,6 +222,8 @@ case "$backend" in bubblewrap) install_bubblewrap command -v bwrap + # disabled AppArmor restrictions on unprivileged user namespaces, which bubblewrap needs to create a new namespace. + # should only be used on ephemeral CI runners, not on persistent hosts. if sysctl kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 fi diff --git a/scripts/ci/prepare-windows-host.ps1 b/scripts/ci/prepare-windows-host.ps1 index d260bd069..76955478d 100644 --- a/scripts/ci/prepare-windows-host.ps1 +++ b/scripts/ci/prepare-windows-host.ps1 @@ -5,8 +5,7 @@ Prepares a Windows host for a backend's artifact-only test suite. .PARAMETER Backend - Matrix backend id (not the handler command), because process-t1 and - process-t3 share a handler but differ in host preparation. + Matrix backend id. .PARAMETER BinaryDirectory Directory holding the downloaded build artifact. diff --git a/scripts/ci/resolve-validation-test-matrix.mjs b/scripts/ci/resolve-validation-test-matrix.mjs index 893faa867..9c73382d2 100644 --- a/scripts/ci/resolve-validation-test-matrix.mjs +++ b/scripts/ci/resolve-validation-test-matrix.mjs @@ -10,7 +10,6 @@ import process from 'node:process'; import { fileURLToPath } from 'node:url'; const FAMILIES = ['windows', 'linux', 'macos']; -const PLANS = ['pr', 'nightly', 'weekly', 'enabled']; const ARM64_UNSUPPORTED_BACKENDS = new Set(['hyperlight', 'microvm']); function assertNonEmptyString(value, label) { @@ -74,10 +73,6 @@ export function validateCatalog(catalog) { if (architecture === 'arm64' && ARM64_UNSUPPORTED_BACKENDS.has(backend)) { throw new Error(`${backend} cannot be scheduled on arm64 (${platform.id})`); } - assertNonEmptyString( - catalog.handlers?.[platform.family]?.[backend], - `${platform.family} handler entry for ${backend}` - ); backends.add(backend); } } @@ -96,11 +91,27 @@ export function validateCatalog(catalog) { throw new Error(`catalog targets do not match the five required build targets`); } + // The catalog's `triggers` keys are the plan list: a plan exists because it + // is declared there. + const triggers = catalog.triggers; + if (triggers == null || typeof triggers !== 'object' || Array.isArray(triggers)) { + throw new Error('catalog triggers must be an object keyed by plan name'); + } + const plans = Object.keys(triggers); + if (plans.length === 0) { + throw new Error('catalog declares no plans under triggers'); + } + // Trigger entries name an OS/backend pair. Architecture expansion happens // later, so a backend is valid here when at least one OS architecture has it. - for (const plan of PLANS) { + for (const plan of plans) { + assertNonEmptyString(plan, 'trigger plan name'); + if (!Array.isArray(triggers[plan])) { + throw new Error(`${plan} must be an array of trigger requests`); + } + const seenRequests = new Set(); - for (const request of catalog.triggers?.[plan] ?? []) { + for (const request of triggers[plan]) { const platform = platforms.get(request.os); if (!platform) { throw new Error(`${plan} references unknown platform ${request.os}`); @@ -120,17 +131,17 @@ export function validateCatalog(catalog) { } } - return { platforms }; + return { platforms, plans }; } export function expandPlan(catalog, plan) { - if (!PLANS.includes(plan)) { - throw new Error(`unsupported plan: ${plan}`); + const { platforms, plans } = validateCatalog(catalog); + if (!plans.includes(plan)) { + throw new Error(`unsupported plan: ${plan} (catalog declares: ${plans.join(', ')})`); } - const { platforms } = validateCatalog(catalog); const combinations = []; - for (const request of catalog.triggers?.[plan] ?? []) { + for (const request of catalog.triggers[plan]) { const platform = platforms.get(request.os); // A trigger is architecture-neutral. Expand it only where the platform's // capability declaration supports the requested backend. @@ -152,8 +163,7 @@ export function expandPlan(catalog, plan) { artifact: details.artifact, pool: details.pool, runner: details.runner, - backend, - command: catalog.handlers[platform.family][backend] + backend }); } } @@ -163,11 +173,7 @@ export function expandPlan(catalog, plan) { } export function resolvePlan(catalog, plan) { - if (!PLANS.includes(plan)) { - throw new Error(`unsupported plan: ${plan}`); - } - - validateCatalog(catalog); + // expandPlan validates the catalog and rejects an unknown plan name. const matrices = Object.fromEntries(FAMILIES.map(family => [family, []])); for (const combination of expandPlan(catalog, plan)) { diff --git a/scripts/ci/validation-test-matrix.json b/scripts/ci/validation-test-matrix.json index 682937da4..bf7b2fc62 100644 --- a/scripts/ci/validation-test-matrix.json +++ b/scripts/ci/validation-test-matrix.json @@ -325,26 +325,6 @@ } } ], - "handlers": { - "windows": { - "process-t1": "process-container", - "process-t3": "process-container", - "isolation-session": "isolation-session", - "wslc": "wslc", - "windows-sandbox": "windows-sandbox", - "microvm": "microvm", - "hyperlight": "hyperlight" - }, - "linux": { - "bubblewrap": "bubblewrap", - "lxc": "lxc", - "microvm": "microvm", - "hyperlight": "hyperlight" - }, - "macos": { - "seatbelt": "seatbelt" - } - }, "triggers": { "pr": [], "nightly": [ @@ -352,7 +332,6 @@ "os": "windows-prerelease-process-container", "backends": [ "process-t1", - "process-t3", "isolation-session", "wslc" ] @@ -408,31 +387,6 @@ } ], "weekly": [], - "enabled": [ - { - "os": "windows-prerelease-process-container", - "backends": [ - "wslc" - ] - }, - { - "os": "windows-25h2", - "backends": [ - "wslc" - ] - }, - { - "os": "windows-24h2", - "backends": [ - "wslc" - ] - }, - { - "os": "windows-23h2", - "backends": [ - "wslc" - ] - } - ] + "enabled": [] } } diff --git a/tests/scripts/README.md b/tests/scripts/README.md index 279579521..f3e3d534a 100644 --- a/tests/scripts/README.md +++ b/tests/scripts/README.md @@ -61,30 +61,32 @@ directly. ### CI dispatch The validation matrix (see `scripts/ci/validation-test-matrix.json` and -`.github/workflows/Validation.Tests.Matrix.Job.yml`) never builds from source. +`.github/workflows/Validation.Tests.Matrix.Job.yml`, documented end to end in +[`docs/ci-validation-infrastructure.md`](../../docs/ci-validation-infrastructure.md)) +never builds from source. It downloads a build artifact, prepares the host, and then hands off to one of -these dispatchers, which map a backend handler to the suites above: +these dispatchers, which map a matrix backend id to the suites above: -| Dispatcher | Platforms | Handlers | -|------------|-----------|----------| -| `run_ci_backend_tests.ps1` | Windows | `process-container` (needs `-ExpectedTier`), `isolation-session`, `windows-sandbox`, `wslc`, `microvm`, `hyperlight` | +| Dispatcher | Platforms | Backend ids | +|------------|-----------|-------------| +| `run_ci_backend_tests.ps1` | Windows | `process-t1`, `process-t3`, `isolation-session`, `windows-sandbox`, `wslc`, `microvm`, `hyperlight` | | `run_ci_backend_tests.sh` | Linux, macOS | `bubblewrap`, `lxc`, `seatbelt`, `microvm`, `hyperlight` | -Pass the **handler** name from the catalog's `handlers` map, not the matrix -backend id (they differ where one handler serves several entries — `process-t1` -and `process-t3` both dispatch to `process-container` and are distinguished by -`-ExpectedTier`): +Pass the backend id exactly as it appears in the catalog — there is no separate +handler name. Ids that share a suite have their own case in the dispatcher: +`process-t1` and `process-t3` both run `WinProcessContainer-Tests.ps1`, which +determines the tier it expects from the host's own `wxc-exec --probe`. ```powershell -tests\scripts\run_ci_backend_tests.ps1 -Backend process-container ` - -BinaryDirectory -Architecture x64 -ExpectedTier T1 +tests\scripts\run_ci_backend_tests.ps1 -Backend process-t1 ` + -BinaryDirectory -Architecture x64 ``` ```bash tests/scripts/run_ci_backend_tests.sh bubblewrap ``` -A handler with no wired suite exits non-zero on purpose, so accidentally +A backend with no wired suite exits non-zero on purpose, so accidentally enabling it in a trigger fails loudly instead of reporting a false success. To see exactly what a plan would schedule without pushing: diff --git a/tests/scripts/run_ci_backend_tests.ps1 b/tests/scripts/run_ci_backend_tests.ps1 index 2d82cc408..adcb916a9 100644 --- a/tests/scripts/run_ci_backend_tests.ps1 +++ b/tests/scripts/run_ci_backend_tests.ps1 @@ -1,11 +1,23 @@ <# .SYNOPSIS Runs a Windows backend test from a downloaded CI artifact. + +.DESCRIPTION +Takes the matrix backend id straight from the catalog, so there is no +id-to-command mapping to keep in sync. #> [CmdletBinding()] param( [Parameter(Mandatory)] - [ValidateSet('process-container', 'isolation-session', 'windows-sandbox', 'wslc', 'microvm', 'hyperlight')] + [ValidateSet( + 'process-t1', + 'process-t3', + 'isolation-session', + 'windows-sandbox', + 'wslc', + 'microvm', + 'hyperlight' + )] [string]$Backend, [Parameter(Mandatory)] @@ -13,10 +25,7 @@ param( [Parameter(Mandatory)] [ValidateSet('x64', 'arm64')] - [string]$Architecture, - - [ValidateSet('T1', 'T3')] - [string]$ExpectedTier + [string]$Architecture ) Set-StrictMode -Version Latest @@ -54,64 +63,52 @@ function Invoke-TestScript { Assert-File -Path $wxc -switch ($Backend) { - 'process-container' { - if (-not $ExpectedTier) { - throw 'Process Container tests require -ExpectedTier.' - } - - $expectedProbeTier = @{ - T1 = 'base-container' - T3 = 'appcontainer-dacl' - }[$ExpectedTier] - - # The existing harness expects separate debug and release layouts. CI - # intentionally tests one release artifact, so stage it in both slots. - $debugDirectory = Join-Path $binaryDirectoryPath 'debug' - $releaseDirectory = Join-Path $binaryDirectoryPath 'release' - New-Item -ItemType Directory -Force -Path $debugDirectory, $releaseDirectory | Out-Null - Copy-Item -LiteralPath $wxc -Destination (Join-Path $debugDirectory 'wxc-exec.exe') -Force - Copy-Item -LiteralPath $wxc -Destination (Join-Path $releaseDirectory 'wxc-exec.exe') -Force - - $uiProbe = Join-Path $binaryDirectoryPath 'wxc-ui-probe.exe' - Assert-File -Path $uiProbe - Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $debugDirectory 'wxc-ui-probe.exe') -Force - Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force - - # Fail before running the suite if the selected image does not provide - # the Process Container tier promised by the matrix entry. - $probe = (& $wxc --probe | Out-String) | ConvertFrom-Json - if ($LASTEXITCODE -ne 0) { - throw "Process Container probe failed with exit code $LASTEXITCODE." - } - if ($probe.tier -ne $expectedProbeTier) { - throw "Expected $ExpectedTier ($expectedProbeTier), but the host selected '$($probe.tier)'." - } +function Invoke-ProcessContainerTests { + # The existing harness expects separate debug and release layouts. CI + # intentionally tests one release artifact, so stage it in both slots. + $debugDirectory = Join-Path $binaryDirectoryPath 'debug' + $releaseDirectory = Join-Path $binaryDirectoryPath 'release' + New-Item -ItemType Directory -Force -Path $debugDirectory, $releaseDirectory | Out-Null + Copy-Item -LiteralPath $wxc -Destination (Join-Path $debugDirectory 'wxc-exec.exe') -Force + Copy-Item -LiteralPath $wxc -Destination (Join-Path $releaseDirectory 'wxc-exec.exe') -Force + + $uiProbe = Join-Path $binaryDirectoryPath 'wxc-ui-probe.exe' + Assert-File -Path $uiProbe + Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $debugDirectory 'wxc-ui-probe.exe') -Force + Copy-Item -LiteralPath $uiProbe -Destination (Join-Path $releaseDirectory 'wxc-ui-probe.exe') -Force + + $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1' + # Skip build and Cargo phases because this job consumes a previously + # built artifact; retain the host and containment behavior phases. + $phases = @( + 'Probes', + 'T3Forced', + 'T1DenyForced', + 'UiMitigationMatrix', + 'GlobalAtomIsolation', + 'DaclDisabled', + 'CrashRecovery' + ) + $global:LASTEXITCODE = 0 + & $script ` + -SkipBuild ` + -SkipReleaseLane ` + -WxcDebug (Join-Path $debugDirectory 'wxc-exec.exe') ` + -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') ` + -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') ` + -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') ` + -Phases $phases + if ($LASTEXITCODE -ne 0) { + throw "Process Container tests failed with exit code $LASTEXITCODE." + } +} - $script = Join-Path $scriptRoot 'WinProcessContainer-Tests.ps1' - # Skip build and Cargo phases because this job consumes a previously - # built artifact; retain the host and containment behavior phases. - $phases = @( - 'Probes', - 'T3Forced', - 'T1DenyForced', - 'UiMitigationMatrix', - 'GlobalAtomIsolation', - 'DaclDisabled', - 'CrashRecovery' - ) - $global:LASTEXITCODE = 0 - & $script ` - -SkipBuild ` - -SkipReleaseLane ` - -WxcDebug (Join-Path $debugDirectory 'wxc-exec.exe') ` - -WxcRelease (Join-Path $releaseDirectory 'wxc-exec.exe') ` - -UiProbeDebug (Join-Path $debugDirectory 'wxc-ui-probe.exe') ` - -UiProbeRelease (Join-Path $releaseDirectory 'wxc-ui-probe.exe') ` - -Phases $phases - if ($LASTEXITCODE -ne 0) { - throw "Process Container tests failed with exit code $LASTEXITCODE." - } +switch ($Backend) { + 'process-t1' { + Invoke-ProcessContainerTests + } + 'process-t3' { + Invoke-ProcessContainerTests } 'isolation-session' { Invoke-TestScript -Path (Join-Path $scriptRoot 'run_isolation_session_tests.ps1') -Arguments @{ @@ -138,7 +135,7 @@ switch ($Backend) { } } 'hyperlight' { - # Keep unwired commands explicit so accidental activation fails loudly. - throw 'The Hyperlight CI handler is not wired to an existing backend test entry point yet.' + # Keep unwired backends explicit so accidental activation fails loudly. + throw 'The Hyperlight CI backend is not wired to an existing test entry point yet.' } } diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index c8a7c3ef3..95eae2dbc 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -2,8 +2,8 @@ set -euo pipefail # Dispatches a downloaded Unix artifact to the repository's existing backend -# test suites. Unsupported handlers fail explicitly rather than reporting a -# false-success placeholder job. +# test suites, keyed by the matrix backend id. Unsupported backends fail +# explicitly rather than reporting a false-success placeholder job. usage() { echo "Usage: $0 " >&2 @@ -24,13 +24,13 @@ case "$backend" in microvm) # Keep unwired commands explicit so accidental activation fails loudly. # Future test script: run_microvm_tests.sh - echo "The MicroVM CI handler is not wired to an artifact-only Linux test entry point yet." >&2 + echo "The MicroVM CI backend is not wired to an artifact-only Linux test entry point yet." >&2 exit 2 ;; hyperlight) # Keep unwired commands explicit so accidental activation fails loudly. # Future test script: run_hyperlight_tests.sh - echo "The Hyperlight CI handler is not wired to an existing backend test entry point yet." >&2 + echo "The Hyperlight CI backend is not wired to an existing test entry point yet." >&2 exit 2 ;; bubblewrap) @@ -53,7 +53,7 @@ case "$backend" in seatbelt) test -x "$binary_directory/mxc-exec-mac" test -x "$binary_directory/unix-test-proxy" - echo "The Seatbelt CI handler is not wired to an existing backend test entry point yet." >&2 + echo "The Seatbelt CI backend is not wired to an existing test entry point yet." >&2 exit 2 ;; *) From f5c161b3709d9201858fef10d7f6ed200b2f1e09 Mon Sep 17 00:00:00 2001 From: Elliot <31219104+theelliotm@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:51:34 -0700 Subject: [PATCH 88/93] Don't greenlight prereq skips Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/scripts/run_ci_backend_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/run_ci_backend_tests.sh b/tests/scripts/run_ci_backend_tests.sh index 95eae2dbc..dd03f1245 100644 --- a/tests/scripts/run_ci_backend_tests.sh +++ b/tests/scripts/run_ci_backend_tests.sh @@ -48,7 +48,7 @@ case "$backend" in mkdir -p "$release_directory" cp -a "$binary_directory/." "$release_directory/" chmod +x "$release_directory/lxc-exec" "$release_directory/unix-test-proxy" - bash "$script_root/run_lxc_all_tests.sh" + MXC_LXC_TESTS_REQUIRE_EXECUTION=1 bash "$script_root/run_lxc_all_tests.sh" ;; seatbelt) test -x "$binary_directory/mxc-exec-mac" From 8752bed921847500a21bd769845542b33a100ab7 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:31:09 -0700 Subject: [PATCH 89/93] moved wxc-ui-probe and wxc-test-driver to optional binaries. removed epel key verification. --- scripts/ci/keys/RPM-GPG-KEY-EPEL-10 | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 scripts/ci/keys/RPM-GPG-KEY-EPEL-10 diff --git a/scripts/ci/keys/RPM-GPG-KEY-EPEL-10 b/scripts/ci/keys/RPM-GPG-KEY-EPEL-10 deleted file mode 100644 index e8b251ef8..000000000 --- a/scripts/ci/keys/RPM-GPG-KEY-EPEL-10 +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBGV4X6kBEAC3eQxgiWuo08uc3mHo4ELux++uqTnYz/tJzEf9Ou3h36WnhumA -Nvs+Ts5h8PBx879Y9/aIX1Z20p1kf6tBCinZnEJu59n+TAAsph0+XQlV1l5YkleK -Z2ff/Fg65k8QcLXWaIGykA/FaKznRiSurGuD6tRGhJw7DawEwBJr8QZSkRUpnH1L -URW97Q/iKrRPiE5VEayE0y8eAL28jIIiFvR+4oJMzvCsRRB/2wYZ2MlJOW91hcYf -mbUoXKOBD5UzsJylu7kj25K/ge8rEJ7KicOOwcdYddxsU3DxGSSfwF8AMagENcm2 -XROeXknjm84A8sNlUkFZBJwfuc7eRTiZGJrnQQVYLrkKj8Mxpq9Ts7hU51TqAWNI -uvGDlJdYNE3D2RMqjMEsZ8ej08Thrib6xslu4NzTBkt+6QNnXL4E3hEgYtoyio60 -GswSz2ulogKg7X4JrNdJYE8/qNowyF3hoVgj5TG1/wQRq+5HlMMOLjgGu9wzLUix -fnVfEUnzaofbrUf4/GabCaeY8xRe4tFQrvzigQ4g+kgwKKnfAeqBmPov0yljkw9z -BYJWR5zvaw0ffg9Ing00KUSaXBXA5jSlgk1603Y+LefY1SlXsTyqohiRvGH6FI77 -HNMo72DwoJfFcYjncZUzKgXWJECR4nhVsdj6pKoOjcQ4aSuyVxtsR86ASQARAQAB -tChGZWRvcmEgKGVwZWwxMCkgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQJOBBMB -CAA4FiEEfY0Vy/xOYmiFkfsmM9mFF+N+0VgFAmV4X6kCGw8FCwkIBwIGFQoJCAsC -BBYCAwECHgECF4AACgkQM9mFF+N+0Vhv/A/+PlhPLSctGRCUEahE+cN4764Acc3p -l40ZYzXRhqR0/Tc1/cSDjlA3qVTc8SPohi5OJXwCyr9EiMqKoyoDN097euqbYpyp -yN/Pj0lBjsXwcpdDtZ21WGeQU0Khb04N68bMtJbDaxeBciTvDDQravZuPPh0m4Rg -Z6myEoa6Aa6EK0hI1Qwi1qIWeRiuEkVT671IaKVETBW5XiUpNBXDAB/L+6DzUF9u -scBzfsUDiPO6NrpYDtV3jwq22y6gWluIct/Ka8brwPbqK2sBfFzrHboRhfqlTGjs -7F9qUGwIQZn/A8iozXZYQ0+JG1bhQyvjA8eN1GOcRpT+O7H7JXN49o6IG2As4+iK -F04+qjqAu2sVfpD8mzM2VubFNllcKKiyCzRYHhSbObRCPzsudDL9GPiXeGGaCuWg -sDkiA1MESvf2tLETAGBs/TziO4GwmXUtlKbRiq1FYm90mVq9mBxPZ/Idn+yZusNB -0O5SXIbI8lYZw5n4XTK4b+byHRBYsOTHiTsGvjTF2Y7oSwW2CVUmL6RZ23mI4qoY -1p5kzRS+GjT1acnTei/FTsOlIKCsjfeHx7uxCkX6xpAD8P3UtLQqfsgH0CL4vSZt -TGO6L1InQlp4ZG3OYIomTKbD3/R0wod3U3dTqdulQMXL895u6OLTY3spY2m2MO2k -p9Dfd2pKuxK9Mys= -=mhQZ ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file From d53b85b60ae0207776a748a615aa45c1dfead069 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:32:21 -0700 Subject: [PATCH 90/93] moved wxc-ui probe and wxc-test-driver to optional binaries. removed epel key verification --- docs/ci-validation-infrastructure.md | 6 +-- scripts/ci/prepare-linux-host.sh | 51 ++++------------------ sdk/node/tests/integration/test-helpers.ts | 10 ++++- 3 files changed, 20 insertions(+), 47 deletions(-) diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index 0c34af3f6..bee99635c 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -203,9 +203,9 @@ message instead of surfacing later as an opaque backend error. - `bubblewrap` — installs `bwrap` (apt/dnf/yum/microdnf) and relaxes `kernel.apparmor_restrict_unprivileged_userns` (ephemeral CI hosts only). -- `lxc` — installs the LXC stack (plus EPEL on RHEL-likes, verified against the - committed key under `scripts/ci/keys/`), reloads the AppArmor profile, starts - and waits for `lxcbr0`, and prints network diagnostics. +- `lxc` — installs the LXC stack, reloads the AppArmor profile, starts and waits + for `lxcbr0`, and prints network diagnostics. On RHEL-likes it first needs + EPEL, because Red Hat dropped LXC after RHEL 7 and ships no replacement. - `microvm` — asserts the NanVix payload exists. - `hyperlight` — no-op. diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index ab59b9c42..ba9ebe6fb 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -25,54 +25,21 @@ apt_update() { fi } -# Repo-committed copy of Fedora's EPEL 10 signing key (see -# https://docs.fedoraproject.org/en-US/security/cryptography/signatures/), -# used to verify the fallback RPM download -epel_gpg_key() { - echo "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/keys/RPM-GPG-KEY-EPEL-10" -} - +# Red Hat ships no third-party content, so epel-release is not in RHEL's own +# repos; the documented install is the release RPM straight from Fedora. EPEL +# packages also routinely link against CRB (CodeReady Builder) content, which +# is shipped but disabled by default. install_epel() { local package_manager="$1" - # Prefer the package manager's own repos first; only reach out to - # Fedora directly as a fallback, and verify the download's GPG - # signature against our committed key before trusting it. - if sudo "$package_manager" install -y epel-release; then - return - fi - echo "epel-release is not available from $package_manager's configured repos; falling back to a signature-verified direct download from Fedora." >&2 - - local gpg_key - gpg_key="$(epel_gpg_key)" - if [[ ! -f "$gpg_key" ]]; then - echo "ERROR: missing committed EPEL GPG key at $gpg_key; refusing to install an unverifiable package." >&2 - exit 1 + if command -v subscription-manager >/dev/null 2>&1; then + sudo subscription-manager repos \ + --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || + echo "WARNING: could not enable the CRB repository; EPEL packages that depend on it may fail to install." >&2 fi - local tmp_dir - tmp_dir="$(mktemp -d)" - # shellcheck disable=SC2064 - trap "rm -rf '$tmp_dir'" RETURN - - local rpm_path="$tmp_dir/epel-release-latest-10.noarch.rpm" - curl -fsSL -o "$rpm_path" \ + sudo "$package_manager" install -y \ https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm - - sudo rpm --import "$gpg_key" - local checksig_output - if ! checksig_output="$(sudo rpm --checksig "$rpm_path" 2>&1)"; then - echo "$checksig_output" >&2 - echo "ERROR: GPG signature verification failed for $rpm_path; aborting install." >&2 - exit 1 - fi - echo "$checksig_output" - if echo "$checksig_output" | grep -qiE 'NOT OK|MISSING KEYS|NOKEY'; then - echo "ERROR: GPG signature verification reported a problem for $rpm_path; aborting install." >&2 - exit 1 - fi - - sudo "$package_manager" install -y "$rpm_path" } install_bubblewrap() { diff --git a/sdk/node/tests/integration/test-helpers.ts b/sdk/node/tests/integration/test-helpers.ts index 1ef6d902d..a22a4be79 100644 --- a/sdk/node/tests/integration/test-helpers.ts +++ b/sdk/node/tests/integration/test-helpers.ts @@ -48,8 +48,6 @@ export function getSdkBinDir(): string { export const EXPECTED_WINDOWS_BINARIES = [ 'wxc-exec.exe', 'wxc-host-prep.exe', - 'wxc-ui-probe.exe', - 'wxc-test-driver.exe', 'winhttp-proxy-shim.exe', 'wxc-test-proxy.exe', 'wxc-windows-sandbox-daemon.exe', @@ -74,6 +72,14 @@ const OPTIONAL_BINARIES = [ 'wxc-wslc-daemon.exe', // Only built with --with-wslc 'plm.exe', // Permissive Learning Mode helper (Windows-only); staged // only when the plm crate is included in the build. + // Test-only binaries. The GitHub build artifact carries them so the + // validation matrix can run the Windows suites from a downloaded artifact, + // and the npm packager copies that whole artifact into bin/ — so they show + // up here. They are not required: no SDK consumer needs them, and the ADO + // package producer filters its artifact through signPattern, which + // deliberately ships only product binaries. + 'wxc-ui-probe.exe', // WinProcessContainer-Tests.ps1 + 'wxc-test-driver.exe', // run_test_configs.ps1 ]; // Combined list of all known binaries across platforms. The npm package From 7af3059c4f0f0a50510f461de9432eb9fdec59b8 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:31:57 -0700 Subject: [PATCH 91/93] testing with baked-in EPEL --- scripts/ci/prepare-linux-host.sh | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index ba9ebe6fb..907ec0eb7 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -25,22 +25,22 @@ apt_update() { fi } -# Red Hat ships no third-party content, so epel-release is not in RHEL's own -# repos; the documented install is the release RPM straight from Fedora. EPEL -# packages also routinely link against CRB (CodeReady Builder) content, which -# is shipped but disabled by default. -install_epel() { - local package_manager="$1" - - if command -v subscription-manager >/dev/null 2>&1; then - sudo subscription-manager repos \ - --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || - echo "WARNING: could not enable the CRB repository; EPEL packages that depend on it may fail to install." >&2 - fi - - sudo "$package_manager" install -y \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm -} +# # Red Hat ships no third-party content, so epel-release is not in RHEL's own +# # repos; the documented install is the release RPM straight from Fedora. EPEL +# # packages also routinely link against CRB (CodeReady Builder) content, which +# # is shipped but disabled by default. +# install_epel() { +# local package_manager="$1" + +# if command -v subscription-manager >/dev/null 2>&1; then +# sudo subscription-manager repos \ +# --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || +# echo "WARNING: could not enable the CRB repository; EPEL packages that depend on it may fail to install." >&2 +# fi + +# sudo "$package_manager" install -y \ +# https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm +# } install_bubblewrap() { if command -v bwrap >/dev/null 2>&1; then @@ -74,13 +74,13 @@ install_lxc() { fi sudo apt-get install -y --no-install-recommends "${packages[@]}" elif command -v dnf >/dev/null 2>&1; then - install_epel dnf + # install_epel dnf sudo dnf install -y lxc lxc-templates dnsmasq iptables elif command -v yum >/dev/null 2>&1; then - install_epel yum + # install_epel yum sudo yum install -y lxc lxc-templates dnsmasq iptables elif command -v microdnf >/dev/null 2>&1; then - install_epel microdnf + # install_epel microdnf sudo microdnf install -y lxc lxc-templates dnsmasq iptables else echo "No supported package manager found to install LXC." >&2 From f7445c6b785a0893e95e6169714d5e337ce48cd2 Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:38:08 -0700 Subject: [PATCH 92/93] re-enabling EPEL download --- scripts/ci/prepare-linux-host.sh | 36 +++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/scripts/ci/prepare-linux-host.sh b/scripts/ci/prepare-linux-host.sh index 907ec0eb7..16eafcfeb 100644 --- a/scripts/ci/prepare-linux-host.sh +++ b/scripts/ci/prepare-linux-host.sh @@ -25,22 +25,20 @@ apt_update() { fi } -# # Red Hat ships no third-party content, so epel-release is not in RHEL's own -# # repos; the documented install is the release RPM straight from Fedora. EPEL -# # packages also routinely link against CRB (CodeReady Builder) content, which -# # is shipped but disabled by default. -# install_epel() { -# local package_manager="$1" - -# if command -v subscription-manager >/dev/null 2>&1; then -# sudo subscription-manager repos \ -# --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || -# echo "WARNING: could not enable the CRB repository; EPEL packages that depend on it may fail to install." >&2 -# fi - -# sudo "$package_manager" install -y \ -# https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm -# } +# Red Hat ships no third-party content, so epel-release is not in RHEL's own +# repos; the documented install is the release RPM straight from Fedora. +install_epel() { + local package_manager="$1" + + if command -v subscription-manager >/dev/null 2>&1; then + sudo subscription-manager repos \ + --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || + echo "WARNING: could not enable the CRB repository; EPEL packages that depend on it may fail to install." >&2 + fi + + sudo "$package_manager" install -y \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm +} install_bubblewrap() { if command -v bwrap >/dev/null 2>&1; then @@ -74,13 +72,13 @@ install_lxc() { fi sudo apt-get install -y --no-install-recommends "${packages[@]}" elif command -v dnf >/dev/null 2>&1; then - # install_epel dnf + install_epel dnf sudo dnf install -y lxc lxc-templates dnsmasq iptables elif command -v yum >/dev/null 2>&1; then - # install_epel yum + install_epel yum sudo yum install -y lxc lxc-templates dnsmasq iptables elif command -v microdnf >/dev/null 2>&1; then - # install_epel microdnf + install_epel microdnf sudo microdnf install -y lxc lxc-templates dnsmasq iptables else echo "No supported package manager found to install LXC." >&2 From 3e9b9a6c92e0da98581392434812e80dad6cd58f Mon Sep 17 00:00:00 2001 From: Elliot Michlin <31219104+theelliotm@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:09:38 -0700 Subject: [PATCH 93/93] Addressing final feedback --- .github/copilot-instructions.md | 2 - .github/workflows/Build.Linux.Job.yml | 25 --------- .github/workflows/Build.Windows.Job.yml | 37 ------------- .../workflows/Validation.Infra.Testing.yml | 38 ------------- .../workflows/Validation.Tests.Matrix.Job.yml | 2 + docs/ci-validation-infrastructure.md | 55 +++++++++++++++++-- 6 files changed, 53 insertions(+), 106 deletions(-) delete mode 100644 .github/workflows/Validation.Infra.Testing.yml diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index e09c726bc..ae4fbf968 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -60,8 +60,6 @@ consumption stay in one workflow run: - `.github/workflows/Validation.Tests.Scheduled.yml` — scheduled entry point. The `nightly` plan runs Mon–Sat; Sunday runs `nightly` *and* `weekly`. `workflow_dispatch` takes a `plan` input to run one on demand. -- `.github/workflows/Validation.Infra.Testing.yml` — push-triggered entry point - for iterating on the validation infrastructure itself, on a dedicated branch. - `.github/workflows/Validation.Tests.Matrix.Job.yml` — workflow-call-only, takes a `plan` input. Its `resolve` job expands the plan into per-family matrices, then the `windows` / `linux` / `macos` jobs each download the diff --git a/.github/workflows/Build.Linux.Job.yml b/.github/workflows/Build.Linux.Job.yml index 9c9abc7e6..66c804c46 100644 --- a/.github/workflows/Build.Linux.Job.yml +++ b/.github/workflows/Build.Linux.Job.yml @@ -119,26 +119,6 @@ jobs: run: cargo build --locked --release --target ${{ matrix.target }} -p unix_test_proxy - # MICROVM NOT WORKING CURRENTLY - # - name: Verify artifact payload - # shell: bash - # working-directory: ${{ github.workspace }} - # run: | - # set -euo pipefail - # bin_dir="src/target/${{ matrix.target }}/release" - # required=(lxc-exec unix-test-proxy) - # if [[ "${{ matrix.arch }}" == "x64" ]]; then - # required+=(nanvixd.elf nanvix_rootfs.img python3.initrd bin/kernel.elf) - # fi - # missing=() - # for file in "${required[@]}"; do - # [[ -f "$bin_dir/$file" ]] || missing+=("$file") - # done - # if (( ${#missing[@]} > 0 )); then - # printf 'Missing artifact files: %s\n' "${missing[*]}" >&2 - # exit 1 - # fi - - name: Upload binaries uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -148,8 +128,3 @@ jobs: src/target/${{ matrix.target }}/release/unix-test-proxy if-no-files-found: error retention-days: 1 - - # src/target/${{ matrix.target }}/release/nanvixd.elf - # src/target/${{ matrix.target }}/release/nanvix_rootfs.img - # src/target/${{ matrix.target }}/release/python3.initrd - # src/target/${{ matrix.target }}/release/bin/kernel.elf diff --git a/.github/workflows/Build.Windows.Job.yml b/.github/workflows/Build.Windows.Job.yml index b61ad79d5..6ad9eb051 100644 --- a/.github/workflows/Build.Windows.Job.yml +++ b/.github/workflows/Build.Windows.Job.yml @@ -71,36 +71,6 @@ jobs: --no-default-features --features "${{ matrix.features }}" - # MICROVM NOT WORKING CURRENTLY - # - name: Verify artifact payload - # shell: pwsh - # run: | - # $binDir = "target\${{ matrix.target }}\release" - # $required = @( - # "wxc-exec.exe", - # "wxc-host-prep.exe", - # "winhttp-proxy-shim.exe", - # "wxc-windows-sandbox-daemon.exe", - # "wxc-windows-sandbox-guest.exe", - # "wxc-test-proxy.exe", - # "wxc-test-driver.exe", - # "wxc-ui-probe.exe", - # "mxc-diagnostic-console.exe", - # "wslcsdk.dll" - # ) - # if ("${{ matrix.arch }}" -eq "x64") { - # $required += @( - # "nanvixd.exe", - # "nanvix_rootfs.img", - # "python3.initrd", - # "bin\kernel.elf" - # ) - # } - # $missing = $required | Where-Object { -not (Test-Path (Join-Path $binDir $_)) } - # if ($missing) { - # throw "Missing artifact files: $($missing -join ', ')" - # } - - name: Upload binaries uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -119,10 +89,3 @@ jobs: src/target/${{ matrix.target }}/release/wxc-test-driver.exe if-no-files-found: error retention-days: 1 - - # src/target/${{ matrix.target }}/release/nanvixd.exe - # src/target/${{ matrix.target }}/release/nanvix_rootfs.img - # src/target/${{ matrix.target }}/release/python3.initrd - # src/target/${{ matrix.target }}/release/bin/kernel.elf - # src/target/${{ matrix.target }}/release/snapshots/kernel.vmem - # src/target/${{ matrix.target }}/release/snapshots/kernel.whp.cbor diff --git a/.github/workflows/Validation.Infra.Testing.yml b/.github/workflows/Validation.Infra.Testing.yml deleted file mode 100644 index ab8c4ffec..000000000 --- a/.github/workflows/Validation.Infra.Testing.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Validation Infrastructure Testing - -on: - push: - branches: - - user/emichlin/validation-infra-lite - -concurrency: - group: validation-infra-pr-tests-${{ github.ref }} - cancel-in-progress: true - -permissions: - actions: read - contents: read - -jobs: - dependency-feed-check: - uses: ./.github/workflows/Dependency.Feed.Check.Job.yml - - windows: - needs: dependency-feed-check - uses: ./.github/workflows/Build.Windows.Job.yml - - linux: - needs: dependency-feed-check - uses: ./.github/workflows/Build.Linux.Job.yml - - macos: - needs: dependency-feed-check - uses: ./.github/workflows/Build.MacOS.Job.yml - - test: - needs: [windows, linux, macos] - uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml - with: - # Exercise every combination currently rolled out, independent of its - # normal PR/nightly/weekly schedule. - plan: nightly diff --git a/.github/workflows/Validation.Tests.Matrix.Job.yml b/.github/workflows/Validation.Tests.Matrix.Job.yml index b1bede25f..9a958d94a 100644 --- a/.github/workflows/Validation.Tests.Matrix.Job.yml +++ b/.github/workflows/Validation.Tests.Matrix.Job.yml @@ -56,6 +56,7 @@ jobs: -Backend '${{ matrix.backend }}' ` -BinaryDirectory (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') *>&1 | Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run backend tests # A backend that hangs (rather than failing) would otherwise burn the @@ -68,6 +69,7 @@ jobs: -BinaryDirectory (Join-Path $env:GITHUB_WORKSPACE 'artifacts\bin') ` -Architecture '${{ matrix.architecture }}' *>&1 | Tee-Object -FilePath (Join-Path $env:RUNNER_TEMP 'mxc-ci.log') -Append + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Upload failure logs if: failure() || cancelled() diff --git a/docs/ci-validation-infrastructure.md b/docs/ci-validation-infrastructure.md index bee99635c..ec0e99486 100644 --- a/docs/ci-validation-infrastructure.md +++ b/docs/ci-validation-infrastructure.md @@ -37,7 +37,7 @@ the individual local test scripts are documented in ### Flow ``` -Validation.Tests.Scheduled.yml (or Validation.Infra.Testing.yml) +Validation.Tests.Scheduled.yml └─ dependency-feed-check ├─ windows / linux / macos → Build.*.Job.yml (upload artifacts) └─ test-nightly / test-weekly → Validation.Tests.Matrix.Job.yml @@ -147,7 +147,7 @@ backend **and** has a non-empty pool. | Plan | Wired to | Contents today | |------|----------|----------------| -| `nightly` | scheduled Mon–Sun, plus `Validation.Infra.Testing.yml` | 4 Windows platforms, 4 Linux platforms | +| `nightly` | scheduled Mon–Sun | 4 Windows platforms, 4 Linux platforms | | `weekly` | scheduled Sunday | empty | | `pr` | *(nothing — `Build.yml` does not call the matrix job)* | empty; reserved for a potential future PR-time subset | | `enabled` | *(nothing — resolvable locally only)* | reserved for testing this infrastructure and rapid iteration | @@ -279,8 +279,8 @@ passing a distinguishing argument later without touching the matrix. ### Wire an unwired backend to a suite Replace the explicit failure in the dispatcher with the suite invocation, add -any host prerequisites, then add the OS/backend pair to a trigger. Verify via -`Validation.Infra.Testing.yml` on your own branch before touching the schedule. +any host prerequisites, then add the OS/backend pair to a trigger. Always verify +by testing it ahead of time. ### Change the schedule @@ -304,6 +304,53 @@ Set the ARM64 `pool` for the platform *and* remove or narrow `hyperlight` and `microvm` on ARM64 outright (x64-only runtimes), and the WSLC dispatcher still refuses non-x64. +## Testing Your Changes to the Validation Infrastructure + +1. Pick a pre-existing trigger or make a custom trigger with the tests you plan + to run in `scripts/ci/validation-test-matrix.json`. +2. Create a workflow file in your branch with the following code, replacing the + branch name and plan name with your branch name and trigger name respectively. +3. Push your changes. + +```yml +name: Validation Infrastructure Testing + +on: + push: + branches: + - # BRANCH NAME HERE + +concurrency: + group: validation-infra-pr-tests-${{ github.ref }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + +jobs: + dependency-feed-check: + uses: ./.github/workflows/Dependency.Feed.Check.Job.yml + + windows: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Windows.Job.yml + + linux: + needs: dependency-feed-check + uses: ./.github/workflows/Build.Linux.Job.yml + + macos: + needs: dependency-feed-check + uses: ./.github/workflows/Build.MacOS.Job.yml + + test: + needs: [windows, linux, macos] + uses: ./.github/workflows/Validation.Tests.Matrix.Job.yml + with: + plan: # YOUR PLAN HERE +``` + ## Important to Note - **A green job does not prove a suite ran.** Several suites (notably