diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4482f871..e3d26a27 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,16 +38,30 @@ jobs: name: Package Linux x86-64 needs: publish-metadata runs-on: ubuntu-22.04 + container: + image: ubuntu:20.04@sha256:c664f8f86ed5a386b0a340d981b8f81714e21a8b9c73f658c4bea56aa179d54a timeout-minutes: 75 steps: - - uses: actions/checkout@v5 - - name: Install dependencies run: | - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes \ - bubblewrap build-essential ca-certificates cmake curl git \ + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + binutils bubblewrap build-essential ca-certificates cmake curl file git \ libicu-dev libssl-dev libz-dev libzstd-dev pkg-config + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | + tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | + tee /etc/apt/sources.list.d/github-cli.list > /dev/null + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install --yes gh + git config --system --add safe.directory '*' + + - uses: actions/checkout@v5 + + - name: Configure safe directory + run: git config --global --add safe.directory '*' - name: Install Rust run: | @@ -59,6 +73,12 @@ jobs: - name: Package Linux release run: ./script/ci-package-release dist/ + - name: Validate Linux release + run: | + ./script/validate-linux-release-artifact \ + dist/autolith-"$GITHUB_REF_NAME"-x86_64-linux.tar.gz \ + x86_64-linux x86_64 /lib64/ld-linux-x86-64.so.2 + - name: Upload Linux release artifacts env: GH_TOKEN: ${{ github.token }} @@ -68,6 +88,135 @@ jobs: dist/autolith-"$GITHUB_REF_NAME"-x86_64-linux.tar.gz.sha256 \ --clobber + package-linux-aarch64: + name: Package Linux aarch64 + needs: publish-metadata + runs-on: ubuntu-24.04-arm + container: + image: ubuntu:20.04@sha256:722ea796ac2d57eeb3627c58a582fc1acc58be51faf815e1bce1682ae5c092f7 + timeout-minutes: 90 + steps: + - name: Install system dependencies + run: | + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + binutils bubblewrap build-essential ca-certificates cmake curl file git \ + libicu-dev libssl-dev libz-dev libzstd-dev pkg-config sbcl + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | + tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | + tee /etc/apt/sources.list.d/github-cli.list > /dev/null + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install --yes gh + git config --system --add safe.directory '*' + + - uses: actions/checkout@v5 + + - name: Configure safe directory + run: git config --global --add safe.directory '*' + + - name: Install Rust + run: | + curl --fail --location --show-error --retry 3 --proto '=https' --tlsv1.2 \ + https://sh.rustup.rs | + sh -s -- -y --default-toolchain 1.97.1 --profile minimal + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Package Linux release + env: + AUTOLITH_HOST_BOOTSTRAP: "1" + run: ./script/ci-package-release dist/ + + - name: Validate Linux release + run: | + ./script/validate-linux-release-artifact \ + dist/autolith-"$GITHUB_REF_NAME"-aarch64-linux.tar.gz \ + aarch64-linux aarch64 /lib/ld-linux-aarch64.so.1 + + - name: Upload Linux release artifacts + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$GITHUB_REF_NAME" \ + dist/autolith-"$GITHUB_REF_NAME"-aarch64-linux.tar.gz \ + dist/autolith-"$GITHUB_REF_NAME"-aarch64-linux.tar.gz.sha256 \ + --clobber + + package-linux-x86_64-musl: + name: Package Linux x86-64 (musl) + needs: publish-metadata + runs-on: ubuntu-22.04 + timeout-minutes: 75 + steps: + - uses: actions/checkout@v5 + + - name: Package and validate Linux musl release + run: | + docker run --rm -e GITHUB_REF_NAME -v "$PWD":/workspace -w /workspace \ + alpine:3.22.5@sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6 \ + sh -c ' + set -eu + apk add --no-cache bash binutils build-base bubblewrap ca-certificates cmake \ + coreutils curl file git grep icu-dev linux-headers openssl-dev pkgconfig \ + sbcl tar zlib-dev zstd-dev + curl --fail --location --show-error --retry 3 --proto "=https" --tlsv1.2 \ + https://sh.rustup.rs | + sh -s -- -y --default-toolchain 1.97.1 --profile minimal + export PATH=/root/.cargo/bin:$PATH + git config --global --add safe.directory "*" + AUTOLITH_HOST_BOOTSTRAP=1 ./script/ci-package-release dist/ + ./script/validate-linux-release-artifact \ + dist/autolith-"$GITHUB_REF_NAME"-x86_64-linux-musl.tar.gz \ + x86_64-linux-musl x86_64 /lib/ld-musl-x86_64.so.1 + ' + + - name: Upload Linux musl release artifacts + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$GITHUB_REF_NAME" \ + dist/autolith-"$GITHUB_REF_NAME"-x86_64-linux-musl.tar.gz \ + dist/autolith-"$GITHUB_REF_NAME"-x86_64-linux-musl.tar.gz.sha256 \ + --clobber + + package-linux-aarch64-musl: + name: Package Linux aarch64 (musl) + needs: publish-metadata + runs-on: ubuntu-24.04-arm + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + + - name: Package and validate Linux musl release + run: | + docker run --rm -e GITHUB_REF_NAME -v "$PWD":/workspace -w /workspace \ + alpine:3.22.5@sha256:2c9d26f410d032d5b1525aa8a873e238b05b90c4ae8618743d4311f0cc827e37 \ + sh -c ' + set -eu + apk add --no-cache bash binutils build-base bubblewrap ca-certificates cmake \ + coreutils curl file git grep icu-dev linux-headers openssl-dev pkgconfig \ + sbcl tar zlib-dev zstd-dev + curl --fail --location --show-error --retry 3 --proto "=https" --tlsv1.2 \ + https://sh.rustup.rs | + sh -s -- -y --default-toolchain 1.97.1 --profile minimal + export PATH=/root/.cargo/bin:$PATH + git config --global --add safe.directory "*" + AUTOLITH_HOST_BOOTSTRAP=1 ./script/ci-package-release dist/ + ./script/validate-linux-release-artifact \ + dist/autolith-"$GITHUB_REF_NAME"-aarch64-linux-musl.tar.gz \ + aarch64-linux-musl aarch64 /lib/ld-musl-aarch64.so.1 + ' + + - name: Upload Linux musl release artifacts + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload "$GITHUB_REF_NAME" \ + dist/autolith-"$GITHUB_REF_NAME"-aarch64-linux-musl.tar.gz \ + dist/autolith-"$GITHUB_REF_NAME"-aarch64-linux-musl.tar.gz.sha256 \ + --clobber package-macos-arm64: name: Package macOS arm64 needs: publish-metadata diff --git a/bin/autolith-release b/bin/autolith-release index 27436f7e..381dc2dc 100755 --- a/bin/autolith-release +++ b/bin/autolith-release @@ -60,6 +60,18 @@ release_tag_valid_p() [[ $1 =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] } +legacy_release_platform_p() +{ + case $1 in + x86_64-linux|arm64-darwin|x86_64-freebsd|x86_64-netbsd|x86_64-openbsd) + return 0 + ;; + *) + return 1 + ;; + esac +} + semantic_version_newer_p() { local candidate=$1 @@ -94,6 +106,7 @@ release_update() { local requested_tag=${1:-} local updated_launcher + local -a install_arguments if [[ -z $requested_tag ]]; then if ! requested_tag=$(release_latest_tag); then @@ -108,9 +121,12 @@ release_update() printf 'Autolith %s is already the newest release.\n' "$release_version" >&2 release_restart fi + install_arguments=(--without-command-link --version "$requested_tag") + if [[ $release_platform == *-linux-musl ]]; then + install_arguments=(--musl "${install_arguments[@]}") + fi if ! AUTOLITH_INSTALL_ROOT="$install_root" \ - "$source_root/script/install" \ - --without-command-link --version "$requested_tag"; then + "$source_root/script/install" "${install_arguments[@]}"; then printf 'Autolith could not install %s. Continuing with %s.\n' \ "${requested_tag#v}" "$release_version" >&2 release_restart @@ -142,7 +158,7 @@ release_images_usable_p() active_manifest=$(dirname -- "$active_core")/manifest.sexp recovery_manifest=$(dirname -- "$recovery_core")/manifest.sexp [[ -r $image_marker ]] || return 1 - [[ $(<"$image_marker") == "$release_tag" ]] || return 1 + [[ $(<"$image_marker") == "$release_tag:$release_platform" ]] || return 1 [[ -f $active_core && -f $active_manifest ]] || return 1 [[ -f $recovery_core && -f $recovery_manifest ]] || return 1 grep -Eq '^\(:ACTIVE-IMAGE :VERSION 1([[:space:]]|$)' \ @@ -169,45 +185,57 @@ release_images_build() marker_directory=$(dirname -- "$image_marker") mkdir -p -- "$marker_directory" temporary_marker=$marker_directory/.release-images.$$ - printf '%s\n' "$release_tag" > "$temporary_marker" + printf '%s:%s\n' "$release_tag" "$release_platform" > "$temporary_marker" chmod 600 "$temporary_marker" mv -f -- "$temporary_marker" "$image_marker" } [[ -n $home ]] || fail "HOME is not set." -platform= +host_platform= os=$(uname -s) arch=$(uname -m) case $os in Linux) case $arch in - x86_64|amd64) platform=x86_64-linux ;; + x86_64|amd64) host_platform=x86_64-linux ;; + aarch64|arm64) host_platform=aarch64-linux ;; esac + if [[ -n $host_platform ]]; then + command -v ldd >/dev/null 2>&1 || + fail "Linux libc could not be identified because ldd is unavailable." + libc_description=$(ldd /bin/sh 2>&1 || true) + if grep -iq musl <<< "$libc_description"; then + host_platform=${host_platform}-musl + elif ! grep -Eiq 'glibc|gnu libc|ld-linux|libc\.so\.6' \ + <<< "$libc_description"; then + fail "Linux libc could not be identified as glibc or musl." + fi + fi ;; Darwin) case $arch in - arm64|aarch64) platform=arm64-darwin ;; + arm64|aarch64) host_platform=arm64-darwin ;; esac ;; FreeBSD) case $arch in - amd64|x86_64) platform=x86_64-freebsd ;; + amd64|x86_64) host_platform=x86_64-freebsd ;; esac ;; NetBSD) case $arch in - amd64|x86_64) platform=x86_64-netbsd ;; + amd64|x86_64) host_platform=x86_64-netbsd ;; esac ;; OpenBSD) case $arch in - amd64|x86_64) platform=x86_64-openbsd ;; + amd64|x86_64) host_platform=x86_64-openbsd ;; esac ;; esac -[[ -n $platform ]] || - fail "binary releases currently support Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only." -if [[ $platform == arm64-darwin ]]; then +[[ -n $host_platform ]] || + fail "binary releases currently support Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only." +if [[ $host_platform == arm64-darwin ]]; then fff_library=$release_root/lib/libfff_c.dylib colorlisp_library=$release_root/lib/libcolorlisp-tree-sitter.dylib else @@ -222,19 +250,29 @@ release_commit=$(release_field commit) semantic_version_valid_p "$release_version" || fail "RELEASE has an invalid version." [[ $release_tag == "v$release_version" ]] || fail "RELEASE has an inconsistent tag." [[ $release_commit =~ ^[0-9a-f]{40}$ ]] || fail "RELEASE has an invalid commit." +if grep -q '^platform=' "$release_record"; then + release_platform=$(release_field platform) + [[ -n $release_platform ]] || fail "RELEASE has an invalid platform." +else + legacy_release_platform_p "$host_platform" || + fail "RELEASE lacks platform identity for this release variant." + release_platform=$host_platform +fi +[[ $release_platform == "$host_platform" ]] || + fail "RELEASE platform $release_platform does not match host $host_platform." [[ -x $sbcl_command ]] || fail "the bundled SBCL runtime is missing." [[ -f $runtime_source_root/version.lisp-expr ]] || fail "the bundled SBCL source is missing." [[ -f $source_root/autolith.asd && -f $source_root/.qlot/setup.lisp ]] || fail "the bundled Autolith source or dependencies are missing." [[ -x $source_root/script/install ]] || fail "the bundled installer is missing." -if [[ $platform == x86_64-linux ]]; then +if [[ $release_platform == *-linux* ]]; then [[ -x $sandbox_helper ]] || fail "the private sandbox helper is missing." fi [[ -f $fff_library ]] || fail "the private search library is missing." [[ -f $colorlisp_library ]] || fail "the private syntax library is missing." command -v git >/dev/null 2>&1 || fail "Git is required." -if [[ $platform == x86_64-linux ]]; then +if [[ $release_platform == *-linux* ]]; then command -v bwrap >/dev/null 2>&1 || fail "Bubblewrap is required." fi @@ -242,7 +280,8 @@ releases_root=$(CDPATH= cd -P -- "$release_root/.." && pwd) install_root=$(CDPATH= cd -P -- "$releases_root/.." && pwd) selected_release_p=false current_selected=$(resolve_path "$install_root/current" 2>/dev/null || true) -if [[ ${release_root##*/} == "$release_tag" && +if [[ (${release_root##*/} == "$release_tag" || + ${release_root##*/} == "$release_tag-$release_platform") && ${releases_root##*/} == releases && "$current_selected" == "$release_root" ]]; then selected_release_p=true @@ -260,7 +299,7 @@ export AUTOLITH_FFF_LIBRARY=$fff_library export COLORLISP_NATIVE_LIBRARY=$colorlisp_library export AUTOLITH_ACTIVE_CORE=$active_core export AUTOLITH_RECOVERY_CORE=$recovery_core -if [[ $platform == x86_64-linux ]]; then +if [[ $release_platform == *-linux* ]]; then export CL_EXEC_SANDBOX_BWRAP CL_EXEC_SANDBOX_BWRAP=$(command -v bwrap) export CL_EXEC_SANDBOX_HELPER=$sandbox_helper @@ -269,8 +308,8 @@ export GIT_OPTIONAL_LOCKS=0 remaining_arguments=("$@") if [[ ${1:-} == --autolith-release-probe ]]; then - printf 'version=%s\ntag=%s\ncommit=%s\nsource=%s\nruntime=%s\n' \ - "$release_version" "$release_tag" "$release_commit" \ + printf 'version=%s\ntag=%s\ncommit=%s\nplatform=%s\nsource=%s\nruntime=%s\n' \ + "$release_version" "$release_tag" "$release_commit" "$release_platform" \ "$source_root" "$sbcl_command" exit 0 fi diff --git a/script/build-release-runtime b/script/build-release-runtime index f0ed5281..7f685919 100755 --- a/script/build-release-runtime +++ b/script/build-release-runtime @@ -28,16 +28,44 @@ launcher_path=$(resolve_path "${BASH_SOURCE[0]}") source_root=$(CDPATH= cd -P -- "$(dirname -- "$launcher_path")/.." && pwd) installation=${1:?usage: build-release-runtime INSTALLATION} bootstrap_version=2.4.0 +host_bootstrap_minimum=2.0.0 +unset AUTOLITH_HOST_BOOTSTRAP || true os=$(uname -s) arch=$(uname -m) host_bootstrap=false case $os in Linux) + command -v ldd >/dev/null 2>&1 || { + printf 'build-release-runtime needs ldd to identify the Linux C library.\n' >&2 + exit 1 + } + ldd_output=$(ldd /bin/sh 2>&1 || true) + if grep -iq musl <<< "$ldd_output"; then + detected_libc=musl + elif grep -Eq 'libc\.so\.6|ld-linux' <<< "$ldd_output"; then + detected_libc=glibc + else + printf 'build-release-runtime could not identify the Linux C library from: %s\n' \ + "$ldd_output" >&2 + exit 1 + fi + if [[ -n ${AUTOLITH_LIBC:-} && ${AUTOLITH_LIBC} != "$detected_libc" ]]; then + printf 'build-release-runtime expected %s but detected %s.\n' \ + "$AUTOLITH_LIBC" "$detected_libc" >&2 + exit 1 + fi case $arch in x86_64|amd64) - bootstrap_arch=x86-64-linux - bootstrap_sha256=50afb9765d6a2f937f609ac33ebe553326347aef23eddd49c46d76456a5b3095 + if [[ $detected_libc == musl ]]; then + host_bootstrap=true + else + bootstrap_arch=x86-64-linux + bootstrap_sha256=50afb9765d6a2f937f609ac33ebe553326347aef23eddd49c46d76456a5b3095 + fi + ;; + aarch64|arm64) + host_bootstrap=true ;; esac ;; @@ -56,7 +84,7 @@ case $os in ;; esac if [[ $host_bootstrap != true && -z ${bootstrap_arch:-} ]]; then - printf 'build-release-runtime currently supports Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64.\n' >&2 + printf 'build-release-runtime currently supports Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64.\n' >&2 exit 1 fi @@ -98,7 +126,32 @@ if [[ $host_bootstrap == true ]]; then printf 'build-release-runtime needs a host SBCL on %s.\n' "$os" >&2 exit 1 fi - printf 'Using host SBCL as the %s bootstrap compiler.\n' "$os" + host_version=$(env -u SBCL_HOME "$host_sbcl" \ + --noinform --no-userinit --no-sysinit --non-interactive \ + --eval '(write-string (lisp-implementation-version))') || { + printf 'build-release-runtime could not inspect the host SBCL on %s.\n' "$os" >&2 + exit 1 + } + if [[ ! $host_version =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(\..*)?$ ]]; then + printf 'build-release-runtime needs a release host SBCL, but found %s.\n' "$host_version" >&2 + exit 1 + fi + host_major=${BASH_REMATCH[1]} + host_minor=${BASH_REMATCH[2]} + host_patch=${BASH_REMATCH[3]} + IFS=. read -r minimum_major minimum_minor minimum_patch <<< "$host_bootstrap_minimum" + if ((host_major < minimum_major || + (host_major == minimum_major && host_minor < minimum_minor) || + (host_major == minimum_major && + host_minor == minimum_minor && + host_patch < minimum_patch))); then + printf 'build-release-runtime needs host SBCL %s or newer on %s, but found %s.\n' \ + "$host_bootstrap_minimum" "$os" "$host_version" >&2 + exit 1 + fi + export AUTOLITH_HOST_BOOTSTRAP=1 + export AUTOLITH_HOST_BOOTSTRAP_MINIMUM=$host_bootstrap_minimum + printf 'Using host SBCL %s as the %s bootstrap compiler.\n' "$host_version" "$os" mkdir -p "$bootstrap_installation/bin" ln -s "$host_sbcl" "$bootstrap_installation/bin/sbcl" else diff --git a/script/build-release-runtime.lisp b/script/build-release-runtime.lisp index 6d84f344..3c5ec008 100644 --- a/script/build-release-runtime.lisp +++ b/script/build-release-runtime.lisp @@ -33,6 +33,31 @@ components) t))) + (host-version-components (value) + "Return the first three numeric components of host SBCL VALUE." + (let ((components + (uiop:split-string value :separator '(#\.)))) + (when (and (>= (length components) 3) + (every (lambda (component) + (and (plusp (length component)) + (every #'digit-char-p component))) + (subseq components 0 3))) + (mapcar #'parse-integer (subseq components 0 3))))) + + (host-version-at-least-p (candidate minimum) + "Return true when host SBCL CANDIDATE satisfies MINIMUM." + (let ((candidate-components (host-version-components candidate)) + (minimum-components (host-version-components minimum))) + (and candidate-components + minimum-components + (loop for candidate-component in candidate-components + for minimum-component in minimum-components + when (> candidate-component minimum-component) + return t + when (< candidate-component minimum-component) + return nil + finally (return t))))) + (sha256-p (value) "Return true when VALUE is a lowercase SHA-256 identity." (and (= (length value) 64) @@ -63,42 +88,39 @@ (uiop:ensure-directory-pathname directory-name)))))) -(check-archive (archive expected-sha256) - "Require ARCHIVE to match EXPECTED-SHA256." - (cond - ((command-available-p "sha256sum") - (run (list "sha256sum" "--check" "--status" "-") - :output nil - :error-output ':output - :directory temporary-root - :input - (make-string-input-stream - (format nil "~A ~A~%" - expected-sha256 - (file-namestring archive))))) - ((command-available-p "shasum") - (run (list "shasum" "-a" "256" "--check" "--status" "-") - :output nil - :error-output ':output - :directory temporary-root - :input - (make-string-input-stream - (format nil "~A ~A~%" - expected-sha256 - (file-namestring archive))))) - ((command-available-p "sha256") - (let ((actual - (string-trim - '(#\Space #\Tab #\Newline #\Return) - (run (list "sha256" "-q" (file-namestring archive)) + (check-archive (archive expected-sha256) + "Require ARCHIVE to match EXPECTED-SHA256." + (let ((actual + (string-trim + '(#\Space #\Tab #\Newline #\Return) + (cond + ((command-available-p "sha256sum") + (first + (uiop:split-string + (run (list "sha256sum" (namestring archive)) + :output ':string + :error-output ':output + :directory temporary-root) + :separator '(#\Space #\Tab)))) + ((command-available-p "shasum") + (first + (uiop:split-string + (run (list "shasum" "-a" "256" + (namestring archive)) :output ':string :error-output ':output - :directory temporary-root)))) - (unless (string-equal actual expected-sha256) - (fail "~A does not match the expected SHA-256." - (file-namestring archive))))) - (t - (fail "sha256sum, shasum, or sha256 is required.")))) + :directory temporary-root) + :separator '(#\Space #\Tab)))) + ((command-available-p "sha256") + (run (list "sha256" "-q" (file-namestring archive)) + :output ':string + :error-output ':output + :directory temporary-root)) + (t + (fail "sha256sum, shasum, or sha256 is required.")))))) + (unless (string-equal actual expected-sha256) + (fail "~A does not match the expected SHA-256." + (file-namestring archive))))) (runtime-version (command) "Return the implementation version reported by SBCL COMMAND." @@ -129,21 +151,22 @@ source-root installation temporary-root bootstrap-installation) (fail "usage: build-release-runtime.lisp SOURCE INSTALLATION TEMP BOOTSTRAP")) - (unless (or (and (string-equal (software-type) "Linux") - (member (string-downcase (machine-type)) - '("x86-64" "x86_64" "amd64") - :test #'string=)) - (and (string-equal (software-type) "Darwin") - (member (string-downcase (machine-type)) - '("arm64" "aarch64") - :test #'string=)) - (and (member (software-type) - '("FreeBSD" "NetBSD" "OpenBSD") - :test #'string-equal) - (member (string-downcase (machine-type)) - '("x86-64" "x86_64" "amd64") - :test #'string=))) - (fail "release runtimes currently support Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only.")) + (unless (or (and (string-equal (software-type) "Linux") + (member (string-downcase (machine-type)) + '("x86-64" "x86_64" "amd64" + "aarch64" "arm64") + :test #'string=)) + (and (string-equal (software-type) "Darwin") + (member (string-downcase (machine-type)) + '("arm64" "aarch64") + :test #'string=)) + (and (member (software-type) + '("FreeBSD" "NetBSD" "OpenBSD") + :test #'string-equal) + (member (string-downcase (machine-type)) + '("x86-64" "x86_64" "amd64") + :test #'string=))) + (fail "release runtimes currently support Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only.")) (let* ((runtime-version (trimmed-file (merge-pathnames "sbcl.version" source-root))) (runtime-sha256 @@ -162,11 +185,18 @@ (fail "sbcl.version is malformed.")) (unless (sha256-p runtime-sha256) (fail "sbcl-source.sha256 is malformed.")) - (unless (or (member (software-type) - '("FreeBSD" "NetBSD" "OpenBSD") - :test #'string-equal) - (string= (runtime-version bootstrap-command) "2.4.0")) - (fail "the bootstrap compiler does not report version 2.4.0.")) + (let* ((bootstrap-version (runtime-version bootstrap-command)) + (host-bootstrap-p + (equal (uiop:getenv "AUTOLITH_HOST_BOOTSTRAP") "1")) + (host-minimum + (or (uiop:getenv "AUTOLITH_HOST_BOOTSTRAP_MINIMUM") + "2.0.0"))) + (if host-bootstrap-p + (unless (host-version-at-least-p bootstrap-version host-minimum) + (fail "the host bootstrap compiler ~A does not satisfy SBCL ~A or newer." + bootstrap-version host-minimum)) + (unless (string= bootstrap-version "2.4.0") + (fail "the bootstrap compiler does not report version 2.4.0.")))) (format t "~&Building the pinned SBCL ~A release runtime.~%" runtime-version) (finish-output) diff --git a/script/install b/script/install index c197b83e..68ffc490 100755 --- a/script/install +++ b/script/install @@ -11,6 +11,7 @@ bin_directory=${AUTOLITH_BIN_DIR:-$home/.local/bin} requested_tag= temporary_root= publish_command_p=true +musl_requested_p=false fail() { @@ -41,24 +42,47 @@ release_latest_tag() printf '%s\n' "$latest_tag" } +legacy_release_platform_p() +{ + case $1 in + x86_64-linux|arm64-darwin|x86_64-freebsd|x86_64-netbsd|x86_64-openbsd) + return 0 + ;; + *) + return 1 + ;; + esac +} + release_installed_p() { release_installed_target=$1 release_installed_expected_tag=$2 + release_installed_expected_platform=$3 + release_installed_record=$release_installed_target/RELEASE [ -x "$release_installed_target/bin/autolith" ] || return 1 - [ -r "$release_installed_target/RELEASE" ] || return 1 + [ -r "$release_installed_record" ] || return 1 grep -Fx "tag=$release_installed_expected_tag" \ - "$release_installed_target/RELEASE" >/dev/null 2>&1 + "$release_installed_record" >/dev/null 2>&1 || return 1 + if grep -q '^platform=' "$release_installed_record"; then + release_installed_platform=$(sed -n 's/^platform=//p' \ + "$release_installed_record") + [ "$release_installed_platform" = "$release_installed_expected_platform" ] + else + legacy_release_platform_p "$release_installed_expected_platform" + fi } check_sha256() { check_checksum=$1 if command -v sha256sum >/dev/null 2>&1; then - sha256sum --check --status "$check_checksum" + sha256sum -c -s "$check_checksum" 2>/dev/null || \ + sha256sum -c "$check_checksum" >/dev/null 2>&1 elif command -v shasum >/dev/null 2>&1; then - shasum -a 256 --check --status "$check_checksum" + shasum -a 256 -c -s "$check_checksum" 2>/dev/null || \ + shasum -a 256 -c "$check_checksum" >/dev/null 2>&1 elif command -v sha256 >/dev/null 2>&1; then check_expected=$(awk '{print $1}' "$check_checksum") check_file=$(awk '{print $2}' "$check_checksum") @@ -73,14 +97,14 @@ check_sha256() publish_links() { publish_target=$1 - publish_tag=$2 + publish_directory=${publish_target##*/} current_temporary=$install_root/.current.$$ command_temporary=$bin_directory/.autolith.$$ if [ -d "$install_root/current" ] && [ ! -L "$install_root/current" ]; then fail "$install_root/current is a directory, not an installation link." fi - ln -s "releases/$publish_tag" "$current_temporary" + ln -s "releases/$publish_directory" "$current_temporary" # GNU mv -T renames over the current link atomically. BSD mv has no -T # and would descend into the resolved directory, so it degrades to a # remove-then-rename pair with a brief unlinked window. @@ -102,13 +126,17 @@ while [ "$#" -gt 0 ]; do requested_tag=$2 shift 2 ;; + --musl) + musl_requested_p=true + shift + ;; --without-command-link) publish_command_p=false shift ;; --help) printf '%s\n' \ - 'usage: install [--version vMAJOR.MINOR.PATCH] [--without-command-link]' \ + 'usage: install [--version vMAJOR.MINOR.PATCH] [--musl] [--without-command-link]' \ '' \ 'Install or update the Autolith binary release.' exit 0 @@ -121,13 +149,28 @@ done [ -n "$home" ] || fail "HOME is not set." platform= +detected_libc= os=$(uname -s) arch=$(uname -m) case $os in Linux) case $arch in x86_64|amd64) platform=x86_64-linux ;; + aarch64|arm64) platform=aarch64-linux ;; esac + if [ -n "$platform" ]; then + command -v ldd >/dev/null 2>&1 || + fail "Linux libc could not be identified because ldd is unavailable." + libc_description=$(ldd /bin/sh 2>&1 || true) + if printf '%s\n' "$libc_description" | grep -iq musl; then + detected_libc=musl + elif printf '%s\n' "$libc_description" | + grep -Eiq 'glibc|gnu libc|ld-linux|libc\.so\.6'; then + detected_libc=glibc + else + fail "Linux libc could not be identified as glibc or musl." + fi + fi ;; Darwin) case $arch in @@ -151,12 +194,30 @@ case $os in ;; esac [ -n "$platform" ] || - fail "binary releases currently support Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only." + fail "binary releases currently support Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only." +requested_libc=${AUTOLITH_LIBC:-} +if [ "$musl_requested_p" = true ]; then + requested_libc=musl +fi +case $requested_libc in + "") ;; + glibc|musl) ;; + *) fail "AUTOLITH_LIBC must be glibc or musl." ;; +esac +if [ "$os" != Linux ] && [ -n "$requested_libc" ]; then + fail "libc selection is supported only on Linux." +fi +if [ -n "$requested_libc" ] && [ "$requested_libc" != "$detected_libc" ]; then + fail "requested libc $requested_libc does not match detected libc $detected_libc." +fi +if [ "$detected_libc" = musl ]; then + platform=${platform}-musl +fi required_commands="bash curl git grep openssl tar" -if [ "$platform" = x86_64-linux ]; then - required_commands="$required_commands bwrap" -fi +case $platform in + *-linux*) required_commands="$required_commands bwrap" ;; +esac for command in $required_commands; do command -v "$command" >/dev/null 2>&1 || fail "$command is required. Nix is the recommended installation path when system dependencies are unavailable." @@ -181,12 +242,19 @@ release_name=autolith-$requested_tag-$platform archive_name=$release_name.tar.gz checksum_name=$archive_name.sha256 releases_root=$install_root/releases -target=$releases_root/$requested_tag +qualified_target=$releases_root/$requested_tag-$platform +legacy_target=$releases_root/$requested_tag +target=$qualified_target mkdir -p -- "$releases_root" chmod 700 "$install_root" "$releases_root" -if release_installed_p "$target" "$requested_tag"; then - publish_links "$target" "$requested_tag" +if release_installed_p "$qualified_target" "$requested_tag" "$platform"; then + target=$qualified_target + publish_links "$target" + printf 'Autolith %s is already installed.\n' "${requested_tag#v}" +elif release_installed_p "$legacy_target" "$requested_tag" "$platform"; then + target=$legacy_target + publish_links "$target" printf 'Autolith %s is already installed.\n' "${requested_tag#v}" else temporary_root=$(mktemp -d "$releases_root/.install.XXXXXX") @@ -206,8 +274,8 @@ else check_sha256 "$checksum_name" ) || fail "the release archive has the wrong SHA-256 identity." tar -xzf "$archive" -C "$temporary_root" - release_installed_p "$extracted" "$requested_tag" || - fail "the release archive has an unexpected layout." + release_installed_p "$extracted" "$requested_tag" "$platform" || + fail "the release archive has an unexpected platform identity or layout." chmod u+w "$extracted" || fail "the verified release directory could not be prepared for publication." @@ -224,7 +292,7 @@ else fi chmod u-w "$target" || fail "the published release permissions could not be restored." - publish_links "$target" "$requested_tag" + publish_links "$target" if [ -n "$stale_target" ] && [ -e "$stale_target" ]; then chmod -R u+w "$stale_target" 2>/dev/null || true rm -rf -- "$stale_target" diff --git a/script/validate-linux-release-artifact b/script/validate-linux-release-artifact new file mode 100755 index 00000000..0ef1fbdd --- /dev/null +++ b/script/validate-linux-release-artifact @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() +{ + printf 'Linux release validation failed: %s\n' "$1" >&2 + exit 1 +} + +[[ $# -eq 4 ]] || + fail "usage: validate-linux-release-artifact ARCHIVE PLATFORM ARCHITECTURE INTERPRETER" + +archive=$1 +expected_platform=$2 +expected_architecture=$3 +expected_interpreter=$4 + +[[ -f $archive ]] || fail "$archive does not exist." +case $expected_architecture in + x86_64) expected_machine='Advanced Micro Devices X86-64' ;; + aarch64) expected_machine='AArch64' ;; + *) fail "unsupported expected architecture $expected_architecture." ;; +esac + +for command in file readelf tar grep sed; do + command -v "$command" >/dev/null 2>&1 || fail "$command is required." +done + +archive_name=$(basename -- "$archive") +release_name=${archive_name%.tar.gz} +[[ $release_name != "$archive_name" ]] || fail "$archive is not a .tar.gz archive." +archive_members=$(tar -tzf "$archive") || fail "$archive cannot be listed." +[[ -n $archive_members ]] || fail "$archive is empty." +while IFS= read -r member; do + [[ -n $member ]] || fail "the archive contains an empty member name." + case "/$member/" in + */../*|*/./*) fail "archive member $member contains an unsafe path component." ;; + esac + case $member in + "$release_name"|"$release_name/"|"$release_name/"*) ;; + *) fail "archive member $member is outside $release_name/." ;; + esac +done <<< "$archive_members" + +temporary_root=$(mktemp -d) +cleanup() +{ + rm -rf -- "$temporary_root" +} +trap cleanup EXIT HUP INT TERM + +release_root=$temporary_root/$release_name +tar -xzf "$archive" -C "$temporary_root" || fail "$archive cannot be extracted." +shopt -s dotglob nullglob +top_level=("$temporary_root"/*) +shopt -u dotglob nullglob +[[ ${#top_level[@]} -eq 1 && ${top_level[0]} == "$release_root" && + -d $release_root && ! -L $release_root ]] || + fail "the archive does not contain exactly one top-level directory named $release_name." + +release_record=$release_root/RELEASE +runtime=$release_root/runtime/bin/sbcl +[[ -r $release_record ]] || fail "RELEASE is missing." +[[ -x $runtime ]] || fail "the bundled SBCL runtime is missing." +platform_count=$(grep -c '^platform=' "$release_record" || true) +[[ $platform_count -eq 1 ]] || + fail "RELEASE must contain exactly one platform field." +platform_record=$(grep '^platform=' "$release_record") +[[ $platform_record == "platform=$expected_platform" ]] || + fail "RELEASE does not identify $expected_platform." + +file_description=$(file --brief "$runtime") +case $expected_architecture in + x86_64) + grep -q 'x86-64' <<< "$file_description" || + fail "the bundled SBCL file identity is not x86-64: $file_description" + ;; + aarch64) + grep -Eq 'ARM aarch64|AArch64' <<< "$file_description" || + fail "the bundled SBCL file identity is not aarch64: $file_description" + ;; +esac + +machine=$(readelf -h "$runtime" | sed -n 's/^[[:space:]]*Machine:[[:space:]]*//p') +[[ $machine == "$expected_machine" ]] || + fail "the bundled SBCL ELF machine is $machine, expected $expected_machine." +interpreter=$(readelf -l "$runtime" | + sed -n 's/.*Requesting program interpreter: \(.*\)]/\1/p') +[[ $interpreter == "$expected_interpreter" ]] || + fail "the bundled SBCL interpreter is $interpreter, expected $expected_interpreter." + +runtime_version=$(env -u SBCL_HOME "$runtime" \ + --noinform --no-userinit --no-sysinit --non-interactive \ + --eval '(write-string (lisp-implementation-version))') +[[ $runtime_version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + fail "the bundled SBCL smoke test returned $runtime_version." +printf 'Validated %s as %s (%s, %s).\n' \ + "$archive_name" "$expected_platform" "$expected_architecture" "$runtime_version" diff --git a/server/release-archive.lisp b/server/release-archive.lisp index d00d57d6..5a732f97 100644 --- a/server/release-archive.lisp +++ b/server/release-archive.lisp @@ -183,16 +183,19 @@ rather than failing, so existence needs the following stat first." nil) (-> release-archive--write-record - (pathname &key (:version string) (:tag string) (:commit string)) + (pathname &key (:version string) (:tag string) (:commit string) + (:platform string)) null) -(defun release-archive--write-record (pathname &key version tag commit) - "Write the strict VERSION, TAG, and COMMIT release record to PATHNAME." +(defun release-archive--write-record + (pathname &key version tag commit platform) + "Write the strict VERSION, TAG, COMMIT, and PLATFORM record to PATHNAME." (with-open-file (stream pathname :direction ':output :if-exists ':supersede :if-does-not-exist ':create :external-format ':utf-8) - (format stream "version=~A~%tag=~A~%commit=~A~%" version tag commit)) + (format stream "version=~A~%tag=~A~%commit=~A~%platform=~A~%" + version tag commit platform)) nil) (-> release-archive--make-temporary-root (pathname) pathname) @@ -226,7 +229,9 @@ rather than failing, so existence needs the following stat first." (list "env" "GIT_CONFIG_NOSYSTEM=1" "GIT_CONFIG_GLOBAL=/dev/null" - "git" "-C" (namestring source-root)) + "git" + "-c" "safe.directory=*" + "-C" (string-right-trim "/" (namestring source-root))) arguments)) (-> release-archive--create-source-identity (pathname string string) null) @@ -236,7 +241,11 @@ rather than failing, so existence needs the following stat first." (release-archive--run (release-archive--identity-git-command source-root - '("init" "--quiet" "--initial-branch=master" "--template="))) + '("init" "--quiet" "--template="))) + (release-archive--run + (release-archive--identity-git-command + source-root + '("symbolic-ref" "HEAD" "refs/heads/master"))) (dolist (setting '(("user.name" "Autolith release build") ("user.email" "release-build@localhost") @@ -271,7 +280,9 @@ rather than failing, so existence needs the following stat first." "TZ=UTC" (format nil "GIT_AUTHOR_DATE=@~A +0000" commit-time) (format nil "GIT_COMMITTER_DATE=@~A +0000" commit-time) - "git" "-C" (namestring source-root) + "git" + "-c" "safe.directory=*" + "-C" (string-right-trim "/" (namestring source-root)) "commit-tree" tree "-m" (format nil "Autolith ~A source" tag))) :output ':string @@ -297,14 +308,56 @@ rather than failing, so existence needs the following stat first." :test #'string=) t)) -(-> release-archive--platform-id (string string) string) -(defun release-archive--platform-id (os architecture) - "Return the canonical release platform identifier for OS and ARCHITECTURE." - (let ((architecture (string-downcase architecture))) +(-> release-archive--linux-libc-output->identity (string) string) +(defun release-archive--linux-libc-output->identity (output) + "Return the Linux C library identity established by ldd OUTPUT." + (cond + ((search "musl" output :test #'char-equal) + "musl") + ((or (search "libc.so.6" output :test #'char-equal) + (search "ld-linux" output :test #'char-equal)) + "glibc") + (t + (error 'release-archive-error + :stage ':prerequisites + :cause (format nil "Could not identify the Linux C library from: ~A" + output))))) + +(-> release-archive--linux-libc () string) +(defun release-archive--linux-libc () + "Return the detected native Linux C library identity." + (unless (release-archive--command-pathname "ldd") + (error 'release-archive-error + :stage ':prerequisites + :cause "ldd is required to identify the Linux C library.")) + (let* ((output + (release-archive--run + (list "sh" "-c" "ldd /bin/sh 2>&1 || true") + :output ':string + :error-output ':output)) + (detected (release-archive--linux-libc-output->identity output)) + (configured (uiop:getenv "AUTOLITH_LIBC"))) + (when (and configured + (plusp (length configured)) + (not (string-equal configured detected))) + (error 'release-archive-error + :stage ':prerequisites + :cause (format nil "AUTOLITH_LIBC names ~A, but this host uses ~A." + configured detected))) + detected)) + +(-> release-archive--platform-id (string string &optional string) string) +(defun release-archive--platform-id (os architecture &optional libc) + "Return the canonical release identifier for OS, ARCHITECTURE, and LIBC." + (let ((architecture (string-downcase architecture)) + (musl-p (and libc (string-equal libc "musl")))) (cond ((and (string-equal os "Linux") (release-archive--x86-64-architecture-p architecture)) - "x86_64-linux") + (if musl-p "x86_64-linux-musl" "x86_64-linux")) + ((and (string-equal os "Linux") + (member architecture '("arm64" "aarch64") :test #'string=)) + (if musl-p "aarch64-linux-musl" "aarch64-linux")) ((and (string-equal os "Darwin") (member architecture '("arm64" "aarch64") :test #'string=)) "arm64-darwin") @@ -320,12 +373,24 @@ rather than failing, so existence needs the following stat first." (t (error 'release-archive-error :stage ':prerequisites - :cause "Binary releases currently support Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only."))))) + :cause "Binary releases currently support Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only."))))) (-> release-archive--platform () string) (defun release-archive--platform () - "Return the canonical release platform identifier." - (release-archive--platform-id (software-type) (machine-type))) + "Return the validated native release platform identifier." + (let* ((os (software-type)) + (libc (and (string-equal os "Linux") + (release-archive--linux-libc))) + (detected (release-archive--platform-id os (machine-type) libc)) + (configured (uiop:getenv "AUTOLITH_RELEASE_PLATFORM"))) + (when (and configured + (plusp (length configured)) + (not (string= configured detected))) + (error 'release-archive-error + :stage ':prerequisites + :cause (format nil "AUTOLITH_RELEASE_PLATFORM names ~A, but this host is ~A." + configured detected))) + detected)) (-> release-archive--validate-platform () null) (defun release-archive--validate-platform () @@ -640,9 +705,9 @@ the managed runtime, matching SBCL source, native libraries, and sandbox helper. (namestring (merge-pathnames (format nil "lib/~A" colorlisp-library-name) release-root)))) - (release-archive--write-record - (merge-pathnames "RELEASE" release-root) - :version version :tag tag :commit commit) + (release-archive--write-record + (merge-pathnames "RELEASE" release-root) + :version version :tag tag :commit commit :platform platform) (format t "~&Creating the internal source identity.~%") (finish-output) (release-archive--create-source-identity diff --git a/server/release-server.lisp b/server/release-server.lisp index 9d22f3ef..ec12b81d 100644 --- a/server/release-server.lisp +++ b/server/release-server.lisp @@ -186,7 +186,9 @@ ;;;; -- Published Releases -- (defparameter *release-server-platform-ids* - '("x86_64-linux" "arm64-darwin" "x86_64-freebsd" "x86_64-netbsd" "x86_64-openbsd") + '("x86_64-linux" "aarch64-linux" + "x86_64-linux-musl" "aarch64-linux-musl" + "arm64-darwin" "x86_64-freebsd" "x86_64-netbsd" "x86_64-openbsd") "Canonical platform identifiers published as binary release archives.") (-> release-server--archive-name (string &optional string) string) diff --git a/tests/release-script-tests.lisp b/tests/release-script-tests.lisp index 2ff41825..9530f6ad 100644 --- a/tests/release-script-tests.lisp +++ b/tests/release-script-tests.lisp @@ -60,15 +60,18 @@ can assert on the exact failure message through the second return value." :output nil) nil) -(-> release-script-tests--record (pathname string) pathname) -(defun release-script-tests--record (pathname tag) - "Write a fixture release record with TAG to PATHNAME." +(-> release-script-tests--record + (pathname string &key (:platform (option string))) + pathname) +(defun release-script-tests--record (pathname tag &key platform) + "Write a fixture release record with TAG and optional PLATFORM to PATHNAME." (release-script-tests--write-file pathname - (format nil "version=~A~%tag=~A~%commit=~A~%" + (format nil "version=~A~%tag=~A~%commit=~A~@[~%platform=~A~]~%" *release-script-tests-version* tag - *release-script-tests-commit*))) + *release-script-tests-commit* + platform))) (-> release-script-tests--fixture-curl () string) (defun release-script-tests--fixture-curl () @@ -76,23 +79,37 @@ can assert on the exact failure message through the second return value." (format nil "#!/bin/sh~%set -eu~%output=~%write_out=~%url=~%while [ \"$#\" -gt 0 ]; do~% case $1 in~% --output) output=$2; shift 2 ;;~% --write-out) write_out=$2; shift 2 ;;~% --retry|--proto|--max-time) shift 2 ;;~% --*) shift ;;~% *) url=$1; shift ;;~% esac~%done~%case $url in~% */latest)~% [ -n \"$write_out\" ]~% printf \"%s\" \"https://example.invalid/releases/${AUTOLITH_TEST_LATEST_TAG:-v0.11.0}\"~% exit 0~% ;;~%esac~%cp \"$AUTOLITH_TEST_RELEASE_FIXTURE/${url##*/}\" \"$output\"~%")) -(-> release-script-tests--install-linux-host-tools (pathname) pathname) -(defun release-script-tests--install-linux-host-tools (directory) - "Install fixture commands reporting and satisfying the binary release target." +(-> release-script-tests--install-linux-host-tools + (pathname &key (:architecture string) (:libc string)) + pathname) +(defun release-script-tests--install-linux-host-tools + (directory &key (architecture "x86_64") (libc "glibc")) + "Install fixture commands reporting and satisfying one Linux release target." (let* ((directory (uiop:ensure-directory-pathname directory)) (uname (merge-pathnames "uname" directory)) + (ldd (merge-pathnames "ldd" directory)) (bwrap (merge-pathnames "bwrap" directory)) (chmod (merge-pathnames "chmod" directory)) (move (merge-pathnames "mv" directory))) (release-script-tests--write-file uname - "#!/bin/sh + (format nil "#!/bin/sh case ${1:-} in -s) printf 'Linux\\n' ;; - -m) printf 'x86_64\\n' ;; + -m) printf '~A\\n' ;; *) exit 64 ;; esac -") +" + architecture)) + (release-script-tests--write-file + ldd + (format nil "#!/bin/sh +printf '~A\\n' +" + (ecase (intern (string-upcase libc) '#:keyword) + (:glibc "libc.so.6 => /lib/libc.so.6 /lib64/ld-linux-x86-64.so.2") + (:musl "musl libc") + (:unknown "unrecognized libc")))) (release-script-tests--write-file bwrap "#!/bin/sh exit 0 ") @@ -133,7 +150,7 @@ else exec /bin/mv \"$@\" fi ") - (dolist (pathname (list uname bwrap chmod move)) + (dolist (pathname (list uname ldd bwrap chmod move)) (release-script-tests--chmod "755" pathname)) directory)) @@ -159,10 +176,11 @@ fi nil) (-> release-script-tests--make-release - (pathname pathname &key (:library-extension string)) + (pathname pathname &key (:library-extension string) + (:platform (option string))) pathname) (defun release-script-tests--make-release - (source-root release-root &key (library-extension "so")) + (source-root release-root &key (library-extension "so") platform) "Create a minimal packaged release fixture below RELEASE-ROOT." (dolist (relative (list "libexec/autolith/.qlot/setup.lisp" @@ -193,7 +211,8 @@ fi "755" (merge-pathnames "libexec/cl-exec-sandbox-helper" release-root)) (release-script-tests--record (merge-pathnames "RELEASE" release-root) - (format nil "v~A" *release-script-tests-version*)) + (format nil "v~A" *release-script-tests-version*) + :platform platform) release-root) (-> release-script-tests--syntax (pathname) null) @@ -210,6 +229,7 @@ fi "script/build-release" "script/build-release-runtime" "script/ci-package-release" + "script/validate-linux-release-artifact" "server/build-in-container")) (release-script-tests--run (list "bash" "-n" (namestring (merge-pathnames relative source-root))) @@ -561,7 +581,8 @@ printf '(:ACTIVE-IMAGE :VERSION 1\\n)\\n' > \"$active/manifest.sexp\" (list "AUTOLITH_NO_UPDATE_CHECK=1" (format nil "PATH=~A" path)))) (release-script-tests--install-linux-host-tools host-bin) - (release-script-tests--make-release source-root release-root) + (release-script-tests--make-release source-root release-root + :platform "x86_64-linux") (let ((output (release-script-tests--run (list (namestring launcher) "--autolith-release-probe") @@ -571,6 +592,7 @@ printf '(:ACTIVE-IMAGE :VERSION 1\\n)\\n' > \"$active/manifest.sexp\" (format nil "version=~A" *release-script-tests-version*) (format nil "tag=v~A" *release-script-tests-version*) (format nil "commit=~A" *release-script-tests-commit*) + "platform=x86_64-linux" (format nil "source=~A" (string-right-trim "/" @@ -586,6 +608,41 @@ printf '(:ACTIVE-IMAGE :VERSION 1\\n)\\n' > \"$active/manifest.sexp\" :separator '(#\Newline #\Return)) :test #'string=) (format nil "release probe reports ~A" line)))) + (release-script-tests--record + (merge-pathnames "RELEASE" release-root) + (format nil "v~A" *release-script-tests-version*) + :platform "aarch64-linux") + (multiple-value-bind (output error-output status) + (release-script-tests--run + (list (namestring launcher) "--autolith-release-probe") + :environment environment + :ignore-error-status t) + (declare (ignore output)) + (test-assert + (and (not (eql status 0)) + (search "RELEASE platform aarch64-linux does not match host x86_64-linux." + error-output)) + "the release launcher rejects a mismatched platform record")) + (release-script-tests--install-linux-host-tools host-bin :libc "musl") + (release-script-tests--record + (merge-pathnames "RELEASE" release-root) + (format nil "v~A" *release-script-tests-version*)) + (multiple-value-bind (output error-output status) + (release-script-tests--run + (list (namestring launcher) "--autolith-release-probe") + :environment environment + :ignore-error-status t) + (declare (ignore output)) + (test-assert + (and (not (eql status 0)) + (search "RELEASE lacks platform identity for this release variant." + error-output)) + "a musl launcher rejects legacy metadata without platform identity")) + (release-script-tests--install-linux-host-tools host-bin) + (release-script-tests--record + (merge-pathnames "RELEASE" release-root) + (format nil "v~A" *release-script-tests-version*) + :platform "x86_64-linux") (let ((library (merge-pathnames "lib/libcolorlisp-tree-sitter.so" release-root))) (delete-file library) @@ -634,7 +691,8 @@ printf '(:ACTIVE-IMAGE :VERSION 1\\n)\\n' > \"$active/manifest.sexp\" (format nil "PATH=~A" path)))) (release-script-tests--install-darwin-host-tools host-bin) (release-script-tests--make-release source-root release-root - :library-extension "dylib") + :library-extension "dylib" + :platform "arm64-darwin") (let ((output (release-script-tests--run (list (namestring launcher) "--autolith-release-probe") @@ -644,6 +702,7 @@ printf '(:ACTIVE-IMAGE :VERSION 1\\n)\\n' > \"$active/manifest.sexp\" (format nil "version=~A" *release-script-tests-version*) (format nil "tag=v~A" *release-script-tests-version*) (format nil "commit=~A" *release-script-tests-commit*) + "platform=arm64-darwin" (format nil "source=~A" (string-right-trim "/" @@ -746,7 +805,7 @@ esac (test-assert (and (not (eql status 0)) (search - "binary releases currently support Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only." + "binary releases currently support Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64 only." output)) "the binary installer rejects unsupported platforms"))) (release-script-tests--install-linux-host-tools fixture-bin) @@ -792,13 +851,14 @@ esac :output nil) (test-assert (probe-file - (merge-pathnames (format nil "releases/~A/bin/autolith" tag) - install-root)) + (merge-pathnames + (format nil "releases/~A-x86_64-linux/bin/autolith" tag) + install-root)) "the installer publishes the requested release") (test-assert (string= (release-script-tests--readlink (merge-pathnames "current" install-root)) - (format nil "releases/~A" tag)) + (format nil "releases/~A-x86_64-linux" tag)) "the installer selects the requested version atomically") (test-assert (string= (release-script-tests--readlink @@ -814,8 +874,10 @@ esac install-root))) (release-script-tests--run (list "cp" "-a" - (namestring (merge-pathnames (format nil "releases/~A/" tag) - install-root)) + (namestring + (merge-pathnames + (format nil "releases/~A-x86_64-linux/" tag) + install-root)) (namestring next-target)) :output nil) (release-script-tests--run @@ -971,19 +1033,32 @@ fi (string-right-trim "/" (namestring install-root))) (format nil "AUTOLITH_BIN_DIR=~A" (string-right-trim "/" (namestring bin-directory)))))) + (multiple-value-bind (output error-output status) + (release-script-tests--run + (list (namestring installer) "--musl" "--version" tag) + :environment base-environment + :ignore-error-status t + :output nil) + (declare (ignore output)) + (test-assert + (and (not (eql status 0)) + (search "libc selection is supported only on Linux." + error-output)) + "the Darwin installer rejects Linux libc selection")) (release-script-tests--run (list (namestring installer) "--version" tag) :environment base-environment :output nil) (test-assert (probe-file - (merge-pathnames (format nil "releases/~A/bin/autolith" tag) - install-root)) + (merge-pathnames + (format nil "releases/~A-arm64-darwin/bin/autolith" tag) + install-root)) "the Darwin installer publishes the requested release") (test-assert (string= (release-script-tests--readlink (merge-pathnames "current" install-root)) - (format nil "releases/~A" tag)) + (format nil "releases/~A-arm64-darwin" tag)) "the Darwin installer selects the requested version") (test-assert (string= (release-script-tests--readlink @@ -1088,7 +1163,7 @@ mv -Tf \"$temporary\" \"$AUTOLITH_INSTALL_ROOT/current\" "(:RECOVERY-IMAGE :VERSION 2)\n") (release-script-tests--write-file (merge-pathnames "autolith/release-images" data-home) - (format nil "~A~%" tag))) + (format nil "~A:x86_64-linux~%" tag))) (let ((path (format nil "~A:~A" (string-right-trim "/" (namestring fixture-bin)) (or (uiop:getenv "PATH") "")))) @@ -1137,6 +1212,201 @@ mv -Tf \"$temporary\" \"$AUTOLITH_INSTALL_ROOT/current\" "the verified updater atomically selects the new release")) nil) +(-> release-script-tests--musl-update-handoff (pathname pathname) null) +(defun release-script-tests--musl-update-handoff (source-root root) + "Exercise musl-preserving update from a platform-qualified installation." + (let* ((tag (format nil "v~A" *release-script-tests-version*)) + (next-tag "v0.12.0") + (platform "x86_64-linux-musl") + (fixture-root (merge-pathnames "musl-update-handoff/" root)) + (install-root (merge-pathnames "installation/" fixture-root)) + (release-root + (merge-pathnames (format nil "releases/~A-~A/" tag platform) + install-root)) + (packaged-source (merge-pathnames "libexec/autolith/" release-root)) + (inner-launcher (merge-pathnames "bin/autolith" packaged-source)) + (bundled-installer (merge-pathnames "script/install" packaged-source)) + (launcher (merge-pathnames "bin/autolith" release-root)) + (data-home (merge-pathnames "data/" fixture-root)) + (fixture-bin (merge-pathnames "fixture-bin/" fixture-root)) + (curl (merge-pathnames "curl" fixture-bin)) + (updated-launcher (merge-pathnames "updated-autolith" fixture-root)) + (log (merge-pathnames "handoff.log" fixture-root))) + (release-script-tests--make-release + source-root release-root :platform platform) + (uiop:ensure-all-directories-exist (list data-home fixture-bin)) + (release-script-tests--install-linux-host-tools fixture-bin :libc "musl") + (release-script-tests--write-file + curl (release-script-tests--fixture-curl)) + (release-script-tests--chmod "755" curl) + (sb-posix:symlink + (format nil "releases/~A-~A" tag platform) + (namestring (merge-pathnames "current" install-root))) + (release-script-tests--write-file + inner-launcher + "#!/bin/sh +exit 76 +") + (release-script-tests--write-file + updated-launcher + "#!/bin/sh +printf 'UPDATED_ARGS=%s\\n' \"$*\" >> \"$AUTOLITH_TEST_LOG\" +") + (release-script-tests--write-file + bundled-installer + "#!/bin/sh +set -eu +printf 'INSTALL_ARGS=%s\\n' \"$*\" >> \"$AUTOLITH_TEST_LOG\" +[ \"$1\" = --musl ] +[ \"$2\" = --without-command-link ] +[ \"$3\" = --version ] +requested=$4 +[ \"$requested\" = \"$AUTOLITH_TEST_LATEST_TAG\" ] +target=$AUTOLITH_INSTALL_ROOT/releases/${requested}-x86_64-linux-musl +mkdir -p \"$target/bin\" +cp \"$AUTOLITH_TEST_UPDATED_LAUNCHER\" \"$target/bin/autolith\" +chmod 755 \"$target/bin/autolith\" +temporary=$AUTOLITH_INSTALL_ROOT/.current.$$ +ln -s \"releases/${requested}-x86_64-linux-musl\" \"$temporary\" +mv -Tf \"$temporary\" \"$AUTOLITH_INSTALL_ROOT/current\" +") + (dolist (pathname (list inner-launcher bundled-installer updated-launcher)) + (release-script-tests--chmod "755" pathname)) + (let* ((active-root (merge-pathnames "autolith/active/" data-home)) + (recovery-root (merge-pathnames "autolith/recovery/" data-home))) + (dolist (pathname (list (merge-pathnames "autolith-active.core" active-root) + (merge-pathnames "autolith-recovery.core" + recovery-root))) + (release-script-tests--write-file pathname "core")) + (release-script-tests--write-file + (merge-pathnames "manifest.sexp" active-root) + "(:ACTIVE-IMAGE :VERSION 1)\n") + (release-script-tests--write-file + (merge-pathnames "manifest.sexp" recovery-root) + "(:RECOVERY-IMAGE :VERSION 2)\n") + (release-script-tests--write-file + (merge-pathnames "autolith/release-images" data-home) + (format nil "~A:~A~%" tag platform))) + (release-script-tests--run + (list (namestring launcher) "resume" "fixture-conversation") + :environment + (list + (format nil "PATH=~A:~A" + (string-right-trim "/" (namestring fixture-bin)) + (or (uiop:getenv "PATH") "")) + (format nil "XDG_DATA_HOME=~A" (namestring data-home)) + (format nil "AUTOLITH_TEST_LOG=~A" (namestring log)) + (format nil "AUTOLITH_TEST_UPDATED_LAUNCHER=~A" + (namestring updated-launcher)) + (format nil "AUTOLITH_TEST_LATEST_TAG=~A" next-tag) + "AUTOLITH_RELEASE_LATEST_URL=https://example.invalid/releases/latest") + :output nil) + (let ((events (uiop:read-file-string log))) + (test-assert + (search (format nil + "INSTALL_ARGS=--musl --without-command-link --version ~A" + next-tag) + events) + "a musl release preserves its variant during self-update") + (test-assert + (search "UPDATED_ARGS=resume fixture-conversation" events) + "the updated musl release receives the original arguments")) + (test-assert + (string= + (release-script-tests--readlink + (merge-pathnames "current" install-root)) + (format nil "releases/~A-~A" next-tag platform)) + "the musl updater selects the platform-qualified target")) + nil) + +(-> release-script-tests--image-marker-platform (pathname pathname) null) +(defun release-script-tests--image-marker-platform (source-root root) + "Exercise saved-image rebuilds when one tag switches Linux libc variants." + (let* ((tag (format nil "v~A" *release-script-tests-version*)) + (fixture-root (merge-pathnames "image-marker-platform/" root)) + (release-root (merge-pathnames "release/" fixture-root)) + (launcher (merge-pathnames "bin/autolith" release-root)) + (inner-launcher + (merge-pathnames "libexec/autolith/bin/autolith" release-root)) + (runtime (merge-pathnames "runtime/bin/sbcl" release-root)) + (fixture-bin (merge-pathnames "fixture-bin/" fixture-root)) + (data-home (merge-pathnames "data/" fixture-root)) + (log (merge-pathnames "sbcl.log" fixture-root)) + (marker (merge-pathnames "autolith/release-images" data-home))) + (release-script-tests--make-release + source-root release-root :platform "x86_64-linux") + (uiop:ensure-all-directories-exist (list fixture-bin data-home)) + (release-script-tests--install-linux-host-tools fixture-bin) + (release-script-tests--write-file + inner-launcher + "#!/bin/sh +exit 0 +") + (release-script-tests--write-file + runtime + "#!/bin/sh +set -eu +printf '%s\\n' \"$*\" >> \"$AUTOLITH_TEST_SBCL_LOG\" +target= +for argument in \"$@\"; do target=$argument; done +case \" $* \" in + *'build-recovery.lisp'*) + mkdir -p \"$(dirname \"$target\")\" + : > \"$target\" + printf '(:RECOVERY-IMAGE :VERSION 2)\\n' > \"$(dirname \"$target\")/manifest.sexp\" + ;; + *'build-active.lisp'*) + mkdir -p \"$(dirname \"$target\")\" + : > \"$target\" + printf '(:ACTIVE-IMAGE :VERSION 1)\\n' > \"$(dirname \"$target\")/manifest.sexp\" + ;; +esac +") + (dolist (pathname (list inner-launcher runtime)) + (release-script-tests--chmod "755" pathname)) + (labels ((environment () + (list + "AUTOLITH_NO_UPDATE_CHECK=1" + (format nil "PATH=~A:~A" + (string-right-trim "/" (namestring fixture-bin)) + (or (uiop:getenv "PATH") "")) + (format nil "XDG_DATA_HOME=~A" (namestring data-home)) + (format nil "AUTOLITH_TEST_SBCL_LOG=~A" (namestring log)))) + + (build-count () + (let ((content (uiop:read-file-string log))) + (loop with start = 0 + for position = (search "--script" content :start2 start) + while position + count t + do (setf start (+ position 8)))))) + (release-script-tests--run + (list (namestring launcher)) + :environment (environment) + :output nil) + (test-assert + (and (string= (string-trim '(#\Newline #\Return) + (uiop:read-file-string marker)) + (format nil "~A:x86_64-linux" tag)) + (= (build-count) 2)) + "the first glibc launch builds and marks both saved images") + (release-script-tests--record + (merge-pathnames "RELEASE" release-root) + tag + :platform "x86_64-linux-musl") + (release-script-tests--install-linux-host-tools fixture-bin :libc "musl") + (release-script-tests--run + (list (namestring launcher)) + :environment (environment) + :output nil) + (test-assert + (and (string= (string-trim '(#\Newline #\Return) + (uiop:read-file-string marker)) + (format nil "~A:x86_64-linux-musl" tag)) + (= (build-count) 4)) + "switching one tag from glibc to musl rebuilds both saved images"))) + nil) + (-> release-script-tests--runtime-adapter (pathname pathname) null) (defun release-script-tests--runtime-adapter (source-root root) "Exercise minimum-version runtime selection in the pinned-runtime adapter." @@ -1391,12 +1661,182 @@ esac (release-script-tests--write-file checksum output))) checksum) +(-> release-script-tests--write-release-archive + (pathname pathname &key (:tag string) (:platform string) + (:record-platform (option string))) + (values pathname pathname)) +(defun release-script-tests--write-release-archive + (release-root fixture-root &key tag platform record-platform) + "Package RELEASE-ROOT as TAG's PLATFORM archive below FIXTURE-ROOT." + (let* ((release-name (format nil "autolith-~A-~A" tag platform)) + (fixture-source (merge-pathnames "source/" fixture-root)) + (fixture-release + (merge-pathnames (format nil "~A/" release-name) fixture-source)) + (archive + (merge-pathnames (format nil "~A.tar.gz" release-name) fixture-root)) + (checksum + (merge-pathnames (format nil "~A.tar.gz.sha256" release-name) + fixture-root))) + (release-script-tests--cleanup fixture-release) + (dolist (pathname (list archive checksum)) + (when (probe-file pathname) + (delete-file pathname))) + (uiop:ensure-all-directories-exist (list fixture-root fixture-source)) + (release-script-tests--run + (list "cp" "-a" (format nil "~A." (namestring release-root)) + (namestring fixture-release)) + :output nil) + (release-script-tests--run + (list "chmod" "-R" "u+w" (namestring fixture-release)) + :output nil) + (release-script-tests--record + (merge-pathnames "RELEASE" fixture-release) + tag + :platform record-platform) + (release-script-tests--chmod "a-w" fixture-release) + (release-script-tests--run + (list "tar" "-czf" (namestring archive) + "-C" (namestring fixture-source) release-name) + :output nil) + (release-script-tests--write-checksum archive checksum) + (values archive checksum))) + +(-> release-script-tests--installer-platform-identity (pathname pathname) null) +(defun release-script-tests--installer-platform-identity (source-root root) + "Exercise installer platform validation and same-tag Linux variant isolation." + (let* ((tag (format nil "v~A" *release-script-tests-version*)) + (release-root (merge-pathnames "identity-release/" root)) + (fixture-root (merge-pathnames "identity-fixture/" root)) + (fixture-bin (merge-pathnames "identity-bin/" root)) + (install-root (merge-pathnames "identity-installation/" root)) + (bin-directory (merge-pathnames "identity-command-bin/" root)) + (curl (merge-pathnames "curl" fixture-bin)) + (installer (merge-pathnames "script/install" source-root))) + (release-script-tests--make-release source-root release-root) + (uiop:ensure-all-directories-exist + (list fixture-root fixture-bin install-root bin-directory)) + (release-script-tests--write-file + curl (release-script-tests--fixture-curl)) + (release-script-tests--chmod "755" curl) + (labels ((environment () + (list + (format nil "PATH=~A:~A" + (string-right-trim "/" (namestring fixture-bin)) + (or (uiop:getenv "PATH") "")) + (format nil "AUTOLITH_TEST_RELEASE_FIXTURE=~A" + (namestring fixture-root)) + "AUTOLITH_RELEASE_BASE_URL=https://example.invalid" + (format nil "AUTOLITH_INSTALL_ROOT=~A" + (string-right-trim "/" (namestring install-root))) + (format nil "AUTOLITH_BIN_DIR=~A" + (string-right-trim "/" (namestring bin-directory))))) + + (install-fails (arguments diagnostic description) + (multiple-value-bind (output error-output status) + (release-script-tests--run + (append (list (namestring installer)) arguments) + :environment (environment) + :ignore-error-status t + :output nil) + (declare (ignore output)) + (test-assert + (and (not (eql status 0)) + (search diagnostic error-output)) + description)))) + (release-script-tests--write-release-archive + release-root fixture-root + :tag tag + :platform "aarch64-linux") + (release-script-tests--install-linux-host-tools + fixture-bin :architecture "aarch64") + (install-fails + (list "--version" tag) + "unexpected platform identity or layout" + "Linux aarch64 rejects a legacy archive without platform identity") + (release-script-tests--write-release-archive + release-root fixture-root + :tag tag + :platform "x86_64-linux" + :record-platform "aarch64-linux") + (release-script-tests--install-linux-host-tools fixture-bin) + (install-fails + (list "--version" tag) + "unexpected platform identity or layout" + "the installer rejects extracted platform metadata that mismatches the archive") + (release-script-tests--write-release-archive + release-root fixture-root + :tag tag + :platform "x86_64-linux" + :record-platform "x86_64-linux") + (release-script-tests--run + (list (namestring installer) "--version" tag) + :environment (environment) + :output nil) + (release-script-tests--write-release-archive + release-root fixture-root + :tag tag + :platform "x86_64-linux-musl") + (release-script-tests--install-linux-host-tools fixture-bin :libc "musl") + (install-fails + (list "--musl" "--version" tag) + "unexpected platform identity or layout" + "musl rejects a legacy archive without platform identity") + (release-script-tests--write-release-archive + release-root fixture-root + :tag tag + :platform "x86_64-linux-musl" + :record-platform "x86_64-linux-musl") + (release-script-tests--install-linux-host-tools fixture-bin) + (install-fails + (list "--musl" "--version" tag) + "requested libc musl does not match detected libc glibc" + "the installer rejects musl selection on a glibc host") + (release-script-tests--install-linux-host-tools fixture-bin :libc "unknown") + (install-fails + (list "--version" tag) + "Linux libc could not be identified as glibc or musl" + "the installer fails closed on unknown Linux libc output") + (release-script-tests--install-linux-host-tools fixture-bin :libc "musl") + (release-script-tests--run + (list (namestring installer) "--musl" "--version" tag) + :environment (environment) + :output nil) + (test-assert + (and + (probe-file + (merge-pathnames + (format nil "releases/~A-x86_64-linux/bin/autolith" tag) + install-root)) + (probe-file + (merge-pathnames + (format nil "releases/~A-x86_64-linux-musl/bin/autolith" tag) + install-root)) + (string= + (release-script-tests--readlink + (merge-pathnames "current" install-root)) + (format nil "releases/~A-x86_64-linux-musl" tag))) + "glibc and musl installations of one tag remain distinct") + (release-script-tests--install-linux-host-tools fixture-bin) + (release-script-tests--run + (list (namestring installer) "--version" tag) + :environment (environment) + :output nil) + (test-assert + (string= + (release-script-tests--readlink + (merge-pathnames "current" install-root)) + (format nil "releases/~A-x86_64-linux" tag)) + "the installer can reselect the glibc variant without collision"))) + nil) + (-> release-script-tests--platform-ids () null) (defun release-script-tests--platform-ids () "Exercise canonical release platform identifiers." (dolist (case '(("Linux" "x86-64" "x86_64-linux") ("Linux" "x86_64" "x86_64-linux") ("Linux" "amd64" "x86_64-linux") + ("Linux" "aarch64" "aarch64-linux") + ("Linux" "arm64" "aarch64-linux") ("Darwin" "arm64" "arm64-darwin") ("Darwin" "aarch64" "arm64-darwin") ("FreeBSD" "amd64" "x86_64-freebsd") @@ -1409,6 +1849,33 @@ esac (test-assert (string= (release-archive--platform-id os architecture) expected) (format nil "~A/~A maps to ~A" os architecture expected)))) + (test-assert + (string= (release-archive--platform-id "Linux" "x86_64" "musl") + "x86_64-linux-musl") + "Linux/x86_64 with musl maps to x86_64-linux-musl") + (test-assert + (string= (release-archive--platform-id "Linux" "aarch64" "musl") + "aarch64-linux-musl") + "Linux/aarch64 with musl maps to aarch64-linux-musl") + (test-assert + (string= (release-archive--linux-libc-output->identity + "/lib/ld-musl-x86_64.so.1") + "musl") + "musl ldd output establishes musl identity") + (test-assert + (string= (release-archive--linux-libc-output->identity + "libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6") + "glibc") + "glibc ldd output establishes glibc identity") + (test-assert + (handler-case + (progn + (release-archive--linux-libc-output->identity "unknown") + nil) + (release-archive-error (condition) + (search "Could not identify the Linux C library" + (release-archive-error-cause condition)))) + "unrecognized ldd output is rejected") (dolist (case '(("Linux" "i686") ("Darwin" "x86_64") ("SunOS" "amd64") @@ -1425,47 +1892,87 @@ esac (search "Binary releases currently support" (release-archive-error-cause condition))))) (format nil "~A/~A is not a release target" os architecture)))) + (let ((old-platform (uiop:getenv "AUTOLITH_RELEASE_PLATFORM"))) + (unwind-protect + (progn + (sb-posix:setenv "AUTOLITH_RELEASE_PLATFORM" "sparc-sunos" 1) + (test-assert + (handler-case + (progn + (release-archive--platform) + nil) + (release-archive-error (condition) + (search "AUTOLITH_RELEASE_PLATFORM names" + (release-archive-error-cause condition)))) + "release platform overrides must match the native host")) + (if old-platform + (sb-posix:setenv "AUTOLITH_RELEASE_PLATFORM" old-platform 1) + (sb-posix:unsetenv "AUTOLITH_RELEASE_PLATFORM")))) + (when (string-equal (software-type) "Linux") + (let* ((old-libc (uiop:getenv "AUTOLITH_LIBC")) + (detected (release-archive--linux-libc)) + (mismatch (if (string= detected "musl") "glibc" "musl"))) + (unwind-protect + (progn + (sb-posix:setenv "AUTOLITH_LIBC" mismatch 1) + (test-assert + (handler-case + (progn + (release-archive--linux-libc) + nil) + (release-archive-error (condition) + (search "but this host uses" + (release-archive-error-cause condition)))) + "release libc overrides must match the native host")) + (if old-libc + (sb-posix:setenv "AUTOLITH_LIBC" old-libc 1) + (sb-posix:unsetenv "AUTOLITH_LIBC"))))) nil) (-> release-script-tests--launcher-bsd (pathname pathname) null) (defun release-script-tests--launcher-bsd (source-root root) "Exercise BSD packaged launcher validation without Bubblewrap." - (dolist (os '("FreeBSD" "NetBSD" "OpenBSD")) - (let* ((release-root - (merge-pathnames (format nil "bsd-launcher-~A/" os) root)) - (launcher (merge-pathnames "bin/autolith" release-root)) - (host-bin (merge-pathnames (format nil "bsd-host-~A/" os) root)) - (path (format nil "~A:~A" - (string-right-trim "/" (namestring host-bin)) - (or (uiop:getenv "PATH") ""))) - (environment - (list "AUTOLITH_NO_UPDATE_CHECK=1" - (format nil "PATH=~A" path)))) - (release-script-tests--write-uname host-bin os "amd64") - (release-script-tests--make-release source-root release-root) - (let ((output + (dolist (spec '(("FreeBSD" "x86_64-freebsd") + ("NetBSD" "x86_64-netbsd") + ("OpenBSD" "x86_64-openbsd"))) + (destructuring-bind (os platform) spec + (let* ((release-root + (merge-pathnames (format nil "bsd-launcher-~A/" os) root)) + (launcher (merge-pathnames "bin/autolith" release-root)) + (host-bin (merge-pathnames (format nil "bsd-host-~A/" os) root)) + (path (format nil "~A:~A" + (string-right-trim "/" (namestring host-bin)) + (or (uiop:getenv "PATH") ""))) + (environment + (list "AUTOLITH_NO_UPDATE_CHECK=1" + (format nil "PATH=~A" path)))) + (release-script-tests--write-uname host-bin os "amd64") + (release-script-tests--make-release + source-root release-root :platform platform) + (let ((output + (release-script-tests--run + (list (namestring launcher) "--autolith-release-probe") + :environment environment))) + (test-assert + (and (search (format nil "version=~A" *release-script-tests-version*) + output) + (search (format nil "platform=~A" platform) output)) + (format nil "the ~A release launcher reports its platform" os))) + (let ((library + (merge-pathnames "lib/libcolorlisp-tree-sitter.so" release-root))) + (delete-file library) + (multiple-value-bind (output error-output status) (release-script-tests--run (list (namestring launcher) "--autolith-release-probe") - :environment environment))) - (test-assert - (search (format nil "version=~A" *release-script-tests-version*) - output) - (format nil "the ~A release launcher probes a packaged release" os))) - (let ((library - (merge-pathnames "lib/libcolorlisp-tree-sitter.so" release-root))) - (delete-file library) - (multiple-value-bind (output error-output status) - (release-script-tests--run - (list (namestring launcher) "--autolith-release-probe") - :environment environment - :ignore-error-status t - :output nil) - (declare (ignore output error-output)) - (test-assert (not (eql status 0)) - (format nil - "the ~A release launcher requires its private syntax library" - os))) - (release-script-tests--write-file library "")))) + :environment environment + :ignore-error-status t + :output nil) + (declare (ignore output error-output)) + (test-assert (not (eql status 0)) + (format nil + "the ~A release launcher requires its private syntax library" + os))) + (release-script-tests--write-file library ""))))) nil) (-> release-script-tests--installer-bsd (pathname pathname) null) @@ -1535,13 +2042,14 @@ esac :output nil) (test-assert (probe-file - (merge-pathnames (format nil "releases/~A/bin/autolith" tag) - install-root)) + (merge-pathnames + (format nil "releases/~A-~A/bin/autolith" tag platform) + install-root)) (format nil "the ~A installer publishes the requested release" os)) (test-assert (string= (release-script-tests--readlink (merge-pathnames "current" install-root)) - (format nil "releases/~A" tag)) + (format nil "releases/~A-~A" tag platform)) (format nil "the ~A installer selects the requested version" os)))))) nil) @@ -1590,7 +2098,7 @@ esac (-> release-script-tests--runtime-bootstrap (pathname pathname) null) (defun release-script-tests--runtime-bootstrap (source-root root) - "Exercise BSD host-SBCL bootstrap selection and unsupported rejection." + "Exercise pinned and host runtime bootstrap selection and validation." (let* ((fixture (merge-pathnames "runtime-bootstrap/" root)) (bin (merge-pathnames "bin/" fixture)) (installation (merge-pathnames "installation/" fixture)) @@ -1598,50 +2106,117 @@ esac (curl-log (merge-pathnames "curl.log" fixture)) (sbcl (merge-pathnames "sbcl" bin)) (curl (merge-pathnames "curl" bin)) + (ldd (merge-pathnames "ldd" bin)) (script (merge-pathnames "script/build-release-runtime" source-root)) (path (format nil "~A:/bin:/usr/bin" (string-right-trim "/" (namestring bin))))) (uiop:ensure-all-directories-exist (list bin installation)) (release-script-tests--write-uname bin "FreeBSD" "amd64") - (release-script-tests--write-file - sbcl - (format nil "#!/bin/sh~%printf '%s\\n' \"$*\" > \"${AUTOLITH_TEST_BOOTSTRAP_LOG:?}\"~%exit 0~%")) - (release-script-tests--write-file - curl - (format nil "#!/bin/sh~%printf 'curl invoked\\n' > \"${AUTOLITH_TEST_CURL_LOG:?}\"~%exit 1~%")) - (dolist (pathname (list sbcl curl)) + (release-script-tests--write-file + sbcl + (format nil "#!/bin/sh~%case \" $* \" in~% *lisp-implementation-version*) printf '%s' \"${AUTOLITH_TEST_SBCL_VERSION:-2.6.6}\"; exit 0 ;;~%esac~%printf '%s\\n' \"$*\" > \"${AUTOLITH_TEST_BOOTSTRAP_LOG:?}\"~%exit 0~%")) + (release-script-tests--write-file + curl + (format nil "#!/bin/sh~%printf 'curl invoked\\n' > \"${AUTOLITH_TEST_CURL_LOG:?}\"~%exit 1~%")) + (release-script-tests--write-file + ldd + "#!/bin/sh +case ${AUTOLITH_TEST_LIBC:-glibc} in + musl) printf 'musl libc\\n' ;; + glibc) printf 'libc.so.6\\n' ;; + *) printf 'unknown libc\\n'; exit 1 ;; +esac +") + (dolist (pathname (list sbcl curl ldd)) (release-script-tests--chmod "755" pathname)) - (multiple-value-bind (output error-output status) - (release-script-tests--run - (list (namestring script) (namestring installation)) - :environment - (list (format nil "PATH=~A" path) - (format nil "AUTOLITH_SBCL=~A" (namestring sbcl)) - (format nil "AUTOLITH_TEST_BOOTSTRAP_LOG=~A" (namestring log)) - (format nil "AUTOLITH_TEST_CURL_LOG=~A" (namestring curl-log))) - :ignore-error-status t) - (declare (ignore error-output)) - (test-assert (zerop status) - "BSD runtime bootstrap uses the host SBCL") - (test-assert (search "Using host SBCL as the FreeBSD bootstrap compiler." - output) - "BSD runtime bootstrap reports the host compiler") - (test-assert (not (probe-file curl-log)) - "BSD runtime bootstrap does not download an official binary") - (test-assert - (and (probe-file log) - (search "build-release-runtime.lisp" (uiop:read-file-string log))) - "BSD runtime bootstrap invokes the runtime builder")) + (labels ((environment (&rest extra) + (append + (list (format nil "PATH=~A" path) + (format nil "AUTOLITH_SBCL=~A" (namestring sbcl)) + (format nil "AUTOLITH_TEST_BOOTSTRAP_LOG=~A" + (namestring log)) + (format nil "AUTOLITH_TEST_CURL_LOG=~A" + (namestring curl-log))) + extra)) + + (run-bootstrap (&rest extra) + (release-script-tests--run + (list (namestring script) (namestring installation)) + :environment (apply #'environment extra) + :ignore-error-status t))) + (multiple-value-bind (output error-output status) + (run-bootstrap) + (declare (ignore error-output)) + (test-assert (zerop status) + "BSD runtime bootstrap uses the host SBCL") + (test-assert + (search "Using host SBCL 2.6.6 as the FreeBSD bootstrap compiler." + output) + "BSD runtime bootstrap reports the validated host compiler") + (test-assert (not (probe-file curl-log)) + "BSD runtime bootstrap does not download an official binary") + (test-assert + (and (probe-file log) + (search "build-release-runtime.lisp" (uiop:read-file-string log))) + "BSD runtime bootstrap invokes the runtime builder")) + (when (probe-file log) + (delete-file log)) + (multiple-value-bind (output error-output status) + (run-bootstrap "AUTOLITH_TEST_SBCL_VERSION=1.9.9") + (let ((diagnostic (concatenate 'string (or output "") + (or error-output "")))) + (test-assert + (and (not (zerop status)) + (search "needs host SBCL 2.0.0 or newer" diagnostic) + (not (probe-file log))) + "host runtime bootstrap rejects an unsupported compiler"))) + (release-script-tests--write-uname bin "Linux" "x86_64") + (multiple-value-bind (output error-output status) + (run-bootstrap "AUTOLITH_HOST_BOOTSTRAP=1") + (let ((diagnostic (concatenate 'string (or output "") + (or error-output "")))) + (test-assert + (and (not (zerop status)) + (search "Installing the pinned SBCL 2.4.0 bootstrap compiler." + diagnostic) + (probe-file curl-log) + (not (probe-file log))) + "Linux x86-64 ignores an external host-bootstrap bypass"))) + (multiple-value-bind (output error-output status) + (run-bootstrap "AUTOLITH_LIBC=musl") + (let ((diagnostic (concatenate 'string (or output "") + (or error-output "")))) + (test-assert + (and (not (zerop status)) + (search "expected musl but detected glibc" diagnostic)) + "runtime bootstrap rejects a mismatched libc override"))) + (multiple-value-bind (output error-output status) + (run-bootstrap "AUTOLITH_TEST_LIBC=unknown") + (let ((diagnostic (concatenate 'string (or output "") + (or error-output "")))) + (test-assert + (and (not (zerop status)) + (search "could not identify the Linux C library" diagnostic)) + "runtime bootstrap rejects unrecognized libc output"))) + (when (probe-file curl-log) + (delete-file curl-log)) + (multiple-value-bind (output error-output status) + (run-bootstrap "AUTOLITH_LIBC=musl" "AUTOLITH_TEST_LIBC=musl") + (declare (ignore error-output)) + (test-assert + (and (zerop status) + (search "Using host SBCL 2.6.6 as the Linux bootstrap compiler." + output) + (not (probe-file curl-log))) + "Linux musl runtime bootstrap uses a validated host compiler")) (release-script-tests--write-uname bin "SunOS" "amd64") (multiple-value-bind (output error-output status) - (release-script-tests--run - (list (namestring script) (namestring installation)) - :environment (list (format nil "PATH=~A" path)) - :ignore-error-status t) - (let ((diagnostic (concatenate 'string (or output "") (or error-output "")))) + (run-bootstrap) + (let ((diagnostic (concatenate 'string (or output "") + (or error-output "")))) (test-assert (and (not (zerop status)) - (search "currently supports Linux x86-64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64" + (search "currently supports Linux x86-64, Linux aarch64, macOS arm64, FreeBSD x86-64, NetBSD x86-64, and OpenBSD x86-64" diagnostic)) "runtime bootstrap rejects unsupported platforms"))) (release-script-tests--write-uname bin "OpenBSD" "amd64") @@ -1652,11 +2227,12 @@ esac (list (format nil "PATH=~A" path) "AUTOLITH_SBCL=/no/such/sbcl") :ignore-error-status t) - (let ((diagnostic (concatenate 'string (or output "") (or error-output "")))) + (let ((diagnostic (concatenate 'string (or output "") + (or error-output "")))) (test-assert (and (not (zerop status)) (search "needs a host SBCL on OpenBSD" diagnostic)) - "BSD runtime bootstrap requires a host SBCL"))) + "BSD runtime bootstrap requires a host SBCL")))) nil)) (-> release-script-tests--archive-helpers (pathname) null) @@ -1667,6 +2243,17 @@ esac (dolist (os '("Darwin" "FreeBSD" "NetBSD" "OpenBSD")) (test-assert (release-archive--gnu-tar-required-p os) (format nil "~A requires GNU tar for reproducible archives" os))) + (let ((record (merge-pathnames "release-record" root))) + (release-archive--write-record + record + :version "0.11.0" + :tag "v0.11.0" + :commit *release-script-tests-commit* + :platform "aarch64-linux-musl") + (test-assert + (search "platform=aarch64-linux-musl" + (uiop:read-file-string record)) + "release records preserve the exact archive platform")) (let ((missing (merge-pathnames "no-sandbox/" root)) (present (merge-pathnames "has-sandbox/" root))) (test-assert (null (release-archive--sandbox-helper missing)) @@ -1764,19 +2351,232 @@ esac (setf (uiop:getenv "PATH") saved))) nil) +(-> release-script-tests--linux-release-validator (pathname pathname) null) +(defun release-script-tests--linux-release-validator (source-root root) + "Exercise fail-closed Linux release artifact validation boundaries." + (let* ((validator + (merge-pathnames "script/validate-linux-release-artifact" source-root)) + (fixture-root + (uiop:ensure-directory-pathname + (merge-pathnames "linux-release-validator/" root))) + (release-name "autolith-v0.11.0-x86_64-linux") + (release-root + (merge-pathnames (format nil "~A/" release-name) fixture-root)) + (release-record (merge-pathnames "RELEASE" release-root)) + (runtime (merge-pathnames "runtime/bin/sbcl" release-root)) + (fixture-bin (merge-pathnames "fixture-bin/" fixture-root)) + (file-command (merge-pathnames "file" fixture-bin)) + (readelf-command (merge-pathnames "readelf" fixture-bin)) + (environment + (list + (format nil "PATH=~A:~A" + (string-right-trim "/" (namestring fixture-bin)) + (or (uiop:getenv "PATH") ""))))) + (release-script-tests--write-file + runtime + "#!/bin/sh +printf '%s' \"${AUTOLITH_TEST_RUNTIME_VERSION:-2.6.6}\" +") + (release-script-tests--write-file + file-command + "#!/bin/sh +printf '%s\\n' \"${AUTOLITH_TEST_FILE_DESCRIPTION:-ELF 64-bit LSB pie executable, x86-64}\" +") + (release-script-tests--write-file + readelf-command + "#!/bin/sh +case $1 in + -h) printf ' Machine: %s\\n' \"${AUTOLITH_TEST_MACHINE:-Advanced Micro Devices X86-64}\" ;; + -l) printf ' [Requesting program interpreter: %s]\\n' \"${AUTOLITH_TEST_INTERPRETER:-/lib64/ld-linux-x86-64.so.2}\" ;; +esac +") + (dolist (pathname (list runtime file-command readelf-command)) + (release-script-tests--chmod "755" pathname)) + (labels ((archive-path (case-name) + (merge-pathnames + (format nil "~A/~A.tar.gz" case-name release-name) + fixture-root)) + + (make-archive (case-name &key extra-top-level) + (let ((archive (archive-path case-name))) + (ensure-directories-exist archive) + (release-script-tests--run + (append (list "tar" "-czf" (namestring archive) + "-C" (namestring fixture-root) release-name) + (when extra-top-level + (list extra-top-level))) + :output nil) + archive)) + + (validate (archive &key extra-environment) + (release-script-tests--run + (list (namestring validator) (namestring archive) + "x86_64-linux" "x86_64" + "/lib64/ld-linux-x86-64.so.2") + :environment (append extra-environment environment) + :ignore-error-status t)) + + (assert-failure (archive diagnostic description + &key extra-environment) + (multiple-value-bind (output error-output status) + (validate archive :extra-environment extra-environment) + (declare (ignore output)) + (test-assert (not (zerop status)) description) + (test-assert (search diagnostic error-output) + (format nil "~A reports ~A" + description diagnostic))))) + (let ((missing (archive-path "missing")) + (unsupported + (release-script-tests--write-file + (archive-path "unsupported") ""))) + (assert-failure missing "does not exist" + "Linux artifact validation rejects a missing archive") + (multiple-value-bind (output error-output status) + (release-script-tests--run + (list (namestring validator) (namestring unsupported) + "sparc64-linux" "sparc64" "/lib/ld-linux.so.2") + :ignore-error-status t) + (declare (ignore output)) + (test-assert (not (zerop status)) + "Linux artifact validation rejects an unsupported architecture") + (test-assert (search "unsupported expected architecture sparc64" + error-output) + "unsupported Linux validation identifies the architecture"))) + (release-script-tests--record + release-record "v0.11.0" :platform "x86_64-linux") + (let ((valid (make-archive "valid"))) + (multiple-value-bind (output error-output status) + (validate valid) + (declare (ignore error-output)) + (test-assert (zerop status) + "Linux artifact validation accepts a matching archive") + (test-assert (search "Validated" output) + "successful Linux artifact validation reports its result")) + (assert-failure valid "ELF machine" + "Linux artifact validation rejects a mismatched ELF machine" + :extra-environment '("AUTOLITH_TEST_MACHINE=SPARC V9")) + (assert-failure valid "interpreter" + "Linux artifact validation rejects a mismatched interpreter" + :extra-environment '("AUTOLITH_TEST_INTERPRETER=/wrong/loader")) + (assert-failure valid "smoke test" + "Linux artifact validation rejects a malformed runtime version" + :extra-environment '("AUTOLITH_TEST_RUNTIME_VERSION=2.6"))) + (release-script-tests--write-file + (merge-pathnames "rogue" fixture-root) "outside release root") + (assert-failure + (make-archive "extra-top-level" :extra-top-level "rogue") + "is outside" "Linux artifact validation rejects extra top-level members") + (release-script-tests--write-file + release-record + "platform=x86_64-linux +platform=aarch64-linux +") + (assert-failure + (make-archive "duplicate-platform") "exactly one platform field" + "Linux artifact validation rejects duplicate platform fields") + (release-script-tests--write-file release-record "platform=aarch64-linux +") + (assert-failure + (make-archive "mismatched-platform") "does not identify x86_64-linux" + "Linux artifact validation rejects mismatched platform metadata"))) + nil) + (-> release-script-tests--github-release-workflow (pathname) null) (defun release-script-tests--github-release-workflow (source-root) "Exercise the GitHub release packaging workflow." - (let ((workflow - (uiop:read-file-string - (merge-pathnames ".github/workflows/release.yml" source-root)))) + (let* ((workflow + (uiop:read-file-string + (merge-pathnames ".github/workflows/release.yml" source-root))) + (workflow-lower (string-downcase workflow)) + (validator + (uiop:read-file-string + (merge-pathnames "script/validate-linux-release-artifact" + source-root)))) (dolist (job '("package-linux-x86_64" + "package-linux-aarch64" + "package-linux-x86_64-musl" + "package-linux-aarch64-musl" "package-macos-arm64" "package-freebsd-x86_64" "package-netbsd-x86_64" "package-openbsd-x86_64")) (test-assert (search job workflow) (format nil "the release workflow packages ~A" job))) + (labels ((job-section (job next-job) + (subseq workflow + (search (format nil " ~A:" job) workflow) + (or (and next-job + (search (format nil " ~A:" next-job) workflow)) + (length workflow))))) + (let ((linux-x86_64 + (job-section "package-linux-x86_64" "package-linux-aarch64")) + (linux-aarch64 + (job-section "package-linux-aarch64" "package-linux-x86_64-musl")) + (linux-x86_64-musl + (job-section "package-linux-x86_64-musl" + "package-linux-aarch64-musl")) + (linux-aarch64-musl + (job-section "package-linux-aarch64-musl" "package-macos-arm64")) + (openbsd + (job-section "package-openbsd-x86_64" nil)) + (script + (uiop:read-file-string + (merge-pathnames "script/ci-package-release" source-root)))) + (dolist (linux (list linux-x86_64 linux-aarch64 + linux-x86_64-musl linux-aarch64-musl)) + (test-assert (search "script/validate-linux-release-artifact" linux) + "each Linux package is validated before upload") + (test-assert (search "binutils" linux) + "each Linux packaging environment installs binutils") + (test-assert (search " file " linux) + "each Linux packaging environment installs file")) + (dolist (validation + `((,linux-x86_64 + "x86_64-linux x86_64 /lib64/ld-linux-x86-64.so.2") + (,linux-aarch64 + "aarch64-linux aarch64 /lib/ld-linux-aarch64.so.1") + (,linux-x86_64-musl + "x86_64-linux-musl x86_64 /lib/ld-musl-x86_64.so.1") + (,linux-aarch64-musl + "aarch64-linux-musl aarch64 /lib/ld-musl-aarch64.so.1"))) + (test-assert (search (second validation) (first validation)) + (format nil "Linux validation checks ~A" + (second validation)))) + (test-assert + (search "image: ubuntu:20.04@sha256:c664f8f86ed5a386b0a340d981b8f81714e21a8b9c73f658c4bea56aa179d54a" + linux-x86_64) + "x86-64 glibc packaging pins the Ubuntu 20.04 platform manifest") + (test-assert + (search "image: ubuntu:20.04@sha256:722ea796ac2d57eeb3627c58a582fc1acc58be51faf815e1bce1682ae5c092f7" + linux-aarch64) + "aarch64 glibc packaging pins the Ubuntu 20.04 platform manifest") + (test-assert (not (search "AUTOLITH_HOST_BOOTSTRAP" linux-x86_64)) + "x86-64 glibc packaging uses the pinned bootstrap") + (dolist (linux (list linux-aarch64 + linux-x86_64-musl linux-aarch64-musl)) + (test-assert (search "AUTOLITH_HOST_BOOTSTRAP" linux) + "non-x86-64-glibc Linux packaging validates host bootstrap")) + (test-assert + (search "alpine:3.22.5@sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6" + linux-x86_64-musl) + "x86-64 musl packaging pins the Alpine platform manifest") + (test-assert + (search "alpine:3.22.5@sha256:2c9d26f410d032d5b1525aa8a873e238b05b90c4ae8618743d4311f0cc827e37" + linux-aarch64-musl) + "aarch64 musl packaging pins the Alpine platform manifest") + (test-assert (search "timeout-minutes: 75" linux-x86_64) + "Linux x86-64 packaging has a 75-minute deadline") + (test-assert (search "gtar--" openbsd) + "OpenBSD packaging installs the default gtar flavor") + (test-assert (not (search " gtar " openbsd)) + "OpenBSD packaging does not pass the ambiguous gtar stem") + (test-assert (search "gtar--" script) + "the packaging script installs the default OpenBSD gtar flavor") + (test-assert (search "gtar-1.35p1" script) + "the packaging script falls back to a pinned OpenBSD gtar") + (test-assert (search "GNU tar (gtar) is required for reproducible release archives." + script) + "the packaging script fails closed when GNU tar is absent"))) (test-assert (search "script/ci-package-release" workflow) "the release workflow uses the shared packaging script") (test-assert (search "vmactions/freebsd-vm@v1" workflow) @@ -1785,32 +2585,28 @@ esac "the release workflow uses the NetBSD VM action") (test-assert (search "vmactions/openbsd-vm@v1" workflow) "the release workflow uses the OpenBSD VM action") - (test-assert (not (search "Wait for the release service" workflow)) - "the release workflow no longer waits for the host builder") - (let ((linux - (subseq workflow - (search "package-linux-x86_64:" workflow) - (search "package-macos-arm64:" workflow))) - (openbsd - (subseq workflow - (search "package-openbsd-x86_64:" workflow))) - (script - (uiop:read-file-string - (merge-pathnames "script/ci-package-release" source-root)))) - (test-assert (search "timeout-minutes: 75" linux) - "Linux packaging has a 75-minute deadline") - (test-assert (search "gtar--" openbsd) - "OpenBSD packaging installs the default gtar flavor") - (test-assert (not (search " gtar " openbsd)) - "OpenBSD packaging does not pass the ambiguous gtar stem") - (test-assert (search "gtar--" script) - "the packaging script installs the default OpenBSD gtar flavor") - (test-assert (search "gtar-1.35p1" script) - "the packaging script falls back to a pinned OpenBSD gtar") - (test-assert (search "GNU tar (gtar) is required for reproducible release archives." - script) - "the packaging script fails closed when GNU tar is absent"))) - nil) + (test-assert (not (search "Wait for the release service" workflow)) + "the release workflow no longer waits for the host builder") + (test-assert (not (search "alpine:latest" workflow-lower)) + "musl packaging never uses an unpinned Alpine latest image") + (test-assert (not (search "AUTOLITH_RELEASE_PLATFORM" workflow)) + "the release workflow relies on detected platform identity") + (test-assert (not (search "static" workflow-lower)) + "the release workflow does not claim static Linux artifacts") + (test-assert (not (search "standalone" workflow-lower)) + "the release workflow does not claim standalone Linux artifacts") + (dolist (needle '("archive_members=$(tar -tzf" + "exactly one top-level directory named" + "exactly one platform field" + "platform_count=$(grep -c '^platform='" + "file --brief" + "readelf -h" + "readelf -l" + "expected_interpreter" + "--eval '(write-string (lisp-implementation-version))'")) + (test-assert (search needle validator) + (format nil "the Linux artifact validator contains ~A" needle)))) + nil) (-> test-release-scripts () null) (defun test-release-scripts () @@ -1823,22 +2619,26 @@ esac (uiop:temporary-directory))))) (unwind-protect (progn - (release-script-tests--syntax source-root) + (release-script-tests--syntax source-root) (release-script-tests--bootstrap-dependency-order source-root root) - (release-script-tests--github-release-workflow source-root) - (release-script-tests--runtime-adapter source-root root) - (release-script-tests--runtime-bootstrap source-root root) - (release-script-tests--source-launcher source-root root) - (release-script-tests--platform-ids) - (release-script-tests--archive-helpers root) - (release-script-tests--portable-copy root) - (release-script-tests--checksum-format root) - (release-script-tests--launcher source-root root) - (release-script-tests--launcher-darwin source-root root) - (release-script-tests--launcher-bsd source-root root) - (release-script-tests--update-handoff source-root root) - (release-script-tests--installer source-root root) - (release-script-tests--installer-darwin source-root root) - (release-script-tests--installer-bsd source-root root)) + (release-script-tests--github-release-workflow source-root) + (release-script-tests--linux-release-validator source-root root) + (release-script-tests--runtime-adapter source-root root) + (release-script-tests--runtime-bootstrap source-root root) + (release-script-tests--source-launcher source-root root) + (release-script-tests--platform-ids) + (release-script-tests--archive-helpers root) + (release-script-tests--portable-copy root) + (release-script-tests--checksum-format root) + (release-script-tests--launcher source-root root) + (release-script-tests--launcher-darwin source-root root) + (release-script-tests--launcher-bsd source-root root) + (release-script-tests--update-handoff source-root root) + (release-script-tests--musl-update-handoff source-root root) + (release-script-tests--image-marker-platform source-root root) + (release-script-tests--installer source-root root) + (release-script-tests--installer-platform-identity source-root root) + (release-script-tests--installer-darwin source-root root) + (release-script-tests--installer-bsd source-root root)) (release-script-tests--cleanup root))) nil) diff --git a/tests/release-server-tests.lisp b/tests/release-server-tests.lisp index a9652801..4ce33c35 100644 --- a/tests/release-server-tests.lisp +++ b/tests/release-server-tests.lisp @@ -314,43 +314,45 @@ (string= (release-server-response-content-type response) "text/plain; charset=utf-8") "checksum responses use the plain-text media type")) - (let* ((tag "v0.10.1") - (directory - (release-server--release-directory configuration tag)) - (archive (release-server--archive-name tag "arm64-darwin"))) - (release-server-tests--write-file - (merge-pathnames archive directory) - "darwin-archive") - (release-server-tests--write-file - (merge-pathnames (format nil "~A.sha256" archive) directory) - "darwin-checksum") - (let ((response - (release-server-route - configuration "GET" - (format nil "/releases/~A/~A" tag archive)))) - (test-assert (= (release-server-response-status response) 200) - "published non-Linux archives are served") - (test-assert - (string= (release-server-response-content-type response) - "application/gzip") - "non-Linux archives use the gzip media type")) - (test-assert - (= (release-server-response-status - (release-server-route - configuration "GET" - (format nil "/releases/~A/~A" - tag - (release-server--archive-name tag "x86_64-freebsd")))) - 404) - "missing platform archives stay unpublished") - (test-assert - (= (release-server-response-status - (release-server-route - configuration "GET" - (format nil "/releases/~A/autolith-~A-sparc-sunos.tar.gz" - tag tag))) - 404) - "unknown platform archives are rejected")) + (let* ((tag "v0.10.1") + (directory + (release-server--release-directory configuration tag))) + (dolist (platform '("aarch64-linux" + "x86_64-linux-musl" + "aarch64-linux-musl" + "arm64-darwin")) + (let ((archive (release-server--archive-name tag platform))) + (release-server-tests--write-file + (merge-pathnames archive directory) + (format nil "~A-archive" platform)) + (release-server-tests--write-file + (merge-pathnames (format nil "~A.sha256" archive) directory) + (format nil "~A-checksum" platform)) + (dolist (name (list archive (format nil "~A.sha256" archive))) + (let ((response + (release-server-route + configuration "GET" + (format nil "/releases/~A/~A" tag name)))) + (test-assert + (= (release-server-response-status response) 200) + (format nil "published ~A artifacts are served" platform)))))) + (test-assert + (= (release-server-response-status + (release-server-route + configuration "GET" + (format nil "/releases/~A/~A" + tag + (release-server--archive-name tag "x86_64-freebsd")))) + 404) + "missing platform archives stay unpublished") + (test-assert + (= (release-server-response-status + (release-server-route + configuration "GET" + (format nil "/releases/~A/autolith-~A-sparc-sunos.tar.gz" + tag tag))) + 404) + "unknown platform archives are rejected")) (test-assert (= (release-server-response-status (release-server-route @@ -646,30 +648,40 @@ (lambda (command &rest arguments) (declare (ignore arguments)) (let* ((url (first (last command))) - (output (nth (1+ (position "--output" command - :test #'string=)) - command))) + (output + (nth (1+ (position "--output" command + :test #'string=)) + command)) + (platform + (find-if + (lambda (candidate) + (search + (release-server--archive-name + fetch-tag candidate) + url)) + *release-server-platform-ids*))) (push url fetched) - (unless (search "x86_64-linux" url) + (unless platform (error "missing")) (release-server-tests--write-artifact (uiop:pathname-directory-pathname output) - fetch-tag "x86_64-linux" "fetched-linux"))))) + fetch-tag platform + (format nil "fetched-~A" platform)))))) (let ((directory (release-builder--fetch-github-assets builder fetch-source))) (test-assert - (and (uiop:file-exists-p - (merge-pathnames - (release-server--archive-name fetch-tag "x86_64-linux") - directory)) - (not (uiop:file-exists-p - (merge-pathnames - (release-server--archive-name fetch-tag "arm64-darwin") - directory))) - (find-if (lambda (url) - (search "lambda-symbolics/autolith" url)) - fetched)) - "the builder fetches GitHub assets from the configured owner/repo"))) + (and + (every + (lambda (platform) + (uiop:file-exists-p + (merge-pathnames + (release-server--archive-name fetch-tag platform) + directory))) + *release-server-platform-ids*) + (find-if (lambda (url) + (search "lambda-symbolics/autolith" url)) + fetched)) + "the builder fetches every recognized GitHub platform asset"))) (let* ((waiting-tag (release-server-tests--source-tag "v0.32.3" "89abcdef0123456789abcdef0123456789abcdef"))