From 64c76faece50f0c2d6af720a0d184b2da6e245ae Mon Sep 17 00:00:00 2001 From: LunaStev Date: Mon, 10 Aug 2026 12:10:42 +0900 Subject: [PATCH] Add reproducible release tooling Signed-off-by: LunaStev --- .github/workflows/ci.yml | 35 ++ .gitignore | 3 + CONTRIBUTING.md | 14 + README.md | 45 +++ tests/xpy/test_release_tool.py | 214 ++++++++++++ x.py | 602 +++++++++++++++++++++++++++++++++ 6 files changed, 913 insertions(+) create mode 100644 tests/xpy/test_release_tool.py create mode 100755 x.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28098b9..404eb89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,33 @@ jobs: - name: Run Clippy run: cargo clippy --locked --all-targets -- -D warnings + release-tooling: + name: Package / Linux amd64 + runs-on: ubuntu-24.04 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install Rust toolchain + run: | + rustup toolchain install stable --profile minimal --target x86_64-unknown-linux-gnu + rustup default stable + + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build release binary + run: python x.py build x86_64-unknown-linux-gnu + + - name: Package and smoke test release binary + run: python x.py package x86_64-unknown-linux-gnu + + - name: Verify release checksum + working-directory: dist + run: sha256sum --check SHA256SUMS + test: name: Test / ${{ matrix.name }} strategy: @@ -53,6 +80,14 @@ jobs: rustup toolchain install stable --profile minimal rustup default stable + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Test release tooling + run: python -m unittest discover -s tests/xpy -v + - name: Run unit and integration tests if: runner.os != 'Windows' run: cargo test --locked diff --git a/.gitignore b/.gitignore index 90998cf..6cc6b36 100644 --- a/.gitignore +++ b/.gitignore @@ -77,10 +77,13 @@ vendor/ # Local caches .cache/ .lock/ +__pycache__/ +*.py[cod] # Build directories CMakeFiles/ build/ +/dist/ cmake-build-debug/ cmake-build-release/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3627df7..cab420f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,7 @@ boundaries or a persistent file format. You need: - a stable Rust toolchain with `rustfmt` and `clippy` +- Python 3.11 or newer for `x.py` release tooling - Git for dependency integration tests - a compatible `wavec` in `PATH` for end-to-end build and run tests @@ -41,6 +42,12 @@ cargo clippy --locked --all-targets -- -D warnings cargo build --locked ``` +The same baseline is available through the repository release tool: + +```sh +python3 x.py check +``` + ## Making a change Create a branch from the current `wavefnd/Vex:master`. Use `feat/` for @@ -62,6 +69,8 @@ Add tests at the same level as the behavior being changed: - parser and policy details belong in unit tests - dependency graph and Git behavior belong in integration tests - compiler invocation changes require dry-run schema and end-to-end smoke tests +- release-tool changes require `python3 -m unittest discover -s tests/xpy -v` +- package changes require archive-content, checksum, and executable smoke tests Git integration tests must use local fixture repositories and must not require external network access. Dependency changes should cover direct and transitive @@ -71,6 +80,11 @@ graphs, exact locked commits, cycles, source/version/name conflicts, and relevan When changing selective update behavior, prove that unrelated locked commits and remote-tracking refs remain unchanged. +Release packages are created with `python3 x.py build` followed by +`python3 x.py package`. Do not hand-edit `dist/` artifacts. The stricter +`python3 x.py release` command is reserved for a clean commit carrying the exact +`v` tag. + ## Pull requests Push your branch to a fork and open a pull request against diff --git a/README.md b/README.md index 3c68597..73320bc 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Vex is designed to sit above `wavec` in the same way Cargo sits above `rustc`: V - Rust toolchain for building Vex from source - `wavec` compatible with the `build --dry-run --error-format=json` schema v1 contract - `git` when using Git dependencies +- Python 3.11 or newer when using the release tooling Vex runs `wavec` from `PATH` by default. Set `VEX_WAVEC=/path/to/wavec` to use a specific compiler binary. @@ -131,6 +132,50 @@ vex check VEX_WAVEC=/opt/wave/bin/wavec vex build --dry-run ``` +## Development and release tooling + +The repository-level `x.py` script is the supported entry point for release +builds and packages. It reads the version from `Cargo.toml`, always builds with +the committed `Cargo.lock`, and writes archives plus `SHA256SUMS` to `dist/`. +Run it with Python 3.11 or newer: + +```sh +# Show the host and every supported release target. +python3 x.py list-targets + +# Run formatting, release-tool tests, Rust tests, Clippy, and a debug build. +python3 x.py check + +# Build and package the native target. +python3 x.py build +python3 x.py package + +# Build or package one or more explicit targets. +python3 x.py build x86_64-unknown-linux-gnu +python3 x.py package x86_64-unknown-linux-gnu +``` + +Archives contain the Vex executable together with `README.md`, `LICENSE`, +`NOTICE`, and `COPYRIGHT`. Their file order, permissions, owners, and timestamps +are normalized. Set `SOURCE_DATE_EPOCH` to an explicit non-negative Unix +timestamp when reproducing an artifact outside the tagged source revision. + +`python3 x.py release [...]` is intentionally stricter than separate +build and package commands. It runs the complete validation suite and succeeds +only when the working tree is clean and `HEAD` has the exact `v` tag. +Cross-target builds still require the corresponding Rust target and native +linker to be installed. `VEX_RELEASE_HOST` exists for release infrastructure +that must override host-target detection; normal development should not set it. + +The existing `Makefile` remains available during the transition, but new +release automation should use `x.py` so local builds and CI share one contract. + +Verify downloaded archives from the directory containing `SHA256SUMS`: + +```sh +sha256sum --check SHA256SUMS +``` + ## License [MPL 2.0 LICENSE](LICENSE) diff --git a/tests/xpy/test_release_tool.py b/tests/xpy/test_release_tool.py new file mode 100644 index 0000000..b44253f --- /dev/null +++ b/tests/xpy/test_release_tool.py @@ -0,0 +1,214 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 + +from __future__ import annotations + +import hashlib +import importlib.util +import os +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import unittest +import zipfile +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] +with (ROOT / "Cargo.toml").open("rb") as manifest_file: + EXPECTED_VERSION = tomllib.load(manifest_file)["package"]["version"] +MODULE_NAME = "vex_release_tool" +SPEC = importlib.util.spec_from_file_location(MODULE_NAME, ROOT / "x.py") +if SPEC is None or SPEC.loader is None: # pragma: no cover - import setup failure + raise RuntimeError("could not load x.py") +release_tool = importlib.util.module_from_spec(SPEC) +sys.modules[MODULE_NAME] = release_tool +SPEC.loader.exec_module(release_tool) + + +class ReleaseToolTests(unittest.TestCase): + def test_load_version_reads_cargo_manifest(self) -> None: + self.assertEqual(release_tool.load_version(), EXPECTED_VERSION) + + def test_load_version_accepts_full_semver(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + manifest = Path(temporary) / "Cargo.toml" + manifest.write_text( + '[package]\nname = "fixture"\nversion = "1.2.3-rc.1+build.7"\n', + encoding="utf-8", + ) + self.assertEqual(release_tool.load_version(manifest), "1.2.3-rc.1+build.7") + + def test_select_targets_deduplicates_without_reordering(self) -> None: + selected = release_tool.select_targets( + [ + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + ] + ) + self.assertEqual( + [target.triple for target in selected], + ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"], + ) + + def test_select_targets_rejects_unknown_target_with_known_targets(self) -> None: + with self.assertRaisesRegex( + release_tool.ReleaseError, + r"(?s)unsupported target: imaginary-target.*Known targets", + ): + release_tool.select_targets(["imaginary-target"]) + + def test_default_target_uses_explicit_host_override(self) -> None: + with mock.patch.dict( + os.environ, + {"VEX_RELEASE_HOST": "aarch64-apple-darwin"}, + clear=False, + ): + selected = release_tool.select_targets([]) + self.assertEqual(selected, [release_tool.SUPPORTED_TARGETS["aarch64-apple-darwin"]]) + + def test_source_date_epoch_rejects_invalid_value(self) -> None: + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "yesterday"}): + with self.assertRaisesRegex( + release_tool.ReleaseError, + "SOURCE_DATE_EPOCH must be a non-negative integer", + ): + release_tool.source_date_epoch() + + def test_tar_archive_is_deterministic_and_complete(self) -> None: + target = release_tool.SUPPORTED_TARGETS["x86_64-unknown-linux-gnu"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.make_package_inputs(root, target, b"linux binary") + stage = release_tool.prepare_stage("0.0.1", target, root=root) + first = root / "first.tar.gz" + second = root / "second.tar.gz" + release_tool.create_tar_archive(stage, first, 1_700_000_000) + for entry in stage.iterdir(): + os.utime(entry, (1_800_000_000, 1_800_000_000)) + release_tool.create_tar_archive(stage, second, 1_700_000_000) + + self.assertEqual(first.read_bytes(), second.read_bytes()) + names, binary = release_tool.read_binary_from_tar( + first, f"{stage.name}/{target.executable_name}" + ) + self.assertEqual( + names, + release_tool.expected_archive_entries(stage.name, target), + ) + self.assertEqual(binary, b"linux binary") + with tarfile.open(first, "r:gz") as packaged: + self.assertEqual( + packaged.getmember(f"{stage.name}/vex").mode, + 0o755, + ) + + def test_zip_archive_is_deterministic_and_complete(self) -> None: + target = release_tool.SUPPORTED_TARGETS["x86_64-pc-windows-msvc"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self.make_package_inputs(root, target, b"windows binary") + stage = release_tool.prepare_stage("0.0.1", target, root=root) + first = root / "first.zip" + second = root / "second.zip" + release_tool.create_zip_archive(stage, first, 1_700_000_000) + release_tool.create_zip_archive(stage, second, 1_700_000_000) + + self.assertEqual(first.read_bytes(), second.read_bytes()) + names, binary = release_tool.read_binary_from_zip( + first, f"{stage.name}/{target.executable_name}" + ) + self.assertEqual( + names, + release_tool.expected_archive_entries(stage.name, target), + ) + self.assertEqual(binary, b"windows binary") + with zipfile.ZipFile(first) as packaged: + mode = packaged.getinfo(f"{stage.name}/vex.exe").external_attr >> 16 + self.assertEqual(mode, 0o100755) + + def test_checksums_are_sorted_and_limited_to_requested_archives(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + dist = Path(temporary) + current_zip = dist / "vex-v0.0.1-z-target.zip" + current_tar = dist / "vex-v0.0.1-a-target.tar.gz" + old_tar = dist / "vex-v0.0.0-old-target.tar.gz" + current_zip.write_bytes(b"zip") + current_tar.write_bytes(b"tar") + old_tar.write_bytes(b"old") + + checksum_path = release_tool.write_checksums( + [current_zip, current_tar], dist + ) + lines = checksum_path.read_text(encoding="utf-8").splitlines() + self.assertEqual( + lines, + [ + f"{hashlib.sha256(b'tar').hexdigest()} {current_tar.name}", + f"{hashlib.sha256(b'zip').hexdigest()} {current_zip.name}", + ], + ) + self.assertNotIn(str(dist), "\n".join(lines)) + + def test_release_requires_version_tag_at_head(self) -> None: + with mock.patch.object( + release_tool, + "capture_command", + return_value="v0.0.1\nother-tag", + ): + release_tool.require_release_tag("0.0.1") + + with mock.patch.object(release_tool, "capture_command", return_value="other-tag"): + with self.assertRaisesRegex( + release_tool.ReleaseError, + "official release must run from tag `v0.0.1`", + ): + release_tool.require_release_tag("0.0.1") + + @staticmethod + def make_package_inputs(root: Path, target: object, binary: bytes) -> None: + executable_name = target.executable_name + binary_path = root / "target" / target.triple / "release" / executable_name + binary_path.parent.mkdir(parents=True) + binary_path.write_bytes(binary) + for document in release_tool.PACKAGE_DOCUMENTS: + (root / document).write_text(f"{document}\n", encoding="utf-8") + + +class ReleaseToolCliTests(unittest.TestCase): + def run_xpy(self, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(ROOT / "x.py"), *arguments], + cwd=ROOT, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + def test_version_output_uses_manifest_version(self) -> None: + result = self.run_xpy("--version") + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), f"x.py {EXPECTED_VERSION}") + + def test_list_targets_reports_all_supported_targets(self) -> None: + result = self.run_xpy("list-targets") + self.assertEqual(result.returncode, 0) + for target in release_tool.SUPPORTED_TARGETS: + self.assertIn(target, result.stdout) + + def test_unknown_target_is_actionable(self) -> None: + result = self.run_xpy("build", "imaginary-target") + self.assertEqual(result.returncode, 1) + self.assertIn("error: unsupported target: imaginary-target", result.stderr) + self.assertIn("Known targets:", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/x.py b/x.py new file mode 100755 index 0000000..e0bb7cb --- /dev/null +++ b/x.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 + +from __future__ import annotations + +import argparse +import datetime as dt +import gzip +import hashlib +import os +import re +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Sequence + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - handled before any command runs + tomllib = None + + +ROOT = Path(__file__).resolve().parent +TARGET_DIR = ROOT / "target" +DIST_DIR = ROOT / "dist" +BINARY_NAME = "vex" +PACKAGE_DOCUMENTS = ("README.md", "LICENSE", "NOTICE", "COPYRIGHT") +CHECKSUM_FILE = "SHA256SUMS" +MINIMUM_ZIP_EPOCH = 315532800 # 1980-01-01T00:00:00Z +MAXIMUM_ZIP_EPOCH = 4354819198 # 2107-12-31T23:59:58Z +VERSION_PATTERN = re.compile( + r"^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$" +) +ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +class ReleaseError(RuntimeError): + """An actionable release-tool failure.""" + + +@dataclass(frozen=True) +class Target: + triple: str + platform: str + architecture: str + archive: str + + @property + def executable_name(self) -> str: + return f"{BINARY_NAME}.exe" if self.platform == "Windows" else BINARY_NAME + + +SUPPORTED_TARGETS = { + target.triple: target + for target in ( + Target("x86_64-unknown-linux-gnu", "Linux", "amd64", "tar.gz"), + Target("aarch64-unknown-linux-gnu", "Linux", "arm64", "tar.gz"), + Target("riscv64gc-unknown-linux-gnu", "Linux", "riscv64", "tar.gz"), + Target("x86_64-pc-windows-msvc", "Windows", "x64", "zip"), + Target("x86_64-apple-darwin", "macOS", "Intel", "tar.gz"), + Target("aarch64-apple-darwin", "macOS", "Apple Silicon", "tar.gz"), + ) +} + + +def status(action: str, message: str) -> None: + print(f"{action:>12} {message}", file=sys.stderr) + + +def command_text(command: Sequence[os.PathLike[str] | str]) -> str: + return shlex.join(str(part) for part in command) + + +def run_command( + command: Sequence[os.PathLike[str] | str], + *, + cwd: Path = ROOT, + env: dict[str, str] | None = None, + capture: bool = False, +) -> subprocess.CompletedProcess[str]: + status("Running", command_text(command)) + try: + return subprocess.run( + [str(part) for part in command], + cwd=cwd, + env=env, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + except FileNotFoundError as error: + raise ReleaseError(f"required tool `{command[0]}` was not found in PATH") from error + except subprocess.CalledProcessError as error: + details = "" + if capture: + details = (error.stderr or error.stdout or "").strip() + suffix = f"\n\nCaused by:\n {details}" if details else "" + raise ReleaseError( + f"command failed with status {error.returncode}: {command_text(command)}{suffix}" + ) from error + + +def capture_command( + command: Sequence[os.PathLike[str] | str], *, cwd: Path = ROOT +) -> str: + return run_command(command, cwd=cwd, capture=True).stdout.strip() + + +def load_version(manifest_path: Path = ROOT / "Cargo.toml") -> str: + if tomllib is None: + raise ReleaseError("Python 3.11 or newer is required to read Cargo.toml") + try: + with manifest_path.open("rb") as manifest_file: + data = tomllib.load(manifest_file) + version = data["package"]["version"] + except (OSError, KeyError, TypeError, tomllib.TOMLDecodeError) as error: + raise ReleaseError(f"could not read package version from `{manifest_path}`: {error}") from error + if not isinstance(version, str) or not VERSION_PATTERN.fullmatch(version): + raise ReleaseError(f"Cargo.toml contains unsupported package version `{version}`") + return version + + +def detect_host_target() -> str: + override = os.environ.get("VEX_RELEASE_HOST") + if override: + return override + output = capture_command(["rustc", "-vV"]) + for line in output.splitlines(): + if line.startswith("host: "): + return line.removeprefix("host: ").strip() + raise ReleaseError("`rustc -vV` did not report a host target") + + +def select_targets(names: Sequence[str]) -> list[Target]: + requested = list(names) if names else [detect_host_target()] + unknown = sorted({name for name in requested if name not in SUPPORTED_TARGETS}) + if unknown: + known = "\n".join(f" {name}" for name in SUPPORTED_TARGETS) + raise ReleaseError( + f"unsupported target{'' if len(unknown) == 1 else 's'}: {', '.join(unknown)}" + f"\n\nKnown targets:\n{known}" + ) + selected: list[Target] = [] + seen: set[str] = set() + for name in requested: + if name not in seen: + selected.append(SUPPORTED_TARGETS[name]) + seen.add(name) + return selected + + +def build_environment(target: Target) -> dict[str, str]: + environment = os.environ.copy() + if target.triple == "riscv64gc-unknown-linux-gnu": + variable = "CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER" + if variable not in environment: + linker = shutil.which("riscv64-linux-gnu-gcc") + if linker is None: + raise ReleaseError( + "RISC-V builds require `riscv64-linux-gnu-gcc` in PATH or " + f"an explicit `{variable}`" + ) + environment[variable] = linker + return environment + + +def build_targets(targets: Iterable[Target]) -> None: + for target in targets: + status("Building", target.triple) + run_command( + [ + "cargo", + "build", + "--locked", + "--release", + "--target", + target.triple, + ], + env=build_environment(target), + ) + + +def package_name(version: str, target: Target) -> str: + return f"{BINARY_NAME}-v{version}-{target.triple}" + + +def source_date_epoch() -> int: + configured = os.environ.get("SOURCE_DATE_EPOCH") + if configured is not None: + try: + value = int(configured) + except ValueError as error: + raise ReleaseError("SOURCE_DATE_EPOCH must be a non-negative integer") from error + if value < 0: + raise ReleaseError("SOURCE_DATE_EPOCH must be a non-negative integer") + return value + + try: + value = int(capture_command(["git", "log", "-1", "--format=%ct"])) + except (ReleaseError, ValueError): + return MINIMUM_ZIP_EPOCH + return max(value, 0) + + +def prepare_stage(version: str, target: Target, *, root: Path = ROOT) -> Path: + source_binary = root / "target" / target.triple / "release" / target.executable_name + if not source_binary.is_file(): + raise ReleaseError( + f"release binary is missing: `{source_binary}`\n" + f"help: run `python3 x.py build {target.triple}` first" + ) + + dist_dir = root / "dist" + stage = dist_dir / package_name(version, target) + if stage.exists(): + shutil.rmtree(stage) + stage.mkdir(parents=True) + + staged_binary = stage / target.executable_name + shutil.copyfile(source_binary, staged_binary) + staged_binary.chmod(0o755) + + for document in PACKAGE_DOCUMENTS: + source = root / document + if not source.is_file(): + raise ReleaseError(f"required package document is missing: `{source}`") + destination = stage / document + shutil.copyfile(source, destination) + destination.chmod(0o644) + + return stage + + +def normalized_tar_info(info: tarfile.TarInfo, epoch: int) -> tarfile.TarInfo: + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + info.mtime = epoch + info.mode = 0o755 if info.isdir() or info.name.endswith(("/vex", "/vex.exe")) else 0o644 + return info + + +def create_tar_archive(stage: Path, archive: Path, epoch: int) -> None: + with archive.open("wb") as raw_archive: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=epoch) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.USTAR_FORMAT) as tar: + entries = [stage, *sorted(stage.iterdir(), key=lambda path: path.name)] + for entry in entries: + arcname = stage.name if entry == stage else f"{stage.name}/{entry.name}" + info = normalized_tar_info(tar.gettarinfo(str(entry), arcname), epoch) + if entry.is_dir(): + tar.addfile(info) + elif entry.is_file(): + with entry.open("rb") as source: + tar.addfile(info, source) + else: + raise ReleaseError(f"unsupported staged entry: `{entry}`") + + +def zip_timestamp(epoch: int) -> tuple[int, int, int, int, int, int]: + supported_epoch = min(max(epoch, MINIMUM_ZIP_EPOCH), MAXIMUM_ZIP_EPOCH) + timestamp = dt.datetime.fromtimestamp(supported_epoch, dt.timezone.utc) + return ( + timestamp.year, + timestamp.month, + timestamp.day, + timestamp.hour, + timestamp.minute, + timestamp.second - timestamp.second % 2, + ) + + +def write_zip_entry( + archive: zipfile.ZipFile, name: str, data: bytes, epoch: int, mode: int +) -> None: + info = zipfile.ZipInfo(name, date_time=zip_timestamp(epoch)) + info.create_system = 3 + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = (mode & 0xFFFF) << 16 + archive.writestr(info, data, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + + +def create_zip_archive(stage: Path, archive: Path, epoch: int) -> None: + with zipfile.ZipFile(archive, mode="w") as zipped: + write_zip_entry(zipped, f"{stage.name}/", b"", epoch, 0o40755) + for entry in sorted(stage.iterdir(), key=lambda path: path.name): + if not entry.is_file(): + raise ReleaseError(f"unsupported staged entry: `{entry}`") + mode = 0o100755 if entry.name in ("vex", "vex.exe") else 0o100644 + write_zip_entry(zipped, f"{stage.name}/{entry.name}", entry.read_bytes(), epoch, mode) + + +def create_archive(stage: Path, target: Target, epoch: int) -> Path: + extension = ".zip" if target.archive == "zip" else ".tar.gz" + archive = stage.parent / f"{stage.name}{extension}" + archive.unlink(missing_ok=True) + if target.archive == "zip": + create_zip_archive(stage, archive, epoch) + else: + create_tar_archive(stage, archive, epoch) + return archive + + +def expected_archive_entries(stage_name: str, target: Target) -> set[str]: + return { + f"{stage_name}/", + f"{stage_name}/{target.executable_name}", + *(f"{stage_name}/{document}" for document in PACKAGE_DOCUMENTS), + } + + +def read_binary_from_tar(archive: Path, member_name: str) -> tuple[set[str], bytes]: + with tarfile.open(archive, "r:gz") as packaged: + members = packaged.getmembers() + names = {member.name.rstrip("/") + ("/" if member.isdir() else "") for member in members} + if any(member.issym() or member.islnk() for member in members): + raise ReleaseError(f"archive contains an unexpected link: `{archive}`") + member = packaged.getmember(member_name) + extracted = packaged.extractfile(member) + if extracted is None: + raise ReleaseError(f"could not read packaged binary `{member_name}`") + return names, extracted.read() + + +def read_binary_from_zip(archive: Path, member_name: str) -> tuple[set[str], bytes]: + with zipfile.ZipFile(archive) as packaged: + names = set(packaged.namelist()) + return names, packaged.read(member_name) + + +def smoke_prefix(target: Target, binary: Path, host: str) -> list[str] | None: + if target.triple == host: + return [str(binary)] + if target.triple == "riscv64gc-unknown-linux-gnu" and host.endswith("-unknown-linux-gnu"): + emulator = shutil.which("qemu-riscv64") + sysroot = Path("/usr/riscv64-linux-gnu") + if emulator and sysroot.is_dir(): + return [emulator, "-L", str(sysroot), str(binary)] + if target.platform == "Windows" and host.endswith("-unknown-linux-gnu"): + emulator = shutil.which("wine") + if emulator: + return [emulator, str(binary)] + return None + + +def strip_ansi(text: str) -> str: + return ANSI_ESCAPE.sub("", text) + + +def smoke_binary(binary_data: bytes, target: Target, version: str, host: str) -> bool: + with tempfile.TemporaryDirectory(prefix="vex-package-smoke-") as temporary: + binary = Path(temporary) / target.executable_name + binary.write_bytes(binary_data) + binary.chmod(0o755) + prefix = smoke_prefix(target, binary, host) + if prefix is None: + status("Skipping", f"execution smoke for {target.triple} on {host}") + return False + + version_result = run_command([*prefix, "--version"], capture=True) + version_output = strip_ansi(version_result.stdout).strip() + if not re.search(rf"\bvex\s+{re.escape(version)}\b", version_output): + raise ReleaseError( + f"packaged binary reported unexpected version `{version_output}`; expected `{version}`" + ) + + help_result = run_command([*prefix, "--help"], capture=True) + if "Vex - Wave package manager" not in strip_ansi(help_result.stdout): + raise ReleaseError("packaged binary help output did not contain the Vex heading") + return True + + +def verify_archive(archive: Path, stage: Path, target: Target, version: str, host: str) -> None: + binary_member = f"{stage.name}/{target.executable_name}" + try: + if target.archive == "zip": + names, binary_data = read_binary_from_zip(archive, binary_member) + else: + names, binary_data = read_binary_from_tar(archive, binary_member) + except (KeyError, OSError, tarfile.TarError, zipfile.BadZipFile) as error: + raise ReleaseError(f"could not verify release archive `{archive}`: {error}") from error + + expected = expected_archive_entries(stage.name, target) + if names != expected: + missing = sorted(expected - names) + extra = sorted(names - expected) + details = [] + if missing: + details.append(f"missing: {', '.join(missing)}") + if extra: + details.append(f"unexpected: {', '.join(extra)}") + raise ReleaseError(f"archive contents are invalid ({'; '.join(details)})") + smoke_binary(binary_data, target, version, host) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as artifact: + while chunk := artifact.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def write_checksums(archives: Iterable[Path], dist_dir: Path = DIST_DIR) -> Path: + selected = sorted(archives, key=lambda path: path.name) + if not selected: + raise ReleaseError("no release archives were provided for checksumming") + checksum_path = dist_dir / CHECKSUM_FILE + content = "".join(f"{sha256(archive)} {archive.name}\n" for archive in selected) + temporary = checksum_path.with_suffix(".tmp") + temporary.write_text(content, encoding="utf-8", newline="\n") + temporary.replace(checksum_path) + return checksum_path + + +def package_targets(targets: Iterable[Target], version: str, host: str) -> list[Path]: + epoch = source_date_epoch() + archives: list[Path] = [] + for target in targets: + status("Packaging", target.triple) + stage = prepare_stage(version, target) + archive: Path | None = None + try: + archive = create_archive(stage, target, epoch) + verify_archive(archive, stage, target, version, host) + except Exception: + if archive is not None: + archive.unlink(missing_ok=True) + raise + finally: + shutil.rmtree(stage, ignore_errors=True) + archives.append(archive) + status("Packaged", str(archive.relative_to(ROOT))) + checksum_path = write_checksums(archives) + status("Checksums", str(checksum_path.relative_to(ROOT))) + return archives + + +def require_clean_tree() -> None: + dirty = capture_command(["git", "status", "--porcelain", "--untracked-files=normal"]) + if dirty: + raise ReleaseError( + "official releases require a clean Git working tree\n" + "help: commit, stash, or remove local changes before running `x.py release`" + ) + + +def require_release_tag(version: str) -> None: + expected = f"v{version}" + tags = capture_command(["git", "tag", "--points-at", "HEAD"]).splitlines() + if expected not in tags: + raise ReleaseError( + f"official release must run from tag `{expected}`\n" + f"help: create and check out the annotated `{expected}` tag" + ) + + +def run_check_suite() -> None: + run_command(["cargo", "fmt", "--check"]) + run_python_tests() + run_command(["cargo", "test", "--locked"]) + run_command(["cargo", "clippy", "--locked", "--all-targets", "--", "-D", "warnings"]) + run_command(["cargo", "build", "--locked"]) + + +def command_check(_: argparse.Namespace) -> None: + run_check_suite() + + +def command_test(_: argparse.Namespace) -> None: + run_python_tests() + run_command(["cargo", "test", "--locked"]) + + +def command_build(args: argparse.Namespace) -> None: + build_targets(select_targets(args.targets)) + + +def command_package(args: argparse.Namespace) -> None: + targets = select_targets(args.targets) + package_targets(targets, load_version(), detect_host_target()) + + +def command_release(args: argparse.Namespace) -> None: + version = load_version() + require_clean_tree() + require_release_tag(version) + targets = select_targets(args.targets) + run_check_suite() + build_targets(targets) + package_targets(targets, version, detect_host_target()) + + +def command_clean(_: argparse.Namespace) -> None: + status("Cleaning", "Cargo build artifacts") + run_command(["cargo", "clean"]) + if DIST_DIR.exists(): + status("Cleaning", str(DIST_DIR.relative_to(ROOT))) + shutil.rmtree(DIST_DIR) + + +def command_list_targets(_: argparse.Namespace) -> None: + try: + host = detect_host_target() + except ReleaseError: + host = "" + print(f"Host: {host}") + print("Supported release targets:") + for target in SUPPORTED_TARGETS.values(): + marker = " (native default)" if target.triple == host else "" + print( + f" {target.triple:<36} {target.platform:<8} " + f"{target.architecture:<13} {target.archive}{marker}" + ) + + +def add_target_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "targets", + nargs="*", + metavar="TARGET", + help="release target triple; defaults to the rustc host", + ) + + +def run_python_tests() -> None: + run_command( + [sys.executable, "-m", "unittest", "discover", "-s", "tests/xpy", "-v"] + ) + + +def create_parser(version: str) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Build and package reproducible Vex release artifacts." + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {version}") + commands = parser.add_subparsers(dest="command") + + check = commands.add_parser("check", help="run the complete local validation suite") + check.set_defaults(handler=command_check) + + test = commands.add_parser("test", help="run release-tool and Rust tests") + test.set_defaults(handler=command_test) + + build = commands.add_parser("build", help="build release binaries") + add_target_arguments(build) + build.set_defaults(handler=command_build) + + package = commands.add_parser("package", help="package existing release binaries") + add_target_arguments(package) + package.set_defaults(handler=command_package) + + release = commands.add_parser( + "release", help="validate, build, and package an official tagged release" + ) + add_target_arguments(release) + release.set_defaults(handler=command_release) + + clean = commands.add_parser("clean", help="remove Cargo and release-tool artifacts") + clean.set_defaults(handler=command_clean) + + list_targets = commands.add_parser("list-targets", help="show supported release targets") + list_targets.set_defaults(handler=command_list_targets) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + if sys.version_info < (3, 11): + print("error: x.py requires Python 3.11 or newer", file=sys.stderr) + return 1 + try: + version = load_version() + except ReleaseError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + parser = create_parser(version) + args = parser.parse_args(argv) + if not hasattr(args, "handler"): + parser.print_help() + return 0 + try: + args.handler(args) + except ReleaseError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())