diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7245c5f..548e225a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -151,7 +151,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Run Rust tests run: cargo test --locked --all-targets --verbose diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fdd0c3c3..777ea990 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,7 +59,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -119,7 +121,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -199,7 +203,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -325,7 +331,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -387,7 +395,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -508,7 +518,9 @@ jobs: env: PYTHONUTF8: "1" PYTHONIOENCODING: "utf-8" - run: python -m py_compile x.py tools/run_tests.py + run: | + python -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python -m unittest tools.test_test_contracts - name: Build release compiler shell: msys2 {0} diff --git a/tests/cases/test101.wave b/tests/cases/test101.wave index 4979fbd5..f94b4980 100644 --- a/tests/cases/test101.wave +++ b/tests/cases/test101.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=asm, asm-contains=ecall|a7 fun do_recv(sockfd: i64, buf: ptr>, src: ptr>, srclen: ptr) -> i64 { var n: i64; asm { diff --git a/tests/cases/test107.wave b/tests/cases/test107.wave index 9bb597a9..92394a70 100644 --- a/tests/cases/test107.wave +++ b/tests/cases/test107.wave @@ -1,4 +1,4 @@ -// wave-test: mode=build, emit=obj +// wave-test: mode=build, runner=compile, emit=obj export(c, "wave_add_i32") fun add_i32(a: i32, b: i32) -> i32 { return a + b; diff --git a/tests/cases/test108.wave b/tests/cases/test108.wave index 24c1a71d..4127421c 100644 --- a/tests/cases/test108.wave +++ b/tests/cases/test108.wave @@ -1,4 +1,4 @@ -// wave-test: mode=build, target=x86_64-pc-windows-gnu, emit=obj, freestanding=true +// wave-test: mode=build, runner=compile, target=x86_64-pc-windows-gnu, emit=obj, freestanding=true const KERNEL_ENTRY_POINT: u64 = 0x200000; const KERNEL_IMAGE_SIZE: u64 = 4; diff --git a/tests/cases/test54.wave b/tests/cases/test54.wave index f27367de..2c77a1f8 100644 --- a/tests/cases/test54.wave +++ b/tests/cases/test54.wave @@ -1,4 +1,4 @@ -// wave-test: host-arch=x86_64, mode=build, target=x86_64-unknown-none-elf, emit=obj, freestanding=true +// wave-test: mode=build, runner=compile, target=x86_64-unknown-none-elf, emit=obj, freestanding=true, object-arch=x86_64, object-bits=64 fun main() { asm { "mov ah, 0x0e" diff --git a/tests/cases/test94.wave b/tests/cases/test94.wave index 8f125259..00fcfdfd 100644 --- a/tests/cases/test94.wave +++ b/tests/cases/test94.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=obj, object-arch=riscv64, object-bits=64, riscv-float-abi=lp64d fun main() { var dummy_ptr: ptr = "dummy"; var ret_val: i64; diff --git a/tests/cases/test96.wave b/tests/cases/test96.wave index 45d99389..2459a319 100644 --- a/tests/cases/test96.wave +++ b/tests/cases/test96.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=asm, asm-contains=ecall|a7 fun factorial_simple(n: i32) -> i32 { var result: i32 = 1; var i: i32 = 1; diff --git a/tests/cases/test99.wave b/tests/cases/test99.wave index fc85d9fd..e5045a9c 100644 --- a/tests/cases/test99.wave +++ b/tests/cases/test99.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=obj, object-arch=riscv64, object-bits=64, riscv-float-abi=lp64d fun syscall_mmap(addr: ptr, length: i64, prot: i64, flags: i64, fd: i64, offset: i64) -> ptr { let r: ptr; asm { diff --git a/tools/run_tests.py b/tools/run_tests.py index a277405c..ae0b8c80 100644 --- a/tools/run_tests.py +++ b/tools/run_tests.py @@ -24,6 +24,19 @@ import tempfile import errno +try: + from tools.test_contracts import ( + normalize_arch, + parse_test_metadata as parse_test_metadata_file, + validate_compiled_artifact, + ) +except ModuleNotFoundError: + from test_contracts import ( + normalize_arch, + parse_test_metadata as parse_test_metadata_file, + validate_compiled_artifact, + ) + ROOT = Path(__file__).resolve().parent.parent TEST_DIR = ROOT / "tests" / "cases" @@ -79,18 +92,7 @@ def resolve_wavec() -> Path: results = [] HOST_OS = platform.system().lower() -HOST_ARCH = platform.machine().lower() - - -def normalize_arch(arch: str) -> str: - aliases = { - "amd64": "x86_64", - "arm64": "aarch64", - } - return aliases.get(arch.lower(), arch.lower()) - - -HOST_ARCH = normalize_arch(HOST_ARCH) +HOST_ARCH = normalize_arch(platform.machine()) TEST_OUTPUT_DIR = Path(tempfile.mkdtemp(prefix="wave-test-output-")) @@ -128,64 +130,13 @@ def iter_test_entries(): def parse_test_metadata(rel_path: str): - path = ROOT / rel_path - meta = { - "host_os": None, - "host_arch": None, - "mode": "run", - "target": None, - "emit": "obj", - "freestanding": False, - "expected_exit": 0, - "udp_input": False, - } - - try: - for line in path.read_text().splitlines(): - stripped = line.strip() - if not stripped.startswith("//"): - if stripped: - break - continue - - marker = "// wave-test:" - if not stripped.startswith(marker): - continue - - body = stripped[len(marker):].strip() - for item in body.split(","): - item = item.strip() - if not item or "=" not in item: - continue - key, value = item.split("=", 1) - key = key.strip() - value = value.strip() - if key == "host-os": - meta["host_os"] = value.lower() - elif key == "host-arch": - meta["host_arch"] = normalize_arch(value) - elif key == "mode": - meta["mode"] = value.lower() - elif key == "target": - meta["target"] = value - elif key == "emit": - meta["emit"] = value.lower() - elif key == "freestanding": - meta["freestanding"] = value.lower() in {"1", "true", "yes"} - elif key == "expected-exit": - meta["expected_exit"] = int(value) - elif key == "udp-input": - meta["udp_input"] = value.lower() in {"1", "true", "yes"} - except OSError: - pass - - return meta + return parse_test_metadata_file(ROOT / rel_path, rel_path) def skip_reason_for_metadata(name: str, rel_path: str): meta = parse_test_metadata(rel_path) - host_os = meta["host_os"] - host_arch = meta["host_arch"] + host_os = meta.host_os + host_arch = meta.host_arch if host_os and host_os != HOST_OS: return f"{name} requires host OS {host_os}, current host is {HOST_OS}" @@ -198,7 +149,7 @@ def skip_reason_for_metadata(name: str, rel_path: str): def command_for_test(name: str, rel_path: str): meta = parse_test_metadata(rel_path) - mode = meta["mode"] + mode = meta.mode if mode == "run": return [str(WAVEC), "run", rel_path] @@ -214,13 +165,13 @@ def command_for_test(name: str, rel_path: str): str(WAVEC), "build", rel_path, - f"--emit={meta['emit']}", + f"--emit={meta.emit}", "--out-dir", str(output_dir), ] - if meta["target"]: - cmd.extend(["--target", meta["target"]]) - if meta["freestanding"]: + if meta.target: + cmd.extend(["--target", meta.target]) + if meta.freestanding: cmd.append("--freestanding") return cmd @@ -318,7 +269,8 @@ def run_and_classify(name, rel_path, cmd): print(f"{CYAN}→ SKIP ({skip_reason}){RESET}\n") return 2 - expected_exit = parse_test_metadata(rel_path)["expected_exit"] + metadata = parse_test_metadata(rel_path) + expected_exit = metadata.expected_exit stdin_data = None if name == "test22.wave": @@ -331,7 +283,7 @@ def run_and_classify(name, rel_path, cmd): return run_test56_server(cmd) try: - if parse_test_metadata(rel_path)["udp_input"]: + if metadata.udp_input: threading.Thread( target=send_udp_test_input, daemon=True @@ -362,6 +314,17 @@ def run_and_classify(name, rel_path, cmd): if expected_exit != 0: print(f"{MAGENTA}→ PASS (expected exit={expected_exit}){RESET}\n") return 3 + artifact_error = validate_compiled_artifact( + name, + ROOT / rel_path, + TEST_OUTPUT_DIR, + metadata, + ) + if artifact_error: + print(f"{RED}→ FAIL (artifact contract){RESET}") + print(artifact_error) + print() + return 0 print(f"{GREEN}→ PASS{RESET}\n") return 1 @@ -412,6 +375,9 @@ def run_and_classify(name, rel_path, cmd): except KeyboardInterrupt: print(f"\n{YELLOW}Interrupted by user.{RESET}") sys.exit(130) +except ValueError as error: + print(f"{RED}invalid wave-test metadata: {error}{RESET}", file=sys.stderr) + sys.exit(2) finally: shutil.rmtree(TEST_OUTPUT_DIR, ignore_errors=True) diff --git a/tools/test_contracts.py b/tools/test_contracts.py new file mode 100644 index 00000000..97b826b0 --- /dev/null +++ b/tools/test_contracts.py @@ -0,0 +1,367 @@ +# This file is part of the Wave language project. +# Copyright (c) 2024–2026 Wave Foundation +# Copyright (c) 2024–2026 LunaStev and contributors +# +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. +# If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# AI TRAINING NOTICE: Prohibited without prior written permission. No use for machine learning or generative AI training, fine-tuning, distillation, embedding, or dataset creation. + +from dataclasses import dataclass +from pathlib import Path +import re + + +ARCH_ALIASES = { + "amd64": "x86_64", + "arm64": "aarch64", +} + +ELF_MACHINES = { + "x86_64": 62, + "aarch64": 183, + "riscv64": 243, +} + +RISCV_FLOAT_ABI_FLAGS = { + "lp64": 0x0, + "lp64f": 0x2, + "lp64d": 0x4, +} + +ARTIFACT_SUFFIXES = { + "obj": ".o", + "asm": ".s", + "ir": ".ll", + "bc": ".bc", +} + + +@dataclass(frozen=True) +class TestMetadata: + host_os: str | None = None + host_arch: str | None = None + mode: str = "run" + runner: str = "native" + target: str | None = None + emit: str = "obj" + freestanding: bool = False + expected_exit: int = 0 + udp_input: bool = False + object_arch: str | None = None + object_bits: int | None = None + riscv_float_abi: str | None = None + asm_contains: tuple[str, ...] = () + asm_not_contains: tuple[str, ...] = () + + +def normalize_arch(arch: str) -> str: + lowered = arch.lower() + return ARCH_ALIASES.get(lowered, lowered) + + +def _parse_bool(key: str, value: str, display_path: str) -> bool: + lowered = value.lower() + if lowered in {"1", "true", "yes"}: + return True + if lowered in {"0", "false", "no"}: + return False + raise ValueError( + f"metadata '{key}' in {display_path} expects true/false, found '{value}'" + ) + + +def _parse_int(key: str, value: str, display_path: str) -> int: + try: + return int(value) + except ValueError as error: + raise ValueError( + f"metadata '{key}' in {display_path} expects an integer, found '{value}'" + ) from error + + +def _parse_patterns(key: str, value: str, display_path: str) -> tuple[str, ...]: + patterns = tuple(pattern.strip() for pattern in value.split("|") if pattern.strip()) + if not patterns: + raise ValueError(f"metadata '{key}' in {display_path} must not be empty") + return patterns + + +def parse_test_metadata(path: Path, display_path: str | None = None) -> TestMetadata: + display = display_path or str(path) + values = {} + marker = "// wave-test:" + + try: + lines = path.read_text().splitlines() + except OSError as error: + raise ValueError(f"failed to read wave-test metadata from {display}: {error}") from error + + for line in lines: + stripped = line.strip() + if not stripped.startswith("//"): + if stripped: + break + continue + if not stripped.startswith(marker): + continue + + body = stripped[len(marker):].strip() + if not body: + raise ValueError(f"empty wave-test metadata in {display}") + + for raw_item in body.split(","): + item = raw_item.strip() + if not item or "=" not in item: + raise ValueError(f"malformed wave-test metadata '{item}' in {display}") + key, value = (part.strip() for part in item.split("=", 1)) + if not key or not value: + raise ValueError(f"malformed wave-test metadata '{item}' in {display}") + if key in values: + raise ValueError(f"duplicate wave-test metadata key '{key}' in {display}") + values[key] = value + + converters = { + "host-os": lambda value: value.lower(), + "host-arch": normalize_arch, + "mode": lambda value: value.lower(), + "runner": lambda value: value.lower(), + "target": str, + "emit": lambda value: value.lower(), + "freestanding": lambda value: _parse_bool("freestanding", value, display), + "expected-exit": lambda value: _parse_int("expected-exit", value, display), + "udp-input": lambda value: _parse_bool("udp-input", value, display), + "object-arch": normalize_arch, + "object-bits": lambda value: _parse_int("object-bits", value, display), + "riscv-float-abi": lambda value: value.lower(), + "asm-contains": lambda value: _parse_patterns("asm-contains", value, display), + "asm-not-contains": lambda value: _parse_patterns( + "asm-not-contains", value, display + ), + } + + converted = {} + field_names = { + "host-os": "host_os", + "host-arch": "host_arch", + "expected-exit": "expected_exit", + "udp-input": "udp_input", + "object-arch": "object_arch", + "object-bits": "object_bits", + "riscv-float-abi": "riscv_float_abi", + "asm-contains": "asm_contains", + "asm-not-contains": "asm_not_contains", + } + for key, value in values.items(): + converter = converters.get(key) + if converter is None: + raise ValueError(f"unsupported wave-test metadata key '{key}' in {display}") + converted[field_names.get(key, key)] = converter(value) + + metadata = TestMetadata(**converted) + compile_keys = { + "target", + "emit", + "freestanding", + "object-arch", + "object-bits", + "riscv-float-abi", + "asm-contains", + "asm-not-contains", + } + if compile_keys.intersection(values) and metadata.runner != "compile": + raise ValueError( + f"compile artifact metadata requires runner 'compile' in {display}" + ) + validate_test_metadata(metadata, display) + return metadata + + +def validate_test_metadata(metadata: TestMetadata, display_path: str) -> None: + if metadata.mode not in {"run", "check", "build"}: + raise ValueError( + f"unsupported wave-test mode '{metadata.mode}' in {display_path}" + ) + if metadata.runner not in {"native", "compile"}: + raise ValueError( + f"unsupported wave-test runner '{metadata.runner}' in {display_path}" + ) + if (metadata.mode == "build") != (metadata.runner == "compile"): + raise ValueError( + f"mode 'build' and runner 'compile' must be used together in {display_path}" + ) + + compile_only_fields = any( + ( + metadata.target, + metadata.freestanding, + metadata.object_arch, + metadata.object_bits, + metadata.riscv_float_abi, + metadata.asm_contains, + metadata.asm_not_contains, + ) + ) + if compile_only_fields and metadata.runner != "compile": + raise ValueError( + f"compile artifact metadata requires runner 'compile' in {display_path}" + ) + if metadata.runner == "compile" and (metadata.host_os or metadata.host_arch): + raise ValueError( + f"compile runner must not depend on the host platform in {display_path}" + ) + if metadata.runner == "compile" and metadata.emit not in ARTIFACT_SUFFIXES: + raise ValueError( + f"unsupported compile artifact emit '{metadata.emit}' in {display_path}" + ) + if metadata.udp_input and (metadata.runner != "native" or metadata.mode != "run"): + raise ValueError( + f"udp-input requires native run mode in {display_path}" + ) + + object_contract = any( + (metadata.object_arch, metadata.object_bits, metadata.riscv_float_abi) + ) + if object_contract and metadata.emit != "obj": + raise ValueError(f"ELF object metadata requires emit=obj in {display_path}") + if metadata.object_arch and metadata.object_arch not in ELF_MACHINES: + raise ValueError( + f"unsupported object architecture '{metadata.object_arch}' in {display_path}" + ) + if metadata.object_bits is not None and metadata.object_bits not in {32, 64}: + raise ValueError( + f"unsupported object bit width '{metadata.object_bits}' in {display_path}" + ) + if metadata.riscv_float_abi: + if metadata.riscv_float_abi not in RISCV_FLOAT_ABI_FLAGS: + raise ValueError( + f"unsupported RISC-V float ABI '{metadata.riscv_float_abi}' in {display_path}" + ) + if metadata.object_arch != "riscv64": + raise ValueError( + f"riscv-float-abi requires object-arch=riscv64 in {display_path}" + ) + + assembly_contract = metadata.asm_contains or metadata.asm_not_contains + if assembly_contract and metadata.emit != "asm": + raise ValueError(f"assembly pattern metadata requires emit=asm in {display_path}") + if metadata.expected_exit != 0 and (object_contract or assembly_contract): + raise ValueError( + f"artifact expectations require a successful compile in {display_path}" + ) + + +def artifact_path_for_test( + name: str, source_path: Path, output_root: Path, metadata: TestMetadata +) -> Path: + suffix = ARTIFACT_SUFFIXES[metadata.emit] + output_dir = output_root / name.replace(" ", "-") + return output_dir / f"{source_path.stem}{suffix}" + + +def read_elf_contract(path: Path): + try: + data = path.read_bytes() + except OSError as error: + return None, f"failed to read artifact {path}: {error}" + + if len(data) < 52 or data[:4] != b"\x7fELF": + return None, f"expected ELF object artifact, found {path}" + + elf_class = data[4] + encoding = data[5] + if encoding == 1: + byteorder = "little" + elif encoding == 2: + byteorder = "big" + else: + return None, f"invalid ELF data encoding {encoding} in {path}" + + flags_offset = {1: 36, 2: 48}.get(elf_class) + if flags_offset is None or len(data) < flags_offset + 4: + return None, f"invalid or truncated ELF header in {path}" + + return { + "bits": {1: 32, 2: 64}[elf_class], + "machine": int.from_bytes(data[18:20], byteorder), + "flags": int.from_bytes(data[flags_offset:flags_offset + 4], byteorder), + }, None + + +def _assembly_code_lines(assembly: str): + for raw_line in assembly.splitlines(): + line = raw_line.split("#", 1)[0].strip() + if not line or line.startswith(".") or line.endswith(":"): + continue + yield line + + +def _assembly_contains(assembly: str, pattern: str) -> bool: + token = re.compile( + rf"(?