From 619ef2f2474bdc243715589a47f6fc8588a3dc74 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:09:39 +0000 Subject: [PATCH 1/8] Unified: Factor out the supported platforms --- unified/platforms.bzl | 9 +++++++++ unified/swift-syntax-rs/BUILD.bazel | 23 +++++++---------------- 2 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 unified/platforms.bzl diff --git a/unified/platforms.bzl b/unified/platforms.bzl new file mode 100644 index 000000000000..29560c75b3c4 --- /dev/null +++ b/unified/platforms.bzl @@ -0,0 +1,9 @@ +"""Shared platform constraint for the unified extractor.""" + +# swift-syntax requires a Swift toolchain, which is only available +# through rules_swift. +UNIFIED_SUPPORTED_PLATFORMS = select({ + "@platforms//os:linux": [], + "@platforms//os:macos": [], + "//conditions:default": ["@platforms//:incompatible"], +}) diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index db451e5b7b9a..8f2a5886fd22 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,6 +1,7 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles") +load("//unified:platforms.bzl", "UNIFIED_SUPPORTED_PLATFORMS") load(":swift_runtime.bzl", "swift_runtime_libs") load(":xcode_transition.bzl", "xcode_transition_swift_library") @@ -11,23 +12,13 @@ swift_runtime_libs( toolchain = "@swift_toolchain_ubuntu22.04//:files", ) -# Targets in this package require a Swift toolchain (Linux or macOS). -# `select()` gives us OR-of-OSes; other platforms get marked incompatible -# so `bazel build/test //...` skips them cleanly. -# -_SWIFT_SUPPORTED_PLATFORMS = select({ - "@platforms//os:linux": [], - "@platforms//os:macos": [], - "//conditions:default": ["@platforms//:incompatible"], -}) - # Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust # targets below link against its `CcInfo`. xcode_transition_swift_library( name = "swift_syntax_ffi", srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], module_name = "SwiftSyntaxFFI", - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, deps = [ "@swift-syntax//:SwiftOperators", "@swift-syntax//:SwiftParser", @@ -44,7 +35,7 @@ rust_library( exclude = ["src/main.rs"], ), edition = "2024", - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, deps = [ ":swift_syntax_ffi", ], @@ -68,7 +59,7 @@ rust_binary( "@platforms//os:linux": [":swift_runtime_libs"], }), edition = "2024", - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, deps = [":swift_syntax_rs"], ) @@ -79,7 +70,7 @@ sh_binary( name = "swift-syntax-parse", srcs = ["swift-syntax-parse.sh"], data = [":swift-syntax-parse.real"], - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, ) # Packaged form for the extractor pack: the wrapper (as `swift-syntax-parse`), @@ -90,7 +81,7 @@ codeql_pkg_runfiles( # original filename. excludes = ["swift-syntax-parse.sh"], exes = [":swift-syntax-parse"], - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, visibility = ["//unified:__pkg__"], ) @@ -103,5 +94,5 @@ rust_test( "@platforms//os:linux": [":swift_runtime_libs"], }), edition = "2024", - target_compatible_with = _SWIFT_SUPPORTED_PLATFORMS, + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, ) From 1d824fec58b97a2c7fd74c54bda0051d3011d935 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:43:26 +0000 Subject: [PATCH 2/8] Unified: Add swift-syntax build dependency to the extractor --- Cargo.lock | 1 + unified/extractor/BUILD.bazel | 13 +++++++++++++ unified/extractor/Cargo.toml | 1 + 3 files changed, 15 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 24479c721561..bebfc2da57cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,6 +449,7 @@ dependencies = [ "rayon", "regex", "serde_json", + "swift-syntax-rs", "tracing", "tracing-subscriber", "yeast", diff --git a/unified/extractor/BUILD.bazel b/unified/extractor/BUILD.bazel index 58b32d9fc5c3..dea196ada0e5 100644 --- a/unified/extractor/BUILD.bazel +++ b/unified/extractor/BUILD.bazel @@ -1,5 +1,12 @@ load("//misc/bazel:rust.bzl", "codeql_rust_binary") load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps") +load("//unified:platforms.bzl", "UNIFIED_SUPPORTED_PLATFORMS") +load("//unified/swift-syntax-rs:swift_runtime.bzl", "swift_runtime_libs") + +swift_runtime_libs( + name = "swift_runtime_libs", + toolchain = "@swift_toolchain_ubuntu22.04//:files", +) exports_files(["Cargo.toml"]) @@ -11,14 +18,20 @@ codeql_rust_binary( "ast_types.yml", "swift_node_types.yml", ], + data = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": [":swift_runtime_libs"], + }), proc_macro_deps = all_crate_deps( proc_macro = True, ), + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, visibility = ["//visibility:public"], deps = all_crate_deps( normal = True, ) + [ "//shared/tree-sitter-extractor", "//shared/yeast", + "//unified/swift-syntax-rs:swift_syntax_rs", ], ) diff --git a/unified/extractor/Cargo.toml b/unified/extractor/Cargo.toml index be333298f1ad..607afc40458b 100644 --- a/unified/extractor/Cargo.toml +++ b/unified/extractor/Cargo.toml @@ -18,3 +18,4 @@ serde_json = "1.0.145" codeql-extractor = { path = "../../shared/tree-sitter-extractor" } yeast = { path = "../../shared/yeast" } +swift-syntax-rs = { path = "../swift-syntax-rs" } From ad1bd59054fad537c58c753846c2d71d05364961 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:56:05 +0000 Subject: [PATCH 3/8] Unified: Call the swift code directly instead of going through a binary --- .../extractor/src/languages/swift/parse.rs | 109 +----------------- unified/extractor/tests/corpus_tests.rs | 22 ---- 2 files changed, 5 insertions(+), 126 deletions(-) diff --git a/unified/extractor/src/languages/swift/parse.rs b/unified/extractor/src/languages/swift/parse.rs index c8633f1a27a5..2abc589332c8 100644 --- a/unified/extractor/src/languages/swift/parse.rs +++ b/unified/extractor/src/languages/swift/parse.rs @@ -1,33 +1,18 @@ -//! Swift front-end parser: shells out to the separate `swift-syntax-parse` -//! binary (which links swift-syntax) to obtain a JSON syntax tree, then adapts -//! that JSON into a `yeast::Ast` via the pure-Rust [`swift_adapter`] module. -//! -//! Running the parser in a separate process keeps the Swift toolchain out of -//! the extractor's own build: the extractor never links Swift, so working on -//! other (e.g. tree-sitter based) languages needs no Swift toolchain. Each call -//! spawns the parser afresh; a longer-lived parser process could be swapped in -//! behind this same seam later without touching the extraction pipeline. - -use std::io::Write; -use std::process::{Command, Stdio}; +//! Swift front-end parser: calls into the `swift-syntax-rs` crate (which links +//! swift-syntax) to obtain a JSON syntax tree, then adapts that JSON into a +//! `yeast::Ast` via the pure-Rust [`swift_adapter`] module. use codeql_extractor::extractor::ParsedTree; use super::swift_adapter; -/// Environment variable naming the `swift-syntax-parse` executable. When unset, -/// the parser is resolved next to the extractor executable, then on `PATH`. -const PARSE_BIN_ENV: &str = "CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE"; - -/// Base name of the `swift-syntax-parse` executable as shipped / looked up. -const PARSE_BIN_NAME: &str = "swift-syntax-parse"; - /// Parse Swift `source` into a [`ParsedTree`] (a raw `yeast::Ast` plus /// side-channel `extra` tokens), ready to be desugared via `run_from_ast`. pub fn parse(source: &[u8]) -> Result { let source = std::str::from_utf8(source).map_err(|e| format!("Swift source is not valid UTF-8: {e}"))?; - let json = run_parser(source)?; + let json = + swift_syntax_rs::parse_to_json(source).map_err(|e| format!("Swift parser failed: {e}"))?; let mut adapted = swift_adapter::json_to_ast(&json)?; adapted.ast.set_source(source.as_bytes().to_vec()); Ok(ParsedTree { @@ -35,87 +20,3 @@ pub fn parse(source: &[u8]) -> Result { extras: adapted.extras, }) } - -/// The `swift-syntax-parse` executable to invoke, resolved in priority order: -/// -/// 1. the `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` override, if set; -/// 2. a copy shipped next to the extractor executable — this is how the CodeQL -/// extractor pack lays it out (`tools//{extractor, -/// swift-syntax-parse}`), so a packaged extractor is self-contained with no -/// environment setup; -/// 3. a bare `swift-syntax-parse`, looked up on `PATH`. -fn parse_bin() -> String { - if let Ok(bin) = std::env::var(PARSE_BIN_ENV) { - if !bin.is_empty() { - return bin; - } - } - if let Ok(exe) = std::env::current_exe() { - if let Some(sibling) = exe.parent().map(|dir| dir.join(PARSE_BIN_NAME)) { - if sibling.is_file() { - return sibling.to_string_lossy().into_owned(); - } - } - } - PARSE_BIN_NAME.to_string() -} - -/// Whether the `swift-syntax-parse` executable can be launched at all. -/// -/// This reports availability of the *executable*, deliberately not whether -/// parsing succeeds: a binary that launches but then crashes or emits invalid -/// JSON is still "available", so callers run and surface the failure rather -/// than silently skipping. Only a genuinely missing/unlaunchable binary (e.g. -/// no Swift toolchain is installed) reports `false`. -pub fn binary_available() -> bool { - match Command::new(parse_bin()) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - { - Ok(mut child) => { - let _ = child.wait(); - true - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => false, - // Any other spawn failure (e.g. a permissions problem) is a genuine - // issue worth surfacing, so treat the parser as available and let the - // caller fail rather than masking it as "unavailable". - Err(_) => true, - } -} - -/// Run the external parser, feeding `source` on stdin and returning its JSON -/// stdout. -fn run_parser(source: &str) -> Result { - let bin = parse_bin(); - let mut child = Command::new(&bin) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| format!("failed to spawn Swift parser `{bin}`: {e}"))?; - - // The parser reads all of stdin before writing any stdout, so writing the - // whole source and then closing stdin (by dropping it) cannot deadlock. - child - .stdin - .take() - .expect("child stdin was piped") - .write_all(source.as_bytes()) - .map_err(|e| format!("failed to write source to Swift parser `{bin}`: {e}"))?; - - let output = child - .wait_with_output() - .map_err(|e| format!("failed to run Swift parser `{bin}`: {e}"))?; - if !output.status.success() { - return Err(format!( - "Swift parser `{bin}` failed ({}): {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )); - } - String::from_utf8(output.stdout) - .map_err(|e| format!("Swift parser produced non-UTF-8 output: {e}")) -} diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index b2700b999b7c..39cbab3194c7 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -20,20 +20,6 @@ fn update_mode_enabled() -> bool { .unwrap_or(false) } -/// Whether the external swift-syntax parser is available. When the parser -/// binary genuinely cannot be found/launched (e.g. no Swift toolchain, and -/// neither `CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE` nor a `swift-syntax-parse` -/// on `PATH`), the corpus test is skipped rather than failed — it cannot run -/// without the Swift-backed parser. -/// -/// Crucially this checks only that the executable *launches*: a parser that is -/// present but crashes, emits invalid JSON, or otherwise regresses is -/// considered available, so the suite runs and fails (rather than silently -/// skipping the very failures CI needs to catch). -fn parser_available() -> bool { - languages::swift_parse::binary_available() -} - /// Parse a corpus `.output` file. The file holds a single test case made of /// three sections separated by `---` delimiter lines: /// @@ -112,14 +98,6 @@ fn collect_corpus_stems(dir: &Path, out: &mut Vec) { #[test] fn test_corpus() { - if !parser_available() { - eprintln!( - "skipping test_corpus: the swift-syntax parser is unavailable \ - (set CODEQL_EXTRACTOR_UNIFIED_SWIFT_SYNTAX_PARSE or put \ - `swift-syntax-parse` on PATH)" - ); - return; - } let update_mode = update_mode_enabled(); let all_languages = languages::all_language_specs(); let corpus_dir = Path::new("tests/corpus"); From 1226971cfba1e69e178d0e9f5dd64f9b51c26ba5 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:19:10 +0000 Subject: [PATCH 4/8] Unified: Remove the `SWIFT`/`SWITFC` environment hacks Just assume `swift`/`swiftc` are on the path. --- unified/swift-syntax-rs/build.rs | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index 6a58ddb4d8da..1056c5a62518 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -20,8 +20,6 @@ fn main() { ] { println!("cargo:rerun-if-changed={}", input.display()); } - println!("cargo:rerun-if-env-changed=SWIFT"); - println!("cargo:rerun-if-env-changed=SWIFTC"); // Build the Swift FFI package as a release dynamic library. // @@ -33,7 +31,7 @@ fn main() { // then fail — at link time, which is fair: they genuinely need Swift (and CI // builds go through Bazel anyway). A Swift toolchain that *is* present but // whose build fails is still surfaced as a hard error below. - let mut command = Command::new(swift_bin()); + let mut command = Command::new("swift"); command .args(["build", "-c", "release"]) .current_dir(&swift_dir); @@ -42,12 +40,11 @@ fn main() { Ok(status) => status, Err(e) => { println!( - "cargo:warning=skipping the Swift FFI build: failed to run `{swift} build`: {e}. \ - Install a Swift toolchain (see https://www.swift.org/install/, e.g. via swiftly) \ - and ensure `swift` is on PATH, or set the `SWIFT` environment variable, to build \ - or test this crate. `cargo check`/`fmt`/`clippy` work without it. The pinned \ - version is in `.swift-version`.", - swift = swift_bin(), + "cargo:warning=skipping the Swift FFI build: failed to run `swift build`: {e}. \ + Install a Swift toolchain (see https://www.swift.org/install/, e.g. via \ + swiftly) and ensure `swift` is on PATH to build or test this crate. \ + `cargo check`/`fmt`/`clippy` work without it. The pinned version is in \ + `.swift-version`." ); return; } @@ -70,7 +67,7 @@ fn main() { /// Query the active Swift toolchain for the directory containing its runtime /// shared libraries (e.g. `libswiftCore.so`). fn swift_runtime_dir() -> Option { - let output = Command::new(swiftc_bin()) + let output = Command::new("swiftc") .arg("-print-target-info") .output() .ok()?; @@ -94,19 +91,6 @@ fn swift_runtime_dir() -> Option { Some(PathBuf::from(resource_path).join(if cfg!(target_os = "macos") { "macosx" } else { "linux" })) } -/// The `swift` driver to invoke: `$SWIFT` if set, otherwise `swift` from `PATH`. -/// This keeps the build tool-agnostic — any Swift install works; no particular -/// version manager is required. -fn swift_bin() -> String { - env::var("SWIFT").unwrap_or_else(|_| "swift".to_string()) -} - -/// The `swiftc` compiler to invoke: `$SWIFTC` if set, otherwise `swiftc` from -/// `PATH`. -fn swiftc_bin() -> String { - env::var("SWIFTC").unwrap_or_else(|_| "swiftc".to_string()) -} - /// Some environments (notably GitHub Codespaces) inject /// `GIT_CONFIG_KEY_0=safe.bareRepository` / `GIT_CONFIG_VALUE_0=explicit`, which /// breaks the cached bare git repositories `swift build` uses. When exactly that From 4e2ac7cf2674cfdb0e1df82c15fe669d044ea9a7 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Thu, 6 Aug 2026 08:58:21 +0000 Subject: [PATCH 5/8] Unified: Clean up remainders of the standalone Swift parser --- unified/BUILD.bazel | 15 +---- unified/extractor/BUILD.bazel | 10 +-- unified/swift-syntax-rs/BUILD.bazel | 61 +++++++------------ unified/swift-syntax-rs/swift-syntax-parse.sh | 12 ---- 4 files changed, 25 insertions(+), 73 deletions(-) delete mode 100755 unified/swift-syntax-rs/swift-syntax-parse.sh diff --git a/unified/BUILD.bazel b/unified/BUILD.bazel index 405500d7d407..1539c7fd2f9d 100644 --- a/unified/BUILD.bazel +++ b/unified/BUILD.bazel @@ -43,19 +43,9 @@ codeql_pkg_files( name = "extractor-arch", exes = [ "//unified/extractor", - ], - prefix = "tools/{CODEQL_PLATFORM}", -) - -# The Swift front-end parser (wrapper + real binary + bundled Swift runtime), -# shipped next to the extractor. Only on platforms where swift-syntax builds -# (Linux/macOS); elsewhere the group is empty so the pack still builds (Swift -# extraction is simply unavailable there). -pkg_filegroup( - name = "swift-syntax-parse-arch", - srcs = select_os( + ] + select_os( + linux = ["//unified/swift-syntax-rs:swift_runtime_libs"], otherwise = [], - posix = ["//unified/swift-syntax-rs:swift-syntax-parse-pkg"], ), prefix = "tools/{CODEQL_PLATFORM}", ) @@ -66,7 +56,6 @@ codeql_pack( ":codeql-extractor-yml", ":dbscheme-group", ":extractor-arch", - ":swift-syntax-parse-arch", "//unified/tools", ], ) diff --git a/unified/extractor/BUILD.bazel b/unified/extractor/BUILD.bazel index dea196ada0e5..8b175d7fc756 100644 --- a/unified/extractor/BUILD.bazel +++ b/unified/extractor/BUILD.bazel @@ -1,12 +1,6 @@ load("//misc/bazel:rust.bzl", "codeql_rust_binary") load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps") load("//unified:platforms.bzl", "UNIFIED_SUPPORTED_PLATFORMS") -load("//unified/swift-syntax-rs:swift_runtime.bzl", "swift_runtime_libs") - -swift_runtime_libs( - name = "swift_runtime_libs", - toolchain = "@swift_toolchain_ubuntu22.04//:files", -) exports_files(["Cargo.toml"]) @@ -19,8 +13,8 @@ codeql_rust_binary( "swift_node_types.yml", ], data = select({ - "@platforms//os:macos": [], - "@platforms//os:linux": [":swift_runtime_libs"], + "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"], + "//conditions:default": [], }), proc_macro_deps = all_crate_deps( proc_macro = True, diff --git a/unified/swift-syntax-rs/BUILD.bazel b/unified/swift-syntax-rs/BUILD.bazel index 8f2a5886fd22..0f04f3e9182c 100644 --- a/unified/swift-syntax-rs/BUILD.bazel +++ b/unified/swift-syntax-rs/BUILD.bazel @@ -1,6 +1,5 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") -load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles") load("//unified:platforms.bzl", "UNIFIED_SUPPORTED_PLATFORMS") load(":swift_runtime.bzl", "swift_runtime_libs") load(":xcode_transition.bzl", "xcode_transition_swift_library") @@ -12,8 +11,18 @@ swift_runtime_libs( toolchain = "@swift_toolchain_ubuntu22.04//:files", ) -# Swift FFI shim: wraps swift-syntax and exposes a small C ABI. The Rust -# targets below link against its `CcInfo`. +# The `$ORIGIN` runpath makes an executable look beside itself instead, so +# we find the Swift runtime libraries we bundle with the extractor. +cc_library( + name = "swift_runtime_rpath", + linkopts = select({ + "@platforms//os:linux": ["-Wl,-rpath,$$ORIGIN"], + "//conditions:default": [], + }), + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, +) + +# Swift FFI shim: wraps swift-syntax and exposes a small C ABI. xcode_transition_swift_library( name = "swift_syntax_ffi", srcs = ["swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift"], @@ -27,7 +36,7 @@ xcode_transition_swift_library( ) # Safe Rust bindings on top of the C ABI. Under Bazel the Swift side comes -# from `:swift_syntax_ffi`; `build.rs` is only used by the `cargo` workflow. +# from `:swift_syntax_ffi`. rust_library( name = "swift_syntax_rs", srcs = glob( @@ -37,61 +46,33 @@ rust_library( edition = "2024", target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, deps = [ + ":swift_runtime_rpath", ":swift_syntax_ffi", ], ) -# The Swift front-end parser. We ship it like `//swift/extractor`: a small shell -# wrapper (`swift-syntax-parse`) sets `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` to its -# own directory and execs the real binary (`swift-syntax-parse.real`); the Swift -# runtime shared libraries are packaged alongside them. `parse.rs` resolves the -# wrapper as a sibling of the extractor executable. +# A debugging aid, for looking at the raw swift-syntax JSON for some input: +# +# echo 'let x = 1' | bazel run //unified/swift-syntax-rs:swift-syntax-parse rust_binary( - name = "swift-syntax-parse.real", + name = "swift-syntax-parse", srcs = ["src/main.rs"], - # Target name carries `.real` (invalid in a crate name), so set it explicitly. - crate_name = "swift_syntax_parse", - # On Linux, carry the toolchain's runtime shared libraries as runfiles so - # they get packaged next to the binary. On macOS the OS provides the Swift - # runtime, so nothing extra is bundled. data = select({ - "@platforms//os:macos": [], "@platforms//os:linux": [":swift_runtime_libs"], + "//conditions:default": [], }), edition = "2024", target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, deps = [":swift_syntax_rs"], ) -# `swift-syntax-parse` wrapper (see `swift-syntax-parse.sh`). Its runfiles carry -# the real binary and the runtime libraries; packaging flattens them into one -# directory. -sh_binary( - name = "swift-syntax-parse", - srcs = ["swift-syntax-parse.sh"], - data = [":swift-syntax-parse.real"], - target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, -) - -# Packaged form for the extractor pack: the wrapper (as `swift-syntax-parse`), -# the real binary, and the runtime libraries, flattened into one directory. -codeql_pkg_runfiles( - name = "swift-syntax-parse-pkg", - # The `.sh` source is shipped as `swift-syntax-parse` (the wrapper); drop the - # original filename. - excludes = ["swift-syntax-parse.sh"], - exes = [":swift-syntax-parse"], - target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, - visibility = ["//unified:__pkg__"], -) - rust_test( name = "swift_syntax_rs_test", size = "small", crate = ":swift_syntax_rs", data = select({ - "@platforms//os:macos": [], "@platforms//os:linux": [":swift_runtime_libs"], + "//conditions:default": [], }), edition = "2024", target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, diff --git a/unified/swift-syntax-rs/swift-syntax-parse.sh b/unified/swift-syntax-rs/swift-syntax-parse.sh deleted file mode 100755 index 811697c820c6..000000000000 --- a/unified/swift-syntax-rs/swift-syntax-parse.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Wrapper that lets the shipped `swift-syntax-parse` find its Swift runtime -# libraries, which are packaged in the same directory as this script (and the -# real binary). Mirrors `swift/extractor/extractor.sh`. -if [[ "$(uname)" == Darwin ]]; then - export DYLD_LIBRARY_PATH=$(dirname "$0") -else - export LD_LIBRARY_PATH=$(dirname "$0") -fi - -exec -a "$0" "$0.real" "$@" From 8d9792ab7d46fae73a451033c2320d095732580a Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Thu, 6 Aug 2026 09:55:59 +0000 Subject: [PATCH 6/8] Unified: Make test runnable under Bazel --- unified/extractor/BUILD.bazel | 52 +++++++++++++++++++++++++ unified/extractor/tests/corpus_tests.rs | 21 +++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/unified/extractor/BUILD.bazel b/unified/extractor/BUILD.bazel index 8b175d7fc756..7959ffe384ad 100644 --- a/unified/extractor/BUILD.bazel +++ b/unified/extractor/BUILD.bazel @@ -1,3 +1,4 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") load("//misc/bazel:rust.bzl", "codeql_rust_binary") load("//misc/bazel/3rdparty/tree_sitter_extractors_deps:defs.bzl", "aliases", "all_crate_deps") load("//unified:platforms.bzl", "UNIFIED_SUPPORTED_PLATFORMS") @@ -29,3 +30,54 @@ codeql_rust_binary( "//unified/swift-syntax-rs:swift_syntax_rs", ], ) + +_TESTS = { + "corpus_tests": { + "data": glob(["tests/corpus/**"]), + "compile_data": [], + "size": "medium", + }, + # `include_str!`s a checked-in `parse_to_json` dump. + "swift_syntax_pipeline": { + "data": [], + "compile_data": glob(["tests/fixtures/**"]), + "size": "small", + }, +} + +[ + rust_test( + name = test_name, + size = spec["size"], + srcs = ["tests/%s.rs" % test_name] + glob(["src/**/*.rs"]), + aliases = aliases(), + compile_data = [ + "ast_types.yml", + "swift_node_types.yml", + ] + spec["compile_data"], + crate_root = "tests/%s.rs" % test_name, + data = spec["data"] + select({ + "@platforms//os:linux": ["//unified/swift-syntax-rs:swift_runtime_libs"], + "//conditions:default": [], + }), + edition = "2024", + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + rustc_flags = ["--cfg=bazel"], + target_compatible_with = UNIFIED_SUPPORTED_PLATFORMS, + deps = all_crate_deps( + normal = True, + ) + [ + "//shared/tree-sitter-extractor", + "//shared/yeast", + "//unified/swift-syntax-rs:swift_syntax_rs", + ], + ) + for test_name, spec in _TESTS.items() +] + +test_suite( + name = "all_tests", + tests = [":%s" % test_name for test_name in _TESTS], +) diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index 39cbab3194c7..319503ce4b63 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -96,11 +96,30 @@ fn collect_corpus_stems(dir: &Path, out: &mut Vec) { } } +#[cfg(bazel)] +fn corpus_dir() -> std::path::PathBuf { + let base = std::path::PathBuf::from( + std::env::var("RUNFILES_DIR").expect("RUNFILES_DIR not set"), + ); + std::fs::read_dir(&base) + .expect("failed to read RUNFILES_DIR") + .filter_map(Result::ok) + .map(|entry| entry.path().join("unified/extractor/tests/corpus")) + .find(|path| path.exists()) + .expect("corpus not found under any runfiles repo root") +} + +#[cfg(not(bazel))] +fn corpus_dir() -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/corpus") +} + + #[test] fn test_corpus() { let update_mode = update_mode_enabled(); let all_languages = languages::all_language_specs(); - let corpus_dir = Path::new("tests/corpus"); + let corpus_dir = corpus_dir(); for lang in all_languages { let output_schema = yeast::node_types_yaml::schema_from_yaml(languages::OUTPUT_AST_SCHEMA) From 808ef1538fb9fff541df76895d8670dd3cd42a1c Mon Sep 17 00:00:00 2001 From: Jeroen Ketema <93738568+jketema@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:13:48 +0000 Subject: [PATCH 7/8] Unified: Ensure extractor can find the Swift `.so` when built with cargo --- unified/extractor/build.rs | 10 ++++++++++ unified/swift-syntax-rs/Cargo.toml | 1 + unified/swift-syntax-rs/build.rs | 2 ++ 3 files changed, 13 insertions(+) create mode 100644 unified/extractor/build.rs diff --git a/unified/extractor/build.rs b/unified/extractor/build.rs new file mode 100644 index 000000000000..bf4053308e9f --- /dev/null +++ b/unified/extractor/build.rs @@ -0,0 +1,10 @@ +fn main() { + if let Ok(dir) = std::env::var("DEP_SWIFTSYNTAXFFI_LIBDIR") { + println!("cargo:rustc-link-search=native={dir}"); + println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); + println!("cargo:rustc-link-arg=-Wl,-rpath,{dir}"); + } + if let Ok(dir) = std::env::var("DEP_SWIFTSYNTAXFFI_RUNTIMEDIR") { + println!("cargo:rustc-link-arg=-Wl,-rpath,{dir}"); + } +} diff --git a/unified/swift-syntax-rs/Cargo.toml b/unified/swift-syntax-rs/Cargo.toml index 6957fb0e50e9..38470341060a 100644 --- a/unified/swift-syntax-rs/Cargo.toml +++ b/unified/swift-syntax-rs/Cargo.toml @@ -4,6 +4,7 @@ description = "Rust wrapper around the swift-syntax package for parsing Swift so version = "0.1.0" authors = ["GitHub"] edition = "2024" +links = "SwiftSyntaxFFI" [lib] name = "swift_syntax_rs" diff --git a/unified/swift-syntax-rs/build.rs b/unified/swift-syntax-rs/build.rs index 1056c5a62518..f1fc4b3df7cd 100644 --- a/unified/swift-syntax-rs/build.rs +++ b/unified/swift-syntax-rs/build.rs @@ -56,11 +56,13 @@ fn main() { println!("cargo:rustc-link-search=native={}", build_dir.display()); println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); println!("cargo:rustc-link-arg=-Wl,-rpath,{}", build_dir.display()); + println!("cargo:libdir={}", build_dir.display()); // The executable also needs to find the Swift runtime libraries at run time. if let Some(runtime) = swift_runtime_dir() { println!("cargo:rustc-link-search=native={}", runtime.display()); println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime.display()); + println!("cargo:runtimedir={}", runtime.display()); } } From c2d938389fe29ebf0e34d0585cc1d587bdb4b225 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Thu, 6 Aug 2026 14:16:58 +0200 Subject: [PATCH 8/8] Address Copilot review comments --- unified/extractor/build.rs | 2 ++ unified/extractor/tests/corpus_tests.rs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/unified/extractor/build.rs b/unified/extractor/build.rs index bf4053308e9f..22db82bc0ff3 100644 --- a/unified/extractor/build.rs +++ b/unified/extractor/build.rs @@ -1,4 +1,6 @@ fn main() { + println!("cargo:rustc-check-cfg=cfg(bazel)"); + if let Ok(dir) = std::env::var("DEP_SWIFTSYNTAXFFI_LIBDIR") { println!("cargo:rustc-link-search=native={dir}"); println!("cargo:rustc-link-lib=dylib=SwiftSyntaxFFI"); diff --git a/unified/extractor/tests/corpus_tests.rs b/unified/extractor/tests/corpus_tests.rs index 319503ce4b63..f0a3c448f12b 100644 --- a/unified/extractor/tests/corpus_tests.rs +++ b/unified/extractor/tests/corpus_tests.rs @@ -114,7 +114,6 @@ fn corpus_dir() -> std::path::PathBuf { std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/corpus") } - #[test] fn test_corpus() { let update_mode = update_mode_enabled();