diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml new file mode 100644 index 0000000..b1bfa70 --- /dev/null +++ b/.github/workflows/on-pr-colima-smoke.yaml @@ -0,0 +1,166 @@ +name: colima backend smoke + +# End-to-end check of `hops local` on the colima backend on macOS. +# Mirrors on-pr-kind-smoke.yaml with --backend colima and kubectl --context colima. +# +# Runner constraints: +# - Colima needs nested virtualization (Lima VM). Pin macos-15-intel — that +# image is known to support nested virt for Colima/Lima on GHA. +# - Do NOT use bare macos-latest alone (tracks arm images; arm64 GHA macOS +# historically fails with HV_UNSUPPORTED for nested virt). +# - Prefer localhost:30500 for registry traffic, not the Colima VM IP +# (macOS 15 Local Network Privacy can block non-root VM-IP access). +# +# Install: brew provides colima/docker/kubectl/helm only. Do not pre-start +# colima here — hops local start --backend colima owns cluster bring-up. +# +# Sizing: hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard +# macos-15-intel runners (~4 CPU / ~14 GiB). Pass explicit smaller sizes +# so the VZ VM can allocate; stop/start resume uses the persisted profile. + +on: + pull_request: + workflow_dispatch: + +jobs: + colima-smoke: + # Nested-virt-capable Intel pin (not bare macos-latest). + runs-on: macos-15-intel + # Nested virt + cold Crossplane pulls can take well over 30m. + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Install colima, docker, kubectl, helm + run: | + set -euxo pipefail + # Tools only — do not `colima start` here so the smoke exercises hops. + brew install colima docker kubectl helm + colima version + docker version --format '{{.Client.Version}}' || docker --version + kubectl version --client + helm version --short + + - name: Build hops + run: cargo build + + - name: Host resources (for sizing) + run: | + set -euxo pipefail + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + + - name: hops local start --backend colima + run: | + set -euxo pipefail + # Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ. + # Memory 10 (not 8): at 8Gi CoreDNS/metrics-server thrash and smoke pods + # sit in ContainerCreating without IPs even after images pull. Leave ~4Gi + # for host macOS + VZ. Disk 40: registry PVC requests 20Gi. + ./target/debug/hops-cli local start --backend colima \ + --cpus 3 --memory 10 --disk 40 + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS (nested virt settles) + run: | + set -euxo pipefail + # After cold start the control plane is still soft: probes time out and + # new pods stick in ContainerCreating. Wait for basics before registry. + kubectl --context colima wait --for=condition=Ready nodes --all --timeout=120s + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl --context colima -n kube-system get pods -o wide || true + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + docker pull public.ecr.aws/docker/library/busybox:stable + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + # Service-name pull: containerd resolves via its certs.d alias. + kubectl --context colima run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # localhost:30500 pull: what provider runtime pods reference. + kubectl --context colima run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # Nested virt: image pull can succeed while CNI/IP assignment lags. + if ! kubectl --context colima wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then + echo "==== smoke pods not Ready ====" + kubectl --context colima get pods smoke-svc-name smoke-localhost -o wide || true + kubectl --context colima describe pod smoke-svc-name smoke-localhost || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-svc-name || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-localhost || true + exit 1 + fi + kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + ./target/debug/hops-cli local stop + ./target/debug/hops-cli local start + # After colima VM stop/start, docker container IDs are gone and pods + # often sit in Error until kubelet recreates them. Wait generously and + # force a rollout restart if Available still stalls. + if ! kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s; then + echo "deployments not Available after resume; restarting..." + kubectl --context colima -n crossplane-system get pods -o wide || true + kubectl --context colima -n crossplane-system rollout restart \ + deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true + kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s + fi + ./target/debug/hops-cli local doctor + + # Capture live cluster state BEFORE destroy so failures are diagnosable. + - name: Debug dump on failure + if: failure() + run: | + set +e + echo "==== host ====" + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + echo "==== colima ====" + colima status || true + colima list || true + echo "==== cluster ====" + kubectl --context colima get nodes,pods -A -o wide || true + echo "==== smoke pods (default) ====" + kubectl --context colima describe pod smoke-svc-name smoke-localhost 2>/dev/null || true + echo "==== crossplane-system ====" + kubectl --context colima describe pods -n crossplane-system || true + kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -100 || true + echo "==== lima ha.stderr / serial ====" + for f in \ + "$HOME/.colima/_lima/colima/ha.stderr.log" \ + "$HOME/.colima/_lima/colima/ha.stdout.log" \ + "$HOME/.colima/_lima/colima/serial.log" \ + "$HOME/.colima/_lima/colima/serialv.log" + do + if [ -f "$f" ]; then + echo "----- $f -----" + tail -n 200 "$f" || true + fi + done + ./target/debug/hops-cli local doctor || true + + - name: hops local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/.github/workflows/on-pr-kind-smoke.yaml b/.github/workflows/on-pr-kind-smoke.yaml new file mode 100644 index 0000000..354d2e6 --- /dev/null +++ b/.github/workflows/on-pr-kind-smoke.yaml @@ -0,0 +1,60 @@ +name: kind backend smoke + +# End-to-end check of `hops local` on the kind backend, which is the CI +# path (ubuntu runners ship kind, docker, kubectl, and helm). Exercises the +# registry NodePort mapping and BOTH containerd certs.d trust names, plus the +# stop/start resume path. + +on: + pull_request: + +jobs: + kind-smoke: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Build hops + run: cargo build + + - name: hops local start --backend kind + run: ./target/debug/hops-cli local start --backend kind + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + docker pull public.ecr.aws/docker/library/busybox:stable + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + # Service-name pull: containerd resolves via its certs.d alias. + kubectl --context kind-hops run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # localhost:30500 pull: what provider runtime pods reference. + kubectl --context kind-hops run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + kubectl --context kind-hops wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=180s + kubectl --context kind-hops delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + ./target/debug/hops-cli local stop + ./target/debug/hops-cli local start + kubectl --context kind-hops -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry --timeout=300s + ./target/debug/hops-cli local doctor + + - name: hops local destroy + run: ./target/debug/hops-cli local destroy diff --git a/README.md b/README.md index c1547e8..a95e79d 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ This tool supports three related workflows: -- Local cluster setup on Colima +- Local cluster setup on colima or kind - Configuration package install/uninstall against the connected cluster - XR observe/manage/adopt/orphan workflows for existing infrastructure For local development, it can also: -- Install and manage Colima +- Install and manage a local cluster backend (colima or kind) - Start a local k8s cluster with Crossplane installed via Helm - Install the Kubernetes and Helm Crossplane providers - Deploy an in-cluster OCI registry (`crossplane-system/registry`) @@ -56,7 +56,7 @@ See "Releases" for available versions and changenotes. - `up` (Upbound CLI, used by `up project build`) - `aws` CLI v2 (used by `local aws` to export profile credentials) -Note: `hops-cli local install` installs `colima` through Homebrew. +Note: `hops-cli local install` installs the selected backend (`colima` or `kind`) through Homebrew. ## Build @@ -87,7 +87,7 @@ hops service --help `hops-cli` is organized into a few command groups: - `local` - - Manage a local Colima-based control plane, install providers, and bootstrap AWS or GitHub provider auth. + - Manage a local control plane (colima or kind backend), install providers, and bootstrap AWS or GitHub provider auth. - `config` - Build, install, reload, and uninstall Crossplane configuration packages against the connected cluster. - `secrets` @@ -192,7 +192,8 @@ Examples: ## Create a Local Control Plane ```bash -# 1) Install Colima (via Homebrew) +# 1) Install the backend (via Homebrew). Defaults to colima on macOS; +# pass --backend kind to use kind on any docker daemon. hops local install # 2) Start local k8s + Crossplane + providers + local registry @@ -211,6 +212,62 @@ hops local zitadel --source-context pat-local --domain auth.ops.com.ai hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0 ``` +### Cluster backends + +`hops local` supports three backends behind the same commands: + +- **colima** — a VM running dockerd + k3s. macOS/Linux; supports `--cpus`, + `--memory`, `--disk`, and `hops local resize`. +- **kind** — cluster nodes as docker containers on any reachable docker + daemon: Docker Desktop, colima's dockerd, or CI runners. No VM of its own, + so sizing flags don't apply (size the docker daemon instead); requires + kind >= v0.27. +- **dory** — [dory](https://augani.github.io/dory)'s built-in k3s, driven + headlessly through the `dory` CLI (`dory k8s enable/disable/status`). + Requires the Dory app running (it provides the engine and forwards + published ports to localhost) and a `dory` CLI with headless k8s support. + hops writes `~/.dory/k8s/registries.yaml` (k3s' native registry trust) and + publishes the registry NodePort at cluster create. The VM is sized in the + Dory app, so hops sizing flags don't apply. + +Select with the global `--backend` flag: + +```bash +hops local start --backend kind +``` + +The chosen backend is persisted to `~/.hops/local/backend` on a successful +start, so later commands (`stop`, `destroy`, `doctor`, package installs) +target the same cluster without the flag. Resolution order: `--backend` flag > +persisted choice > existing cluster detection (colima wins) > platform +default (macOS: colima, otherwise kind). + +Unless `--context` is given, kubectl commands automatically use the backend's +kubeconfig context (`colima`, `kind-hops`, or `dory`), regardless of your +current-context. For dory, hops also prepends `~/.kube/dory-config` (where +dory keeps its kubeconfig) to `KUBECONFIG` for its own kubectl/helm calls. + +#### Using dory + +With a `dory` CLI that supports `dory k8s enable` (headless Kubernetes), +use the native backend: + +```bash +hops local start --backend dory +``` + +For `hops provider install` / `hops config install` builds, point your docker +CLI at dory's engine (`export DOCKER_HOST=unix://$HOME/.dory/engine.sock`) so +image builds/pushes land on the daemon that reaches the registry. + +Without the headless CLI, dory still exposes a real docker socket, so the +kind backend works against it: + +```bash +docker context use dory # or: export DOCKER_HOST=unix://$HOME/.dory/dory.sock +hops local start --backend kind +``` + ### Local provider setup and auth `hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only. diff --git a/src/commands/auth/bootstrap.rs b/src/commands/auth/bootstrap.rs index fdc5e35..7368483 100644 --- a/src/commands/auth/bootstrap.rs +++ b/src/commands/auth/bootstrap.rs @@ -66,7 +66,10 @@ pub fn run(args: &BootstrapArgs) -> Result<(), Box> { fn write_secret(path: &PathBuf, value: &str, force: bool) -> Result<(), Box> { if path.exists() && !force { - log::info!(" skip {}: already present (use --force to overwrite)", path.display()); + log::info!( + " skip {}: already present (use --force to overwrite)", + path.display() + ); return Ok(()); } if let Some(parent) = path.parent() { @@ -151,11 +154,7 @@ mod tests { "no lowercase: {}", pwd ); - assert!( - pwd.chars().any(|c| c.is_ascii_digit()), - "no digit: {}", - pwd - ); + assert!(pwd.chars().any(|c| c.is_ascii_digit()), "no digit: {}", pwd); assert!( pwd.chars().any(|c| !c.is_ascii_alphanumeric()), "no symbol: {}", diff --git a/src/commands/config/install.rs b/src/commands/config/install.rs index fb6385f..dd396c2 100644 --- a/src/commands/config/install.rs +++ b/src/commands/config/install.rs @@ -1,14 +1,12 @@ +use crate::commands::local::backend::{self, Backend}; +use crate::commands::local::package_install::run_watch; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout, ensure_registry, image_config_name, parse_docker_push_digest, parse_repo_spec, resolve_repo_install_target, rewrite_registry, rewrite_registry_with_tag, sanitize_name_component, short_hash, split_ref, strip_registry, - unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH, -}; -use crate::commands::local::package_install::run_watch; -use crate::commands::local::{ - kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output, sync_registry_hosts_entry, - HOPS_KUBE_CONTEXT_ENV, + unique_suffix, RepoInstallTarget, RepoSpec, REGISTRY_PULL, REGISTRY_PUSH, }; +use crate::commands::local::{kubectl_apply_stdin, kubectl_command, run_cmd, run_cmd_output}; use clap::Args; use flate2::read::GzDecoder; use serde::Deserialize; @@ -43,6 +41,10 @@ pub struct ConfigArgs { #[arg(long)] pub context: Option, + /// Local cluster backend whose node should be wired for local package pulls. + #[arg(long, value_enum)] + pub backend: Option, + /// Watch the project directory for changes and re-run install automatically #[arg(long, conflicts_with = "repo")] pub watch: bool, @@ -108,17 +110,22 @@ struct PackageResource { } pub fn run(args: &ConfigArgs) -> Result<(), Box> { - if let Some(ctx) = &args.context { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::activate(args.backend, args.context.as_deref()); match (args.repo.as_deref(), args.version.as_deref()) { (Some(repo), Some(version)) => { apply_repo_version(repo, version, args.skip_dependency_resolution) } - (Some(repo), None) => run_repo_install(repo, args.skip_dependency_resolution), + (Some(repo), None) => run_repo_install( + repo, + args.skip_dependency_resolution, + backend, + args.backend, + args.context.as_deref(), + ), (None, _) => { let path = args.path.as_deref().unwrap_or("."); + prepare_local_registry(backend, args.backend, args.context.as_deref())?; run_local_path(path, args.skip_dependency_resolution)?; if args.watch { @@ -137,10 +144,19 @@ pub fn run(args: &ConfigArgs) -> Result<(), Box> { fn run_repo_install( repo: &str, skip_dependency_resolution: bool, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { - RepoInstallTarget::SourceBuild => run_repo_clone(&spec, skip_dependency_resolution), + RepoInstallTarget::SourceBuild => run_repo_clone( + &spec, + skip_dependency_resolution, + backend, + backend_flag, + context, + ), RepoInstallTarget::PublishedVersion(version) => { apply_repo_version_spec(&spec, &version, skip_dependency_resolution) } @@ -150,11 +166,24 @@ fn run_repo_install( fn run_repo_clone( spec: &RepoSpec, skip_dependency_resolution: bool, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { - let cache_path = ensure_cached_repo_checkout(&spec)?; + let cache_path = ensure_cached_repo_checkout(spec)?; + prepare_local_registry(backend, backend_flag, context)?; run_local_path(&cache_path.to_string_lossy(), skip_dependency_resolution) } +fn prepare_local_registry( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + ensure_registry()?; + backend::wire_local_registry_for_target(backend, backend_flag, context) +} + fn apply_repo_version_spec( spec: &RepoSpec, version: &str, @@ -216,18 +245,12 @@ fn apply_repo_version( apply_repo_version_spec(&spec, version, skip_dependency_resolution) } -fn run_local_path( - path: &str, - skip_dependency_resolution: bool, -) -> Result<(), Box> { +fn run_local_path(path: &str, skip_dependency_resolution: bool) -> Result<(), Box> { let dir = Path::new(path); if !dir.is_dir() { return Err(format!("{} is not a directory", path).into()); } - ensure_registry()?; - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; - // Build the Crossplane package log::info!("Building Crossplane package in {}...", path); let status = Command::new("up") @@ -246,7 +269,7 @@ fn run_local_path( let packages: Vec<_> = fs::read_dir(&output_dir) .map_err(|e| format!("Failed to read {}: {}", output_dir.display(), e))? .filter_map(|entry| entry.ok()) - .filter(|entry| entry.path().extension().map_or(false, |ext| ext == "uppkg")) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "uppkg")) .collect(); if packages.is_empty() { @@ -1047,6 +1070,15 @@ spec: assert!(!without_skip.contains("skipDependencyResolution: true")); } + #[test] + fn local_registry_wiring_skips_foreign_context_without_backend_flag() { + assert!(!backend::should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + #[test] fn package_source_strips_tag_and_digest() { assert_eq!( diff --git a/src/commands/local/backend/colima.rs b/src/commands/local/backend/colima.rs new file mode 100644 index 0000000..15534eb --- /dev/null +++ b/src/commands/local/backend/colima.rs @@ -0,0 +1,523 @@ +//! Colima backend: VM + dockerd + k3s (docker runtime). + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL}; +use crate::commands::local::{run_cmd, run_cmd_output, wait_for_kubernetes}; +use dialoguer::Confirm; +use serde::Deserialize; +use std::error::Error; +use std::io::{IsTerminal, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +const DEFAULT_CPUS: u32 = 8; +const DEFAULT_MEMORY_GIB: u32 = 16; +const DEFAULT_DISK_GIB: u32 = 60; +const GIB: u64 = 1024 * 1024 * 1024; + +pub fn install() -> Result<(), Box> { + log::info!("Installing Colima via Homebrew..."); + run_cmd("brew", &["install", "colima"])?; + log::info!("Colima installed successfully"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling Colima..."); + run_cmd("brew", &["uninstall", "colima"])?; + log::info!("Colima uninstalled"); + Ok(()) +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping Colima..."); + run_cmd("colima", &["stop"])?; + log::info!("Colima stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Destroying Colima VM..."); + run_cmd("colima", &["delete", "--force"])?; + log::info!("Colima VM destroyed"); + Ok(()) +} + +pub fn reset() -> Result<(), Box> { + log::info!("Resetting Colima Kubernetes..."); + run_cmd("colima", &["kubernetes", "reset"])?; + log::info!("Colima Kubernetes reset complete"); + Ok(()) +} + +pub fn start(size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { + let instance = colima_instance()?; + validate_requested_size(size, instance.as_ref())?; + start_or_resize_colima(size, assume_yes, instance.as_ref()) +} + +pub fn resize(size: &SizeArgs) -> Result<(), Box> { + if !size.any_set() { + return Err("Specify at least one of --cpus, --memory, or --disk".into()); + } + + let instance = colima_instance()?; + let instance = instance + .as_ref() + .ok_or("No Colima instance exists yet; use `hops local start` to create one")?; + + validate_requested_size(size, Some(instance))?; + resize_existing_colima(size, Some(instance)) +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +struct ColimaInstance { + #[serde(default)] + status: String, + #[serde(default)] + cpus: Option, + #[serde(default)] + memory: Option, + #[serde(default)] + disk: Option, +} + +impl ColimaInstance { + fn is_running(&self) -> bool { + self.status.eq_ignore_ascii_case("running") + } + + fn memory_gib(&self) -> Option { + self.memory.map(bytes_to_gib) + } + + fn disk_gib(&self) -> Option { + self.disk.map(bytes_to_gib) + } +} + +fn start_or_resize_colima( + size: &SizeArgs, + assume_yes: bool, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + let is_running = instance.map(ColimaInstance::is_running).unwrap_or(false); + + if is_running && size.any_set() { + let changes = requested_size_changes(size, instance.expect("checked is_running")); + if !changes.is_empty() { + confirm_running_resize(size, assume_yes, &changes)?; + resize_existing_colima(size, instance)?; + return Ok(()); + } + + log::info!("Requested Colima size already matches the running VM"); + } + + log::info!("Starting Colima with Kubernetes..."); + + let start_size = if is_running { + SizeArgs::default() + } else { + size.clone() + }; + let include_defaults = instance.is_none(); + start_colima(&start_size, include_defaults) +} + +fn resize_existing_colima( + size: &SizeArgs, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + if instance.map(ColimaInstance::is_running).unwrap_or(false) { + log::info!("Stopping Colima to apply requested size..."); + run_cmd("colima", &["stop"])?; + } + + log::info!("Starting Colima with requested size..."); + start_colima(size, false) +} + +fn start_colima(size: &SizeArgs, include_defaults: bool) -> Result<(), Box> { + let args = colima_start_args(size, include_defaults); + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + run_cmd("colima", &refs) +} + +fn colima_start_args(size: &SizeArgs, include_defaults: bool) -> Vec { + let mut args = vec!["start".to_string(), "--kubernetes".to_string()]; + + if let Some(cpus) = size.cpus.or(include_defaults.then_some(DEFAULT_CPUS)) { + args.push("--cpus".to_string()); + args.push(cpus.to_string()); + } + if let Some(memory) = size + .memory + .or(include_defaults.then_some(DEFAULT_MEMORY_GIB)) + { + args.push("--memory".to_string()); + args.push(memory.to_string()); + } + if let Some(disk) = size.disk.or(include_defaults.then_some(DEFAULT_DISK_GIB)) { + args.push("--disk".to_string()); + args.push(disk.to_string()); + } + + args +} + +fn confirm_running_resize( + size: &SizeArgs, + assume_yes: bool, + changes: &[String], +) -> Result<(), Box> { + if assume_yes { + return Ok(()); + } + + let change_text = changes.join(", "); + let resize_command = format!("hops local resize{}", size.command_suffix()); + let start_command = format!("hops local start{} --yes", size.command_suffix()); + + if !std::io::stdin().is_terminal() { + return Err(format!( + "Colima is already running with different size ({change_text}). Run `{resize_command}` first, or rerun `{start_command}` to stop and resize automatically." + ) + .into()); + } + + let confirmed = Confirm::new() + .with_prompt(format!( + "Colima is already running with different size ({change_text}). Stop and restart it now?" + )) + .default(false) + .interact()?; + + if confirmed { + Ok(()) + } else { + Err(format!( + "Colima size was not changed. Run `{resize_command}` first, then rerun `hops local start`." + ) + .into()) + } +} + +fn validate_requested_size( + size: &SizeArgs, + instance: Option<&ColimaInstance>, +) -> Result<(), Box> { + if let (Some(requested), Some(current)) = + (size.disk, instance.and_then(ColimaInstance::disk_gib)) + { + if requested < current { + return Err(format!( + "Colima disk cannot be shrunk from {current}GiB to {requested}GiB. Use --disk {current} or larger, or destroy and recreate the VM." + ) + .into()); + } + } + + Ok(()) +} + +fn requested_size_changes(size: &SizeArgs, instance: &ColimaInstance) -> Vec { + let mut changes = Vec::new(); + + if let Some(requested) = size.cpus { + match instance.cpus { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("cpus {current} -> {requested}")), + None => changes.push(format!("cpus unknown -> {requested}")), + } + } + if let Some(requested) = size.memory { + match instance.memory_gib() { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("memory {current}GiB -> {requested}GiB")), + None => changes.push(format!("memory unknown -> {requested}GiB")), + } + } + if let Some(requested) = size.disk { + match instance.disk_gib() { + Some(current) if requested == current => {} + Some(current) => changes.push(format!("disk {current}GiB -> {requested}GiB")), + None => changes.push(format!("disk unknown -> {requested}GiB")), + } + } + + changes +} + +/// Whether a Colima instance exists (running or stopped). Missing binary or +/// failing command reads as "no instance". +pub fn instance_exists() -> bool { + matches!(colima_instance(), Ok(Some(_))) +} + +fn colima_instance() -> Result, Box> { + let output = match run_cmd_output("colima", &["list", "--json"]) { + Ok(output) => output, + Err(_) => return Ok(None), + }; + + parse_colima_list(&output) +} + +fn parse_colima_list(output: &str) -> Result, Box> { + let trimmed = output.trim(); + if trimmed.is_empty() { + return Ok(None); + } + + if let Ok(instance) = serde_json::from_str::(trimmed) { + return Ok(Some(instance)); + } + + if let Ok(instances) = serde_json::from_str::>(trimmed) { + return Ok(instances.into_iter().next()); + } + + for line in trimmed + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + if let Ok(instance) = serde_json::from_str::(line) { + return Ok(Some(instance)); + } + } + + Err("Unable to parse `colima list --json` output".into()) +} + +fn bytes_to_gib(bytes: u64) -> u32 { + (bytes / GIB) as u32 +} + +/// Add the cluster-internal registry to Docker's insecure-registries list +/// inside the Colima VM. Docker defaults to HTTPS for non-localhost registries; +/// our in-cluster registry speaks plain HTTP. +pub fn configure_docker_insecure_registry() -> Result<(), Box> { + let config = run_cmd_output("colima", &["ssh", "--", "cat", "/etc/docker/daemon.json"])?; + + if config.contains("insecure-registries") { + return Ok(()); + } + + log::info!("Configuring Docker for insecure local registry..."); + + // Insert the insecure-registries key before the final closing brace. + let new_config = if let Some(pos) = config.rfind('}') { + let prefix = config[..pos].trim_end(); + format!( + "{},\n \"insecure-registries\": [\"{}\"]\n}}\n", + prefix, REGISTRY_PULL + ) + } else { + return Err("Invalid daemon.json: no closing brace".into()); + }; + + let mut child = Command::new("colima") + .args(["ssh", "--", "sudo", "tee", "/etc/docker/daemon.json"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(new_config.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err("Failed to write Docker daemon.json".into()); + } + + log::info!("Restarting Docker daemon..."); + run_cmd( + "colima", + &["ssh", "--", "sudo", "systemctl", "restart", "docker"], + )?; + + // Wait for Docker to come back. + for _ in 0..30 { + if run_cmd_output("docker", &["info"]).is_ok() { + // Docker restart can temporarily disrupt the Kubernetes API. + wait_for_kubernetes()?; + return Ok(()); + } + thread::sleep(Duration::from_secs(2)); + } + Err("Docker did not come back after restart".into()) +} + +/// Ensure Colima's /etc/hosts maps the registry hostname to the current +/// ClusterIP so the kubelet's docker daemon can resolve pull refs. +pub fn sync_hosts_entry(cluster_ip: &str) -> Result<(), Box> { + let hostname = REGISTRY_HOSTNAME; + let current_ip = run_cmd_output( + "colima", + &[ + "ssh", + "--", + "sh", + "-c", + &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), + ], + ) + .unwrap_or_default(); + if current_ip.trim() == cluster_ip { + return Ok(()); + } + + log::info!("Updating hosts entry: {} -> {}", hostname, cluster_ip); + + let escaped_host = hostname.replace('.', "\\."); + run_cmd( + "colima", + &[ + "ssh", + "--", + "sudo", + "sed", + "-i", + &format!("/{}/d", escaped_host), + "/etc/hosts", + ], + )?; + run_cmd( + "colima", + &[ + "ssh", + "--", + "sudo", + "sh", + "-c", + &format!("echo '{} {}' >> /etc/hosts", cluster_ip, hostname), + ], + )?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn instance(status: &str, cpus: u32, memory_gib: u32, disk_gib: u32) -> ColimaInstance { + ColimaInstance { + status: status.to_string(), + cpus: Some(cpus), + memory: Some(memory_gib as u64 * GIB), + disk: Some(disk_gib as u64 * GIB), + } + } + + #[test] + fn colima_start_args_use_hops_defaults_for_new_profiles() { + let args = colima_start_args(&SizeArgs::default(), true); + + assert_eq!( + args, + vec![ + "start", + "--kubernetes", + "--cpus", + "8", + "--memory", + "16", + "--disk", + "60" + ] + ); + } + + #[test] + fn colima_start_args_pass_only_requested_size_for_existing_profiles() { + let size = SizeArgs { + cpus: Some(12), + memory: Some(32), + disk: None, + }; + + let args = colima_start_args(&size, false); + + assert_eq!( + args, + vec!["start", "--kubernetes", "--cpus", "12", "--memory", "32"] + ); + } + + #[test] + fn requested_size_changes_compare_only_explicit_fields() { + let current = instance("Running", 8, 16, 60); + let size = SizeArgs { + cpus: None, + memory: Some(32), + disk: None, + }; + + assert_eq!( + requested_size_changes(&size, ¤t), + vec!["memory 16GiB -> 32GiB"] + ); + } + + #[test] + fn requested_size_changes_treat_missing_current_value_as_change() { + let current = ColimaInstance { + status: "Running".to_string(), + cpus: None, + memory: None, + disk: None, + }; + let size = SizeArgs { + cpus: Some(12), + memory: None, + disk: None, + }; + + assert_eq!( + requested_size_changes(&size, ¤t), + vec!["cpus unknown -> 12"] + ); + } + + #[test] + fn parse_colima_list_accepts_single_object() { + let output = r#"{"name":"default","status":"Stopped","arch":"aarch64","cpus":8,"memory":17179869184,"disk":64424509440,"runtime":"docker+k3s"}"#; + + let parsed = parse_colima_list(output).expect("parse").expect("instance"); + + assert_eq!(parsed.status, "Stopped"); + assert_eq!(parsed.cpus, Some(8)); + assert_eq!(parsed.memory_gib(), Some(16)); + assert_eq!(parsed.disk_gib(), Some(60)); + } + + #[test] + fn parse_colima_list_accepts_array_output() { + let output = r#"[{"status":"Running","cpus":12,"memory":34359738368,"disk":107374182400}]"#; + + let parsed = parse_colima_list(output).expect("parse").expect("instance"); + + assert!(parsed.is_running()); + assert_eq!(parsed.cpus, Some(12)); + assert_eq!(parsed.memory_gib(), Some(32)); + assert_eq!(parsed.disk_gib(), Some(100)); + } + + #[test] + fn validate_requested_size_rejects_disk_shrink() { + let current = instance("Stopped", 8, 16, 100); + let size = SizeArgs { + cpus: None, + memory: None, + disk: Some(60), + }; + + let err = validate_requested_size(&size, Some(¤t)).expect_err("disk shrink"); + + assert!(err.to_string().contains("cannot be shrunk")); + } +} diff --git a/src/commands/local/backend/dory.rs b/src/commands/local/backend/dory.rs new file mode 100644 index 0000000..1c0dfed --- /dev/null +++ b/src/commands/local/backend/dory.rs @@ -0,0 +1,741 @@ +//! dory backend: k3s in a container on dory's shared-VM dockerd +//! (https://augani.github.io/dory), driven headlessly through the `dory` CLI +//! (`dory k8s enable|disable|status`) and the engine docker socket. +//! +//! Registry plumbing differs from colima/kind because the cluster container +//! has create-time config shared with the Dory app: +//! - published NodePorts are static config: hops writes `~/.dory/k8s/ports` +//! BEFORE `dory k8s enable`, so GUI-side recreates keep the same port. +//! - trust is static config: hops writes `~/.dory/k8s/registries.yaml` +//! BEFORE `dory k8s enable`; dory bind-mounts it and k3s reads it at boot, +//! aliasing both pull names to the registry Service hostname over HTTP. +//! - name resolution is dynamic: `wire_registry` syncs the hostname -> +//! ClusterIP in the node container's /etc/hosts on every start (same +//! re-wire-on-start model as the other backends). +//! - the host reaches `localhost:30500` because `dory k8s enable --publish +//! 30500:30500` publishes the NodePort and the Dory app's port forwarder +//! maps published ports to host loopback. + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH}; +use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use std::error::Error; +use std::path::PathBuf; + +const NODE_CONTAINER: &str = "dory-k8s"; +/// hops' registry NodePort, published on the cluster container at create. +const REGISTRY_PORT_PUBLISH: &str = "30500:30500"; +const REGISTRIES_BEGIN: &str = "# BEGIN hops-managed (do not edit inside)"; +const REGISTRIES_END: &str = "# END hops-managed"; + +fn home() -> Result> { + Ok(PathBuf::from(std::env::var("HOME").map_err(|_| { + "HOME is not set; unable to locate dory's state directory" + })?)) +} + +fn engine_socket() -> Result> { + Ok(home()?.join(".dory/engine.sock")) +} + +/// dory's side-file kubeconfig (context name `dory`). Current dory also +/// merges the context into ~/.kube/config at enable time; the side file is +/// the pre-merge fallback and dory's own `--kubeconfig` input. +pub fn kubeconfig_path() -> Option { + home() + .ok() + .map(|h| h.join(".kube/dory-config").to_string_lossy().into_owned()) +} + +fn registries_yaml_path() -> Result> { + Ok(home()?.join(".dory/k8s/registries.yaml")) +} + +fn ports_file_path() -> Result> { + Ok(home()?.join(".dory/k8s/ports")) +} + +/// k3s' native registry config: alias both pull names to the registry +/// Service hostname over plain HTTP. The hostname resolves through the +/// node's /etc/hosts, which `wire_registry` keeps pointed at the ClusterIP. +fn registries_yaml() -> String { + format!( + "# Written by `hops local start --backend dory`.\n\ + # k3s reads this at boot; edits require `hops local reset`.\n\ + # Hops manages only the marked block; keep user mirrors outside it.\n\ + mirrors:\n\ + {block}", + block = registries_yaml_block(), + ) +} + +fn registries_yaml_block() -> String { + format!( + " {begin}\n\ + \x20\x20# hops: mirror {pull}\n\ + \x20\x20\"{pull}\":\n\ + \x20\x20 endpoint:\n\ + \x20\x20 - \"http://{pull}\"\n\ + \x20\x20# hops: mirror {push}\n\ + \x20\x20\"{push}\":\n\ + \x20\x20 endpoint:\n\ + \x20\x20 - \"http://{pull}\"\n\ + \x20\x20{end}\n", + begin = REGISTRIES_BEGIN, + end = REGISTRIES_END, + pull = REGISTRY_PULL, + push = REGISTRY_PUSH, + ) +} + +fn legacy_registries_yaml() -> String { + format!( + "# Written by `hops local start --backend dory`.\n\ + # k3s reads this at boot; edits require `hops local reset`.\n\ + mirrors:\n\ + \x20 \"{pull}\":\n\ + \x20 endpoint:\n\ + \x20 - \"http://{pull}\"\n\ + \x20 \"{push}\":\n\ + \x20 endpoint:\n\ + \x20 - \"http://{pull}\"\n", + pull = REGISTRY_PULL, + push = REGISTRY_PUSH, + ) +} + +/// Run docker against dory's engine socket (the daemon the Dory app manages). +fn engine_docker(args: &[&str]) -> Result<(), Box> { + let sock = format!("unix://{}", engine_socket()?.display()); + let mut full = vec!["-H", sock.as_str()]; + full.extend_from_slice(args); + run_cmd("docker", &full) +} + +fn engine_docker_output(args: &[&str]) -> Result> { + let sock = format!("unix://{}", engine_socket()?.display()); + let mut full = vec!["-H", sock.as_str()]; + full.extend_from_slice(args); + run_cmd_output("docker", &full) +} + +pub fn install() -> Result<(), Box> { + log::info!("Installing Dory via Homebrew..."); + run_cmd("brew", &["install", "--cask", "Augani/dory/dory"])?; + log::info!("Dory installed; launch the Dory app once so it provisions its engine"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling Dory..."); + run_cmd("brew", &["uninstall", "--cask", "dory"])?; + log::info!("Dory uninstalled"); + Ok(()) +} + +pub fn start(size: &SizeArgs) -> Result<(), Box> { + if size.any_set() { + return Err(format!( + "the dory backend's VM is sized by the Dory app, not hops; drop{}", + size.command_suffix() + ) + .into()); + } + + preflight()?; + write_ports_file()?; + write_registries_yaml()?; + + // Creates, restarts, or reuses the dory-k8s container as needed. If the + // running container has create-time config drift, dory exits 3 rather than + // destroying state; surface that plus the hops reset path. + run_dory_enable(false) +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping dory k8s node '{}'...", NODE_CONTAINER); + engine_docker(&["stop", NODE_CONTAINER])?; + log::info!("dory cluster stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Deleting dory k8s cluster..."); + run_cmd("dory", &["k8s", "disable"])?; + log::info!("dory cluster deleted"); + Ok(()) +} + +/// The cluster container IS the cluster, so reset means recreate. +pub fn reset() -> Result<(), Box> { + preflight()?; + run_cmd("dory", &["k8s", "disable"])?; + write_ports_file()?; + write_registries_yaml()?; + run_dory_enable(true) +} + +pub fn resize(_size: &SizeArgs) -> Result<(), Box> { + Err("the dory backend has no hops-managed VM to resize; \ + adjust resources in the Dory app instead" + .into()) +} + +/// Whether the hops-relevant dory cluster exists (running or stopped). +/// Missing app/engine/CLI reads as "no cluster". +pub fn cluster_exists() -> bool { + let Ok(sock) = engine_socket() else { + return false; + }; + if !sock.exists() { + return false; + } + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]).is_ok() +} + +fn preflight() -> Result<(), Box> { + if !command_exists("dory") { + return Err("the `dory` CLI is not on PATH; link it from the dory repo \ + (`ln -sf /scripts/dory /opt/homebrew/bin/dory`)" + .into()); + } + let sock = engine_socket()?; + if !sock.exists() { + return Err(format!( + "dory's engine socket ({}) is missing; launch the Dory app and wait for its engine to start", + sock.display() + ) + .into()); + } + if !command_exists("docker") { + return Err("docker CLI not found; install it (dory provides the daemon)".into()); + } + Ok(()) +} + +/// Write the static registry trust config read by k3s at boot. Must exist +/// before `dory k8s enable` because dory binds it at container create. +fn write_registries_yaml() -> Result<(), Box> { + let path = registries_yaml_path()?; + let current = match std::fs::read_to_string(&path) { + Ok(content) => Some(content), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, + Err(err) => return Err(err.into()), + }; + let desired = merge_registries_yaml(current.as_deref())?; + if current.as_deref() == Some(desired.as_str()) { + return Ok(()); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + log::info!("Writing k3s registry config: {}", path.display()); + std::fs::write(&path, desired)?; + Ok(()) +} + +fn write_ports_file() -> Result<(), Box> { + let path = ports_file_path()?; + let current = match std::fs::read_to_string(&path) { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(err) => return Err(err.into()), + }; + let desired = ports_file_with_publish(¤t); + if current == desired { + return Ok(()); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + log::info!( + "Ensuring dory k8s port config includes {}: {}", + REGISTRY_PORT_PUBLISH, + path.display() + ); + std::fs::write(&path, desired)?; + Ok(()) +} + +/// Seconds after the Dory app (re)creates its engine socket during which it is +/// still provisioning the engine. A dockerd restart at the end of that window +/// SIGTERMs every container — including a k3s node enabled meanwhile, which +/// reports Ready and then dies under the bootstrap (observed ~90s on Dory +/// 0.2.0; padded for slower machines). +const ENGINE_LAUNCH_WINDOW_SECS: u64 = 180; + +/// Age of the current engine session: the app recreates the engine socket at +/// launch, so its mtime marks when provisioning began. None when unreadable. +fn engine_session_age() -> Option { + let sock = engine_socket().ok()?; + let modified = std::fs::metadata(&sock).ok()?.modified().ok()?; + std::time::SystemTime::now().duration_since(modified).ok() +} + +/// Time left inside the app's provisioning window for a given engine session +/// age; None once the window has passed. +fn launch_window_remaining(age: std::time::Duration) -> Option { + std::time::Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS) + .checked_sub(age) + .filter(|remaining| !remaining.is_zero()) +} + +fn node_running() -> bool { + engine_docker_output(&["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER]) + .map(|state| state.trim() == "true") + .unwrap_or(false) +} + +/// Hold a freshly-enabled cluster under observation while the Dory app may +/// still be provisioning its engine, re-enabling if the engine restart takes +/// the node down. Immediate no-op when the window has already passed, so +/// steady-state starts pay one container inspect and nothing more. +fn hold_through_engine_launch_window() -> Result<(), Box> { + let in_window = |age: Option| { + age.map(|a| launch_window_remaining(a).is_some()) + .unwrap_or(false) + }; + if in_window(engine_session_age()) { + log::info!( + "Dory engine session is younger than {}s; watching the k8s node through the app's provisioning window...", + ENGINE_LAUNCH_WINDOW_SECS + ); + } + let mut reenables = 0; + loop { + match (node_running(), in_window(engine_session_age())) { + (true, false) => return Ok(()), + (true, true) => {} + (false, _) => { + if reenables >= 3 { + return Err("the dory engine keeps stopping the k8s node during app startup; \ + wait for the Dory app to finish provisioning, then re-run `hops local start --backend dory`" + .into()); + } + reenables += 1; + log::warn!( + "dory engine restart stopped the k8s node; re-enabling ({}/3)...", + reenables + ); + let args = dory_enable_args(false); + run_cmd("dory", &args)?; + } + } + std::thread::sleep(std::time::Duration::from_secs(3)); + } +} + +fn run_dory_enable(recreate: bool) -> Result<(), Box> { + let args = dory_enable_args(recreate); + match run_cmd("dory", &args) { + Ok(()) => hold_through_engine_launch_window(), + Err(err) if !recreate && err.to_string().contains("exit status: 3") => Err(format!( + "{}\nhint: run `hops local reset --backend dory` to recreate the dory cluster and apply create-time config drift", + err + ) + .into()), + Err(err) => Err(err), + } +} + +fn dory_enable_args(recreate: bool) -> Vec<&'static str> { + let mut args = vec!["k8s", "enable"]; + if recreate { + args.push("--recreate"); + } + args.extend(["--publish", REGISTRY_PORT_PUBLISH]); + args +} + +fn merge_registries_yaml(existing: Option<&str>) -> Result> { + let Some(existing) = existing else { + return Ok(registries_yaml()); + }; + if existing.trim().is_empty() || existing == legacy_registries_yaml() { + return Ok(registries_yaml()); + } + + match ( + existing.find(REGISTRIES_BEGIN), + existing.find(REGISTRIES_END), + ) { + (Some(begin), Some(end)) if begin <= end => replace_registries_block(existing, begin, end), + (Some(_), Some(_)) | (Some(_), None) | (None, Some(_)) => { + Err(format!("malformed dory registries.yaml managed block; expected `{REGISTRIES_BEGIN}` before `{REGISTRIES_END}`").into()) + } + (None, None) => insert_registries_block(existing), + } +} + +fn replace_registries_block( + existing: &str, + begin: usize, + end: usize, +) -> Result> { + let line_start = existing[..begin].rfind('\n').map_or(0, |idx| idx + 1); + let line_end = existing[end..] + .find('\n') + .map_or(existing.len(), |idx| end + idx + 1); + let mut merged = String::with_capacity(existing.len() + registries_yaml_block().len()); + merged.push_str(&existing[..line_start]); + merged.push_str(®istries_yaml_block()); + merged.push_str(&existing[line_end..]); + Ok(merged) +} + +fn insert_registries_block(existing: &str) -> Result> { + if contains_hops_mirror_key(existing) { + return Err("dory registries.yaml already contains hops registry mirror entries without managed markers; remove those entries or wrap them in the hops-managed block" + .into()); + } + + let mut merged = ensure_trailing_newline(existing); + if let Some(insert_at) = mirrors_line_insert_position(&merged) { + merged.insert_str(insert_at, ®istries_yaml_block()); + return Ok(merged); + } + + merged.push_str("mirrors:\n"); + merged.push_str(®istries_yaml_block()); + Ok(merged) +} + +fn contains_hops_mirror_key(content: &str) -> bool { + content.contains(&format!("\"{}\":", REGISTRY_PULL)) + || content.contains(&format!("\"{}\":", REGISTRY_PUSH)) +} + +fn mirrors_line_insert_position(content: &str) -> Option { + let mut offset = 0; + for line in content.split_inclusive('\n') { + let without_newline = line.trim_end_matches('\n').trim_end_matches('\r'); + if is_top_level_mirrors_line(without_newline) { + return Some(offset + line.len()); + } + offset += line.len(); + } + None +} + +fn is_top_level_mirrors_line(line: &str) -> bool { + line.starts_with("mirrors:") && line.split('#').next().unwrap_or("").trim_end() == "mirrors:" +} + +fn ensure_trailing_newline(content: &str) -> String { + let mut out = content.to_string(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out +} + +#[derive(Debug, PartialEq, Eq)] +struct PortPublish { + host: u16, + container: u16, + proto: String, +} + +fn ports_file_with_publish(existing: &str) -> String { + if ports_file_contains_publish(existing, REGISTRY_PORT_PUBLISH) { + return existing.to_string(); + } + + let mut out = ensure_trailing_newline(existing); + out.push_str(REGISTRY_PORT_PUBLISH); + out.push('\n'); + out +} + +fn ports_file_contains_publish(existing: &str, desired: &str) -> bool { + let Some(desired) = parse_port_publish(desired) else { + return false; + }; + existing + .lines() + .filter_map(parse_port_publish) + .any(|port| port == desired) +} + +fn parse_port_publish(line: &str) -> Option { + let cleaned: String = line + .split('#') + .next() + .unwrap_or("") + .chars() + .filter(|c| !c.is_whitespace()) + .collect(); + if cleaned.is_empty() { + return None; + } + let (ports, proto) = cleaned + .split_once('/') + .map_or((cleaned.as_str(), "tcp"), |(ports, proto)| (ports, proto)); + if proto != "tcp" && proto != "udp" { + return None; + } + let (host, container) = ports.split_once(':')?; + let host = host.parse().ok().filter(|port| *port > 0)?; + let container = container.parse().ok().filter(|port| *port > 0)?; + Some(PortPublish { + host, + container, + proto: proto.to_string(), + }) +} + +/// Keep the node container's /etc/hosts pointing the registry hostname at +/// the current ClusterIP (docker regenerates /etc/hosts on restart, and the +/// ClusterIP changes if the Service is recreated — hence re-run per start). +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + let hostname = REGISTRY_HOSTNAME; + let current_ip = engine_docker_output(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), + ]) + .unwrap_or_default(); + if current_ip.trim() == cluster_ip { + return Ok(()); + } + + log::info!("Updating node hosts entry: {} -> {}", hostname, cluster_ip); + + // /etc/hosts is a bind mount inside the container: `sed -i` fails with + // "Resource busy" (rename over a mount point), so rewrite it in place. + engine_docker(&[ + "exec", + NODE_CONTAINER, + "sh", + "-c", + &format!( + "awk '$2 != \"{host}\"' /etc/hosts > /tmp/hosts.new && \ + cat /tmp/hosts.new > /etc/hosts && rm -f /tmp/hosts.new && \ + echo '{ip} {host}' >> /etc/hosts", + host = hostname, + ip = cluster_ip + ), + ])?; + + Ok(()) +} + +/// Make `--context dory` resolvable. Current dory merges the context into +/// ~/.kube/config at enable time, so normally there is nothing to do — +/// mutating KUBECONFIG for every child process is then pure noise. Older +/// dory versions only write the side file; for those, prepend it to +/// KUBECONFIG (preserving whatever the user already has). +pub fn export_kubeconfig_env() { + if effective_kubeconfig_has_dory_context() { + return; + } + let Some(dory_cfg) = kubeconfig_path() else { + return; + }; + let existing = std::env::var("KUBECONFIG").unwrap_or_default(); + if existing.split(':').any(|p| p == dory_cfg) { + return; + } + let rest = if existing.is_empty() { + match home() { + Ok(h) => h.join(".kube/config").to_string_lossy().into_owned(), + Err(_) => return, + } + } else { + existing + }; + std::env::set_var("KUBECONFIG", format!("{}:{}", dory_cfg, rest)); +} + +/// Whether the kubeconfig(s) kubectl will read without our help — the +/// $KUBECONFIG chain when set, else ~/.kube/config — already define a +/// `dory` entry (i.e. dory's kubectl-merge ran against a file in scope). +fn effective_kubeconfig_has_dory_context() -> bool { + let paths: Vec = match std::env::var("KUBECONFIG") { + Ok(chain) if !chain.is_empty() => chain.split(':').map(PathBuf::from).collect(), + _ => match home() { + Ok(h) => vec![h.join(".kube/config")], + Err(_) => return false, + }, + }; + paths.iter().any(|path| { + std::fs::read_to_string(path) + .map(|content| has_dory_entry(&content)) + .unwrap_or(false) + }) +} + +/// Line-anchored scan for a kubeconfig entry named `dory` — the mapping +/// form (`name: dory`, as kubectl writes context/cluster names) or the +/// sequence-item form (`- name: dory`, the users list). `name: dory-prod` +/// or `username: dory` must not count. +fn has_dory_entry(kubeconfig: &str) -> bool { + kubeconfig.lines().any(|line| { + let trimmed = line.trim(); + trimmed == "name: dory" || trimmed == "- name: dory" + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn launch_window_remaining_covers_only_the_provisioning_window() { + use std::time::Duration; + + assert_eq!( + launch_window_remaining(Duration::ZERO), + Some(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)) + ); + assert_eq!( + launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS - 1)), + Some(Duration::from_secs(1)) + ); + assert_eq!( + launch_window_remaining(Duration::from_secs(ENGINE_LAUNCH_WINDOW_SECS)), + None + ); + assert_eq!(launch_window_remaining(Duration::from_secs(3600)), None); + } + + #[test] + fn has_dory_entry_matches_mapping_and_sequence_forms_only() { + let merged = "contexts:\n- context:\n cluster: dory\n user: dory\n name: dory\n"; + let users_list = "users:\n- name: dory\n user: {}\n"; + let near_misses = "name: dory-prod\nusername: dory\n# name: dory\nfullname: dory\n"; + + assert!(has_dory_entry(merged)); + assert!(has_dory_entry(users_list)); + assert!(!has_dory_entry(near_misses)); + assert!(!has_dory_entry("")); + } + + #[test] + fn registries_yaml_aliases_both_pull_names_to_the_service_over_http() { + let yaml = registries_yaml(); + + assert!(yaml.contains(REGISTRIES_BEGIN)); + assert!(yaml.contains(REGISTRIES_END)); + assert!(yaml.contains("\"registry.crossplane-system.svc.cluster.local:5000\":")); + assert!(yaml.contains("\"localhost:30500\":")); + // Both mirrors resolve to the same HTTP endpoint on the Service name. + assert_eq!( + yaml.matches("- \"http://registry.crossplane-system.svc.cluster.local:5000\"") + .count(), + 2 + ); + assert!(!yaml.contains("https://")); + } + + #[test] + fn registries_yaml_replaces_only_hops_managed_block() { + let existing = format!( + "configs:\n example: value\nmirrors:\n {begin}\n old: value\n {end}\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\n", + begin = REGISTRIES_BEGIN, + end = REGISTRIES_END, + ); + + let merged = merge_registries_yaml(Some(&existing)).unwrap(); + + assert!(merged.starts_with("configs:\n example: value\nmirrors:\n")); + assert!(merged.contains(" \"user.local:5000\":\n")); + assert!(!merged.contains("old: value")); + assert!(merged.contains(&format!(" \"{}\":", REGISTRY_PULL))); + assert_eq!(merged.matches(REGISTRIES_BEGIN).count(), 1); + assert_eq!(merged.matches(REGISTRIES_END).count(), 1); + } + + #[test] + fn registries_yaml_inserts_block_under_existing_mirrors_key() { + let existing = "mirrors:\n \"user.local:5000\":\n endpoint:\n - \"http://user.local:5000\"\nconfigs:\n another: value\n"; + + let merged = merge_registries_yaml(Some(existing)).unwrap(); + + let block_pos = merged.find(REGISTRIES_BEGIN).unwrap(); + let user_pos = merged.find("\"user.local:5000\"").unwrap(); + assert!(block_pos < user_pos); + assert!(merged.contains("configs:\n another: value\n")); + } + + #[test] + fn registries_yaml_appends_mirrors_section_when_missing() { + let existing = "configs:\n example: value\n"; + + let merged = merge_registries_yaml(Some(existing)).unwrap(); + + assert!(merged.starts_with(existing)); + assert!(merged.contains("\nmirrors:\n")); + assert!(merged.contains(REGISTRIES_BEGIN)); + } + + #[test] + fn registries_yaml_upgrades_legacy_hops_file_to_managed_block() { + let merged = merge_registries_yaml(Some(&legacy_registries_yaml())).unwrap(); + + assert_eq!(merged, registries_yaml()); + assert_eq!( + merged.matches(&format!("\"{}\":", REGISTRY_PULL)).count(), + 1 + ); + assert_eq!( + merged.matches(&format!("\"{}\":", REGISTRY_PUSH)).count(), + 1 + ); + } + + #[test] + fn ports_file_appends_registry_port_without_touching_existing_lines() { + let existing = "# user port\n8080:80/udp\n"; + + let merged = ports_file_with_publish(existing); + + assert_eq!(merged, "# user port\n8080:80/udp\n30500:30500\n"); + } + + #[test] + fn ports_file_absent_writes_only_registry_port() { + assert_eq!(ports_file_with_publish(""), "30500:30500\n"); + } + + #[test] + fn ports_file_treats_tcp_variant_as_already_present() { + let existing = " 30500 : 30500 / tcp # registry\n"; + + assert_eq!(ports_file_with_publish(existing), existing); + } + + #[test] + fn dory_enable_args_only_recreate_for_reset_path() { + assert_eq!( + dory_enable_args(false), + vec!["k8s", "enable", "--publish", REGISTRY_PORT_PUBLISH] + ); + assert_eq!( + dory_enable_args(true), + vec![ + "k8s", + "enable", + "--recreate", + "--publish", + REGISTRY_PORT_PUBLISH + ] + ); + } + + #[test] + fn start_rejects_size_flags() { + let size = SizeArgs { + cpus: Some(4), + memory: None, + disk: None, + }; + + let err = start(&size).expect_err("size flags must be rejected"); + + assert!(err.to_string().contains("--cpus 4")); + assert!(err.to_string().contains("Dory app")); + } +} diff --git a/src/commands/local/backend/kind.rs b/src/commands/local/backend/kind.rs new file mode 100644 index 0000000..2a22f62 --- /dev/null +++ b/src/commands/local/backend/kind.rs @@ -0,0 +1,329 @@ +//! kind backend: docker containers as nodes, containerd runtime. +//! +//! Works against any reachable docker daemon — Docker Desktop, colima's +//! dockerd, dory's dockerd, or a CI runner's. Registry trust is wired through +//! containerd's certs.d (`config_path` is enabled by default in kind node +//! images since v0.27.0), written after cluster creation; containerd reads +//! certs.d per-pull, so no restart is needed. + +use super::SizeArgs; +use crate::commands::local::package_install::{REGISTRY_PULL, REGISTRY_PUSH}; +use crate::commands::local::{command_exists, run_cmd, run_cmd_output}; +use std::error::Error; +use std::io::Write; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +pub const CLUSTER_NAME: &str = "hops"; +const NODE_CONTAINER: &str = "hops-control-plane"; + +/// kind node images before v0.27.0 ship containerd 1.x without certs.d +/// `config_path` enabled, so our hosts.toml files would be ignored. +const MIN_KIND_VERSION: (u32, u32) = (0, 27); + +const KIND_CONFIG: &str = r#"kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30500 + hostPort: 30500 + listenAddress: "127.0.0.1" +"#; + +pub fn install() -> Result<(), Box> { + log::info!("Installing kind via Homebrew..."); + run_cmd("brew", &["install", "kind"])?; + log::info!("kind installed successfully"); + Ok(()) +} + +pub fn uninstall() -> Result<(), Box> { + log::info!("Uninstalling kind..."); + run_cmd("brew", &["uninstall", "kind"])?; + log::info!("kind uninstalled"); + Ok(()) +} + +pub fn start(size: &SizeArgs) -> Result<(), Box> { + if size.any_set() { + return Err(format!( + "the kind backend has no VM to size; drop{} (resources are governed by the docker daemon kind runs on)", + size.command_suffix() + ) + .into()); + } + + preflight()?; + + if !cluster_exists() { + return create_cluster(); + } + + if node_running() { + log::info!("kind cluster '{}' is already running", CLUSTER_NAME); + return Ok(()); + } + + // kind has no start/stop; the node is a docker container. Restarting a + // single-node cluster is reliable in practice but not guaranteed by kind. + log::info!("Starting stopped kind node '{}'...", NODE_CONTAINER); + run_cmd("docker", &["start", NODE_CONTAINER])?; + wait_for_api_after_restart() +} + +pub fn stop() -> Result<(), Box> { + log::info!("Stopping kind node '{}'...", NODE_CONTAINER); + run_cmd("docker", &["stop", NODE_CONTAINER])?; + log::info!("kind cluster stopped"); + Ok(()) +} + +pub fn destroy() -> Result<(), Box> { + log::info!("Deleting kind cluster '{}'...", CLUSTER_NAME); + run_cmd("kind", &["delete", "cluster", "--name", CLUSTER_NAME])?; + log::info!("kind cluster deleted"); + Ok(()) +} + +/// kind's node container IS the cluster, so reset means recreate. +pub fn reset() -> Result<(), Box> { + preflight()?; + if cluster_exists() { + destroy()?; + } + create_cluster() +} + +pub fn resize(_size: &SizeArgs) -> Result<(), Box> { + Err( + "kind clusters have no VM to resize; adjust the docker daemon's resources, \ + or `hops local destroy && hops local start` to recreate the cluster" + .into(), + ) +} + +/// Whether the hops kind cluster exists (running or stopped). Missing binary +/// or failing command reads as "no cluster". +pub fn cluster_exists() -> bool { + if !command_exists("kind") { + return false; + } + run_cmd_output("kind", &["get", "clusters"]) + .map(|out| out.lines().any(|line| line.trim() == CLUSTER_NAME)) + .unwrap_or(false) +} + +fn node_running() -> bool { + run_cmd_output( + "docker", + &["inspect", "-f", "{{.State.Running}}", NODE_CONTAINER], + ) + .map(|out| out.trim() == "true") + .unwrap_or(false) +} + +fn preflight() -> Result<(), Box> { + if !command_exists("kind") { + return Err( + "kind is not installed; run `hops local install --backend kind` or `brew install kind`" + .into(), + ); + } + + let version_output = run_cmd_output("kind", &["version"])?; + match parse_kind_version(&version_output) { + Some(version) if version >= MIN_KIND_VERSION => {} + Some((major, minor)) => { + return Err(format!( + "kind v{major}.{minor} is too old: node images before v{}.{} lack containerd \ + certs.d support needed for the local registry. Upgrade with `brew upgrade kind`.", + MIN_KIND_VERSION.0, MIN_KIND_VERSION.1 + ) + .into()); + } + None => log::warn!( + "Unable to parse `kind version` output ({}); continuing", + version_output.trim() + ), + } + + if run_cmd_output("docker", &["info", "--format", "{{.ServerVersion}}"]).is_err() { + return Err( + "no reachable docker daemon; start Docker Desktop / colima / dory \ + (or point DOCKER_HOST / `docker context use` at one) and retry" + .into(), + ); + } + + Ok(()) +} + +fn create_cluster() -> Result<(), Box> { + log::info!("Creating kind cluster '{}'...", CLUSTER_NAME); + let mut child = Command::new("kind") + .args(["create", "cluster", "--name", CLUSTER_NAME, "--config", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(KIND_CONFIG.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err(format!("kind create cluster exited with {}", status).into()); + } + Ok(()) +} + +fn wait_for_api_after_restart() -> Result<(), Box> { + log::info!("Waiting for Kubernetes API..."); + for _ in 0..24 { + if run_cmd_output("kubectl", &["cluster-info"]).is_ok() { + return Ok(()); + } + thread::sleep(Duration::from_secs(5)); + } + Err( + "Kubernetes API did not come back after restarting the kind node; \ + run `hops local reset` to recreate the cluster" + .into(), + ) +} + +/// Alias both registry pull names to the registry Service's ClusterIP via +/// containerd certs.d files on the node. `localhost:30500` is what provider +/// runtime pods reference; aliasing it here means the name never depends on +/// kube-proxy's localhost-NodePort behavior. Files live on the node's +/// writable layer, so they survive docker stop/start (unlike /etc/hosts, +/// which docker regenerates). +pub fn wire_registry(cluster_ip: &str) -> Result<(), Box> { + for name in [REGISTRY_PULL, REGISTRY_PUSH] { + write_hosts_toml(name, cluster_ip)?; + } + Ok(()) +} + +fn hosts_toml(cluster_ip: &str) -> String { + format!( + "[host.\"http://{}:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n", + cluster_ip + ) +} + +fn write_hosts_toml(registry_name: &str, cluster_ip: &str) -> Result<(), Box> { + let dir = format!("/etc/containerd/certs.d/{}", registry_name); + let path = format!("{}/hosts.toml", dir); + let desired = hosts_toml(cluster_ip); + + let current = + run_cmd_output("docker", &["exec", NODE_CONTAINER, "cat", &path]).unwrap_or_default(); + if current == desired { + return Ok(()); + } + + log::info!( + "Wiring containerd registry alias: {} -> {}:5000", + registry_name, + cluster_ip + ); + + let mut child = Command::new("docker") + .args([ + "exec", + "-i", + NODE_CONTAINER, + "sh", + "-c", + &format!("mkdir -p '{}' && cat > '{}'", dir, path), + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(desired.as_bytes())?; + } + let status = child.wait()?; + if !status.success() { + return Err(format!("failed to write {} on kind node", path).into()); + } + Ok(()) +} + +fn parse_kind_version(output: &str) -> Option<(u32, u32)> { + // Typical output: "kind v0.32.0 go1.23.4 darwin/arm64" + output.split_whitespace().find_map(|token| { + let token = token.strip_prefix('v').unwrap_or(token); + let mut parts = token.split('.'); + let major: u32 = parts.next()?.parse().ok()?; + let minor: u32 = parts.next()?.parse().ok()?; + Some((major, minor)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_config_pins_registry_nodeport_to_localhost() { + assert!(KIND_CONFIG.contains("containerPort: 30500")); + assert!(KIND_CONFIG.contains("hostPort: 30500")); + assert!(KIND_CONFIG.contains("listenAddress: \"127.0.0.1\"")); + assert!(KIND_CONFIG.contains("role: control-plane")); + } + + #[test] + fn hosts_toml_aliases_to_cluster_ip_over_http() { + let toml = hosts_toml("10.43.12.7"); + + assert_eq!( + toml, + "[host.\"http://10.43.12.7:5000\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n" + ); + } + + #[test] + fn parse_kind_version_reads_standard_output() { + assert_eq!( + parse_kind_version("kind v0.32.0 go1.23.4 darwin/arm64"), + Some((0, 32)) + ); + assert_eq!( + parse_kind_version("kind v0.27.0 go1.22 linux/amd64"), + Some((0, 27)) + ); + } + + #[test] + fn parse_kind_version_handles_unexpected_output() { + assert_eq!(parse_kind_version("something unparseable"), None); + assert_eq!(parse_kind_version(""), None); + } + + #[test] + fn old_kind_versions_fail_the_minimum_check() { + let version = parse_kind_version("kind v0.26.0 go1.22 linux/amd64").unwrap(); + assert!(version < MIN_KIND_VERSION); + + let new_enough = parse_kind_version("kind v0.27.0 go1.22 linux/amd64").unwrap(); + assert!(new_enough >= MIN_KIND_VERSION); + } + + #[test] + fn start_rejects_size_flags() { + let size = SizeArgs { + cpus: Some(4), + memory: None, + disk: None, + }; + + let err = start(&size).expect_err("size flags must be rejected"); + + assert!(err.to_string().contains("--cpus 4")); + assert!(err.to_string().contains("no VM to size")); + } +} diff --git a/src/commands/local/backend/mod.rs b/src/commands/local/backend/mod.rs new file mode 100644 index 0000000..2e2427f --- /dev/null +++ b/src/commands/local/backend/mod.rs @@ -0,0 +1,541 @@ +//! Local-cluster backend abstraction. +//! +//! Abstracts the node/VM-level operations that differ between local cluster +//! providers (lifecycle, sizing, registry trust). Everything kubectl/helm +//! shaped lives outside this module and is backend-agnostic. + +mod colima; +mod dory; +mod kind; + +use super::{local_state_dir, run_cmd_output, HOPS_KUBE_CONTEXT_ENV}; +use clap::Args; +use std::error::Error; +use std::fmt; +use std::process::Command; +use std::str::FromStr; + +/// Sizing flags for backends with a resizable VM. +#[derive(Args, Debug, Clone, Default, PartialEq, Eq)] +pub struct SizeArgs { + /// Number of CPUs to allocate to the cluster VM (colima backend only). + #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] + pub cpus: Option, + + /// Memory to allocate to the cluster VM, in GiB (colima backend only). + #[arg(long, value_name = "GIB")] + pub memory: Option, + + /// Disk size to allocate to the cluster VM, in GiB (colima backend only). + #[arg(long, value_name = "GIB")] + pub disk: Option, +} + +impl SizeArgs { + pub fn any_set(&self) -> bool { + self.cpus.is_some() || self.memory.is_some() || self.disk.is_some() + } + + pub fn command_suffix(&self) -> String { + let mut parts = Vec::new(); + if let Some(cpus) = self.cpus { + parts.push(format!("--cpus {}", cpus)); + } + if let Some(memory) = self.memory { + parts.push(format!("--memory {}", memory)); + } + if let Some(disk) = self.disk { + parts.push(format!("--disk {}", disk)); + } + + if parts.is_empty() { + String::new() + } else { + format!(" {}", parts.join(" ")) + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum Backend { + /// VM + dockerd + k3s (macOS/Linux) + Colima, + /// docker containers as nodes; works on any docker daemon + /// (Docker Desktop, colima, dory, CI runners) + Kind, + /// k3s on dory's shared-VM engine, via the `dory` CLI + Dory, +} + +impl Backend { + /// Human-facing backend name (also the persisted spelling). + pub fn name(self) -> &'static str { + match self { + Backend::Colima => "colima", + Backend::Kind => "kind", + Backend::Dory => "dory", + } + } + + /// kubeconfig context name this backend's cluster registers under. + pub fn kube_context(self) -> &'static str { + match self { + Backend::Colima => "colima", + Backend::Kind => "kind-hops", + Backend::Dory => "dory", + } + } + + pub fn install(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::install(), + Backend::Kind => kind::install(), + Backend::Dory => dory::install(), + } + } + + pub fn uninstall(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::uninstall(), + Backend::Kind => kind::uninstall(), + Backend::Dory => dory::uninstall(), + } + } + + /// Whether this backend's local cluster/VM exists, running or stopped. + pub fn cluster_exists(self) -> bool { + match self { + Backend::Colima => colima::instance_exists(), + Backend::Kind => kind::cluster_exists(), + Backend::Dory => dory::cluster_exists(), + } + } + + /// Bring the cluster up (create, start, or resize as needed). Does not + /// wait for the Kubernetes API; callers follow with `wait_for_kubernetes`. + pub fn start(self, size: &SizeArgs, assume_yes: bool) -> Result<(), Box> { + match self { + Backend::Colima => colima::start(size, assume_yes), + Backend::Kind => kind::start(size), + Backend::Dory => dory::start(size), + } + } + + pub fn stop(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::stop(), + Backend::Kind => kind::stop(), + Backend::Dory => dory::stop(), + } + } + + pub fn destroy(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::destroy(), + Backend::Kind => kind::destroy(), + Backend::Dory => dory::destroy(), + } + } + + pub fn reset(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::reset(), + Backend::Kind => kind::reset(), + Backend::Dory => dory::reset(), + } + } + + pub fn resize(self, size: &SizeArgs) -> Result<(), Box> { + match self { + Backend::Colima => colima::resize(size), + Backend::Kind => kind::resize(size), + Backend::Dory => dory::resize(size), + } + } + + /// Make the node runtime trust the in-cluster registry over HTTP. + /// Runs before any images exist; must be safe to re-run. + pub fn ensure_registry_trust(self) -> Result<(), Box> { + match self { + Backend::Colima => colima::configure_docker_insecure_registry(), + // containerd trust is per-name via certs.d, written in + // wire_registry once the registry Service's ClusterIP is known. + Backend::Kind => Ok(()), + // trust is static registries.yaml, written by dory::start before + // the cluster boots (k3s reads it at boot). + Backend::Dory => Ok(()), + } + } + + /// Point the node at the registry Service's current ClusterIP so pulls of + /// both registry names resolve. Idempotent; re-run on every start because + /// the ClusterIP changes if the Service is recreated. + pub fn wire_registry(self, cluster_ip: &str) -> Result<(), Box> { + match self { + Backend::Colima => colima::sync_hosts_entry(cluster_ip), + Backend::Kind => kind::wire_registry(cluster_ip), + Backend::Dory => dory::wire_registry(cluster_ip), + } + } +} + +impl fmt::Display for Backend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.name()) + } +} + +impl FromStr for Backend { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim() { + "colima" => Ok(Backend::Colima), + "kind" => Ok(Backend::Kind), + "dory" => Ok(Backend::Dory), + other => Err(format!( + "unknown backend '{}' (expected colima, kind, or dory)", + other + )), + } + } +} + +const BACKEND_FILE: &str = "backend"; + +pub fn platform_default() -> Backend { + if cfg!(target_os = "macos") { + Backend::Colima + } else { + Backend::Kind + } +} + +/// Resolve which backend to operate on: explicit flag > preference persisted +/// by the last successful start > detection of an existing cluster (colima +/// wins for back-compat with pre-backend installs) > platform default. +pub fn resolve(flag: Option) -> Backend { + resolve_from( + flag, + persisted(), + colima::instance_exists, + kind::cluster_exists, + dory::cluster_exists, + platform_default() == Backend::Colima, + ) +} + +/// Resolve the backend once and activate the kube-targeting environment for +/// child kubectl/helm processes. +pub fn activate(flag: Option, context: Option<&str>) -> Backend { + let backend = resolve(flag); + + if backend == Backend::Dory { + dory::export_kubeconfig_env(); + } + + let explicit_context = context.filter(|ctx| !ctx.is_empty()); + let backend_context_exists = if explicit_context.is_none() { + kube_context_exists(backend.kube_context()) + } else { + false + }; + + match kube_context_export(backend, explicit_context, backend_context_exists) { + KubeContextExport::Set(ctx) => std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx), + KubeContextExport::Unset { missing_context } => { + std::env::remove_var(HOPS_KUBE_CONTEXT_ENV); + log::warn!( + "Kubernetes context '{}' for backend '{}' was not found; using kubeconfig current-context. Pass --context to target a specific cluster.", + missing_context, + backend.name() + ); + } + } + + backend +} + +fn resolve_from( + flag: Option, + persisted: Option, + colima_detected: impl FnOnce() -> bool, + kind_detected: impl FnOnce() -> bool, + dory_detected: impl FnOnce() -> bool, + macos: bool, +) -> Backend { + if let Some(backend) = flag { + return backend; + } + if let Some(backend) = persisted { + return backend; + } + if colima_detected() { + return Backend::Colima; + } + if kind_detected() { + return Backend::Kind; + } + if dory_detected() { + return Backend::Dory; + } + if macos { + Backend::Colima + } else { + Backend::Kind + } +} + +fn persisted() -> Option { + let path = local_state_dir().ok()?.join(BACKEND_FILE); + std::fs::read_to_string(path).ok()?.parse().ok() +} + +/// Record the backend so later invocations (stop, destroy, doctor, installs) +/// target the same cluster without re-detection. +pub fn persist(backend: Backend) -> Result<(), Box> { + let dir = local_state_dir()?; + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join(BACKEND_FILE), format!("{}\n", backend.name()))?; + Ok(()) +} + +/// Drop the persisted preference (used by uninstall; destroy keeps it). +pub fn clear_persisted() -> Result<(), Box> { + match std::fs::remove_file(local_state_dir()?.join(BACKEND_FILE)) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err.into()), + } +} + +/// Fetch the in-cluster registry Service's ClusterIP. +fn registry_cluster_ip() -> Result> { + let cluster_ip = run_cmd_output( + "kubectl", + &[ + "get", + "svc", + "registry", + "-n", + "crossplane-system", + "-o", + "jsonpath={.spec.clusterIP}", + ], + )?; + let cluster_ip = cluster_ip.trim().to_string(); + if cluster_ip.is_empty() { + return Err("Service crossplane-system/registry has no ClusterIP".into()); + } + Ok(cluster_ip) +} + +/// Wire node-level registry pulls for a backend: fetch the registry Service +/// ClusterIP and apply the backend-specific trust/aliasing. +pub fn wire_local_registry(backend: Backend) -> Result<(), Box> { + backend.wire_registry(®istry_cluster_ip()?) +} + +pub fn should_wire_local_registry( + backend_flag: Option, + context: Option<&str>, + backend: Backend, +) -> bool { + if backend_flag.is_some() { + return true; + } + + match context.filter(|ctx| !ctx.is_empty()) { + Some(ctx) => ctx == backend.kube_context(), + None => true, + } +} + +pub fn wire_local_registry_for_target( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + if should_wire_local_registry(backend_flag, context, backend) { + return wire_local_registry(backend); + } + + log::warn!( + "registry node wiring skipped: explicit --context does not match a selected backend" + ); + Ok(()) +} + +#[derive(Debug, PartialEq, Eq)] +enum KubeContextExport<'a> { + Set(&'a str), + Unset { missing_context: &'static str }, +} + +fn kube_context_export<'a>( + backend: Backend, + explicit_context: Option<&'a str>, + backend_context_exists: bool, +) -> KubeContextExport<'a> { + if let Some(ctx) = explicit_context { + return KubeContextExport::Set(ctx); + } + + let backend_context = backend.kube_context(); + if backend_context_exists { + KubeContextExport::Set(backend_context) + } else { + KubeContextExport::Unset { + missing_context: backend_context, + } + } +} + +fn kube_context_exists(context: &str) -> bool { + let output = Command::new("kubectl") + .args(["config", "get-contexts", "-o", "name"]) + .output(); + + let Ok(output) = output else { + return false; + }; + if !output.status.success() { + return false; + } + + String::from_utf8_lossy(&output.stdout) + .lines() + .any(|line| line.trim() == context) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn no_detect() -> bool { + false + } + + #[test] + fn flag_beats_persisted_and_detection() { + let resolved = resolve_from( + Some(Backend::Kind), + Some(Backend::Colima), + || true, + no_detect, + no_detect, + true, + ); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn persisted_beats_detection() { + let resolved = resolve_from( + None, + Some(Backend::Kind), + || true, + no_detect, + no_detect, + true, + ); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn colima_detection_beats_kind_detection() { + let resolved = resolve_from(None, None, || true, || true, || true, false); + + assert_eq!(resolved, Backend::Colima); + } + + #[test] + fn dory_detection_used_when_no_colima_or_kind() { + let resolved = resolve_from(None, None, no_detect, no_detect, || true, true); + + assert_eq!(resolved, Backend::Dory); + } + + #[test] + fn kind_detection_used_when_no_colima() { + let resolved = resolve_from(None, None, no_detect, || true, no_detect, true); + + assert_eq!(resolved, Backend::Kind); + } + + #[test] + fn platform_default_when_nothing_detected() { + assert_eq!( + resolve_from(None, None, no_detect, no_detect, no_detect, true), + Backend::Colima + ); + assert_eq!( + resolve_from(None, None, no_detect, no_detect, no_detect, false), + Backend::Kind + ); + } + + #[test] + fn backend_name_round_trips_through_from_str() { + for backend in [Backend::Colima, Backend::Kind, Backend::Dory] { + assert_eq!(backend.name().parse::().unwrap(), backend); + } + assert!("podman".parse::().is_err()); + } + + #[test] + fn explicit_context_is_exported_even_when_backend_context_is_absent() { + assert_eq!( + kube_context_export(Backend::Colima, Some("foreign"), false), + KubeContextExport::Set("foreign") + ); + } + + #[test] + fn backend_context_is_exported_only_when_present() { + assert_eq!( + kube_context_export(Backend::Kind, None, true), + KubeContextExport::Set("kind-hops") + ); + assert_eq!( + kube_context_export(Backend::Kind, None, false), + KubeContextExport::Unset { + missing_context: "kind-hops" + } + ); + } + + #[test] + fn registry_wiring_allowed_without_explicit_context() { + assert!(should_wire_local_registry(None, None, Backend::Colima)); + } + + #[test] + fn registry_wiring_skips_foreign_explicit_context_without_backend_flag() { + assert!(!should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + + #[test] + fn registry_wiring_allowed_when_context_matches_backend() { + assert!(should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Kind + )); + } + + #[test] + fn registry_wiring_allowed_when_backend_is_explicit() { + assert!(should_wire_local_registry( + Some(Backend::Colima), + Some("foreign"), + Backend::Colima + )); + } +} diff --git a/src/commands/local/destroy.rs b/src/commands/local/destroy.rs index 8638986..c860228 100644 --- a/src/commands/local/destroy.rs +++ b/src/commands/local/destroy.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Destroying Colima VM..."); - run_cmd("colima", &["delete", "--force"])?; - log::info!("Colima VM destroyed"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.destroy() } diff --git a/src/commands/local/doctor.rs b/src/commands/local/doctor.rs index e7f8967..248c588 100644 --- a/src/commands/local/doctor.rs +++ b/src/commands/local/doctor.rs @@ -220,20 +220,17 @@ fn check_provider(d: &mut Doctor, e: &ProviderExpectation) { }, ); - let pc_ok = exists(&[ - "get", - e.pc_resource, - e.pc_name, - "-n", - e.pc_namespace, - ]); + let pc_ok = exists(&["get", e.pc_resource, e.pc_name, "-n", e.pc_namespace]); d.check( "ProviderConfig present", pc_ok, if pc_ok { String::new() } else { - format!("{}/{} missing in namespace {}", e.pc_resource, e.pc_name, e.pc_namespace) + format!( + "{}/{} missing in namespace {}", + e.pc_resource, e.pc_name, e.pc_namespace + ) }, ); } @@ -244,7 +241,10 @@ fn provider_condition(provider: &str, cond: &str) -> String { "provider.pkg.crossplane.io", provider, "-o", - &format!("jsonpath={{.status.conditions[?(@.type==\"{}\")].status}}", cond), + &format!( + "jsonpath={{.status.conditions[?(@.type==\"{}\")].status}}", + cond + ), ]) .unwrap_or_default() } @@ -253,7 +253,11 @@ fn cond_detail(cond: &str, status: &str) -> String { if status == "True" { String::new() } else { - format!("{}={}", cond, if status.is_empty() { "" } else { status }) + format!( + "{}={}", + cond, + if status.is_empty() { "" } else { status } + ) } } diff --git a/src/commands/local/install.rs b/src/commands/local/install.rs index 0715c89..fabbb65 100644 --- a/src/commands/local/install.rs +++ b/src/commands/local/install.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Installing Colima via Homebrew..."); - run_cmd("brew", &["install", "colima"])?; - log::info!("Colima installed successfully"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.install() } diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index 6adfca6..abb27ac 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -1,4 +1,5 @@ mod aws; +pub mod backend; mod cloudflare; mod destroy; mod doctor; @@ -33,22 +34,53 @@ pub const PROVIDER_INSTALL_MANAGED_BY: &str = "hops-provider-install"; /// Env var checked by kubectl helpers to inject `--context `. pub const HOPS_KUBE_CONTEXT_ENV: &str = "HOPS_KUBE_CONTEXT"; -/// Build the kubectl args prefix. Returns `["--context", ctx]` when the env var -/// is set, or an empty vec otherwise. -fn kubectl_context_args() -> Vec { - match std::env::var(HOPS_KUBE_CONTEXT_ENV) { - Ok(ctx) if !ctx.is_empty() => vec!["--context".to_string(), ctx], - _ => vec![], - } +fn kube_context_from_env() -> Option { + std::env::var(HOPS_KUBE_CONTEXT_ENV) + .ok() + .filter(|ctx| !ctx.is_empty()) } /// Prepend `--context` to a kubectl arg slice when configured. fn with_kube_context(args: &[&str]) -> Vec { - let mut out = kubectl_context_args(); + let ctx = kube_context_from_env(); + with_kube_context_value(args, ctx.as_deref()) +} + +fn with_kube_context_value(args: &[&str], context: Option<&str>) -> Vec { + let mut out = match context { + Some(ctx) if !ctx.is_empty() => vec!["--context".to_string(), ctx.to_string()], + _ => vec![], + }; out.extend(args.iter().map(|s| s.to_string())); out } +fn with_helm_kube_context(args: &[&str]) -> Vec { + let ctx = kube_context_from_env(); + with_helm_kube_context_value(args, ctx.as_deref()) +} + +fn with_helm_kube_context_value(args: &[&str], context: Option<&str>) -> Vec { + let Some(ctx) = context.filter(|ctx| !ctx.is_empty()) else { + return args.iter().map(|s| s.to_string()).collect(); + }; + if args.first() == Some(&"repo") { + return args.iter().map(|s| s.to_string()).collect(); + } + + let mut out = Vec::with_capacity(args.len() + 2); + if let Some((command, rest)) = args.split_first() { + out.push((*command).to_string()); + out.push("--kube-context".to_string()); + out.push(ctx.to_string()); + out.extend(rest.iter().map(|s| s.to_string())); + } else { + out.push("--kube-context".to_string()); + out.push(ctx.to_string()); + } + out +} + /// Build a `Command` for kubectl with `--context` injected when configured. pub fn kubectl_command(args: &[&str]) -> Command { let full = with_kube_context(args); @@ -63,21 +95,29 @@ pub struct LocalArgs { pub command: LocalCommands, /// Kubernetes context to use for all kubectl commands (e.g. "colima"). - /// Global: applies to every `hops local` subcommand and may be given before - /// or after the subcommand. + /// Defaults to the resolved backend's own context. Global: applies to + /// every `hops local` subcommand and may be given before or after the + /// subcommand. #[arg(long, global = true)] pub context: Option, + + /// Local cluster backend to target. Defaults to the backend persisted by + /// the last successful `hops local start`, else an existing cluster if + /// one is detected, else the platform default (macOS: colima, otherwise + /// kind). + #[arg(long, global = true, value_enum)] + pub backend: Option, } #[derive(Subcommand, Debug)] pub enum LocalCommands { - /// Install Colima via Homebrew + /// Install the local cluster backend (colima or kind) via Homebrew Install, - /// Reset local Colima Kubernetes state + /// Reset local Kubernetes state (colima: k8s reset; kind: recreate cluster) Reset, /// Start local k8s cluster with Crossplane and providers Start(start::StartArgs), - /// Resize the local Colima VM without destroying cluster state + /// Resize the local cluster VM without destroying cluster state (colima only) Resize(resize::ResizeArgs), /// Check what `hops local start` set up and report drift Doctor, @@ -93,34 +133,35 @@ pub enum LocalCommands { Listmonk(listmonk::ListmonkArgs), /// Stop the local cluster Stop, - /// Destroy the local cluster VM + /// Destroy the local cluster Destroy, - /// Uninstall Colima - Uninstall, + /// Uninstall the local cluster backend + Uninstall(uninstall::UninstallArgs), } pub fn run(args: &LocalArgs) -> Result<(), Box> { - // Plumb --context through the same env channel the kubectl helpers read, so - // every subcommand's kubectl calls target the chosen context. - if let Some(ctx) = &args.context { - if !ctx.is_empty() { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } - } + let explicit_context = args.context.as_deref().filter(|ctx| !ctx.is_empty()); + let install_backend = matches!(&args.command, LocalCommands::Install) + .then(|| args.backend.unwrap_or_else(backend::platform_default)); + let activation_flag = install_backend.or(args.backend); + let install_context = install_backend.map(backend::Backend::kube_context); + let activation_context = explicit_context.or(install_context); + let backend = backend::activate(activation_flag, activation_context); + match &args.command { - LocalCommands::Install => install::run(), - LocalCommands::Reset => reset::run(), - LocalCommands::Start(start_args) => start::run(start_args), - LocalCommands::Resize(resize_args) => resize::run(resize_args), + LocalCommands::Install => install::run(backend), + LocalCommands::Reset => reset::run(backend), + LocalCommands::Start(start_args) => start::run(backend, start_args), + LocalCommands::Resize(resize_args) => resize::run(backend, resize_args), LocalCommands::Doctor => doctor::run(), LocalCommands::Aws(aws_args) => aws::run(aws_args), LocalCommands::Cloudflare(cloudflare_args) => cloudflare::run(cloudflare_args), LocalCommands::Github(github_args) => github::run(github_args), LocalCommands::Zitadel(zitadel_args) => zitadel::run(zitadel_args), LocalCommands::Listmonk(listmonk_args) => listmonk::run(listmonk_args), - LocalCommands::Stop => stop::run(), - LocalCommands::Destroy => destroy::run(), - LocalCommands::Uninstall => uninstall::run(), + LocalCommands::Stop => stop::run(backend), + LocalCommands::Destroy => destroy::run(backend), + LocalCommands::Uninstall(uninstall_args) => uninstall::run(backend, uninstall_args), } } @@ -132,11 +173,17 @@ pub fn run_cmd(program: &str, args: &[&str]) -> Result<(), Box> { let refs: Vec<&str> = full.iter().map(|s| s.as_str()).collect(); return run_cmd_with_logged_args(program, &refs, &refs); } + if program == "helm" { + let full = with_helm_kube_context(args); + let refs: Vec<&str> = full.iter().map(|s| s.as_str()).collect(); + return run_cmd_with_logged_args(program, &refs, &refs); + } run_cmd_with_logged_args(program, args, args) } /// Run an external command and capture stdout. -/// For kubectl commands, automatically injects `--context` when configured. +/// For kubectl/helm commands, automatically injects the active context when +/// configured. pub fn run_cmd_output(program: &str, args: &[&str]) -> Result> { if program == "kubectl" { let full = with_kube_context(args); @@ -148,6 +195,16 @@ pub fn run_cmd_output(program: &str, args: &[&str]) -> Result Result> Ok(local_state_dir()?.join(REPO_CACHE_DIR).join(org).join(repo)) } -fn local_state_dir() -> Result> { +pub(crate) fn local_state_dir() -> Result> { let home = std::env::var("HOME") .map_err(|_| "HOME is not set; unable to determine local state directory")?; Ok(Path::new(&home).join(LOCAL_STATE_DIR)) } -fn command_exists(program: &str) -> bool { +pub(crate) fn command_exists(program: &str) -> bool { Command::new("sh") .args(["-c", &format!("command -v {} >/dev/null 2>&1", program)]) .status() @@ -195,94 +252,74 @@ fn command_exists(program: &str) -> bool { .unwrap_or(false) } -/// Ensure Colima's /etc/hosts maps a service hostname to the current ClusterIP. -pub fn sync_registry_hosts_entry( - namespace: &str, - service: &str, - hostname: &str, -) -> Result<(), Box> { - let cluster_ip = run_cmd_output( - "kubectl", - &[ - "get", - "svc", - service, - "-n", - namespace, - "-o", - "jsonpath={.spec.clusterIP}", - ], - )?; - let cluster_ip = cluster_ip.trim(); - if cluster_ip.is_empty() { - return Err(format!("Service {}/{} has no ClusterIP", namespace, service).into()); - } - - let current_ip = run_cmd_output( - "colima", - &[ - "ssh", - "--", - "sh", - "-c", - &format!("awk '$2 == \"{}\" {{print $1; exit}}' /etc/hosts", hostname), - ], - ) - .unwrap_or_default(); - if current_ip.trim() == cluster_ip { - return Ok(()); +/// Poll until the Kubernetes API server is reachable. +pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { + log::info!("Waiting for Kubernetes API..."); + // ~10 minutes — nested-virt apiserver can stay overloaded after package install. + for _ in 0..120 { + let result = run_cmd_output("kubectl", &["get", "--raw", "/readyz"]); + if result.is_ok() { + return Ok(()); + } + // Fall back to a cheap list if /readyz is denied on some setups. + if run_cmd_output("kubectl", &["get", "ns", "default"]).is_ok() { + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_secs(5)); } - - log::info!("Updating hosts entry: {} -> {}", hostname, cluster_ip); - - let escaped_host = hostname.replace('.', "\\."); - run_cmd( - "colima", - &[ - "ssh", - "--", - "sudo", - "sed", - "-i", - &format!("/{}/d", escaped_host), - "/etc/hosts", - ], - )?; - run_cmd( - "colima", - &[ - "ssh", - "--", - "sudo", - "sh", - "-c", - &format!("echo '{} {}' >> /etc/hosts", cluster_ip, hostname), - ], - )?; - - Ok(()) + Err("Timed out waiting for Kubernetes API".into()) } /// Pipe a YAML string into `kubectl apply -f -`. /// Automatically injects `--context` when configured. +/// +/// Uses `--validate=false` so a slow/overloaded API server (common on nested +/// virt CI while Crossplane is warming) does not fail the apply solely because +/// OpenAPI schema download timed out. Retries a few times for transient +/// connection errors. pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box> { - let full = with_kube_context(&["apply", "-f", "-"]); - let mut child = Command::new("kubectl") - .args(&full) - .stdin(Stdio::piped()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn()?; + let full = with_kube_context(&["apply", "--validate=false", "-f", "-"]); + let mut last_status = None; + // Nested-virt CI (colima/GHA) can lose the apiserver for minutes after + // Crossplane/provider install (TLS handshake timeouts). Retry with backoff + // and re-probe the API between attempts. + const ATTEMPTS: u32 = 12; - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(yaml.as_bytes())?; - } + for attempt in 1..=ATTEMPTS { + let mut child = Command::new("kubectl") + .args(&full) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; - let status = child.wait()?; - if !status.success() { - return Err(format!("kubectl apply exited with {}", status).into()); + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(yaml.as_bytes())?; + } + + let status = child.wait()?; + if status.success() { + return Ok(()); + } + last_status = Some(status); + log::warn!( + "kubectl apply failed (attempt {}/{}, status {}); waiting for API...", + attempt, + ATTEMPTS, + status + ); + // Best-effort API recovery before the next apply. + let _ = wait_for_kubernetes(); + std::thread::sleep(std::time::Duration::from_secs(10)); } - Ok(()) + + Err(format!( + "kubectl apply exited with {} after retries", + last_status + .map(|s| s.to_string()) + .unwrap_or_else(|| "unknown".into()) + ) + .into()) } /// Apply a JSON merge patch with `kubectl patch --type merge`. @@ -313,3 +350,48 @@ pub fn kubectl_patch_merge( let logged_refs: Vec<&str> = full_logged.iter().map(|s| s.as_str()).collect(); run_cmd_with_logged_args("kubectl", &args_refs, &logged_refs) } + +#[cfg(test)] +mod tests { + use super::*; + + fn strings(args: Vec) -> Vec { + args + } + + #[test] + fn kubectl_args_prepend_context() { + assert_eq!( + strings(with_kube_context_value(&["get", "pods"], Some("kind-hops"))), + vec!["--context", "kind-hops", "get", "pods"] + ); + } + + #[test] + fn helm_upgrade_injects_kube_context_after_subcommand() { + assert_eq!( + strings(with_helm_kube_context_value( + &["upgrade", "--install", "crossplane"], + Some("kind-hops") + )), + vec![ + "upgrade", + "--kube-context", + "kind-hops", + "--install", + "crossplane" + ] + ); + } + + #[test] + fn helm_repo_commands_skip_kube_context() { + assert_eq!( + strings(with_helm_kube_context_value( + &["repo", "update", "crossplane-stable"], + Some("kind-hops") + )), + vec!["repo", "update", "crossplane-stable"] + ); + } +} diff --git a/src/commands/local/package_install.rs b/src/commands/local/package_install.rs index 28adf1c..dd25544 100644 --- a/src/commands/local/package_install.rs +++ b/src/commands/local/package_install.rs @@ -231,7 +231,10 @@ pub fn parse_repo_install_choice(input: &str) -> Result) -> Option { +pub fn resolve_published_version_input( + input: &str, + default_version: Option<&str>, +) -> Option { let trimmed = input.trim(); if !trimmed.is_empty() { return Some(trimmed.to_string()); @@ -394,7 +397,9 @@ fn clone_repo_into_cache( "Cloning {} into local cache at {}{}...", clone_url, cache_path.display(), - branch.map(|b| format!(" (branch {})", b)).unwrap_or_default() + branch + .map(|b| format!(" (branch {})", b)) + .unwrap_or_default() ); let mut args: Vec<&str> = vec!["clone"]; if let Some(b) = branch { @@ -405,10 +410,7 @@ fn clone_repo_into_cache( Ok(()) } -fn refresh_cached_repo( - cache_path: &Path, - branch: Option<&str>, -) -> Result<(), Box> { +fn refresh_cached_repo(cache_path: &Path, branch: Option<&str>) -> Result<(), Box> { let cache_path_str = cache_path.to_string_lossy().to_string(); run_cmd( "git", @@ -445,11 +447,7 @@ fn should_ignore_path(path: &Path) -> bool { /// Watch `path` for filesystem events and invoke `rebuild` after a debounced /// quiet period. Loops until the watcher channel closes (typically Ctrl+C). -pub fn run_watch( - path: &str, - debounce_secs: u64, - mut rebuild: F, -) -> Result<(), Box> +pub fn run_watch(path: &str, debounce_secs: u64, mut rebuild: F) -> Result<(), Box> where F: FnMut() -> Result<(), Box>, { @@ -457,8 +455,8 @@ where let debounce = Duration::from_secs(debounce_secs); let (tx, rx) = mpsc::channel(); - let mut watcher = notify::recommended_watcher(move |res: notify::Result| { - match res { + let mut watcher = + notify::recommended_watcher(move |res: notify::Result| match res { Ok(event) => { let dominated_by_ignored = event.paths.iter().all(|p| should_ignore_path(p)); log::debug!( @@ -472,8 +470,7 @@ where } } Err(e) => log::debug!("watch error: {:?}", e), - } - })?; + })?; watcher.watch(&dir, RecursiveMode::Recursive)?; log::info!( diff --git a/src/commands/local/reset.rs b/src/commands/local/reset.rs index 6dbf74e..1f4c005 100644 --- a/src/commands/local/reset.rs +++ b/src/commands/local/reset.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Resetting Colima Kubernetes..."); - run_cmd("colima", &["kubernetes", "reset"])?; - log::info!("Colima Kubernetes reset complete"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.reset() } diff --git a/src/commands/local/resize.rs b/src/commands/local/resize.rs index 19a987a..2967efd 100644 --- a/src/commands/local/resize.rs +++ b/src/commands/local/resize.rs @@ -1,15 +1,15 @@ -use super::start::{resize_colima, ColimaSizeArgs}; +use super::backend::{Backend, SizeArgs}; use clap::Args; use std::error::Error; #[derive(Args, Debug, Clone)] pub struct ResizeArgs { #[command(flatten)] - pub size: ColimaSizeArgs, + pub size: SizeArgs, } -pub fn run(args: &ResizeArgs) -> Result<(), Box> { - resize_colima(&args.size)?; - log::info!("Colima resize complete"); +pub fn run(backend: Backend, args: &ResizeArgs) -> Result<(), Box> { + backend.resize(&args.size)?; + log::info!("Resize complete"); Ok(()) } diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 55bac80..b151675 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -1,10 +1,7 @@ -use super::{kubectl_apply_stdin, run_cmd, run_cmd_output, sync_registry_hosts_entry}; +use super::backend::{self, SizeArgs}; +use super::{kubectl_apply_stdin, run_cmd, run_cmd_output, wait_for_kubernetes}; use clap::Args; -use dialoguer::Confirm; -use serde::Deserialize; use std::error::Error; -use std::io::{IsTerminal, Write}; -use std::process::{Command, Stdio}; use std::thread; use std::time::Duration; @@ -18,107 +15,45 @@ const PC_HELM: &str = include_str!("../../../bootstrap/helm/pc.yaml"); const PC_K8S: &str = include_str!("../../../bootstrap/k8s/pc.yaml"); const REGISTRY: &str = include_str!("../../../bootstrap/registry/registry.yaml"); -/// Cluster-internal hostname for the package registry. -const REGISTRY_HOST: &str = "registry.crossplane-system.svc.cluster.local:5000"; -const REGISTRY_HOSTNAME: &str = "registry.crossplane-system.svc.cluster.local"; - -const DEFAULT_CPUS: u32 = 8; -const DEFAULT_MEMORY_GIB: u32 = 16; -const DEFAULT_DISK_GIB: u32 = 60; -const GIB: u64 = 1024 * 1024 * 1024; - #[derive(Args, Debug, Clone)] pub struct StartArgs { #[command(flatten)] - pub size: ColimaSizeArgs, + pub size: SizeArgs, - /// Stop and restart a running Colima VM without prompting when requested size differs. + /// Stop and restart a running cluster VM without prompting when requested size differs. #[arg(long)] pub yes: bool, } -#[derive(Args, Debug, Clone, Default, PartialEq, Eq)] -pub struct ColimaSizeArgs { - /// Number of CPUs to allocate to the Colima VM. - #[arg(long = "cpus", visible_alias = "cpu", value_name = "N")] - pub cpus: Option, - - /// Memory to allocate to the Colima VM, in GiB. - #[arg(long, value_name = "GIB")] - pub memory: Option, - - /// Disk size to allocate to the Colima VM, in GiB. - #[arg(long, value_name = "GIB")] - pub disk: Option, -} - -impl ColimaSizeArgs { - pub fn any_set(&self) -> bool { - self.cpus.is_some() || self.memory.is_some() || self.disk.is_some() - } - - pub fn command_suffix(&self) -> String { - let mut parts = Vec::new(); - if let Some(cpus) = self.cpus { - parts.push(format!("--cpus {}", cpus)); - } - if let Some(memory) = self.memory { - parts.push(format!("--memory {}", memory)); - } - if let Some(disk) = self.disk { - parts.push(format!("--disk {}", disk)); - } - - if parts.is_empty() { - String::new() - } else { - format!(" {}", parts.join(" ")) - } - } -} - -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] -pub(crate) struct ColimaInstance { - #[serde(default)] - status: String, - #[serde(default)] - cpus: Option, - #[serde(default)] - memory: Option, - #[serde(default)] - disk: Option, -} - -impl ColimaInstance { - fn is_running(&self) -> bool { - self.status.eq_ignore_ascii_case("running") - } - - fn memory_gib(&self) -> Option { - self.memory.map(bytes_to_gib) - } - - fn disk_gib(&self) -> Option { - self.disk.map(bytes_to_gib) - } -} - -pub fn run(args: &StartArgs) -> Result<(), Box> { - let instance = colima_instance()?; - validate_requested_size(&args.size, instance.as_ref())?; +pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box> { + // 1. Bring the backend cluster up + backend.start(&args.size, args.yes)?; - // 1. Start Colima with Kubernetes - start_or_resize_colima(args, instance.as_ref())?; + // Remember the choice so stop/destroy/doctor and package installs target + // this backend without needing the flag again. + backend::persist(backend)?; // 2. Wait for the Kubernetes API to become reachable. - // Colima may return immediately ("already running") before the + // The backend may return immediately ("already running") before the // API server is ready, or a fresh start needs time to initialise. wait_for_kubernetes()?; - // 3. Configure Docker in the VM to allow HTTP pulls from the - // cluster-internal registry. Without this the kubelet's Docker - // daemon defaults to HTTPS and fails. - configure_docker_insecure_registry()?; + // 3. Make the node runtime trust the cluster-internal registry over HTTP. + backend.ensure_registry_trust()?; + + // After a docker daemon restart (colima path), re-confirm the node is Ready + // so Crossplane pods are not stuck Pending on NotReady. + log::info!("Waiting for Kubernetes nodes to be Ready..."); + run_cmd( + "kubectl", + &[ + "wait", + "--for=condition=Ready", + "nodes", + "--all", + "--timeout=300s", + ], + )?; // 4. Add Crossplane Helm repo log::info!("Adding Crossplane Helm repo..."); @@ -131,9 +66,16 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { "https://charts.crossplane.io/stable", ], )?; - run_cmd("helm", &["repo", "update"])?; + // Update only our repo — a bare `helm repo update` fails outright when any + // unrelated repo in the user's helm config has gone stale. + run_cmd("helm", &["repo", "update", "crossplane-stable"])?; // 5. Install Crossplane + // + // Do not use helm --wait: on nested-virt CI (colima/GHA) image pulls and + // scheduling can exceed helm's single wait window, and a failed --wait + // leaves us without structured kubectl diagnostics. Apply the chart, then + // poll deployments ourselves with a longer budget + failure dumps. log::info!("Installing Crossplane..."); run_cmd( "helm", @@ -145,15 +87,25 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { "-n", "crossplane-system", "--create-namespace", - "--wait", "--timeout", "5m", ], )?; - // 6. Wait for Crossplane deployment + // 6. Wait for Crossplane core deployment. + // rbac-manager can flap under nested-virt resource pressure; the core + // controller is what providers need. Best-effort wait for rbac-manager. log::info!("Waiting for Crossplane to be ready..."); - wait_for_deployment("crossplane-system", "crossplane")?; + wait_for_deployment_with_diagnostics("crossplane-system", "crossplane")?; + if let Err(e) = wait_for_deployment_attempts("crossplane-system", "crossplane-rbac-manager", 36) + { + log::warn!( + "crossplane-rbac-manager not Available yet ({e}); continuing — core Crossplane is ready" + ); + } + + // API can be briefly overloaded right after Crossplane becomes leader. + wait_for_kubernetes()?; // 7. Deploy per-provider DRCs (each pins its own cluster-admin SA) log::info!("Applying DeploymentRuntimeConfigs (per-provider)..."); @@ -170,297 +122,61 @@ pub fn run(args: &StartArgs) -> Result<(), Box> { wait_for_crd("providerconfigs.helm.m.crossplane.io")?; wait_for_crd("providerconfigs.kubernetes.m.crossplane.io")?; + // Re-check API before ProviderConfigs (openapi validation can time out if + // the apiserver is still busy — apply also uses --validate=false). + wait_for_kubernetes()?; + // 10. Apply ProviderConfigs log::info!("Applying ProviderConfigs..."); kubectl_apply_stdin(PC_HELM)?; kubectl_apply_stdin(PC_K8S)?; // 11. Deploy local OCI registry for Crossplane packages + // + // Provider install can leave the apiserver briefly unresponsive; re-wait + // and best-effort pre-pull registry:2 so the pod is not cold-started. + wait_for_kubernetes()?; + log::info!("Pre-pulling registry:2 (best effort)..."); + let _ = run_cmd("docker", &["pull", "registry:2"]); log::info!("Deploying local package registry..."); kubectl_apply_stdin(REGISTRY)?; - wait_for_deployment("crossplane-system", "registry")?; + // Nested virt: image pull + schedule for the registry can exceed the + // default ~5m wait used for lighter resources. + wait_for_deployment_with_diagnostics("crossplane-system", "registry")?; - // 12. Map the registry's cluster-internal hostname to its ClusterIP - // inside the VM so the kubelet can resolve it. - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; + // 12. Point the node at the registry Service's ClusterIP so pulls of the + // cluster-internal registry names resolve. + backend::wire_local_registry(backend)?; log::info!("Local environment is ready"); Ok(()) } -fn start_or_resize_colima( - args: &StartArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - let is_running = instance.map(ColimaInstance::is_running).unwrap_or(false); - - if is_running && args.size.any_set() { - let changes = requested_size_changes(&args.size, instance.expect("checked is_running")); - if !changes.is_empty() { - confirm_running_resize(args, &changes)?; - resize_existing_colima(&args.size, instance)?; - return Ok(()); - } - - log::info!("Requested Colima size already matches the running VM"); - } - - log::info!("Starting Colima with Kubernetes..."); - - let size = if is_running { - ColimaSizeArgs::default() - } else { - args.size.clone() - }; - let include_defaults = instance.is_none(); - start_colima(&size, include_defaults) -} - -pub(crate) fn resize_colima(size: &ColimaSizeArgs) -> Result<(), Box> { - if !size.any_set() { - return Err("Specify at least one of --cpus, --memory, or --disk".into()); - } - - let instance = colima_instance()?; - let instance = instance - .as_ref() - .ok_or("No Colima instance exists yet; use `hops local start` to create one")?; - - validate_requested_size(size, Some(instance))?; - resize_existing_colima(size, Some(instance)) -} - -fn resize_existing_colima( - size: &ColimaSizeArgs, - instance: Option<&ColimaInstance>, -) -> Result<(), Box> { - if instance.map(ColimaInstance::is_running).unwrap_or(false) { - log::info!("Stopping Colima to apply requested size..."); - run_cmd("colima", &["stop"])?; - } - - log::info!("Starting Colima with requested size..."); - start_colima(size, false) -} - -fn start_colima(size: &ColimaSizeArgs, include_defaults: bool) -> Result<(), Box> { - let args = colima_start_args(size, include_defaults); - let refs: Vec<&str> = args.iter().map(String::as_str).collect(); - run_cmd("colima", &refs) -} - -fn colima_start_args(size: &ColimaSizeArgs, include_defaults: bool) -> Vec { - let mut args = vec!["start".to_string(), "--kubernetes".to_string()]; - - if let Some(cpus) = size.cpus.or(include_defaults.then_some(DEFAULT_CPUS)) { - args.push("--cpus".to_string()); - args.push(cpus.to_string()); - } - if let Some(memory) = size - .memory - .or(include_defaults.then_some(DEFAULT_MEMORY_GIB)) - { - args.push("--memory".to_string()); - args.push(memory.to_string()); - } - if let Some(disk) = size.disk.or(include_defaults.then_some(DEFAULT_DISK_GIB)) { - args.push("--disk".to_string()); - args.push(disk.to_string()); - } - - args -} - -fn confirm_running_resize(args: &StartArgs, changes: &[String]) -> Result<(), Box> { - if args.yes { - return Ok(()); - } - - let change_text = changes.join(", "); - let resize_command = format!("hops local resize{}", args.size.command_suffix()); - let start_command = format!("hops local start{} --yes", args.size.command_suffix()); - - if !std::io::stdin().is_terminal() { - return Err(format!( - "Colima is already running with different size ({change_text}). Run `{resize_command}` first, or rerun `{start_command}` to stop and resize automatically." - ) - .into()); - } - - let confirmed = Confirm::new() - .with_prompt(format!( - "Colima is already running with different size ({change_text}). Stop and restart it now?" - )) - .default(false) - .interact()?; - - if confirmed { - Ok(()) - } else { - Err(format!( - "Colima size was not changed. Run `{resize_command}` first, then rerun `hops local start`." - ) - .into()) - } +/// Poll until a deployment's Available condition is True (~5 minutes). +fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box> { + wait_for_deployment_attempts(namespace, name, 60) } -fn validate_requested_size( - size: &ColimaSizeArgs, - instance: Option<&ColimaInstance>, +/// Longer wait used for Crossplane on cold nested-virt runners (~15 minutes). +fn wait_for_deployment_with_diagnostics( + namespace: &str, + name: &str, ) -> Result<(), Box> { - if let (Some(requested), Some(current)) = - (size.disk, instance.and_then(ColimaInstance::disk_gib)) - { - if requested < current { - return Err(format!( - "Colima disk cannot be shrunk from {current}GiB to {requested}GiB. Use --disk {current} or larger, or destroy and recreate the VM." - ) - .into()); - } - } - - Ok(()) -} - -fn requested_size_changes(size: &ColimaSizeArgs, instance: &ColimaInstance) -> Vec { - let mut changes = Vec::new(); - - if let Some(requested) = size.cpus { - match instance.cpus { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("cpus {current} -> {requested}")), - None => changes.push(format!("cpus unknown -> {requested}")), - } - } - if let Some(requested) = size.memory { - match instance.memory_gib() { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("memory {current}GiB -> {requested}GiB")), - None => changes.push(format!("memory unknown -> {requested}GiB")), - } - } - if let Some(requested) = size.disk { - match instance.disk_gib() { - Some(current) if requested == current => {} - Some(current) => changes.push(format!("disk {current}GiB -> {requested}GiB")), - None => changes.push(format!("disk unknown -> {requested}GiB")), - } - } - - changes -} - -pub(crate) fn colima_instance() -> Result, Box> { - let output = match run_cmd_output("colima", &["list", "--json"]) { - Ok(output) => output, - Err(_) => return Ok(None), - }; - - parse_colima_list(&output) -} - -fn parse_colima_list(output: &str) -> Result, Box> { - let trimmed = output.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - if let Ok(instance) = serde_json::from_str::(trimmed) { - return Ok(Some(instance)); - } - - if let Ok(instances) = serde_json::from_str::>(trimmed) { - return Ok(instances.into_iter().next()); - } - - for line in trimmed - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - { - if let Ok(instance) = serde_json::from_str::(line) { - return Ok(Some(instance)); - } - } - - Err("Unable to parse `colima list --json` output".into()) -} - -fn bytes_to_gib(bytes: u64) -> u32 { - (bytes / GIB) as u32 -} - -/// Add the cluster-internal registry to Docker's insecure-registries list -/// inside the Colima VM. Docker defaults to HTTPS for non-localhost registries; -/// our in-cluster registry speaks plain HTTP. -fn configure_docker_insecure_registry() -> Result<(), Box> { - let config = run_cmd_output("colima", &["ssh", "--", "cat", "/etc/docker/daemon.json"])?; - - if config.contains("insecure-registries") { - return Ok(()); - } - - log::info!("Configuring Docker for insecure local registry..."); - - // Insert the insecure-registries key before the final closing brace. - let new_config = if let Some(pos) = config.rfind('}') { - let prefix = config[..pos].trim_end(); - format!( - "{},\n \"insecure-registries\": [\"{}\"]\n}}\n", - prefix, REGISTRY_HOST - ) - } else { - return Err("Invalid daemon.json: no closing brace".into()); - }; - - let mut child = Command::new("colima") - .args(["ssh", "--", "sudo", "tee", "/etc/docker/daemon.json"]) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn()?; - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(new_config.as_bytes())?; - } - let status = child.wait()?; - if !status.success() { - return Err("Failed to write Docker daemon.json".into()); - } - - log::info!("Restarting Docker daemon..."); - run_cmd( - "colima", - &["ssh", "--", "sudo", "systemctl", "restart", "docker"], - )?; - - // Wait for Docker to come back. - for _ in 0..30 { - if run_cmd_output("docker", &["info"]).is_ok() { - // Docker restart can temporarily disrupt the Kubernetes API. - wait_for_kubernetes()?; - return Ok(()); - } - thread::sleep(Duration::from_secs(2)); - } - Err("Docker did not come back after restart".into()) -} - -/// Poll until the Kubernetes API server is reachable. -fn wait_for_kubernetes() -> Result<(), Box> { - log::info!("Waiting for Kubernetes API..."); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["cluster-info"]); - if result.is_ok() { - return Ok(()); + match wait_for_deployment_attempts(namespace, name, 180) { + Ok(()) => Ok(()), + Err(e) => { + dump_namespace_diagnostics(namespace); + Err(e) } - thread::sleep(Duration::from_secs(5)); } - Err("Timed out waiting for Kubernetes API".into()) } -/// Poll until a deployment's Available condition is True. -fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box> { - for _ in 0..60 { +fn wait_for_deployment_attempts( + namespace: &str, + name: &str, + attempts: u32, +) -> Result<(), Box> { + for i in 0..attempts { let output = run_cmd_output( "kubectl", &[ @@ -480,142 +196,70 @@ fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box } } + // Periodic progress so CI logs show the wait is alive. + if i > 0 && i % 12 == 0 { + log::info!( + "Still waiting for deployment {}/{} ({}s elapsed)...", + namespace, + name, + i * 5 + ); + let _ = run_cmd( + "kubectl", + &["get", "pods", "-n", namespace, "-o", "wide"], + ); + } + thread::sleep(Duration::from_secs(5)); } Err(format!("Timed out waiting for deployment {}/{}", namespace, name).into()) } -/// Poll until a CRD exists in the cluster. +fn dump_namespace_diagnostics(namespace: &str) { + log::error!("Diagnostics for namespace {} after readiness timeout:", namespace); + let _ = run_cmd("kubectl", &["get", "pods", "-n", namespace, "-o", "wide"]); + let _ = run_cmd("kubectl", &["describe", "pods", "-n", namespace]); + let _ = run_cmd( + "kubectl", + &[ + "get", + "events", + "-n", + namespace, + "--sort-by=.lastTimestamp", + ], + ); + let _ = run_cmd("kubectl", &["get", "nodes", "-o", "wide"]); +} + +/// Poll until a CRD exists **and** is Established (API serves the kind). +/// +/// Merely creating the CRD object is not enough: under load the apiserver can +/// return the CRD while discovery still lacks the kind, causing +/// `no matches for kind "ProviderConfig"` on the next apply. fn wait_for_crd(crd: &str) -> Result<(), Box> { log::info!("Waiting for CRD {}...", crd); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["get", "crd", crd]); - if result.is_ok() { - return Ok(()); + for _ in 0..120 { + let exists = run_cmd_output("kubectl", &["get", "crd", crd]).is_ok(); + if exists { + let established = run_cmd_output( + "kubectl", + &[ + "get", + "crd", + crd, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Established\")].status}", + ], + ) + .unwrap_or_default(); + if established.trim() == "True" { + // Brief settle so discovery caches pick up the new kind. + thread::sleep(Duration::from_secs(2)); + return Ok(()); + } } thread::sleep(Duration::from_secs(5)); } - Err(format!("Timed out waiting for CRD {}", crd).into()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn instance(status: &str, cpus: u32, memory_gib: u32, disk_gib: u32) -> ColimaInstance { - ColimaInstance { - status: status.to_string(), - cpus: Some(cpus), - memory: Some(memory_gib as u64 * GIB), - disk: Some(disk_gib as u64 * GIB), - } - } - - #[test] - fn colima_start_args_use_hops_defaults_for_new_profiles() { - let args = colima_start_args(&ColimaSizeArgs::default(), true); - - assert_eq!( - args, - vec![ - "start", - "--kubernetes", - "--cpus", - "8", - "--memory", - "16", - "--disk", - "60" - ] - ); - } - - #[test] - fn colima_start_args_pass_only_requested_size_for_existing_profiles() { - let size = ColimaSizeArgs { - cpus: Some(12), - memory: Some(32), - disk: None, - }; - - let args = colima_start_args(&size, false); - - assert_eq!( - args, - vec!["start", "--kubernetes", "--cpus", "12", "--memory", "32"] - ); - } - - #[test] - fn requested_size_changes_compare_only_explicit_fields() { - let current = instance("Running", 8, 16, 60); - let size = ColimaSizeArgs { - cpus: None, - memory: Some(32), - disk: None, - }; - - assert_eq!( - requested_size_changes(&size, ¤t), - vec!["memory 16GiB -> 32GiB"] - ); - } - - #[test] - fn requested_size_changes_treat_missing_current_value_as_change() { - let current = ColimaInstance { - status: "Running".to_string(), - cpus: None, - memory: None, - disk: None, - }; - let size = ColimaSizeArgs { - cpus: Some(12), - memory: None, - disk: None, - }; - - assert_eq!( - requested_size_changes(&size, ¤t), - vec!["cpus unknown -> 12"] - ); - } - - #[test] - fn parse_colima_list_accepts_single_object() { - let output = r#"{"name":"default","status":"Stopped","arch":"aarch64","cpus":8,"memory":17179869184,"disk":64424509440,"runtime":"docker+k3s"}"#; - - let parsed = parse_colima_list(output).expect("parse").expect("instance"); - - assert_eq!(parsed.status, "Stopped"); - assert_eq!(parsed.cpus, Some(8)); - assert_eq!(parsed.memory_gib(), Some(16)); - assert_eq!(parsed.disk_gib(), Some(60)); - } - - #[test] - fn parse_colima_list_accepts_array_output() { - let output = r#"[{"status":"Running","cpus":12,"memory":34359738368,"disk":107374182400}]"#; - - let parsed = parse_colima_list(output).expect("parse").expect("instance"); - - assert!(parsed.is_running()); - assert_eq!(parsed.cpus, Some(12)); - assert_eq!(parsed.memory_gib(), Some(32)); - assert_eq!(parsed.disk_gib(), Some(100)); - } - - #[test] - fn validate_requested_size_rejects_disk_shrink() { - let current = instance("Stopped", 8, 16, 100); - let size = ColimaSizeArgs { - cpus: None, - memory: None, - disk: Some(60), - }; - - let err = validate_requested_size(&size, Some(¤t)).expect_err("disk shrink"); - - assert!(err.to_string().contains("cannot be shrunk")); - } + Err(format!("Timed out waiting for CRD {} to be Established", crd).into()) } diff --git a/src/commands/local/stop.rs b/src/commands/local/stop.rs index 16ffa2d..4fa2bf5 100644 --- a/src/commands/local/stop.rs +++ b/src/commands/local/stop.rs @@ -1,9 +1,6 @@ -use super::run_cmd; +use super::backend::Backend; use std::error::Error; -pub fn run() -> Result<(), Box> { - log::info!("Stopping Colima..."); - run_cmd("colima", &["stop"])?; - log::info!("Colima stopped"); - Ok(()) +pub fn run(backend: Backend) -> Result<(), Box> { + backend.stop() } diff --git a/src/commands/local/uninstall.rs b/src/commands/local/uninstall.rs index d84e1b0..6cc4a4e 100644 --- a/src/commands/local/uninstall.rs +++ b/src/commands/local/uninstall.rs @@ -1,21 +1,171 @@ -use super::run_cmd; +use super::backend::Backend; +use clap::Args; use std::error::Error; use std::io::{self, Write}; -pub fn run() -> Result<(), Box> { - print!("Uninstall Colima? This will remove the binary. [y/N] "); - io::stdout().flush()?; +#[derive(Args, Debug)] +pub struct UninstallArgs { + /// Uninstall the backend binary even if its local cluster still exists. + #[arg(long)] + pub force: bool, +} - let mut input = String::new(); - io::stdin().read_line(&mut input)?; +pub fn run(backend: Backend, args: &UninstallArgs) -> Result<(), Box> { + run_inner( + backend, + args.force, + Backend::cluster_exists, + Backend::uninstall, + super::backend::clear_persisted, + confirm_uninstall, + ) +} + +fn run_inner( + backend: Backend, + force: bool, + cluster_exists: ClusterExists, + uninstall: Uninstall, + clear_persisted: ClearPersisted, + confirm: Confirm, +) -> Result<(), Box> +where + ClusterExists: FnOnce(Backend) -> bool, + Uninstall: FnOnce(Backend) -> Result<(), Box>, + ClearPersisted: FnOnce() -> Result<(), Box>, + Confirm: FnOnce(Backend) -> Result>, +{ + if !force && cluster_exists(backend) { + return Err("destroy the cluster first: `hops local destroy` (or pass --force to uninstall the backend binary anyway)".into()); + } - if input.trim().eq_ignore_ascii_case("y") { - log::info!("Uninstalling Colima..."); - run_cmd("brew", &["uninstall", "colima"])?; - log::info!("Colima uninstalled"); + if confirm(backend)? { + uninstall(backend)?; + clear_persisted()?; } else { log::info!("Uninstall cancelled"); } Ok(()) } + +fn confirm_uninstall(backend: Backend) -> Result> { + print!( + "Uninstall {}? This will remove the binary. [y/N] ", + backend.name() + ); + io::stdout().flush()?; + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + + Ok(input.trim().eq_ignore_ascii_case("y")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::rc::Rc; + + #[test] + fn refuses_uninstall_when_cluster_exists_without_force() { + let events = Rc::new(RefCell::new(Vec::new())); + let err = run_inner( + Backend::Kind, + false, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("cluster_exists"); + true + } + }, + |_| { + events.borrow_mut().push("uninstall"); + Ok(()) + }, + || { + events.borrow_mut().push("clear"); + Ok(()) + }, + |_| { + events.borrow_mut().push("confirm"); + Ok(true) + }, + ) + .expect_err("cluster guard should refuse uninstall"); + + assert!(err.to_string().contains("destroy the cluster first")); + assert_eq!(&*events.borrow(), &["cluster_exists"]); + } + + #[test] + fn force_uninstalls_without_cluster_probe_and_clears_after_success() { + let events = Rc::new(RefCell::new(Vec::new())); + run_inner( + Backend::Kind, + true, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("cluster_exists"); + true + } + }, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("uninstall"); + Ok(()) + } + }, + { + let events = Rc::clone(&events); + move || { + events.borrow_mut().push("clear"); + Ok(()) + } + }, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("confirm"); + Ok(true) + } + }, + ) + .expect("force should allow uninstall"); + + assert_eq!(&*events.borrow(), &["confirm", "uninstall", "clear"]); + } + + #[test] + fn failed_uninstall_does_not_clear_persisted_backend() { + let events = Rc::new(RefCell::new(Vec::new())); + let err = run_inner( + Backend::Kind, + false, + |_| false, + { + let events = Rc::clone(&events); + move |_| { + events.borrow_mut().push("uninstall"); + Err("brew failed".into()) + } + }, + { + let events = Rc::clone(&events); + move || { + events.borrow_mut().push("clear"); + Ok(()) + } + }, + |_| Ok(true), + ) + .expect_err("uninstall failure should propagate"); + + assert_eq!(err.to_string(), "brew failed"); + assert_eq!(&*events.borrow(), &["uninstall"]); + } +} diff --git a/src/commands/provider/install.rs b/src/commands/provider/install.rs index 321b0d6..c45a3e6 100644 --- a/src/commands/provider/install.rs +++ b/src/commands/provider/install.rs @@ -1,11 +1,11 @@ +use crate::commands::local::backend::{self, Backend}; use crate::commands::local::package_install::{ docker_arch, ensure_cached_repo_checkout_at, ensure_registry, parse_repo_spec, resolve_repo_install_target, run_watch, sanitize_name_component, RepoInstallTarget, RepoSpec, - REGISTRY_HOSTNAME, REGISTRY_PULL, REGISTRY_PUSH, + REGISTRY_PULL, REGISTRY_PUSH, }; use crate::commands::local::{ - kubectl_apply_stdin, run_cmd, run_cmd_output, sync_registry_hosts_entry, - HOPS_KUBE_CONTEXT_ENV, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, + kubectl_apply_stdin, run_cmd, run_cmd_output, MANAGED_BY_LABEL, PROVIDER_INSTALL_MANAGED_BY, }; use clap::Args; use serde::Deserialize; @@ -53,6 +53,10 @@ pub struct ProviderInstallArgs { #[arg(long)] pub context: Option, + /// Local cluster backend whose node should be wired for local package pulls. + #[arg(long, value_enum)] + pub backend: Option, + /// Watch the project directory for changes and re-run install automatically #[arg(long, conflicts_with = "repo")] pub watch: bool, @@ -73,9 +77,7 @@ struct PackageMetadata { } pub fn run(args: &ProviderInstallArgs) -> Result<(), Box> { - if let Some(ctx) = &args.context { - std::env::set_var(HOPS_KUBE_CONTEXT_ENV, ctx); - } + let backend = backend::activate(args.backend, args.context.as_deref()); match (args.repo.as_deref(), args.version.as_deref()) { (Some(repo), Some(version)) => { @@ -86,10 +88,14 @@ pub fn run(args: &ProviderInstallArgs) -> Result<(), Box> { args.skip_dependency_resolution, args.version_prefix.as_deref(), args.branch.as_deref(), + backend, + args.backend, + args.context.as_deref(), ), (None, _) => { let path = args.path.as_deref().unwrap_or("."); let prefix = args.version_prefix.clone(); + prepare_local_registry(backend, args.backend, args.context.as_deref())?; run_local_path(path, args.skip_dependency_resolution, prefix.as_deref())?; if args.watch { @@ -111,11 +117,15 @@ fn run_repo_install( skip_dependency_resolution: bool, version_prefix: Option<&str>, branch: Option<&str>, + backend: Backend, + backend_flag: Option, + context: Option<&str>, ) -> Result<(), Box> { let spec = parse_repo_spec(repo)?; match resolve_repo_install_target(&spec)? { RepoInstallTarget::SourceBuild => { let cache_path = ensure_cached_repo_checkout_at(&spec, branch)?; + prepare_local_registry(backend, backend_flag, context)?; run_local_path( &cache_path.to_string_lossy(), skip_dependency_resolution, @@ -128,6 +138,15 @@ fn run_repo_install( } } +fn prepare_local_registry( + backend: Backend, + backend_flag: Option, + context: Option<&str>, +) -> Result<(), Box> { + ensure_registry()?; + backend::wire_local_registry_for_target(backend, backend_flag, context) +} + fn apply_repo_version( repo: &str, version: &str, @@ -154,11 +173,7 @@ fn apply_repo_version_spec( sanitize_name_component(&spec.repo) ); - log::info!( - "Applying Provider '{}' from {}", - provider_name, - package_ref - ); + log::info!("Applying Provider '{}' from {}", provider_name, package_ref); let providers_json = run_cmd_output( "kubectl", &["get", "providers.pkg.crossplane.io", "-o", "json"], @@ -189,9 +204,6 @@ fn run_local_path( let provider_name = read_provider_name(dir)?; log::info!("Provider package name: {}", provider_name); - ensure_registry()?; - sync_registry_hosts_entry("crossplane-system", "registry", REGISTRY_HOSTNAME)?; - // Resolve the existing upstream Provider before building so we can: // 1. carry the upstream package URL into the new `spec.package` (Crossplane // records the URL-without-tag as the Lock Source; deps declared as @@ -228,10 +240,7 @@ fn run_local_path( &local_image_path_for_tag, )?; - let push_xpkg_ref = format!( - "{}/hops-ops/{}:{}", - REGISTRY_PUSH, provider_name, dev_tag - ); + let push_xpkg_ref = format!("{}/hops-ops/{}:{}", REGISTRY_PUSH, provider_name, dev_tag); let local_pull_xpkg_path = format!("{}/hops-ops/{}", REGISTRY_PULL, provider_name); log::info!("Pushing xpkg to {}...", push_xpkg_ref); crossplane_xpkg_push(&xpkg_path, &push_xpkg_ref)?; @@ -277,11 +286,7 @@ fn read_provider_name(dir: &Path) -> Result> { let name = parsed.metadata.name.trim().to_string(); if name.is_empty() { - return Err(format!( - "{} has no metadata.name", - crossplane_yaml.display() - ) - .into()); + return Err(format!("{} has no metadata.name", crossplane_yaml.display()).into()); } Ok(name) } @@ -293,7 +298,10 @@ fn ensure_build_submodule(dir: &Path) -> Result<(), Box> { } if !dir.join(".gitmodules").is_file() { - log::debug!("No .gitmodules in {}; skipping submodule init", dir.display()); + log::debug!( + "No .gitmodules in {}; skipping submodule init", + dir.display() + ); return Ok(()); } @@ -344,28 +352,19 @@ fn find_xpkg_for_provider( .filter_map(|e| e.ok()) .filter(|e| { let p = e.path(); - p.extension().map_or(false, |ext| ext == "xpkg") + p.extension().is_some_and(|ext| ext == "xpkg") && p.file_name() .and_then(|n| n.to_str()) - .map_or(false, |n| n.starts_with(&prefix)) + .is_some_and(|n| n.starts_with(&prefix)) }) .map(|e| e.path()) .collect(); if candidates.is_empty() { - return Err(format!( - "no {}*.xpkg found in {}", - prefix, - xpkg_dir.display() - ) - .into()); + return Err(format!("no {}*.xpkg found in {}", prefix, xpkg_dir.display()).into()); } - candidates.sort_by_key(|p| { - fs::metadata(p) - .and_then(|m| m.modified()) - .ok() - }); + candidates.sort_by_key(|p| fs::metadata(p).and_then(|m| m.modified()).ok()); Ok(candidates.pop().unwrap()) } @@ -544,7 +543,10 @@ struct TagsListResponse { fn is_full_semver(s: &str) -> bool { let trimmed = s.strip_prefix('v').unwrap_or(s); let parts: Vec<&str> = trimmed.splitn(3, '.').collect(); - parts.len() == 3 && parts.iter().all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) + parts.len() == 3 + && parts + .iter() + .all(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) } /// Parse a `vN` (or `N`) bare-major version string, e.g. `v1` -> `Some(1)`. @@ -1104,11 +1106,8 @@ mod tests { #[test] fn build_runtime_config_yaml_overrides_image_when_provided() { - let with_image = build_runtime_config_yaml( - "p-runtime", - "p", - Some("registry.example/p-arm64:dev-abc"), - ); + let with_image = + build_runtime_config_yaml("p-runtime", "p", Some("registry.example/p-arm64:dev-abc")); assert!(with_image.contains("name: package-runtime")); assert!(with_image.contains("image: registry.example/p-arm64:dev-abc")); @@ -1119,7 +1118,6 @@ mod tests { assert!(without.contains("app.kubernetes.io/managed-by: hops-provider-install")); } - #[test] fn local_runtime_image_ref_uses_nodeport_registry() { let image = local_runtime_image_ref("provider-helm", "arm64", "v1.999.3"); @@ -1130,6 +1128,15 @@ mod tests { assert!(!image.contains(REGISTRY_PULL)); } + #[test] + fn local_registry_wiring_skips_foreign_context_without_backend_flag() { + assert!(!backend::should_wire_local_registry( + None, + Some("kind-hops"), + Backend::Colima + )); + } + #[test] fn build_cluster_role_binding_yaml_targets_service_account() { let yaml = build_cluster_role_binding_yaml("p-cluster-admin", "p"); @@ -1165,7 +1172,10 @@ mod tests { fn assert_no_existing_provider_error(err: Box, provider_name: &str) { let msg = err.to_string(); assert!( - msg.contains(&format!("no existing Provider matching '{}'", provider_name)), + msg.contains(&format!( + "no existing Provider matching '{}'", + provider_name + )), "missing 'no existing Provider matching' phrase: {}", msg ); @@ -1183,9 +1193,8 @@ mod tests { #[test] fn resolve_provider_target_errors_when_list_is_empty() { - let err = - resolve_provider_target("provider-helm", "{\"items\":[]}", TEST_LOCAL_REGISTRY) - .expect_err("expected no-existing-Provider error"); + let err = resolve_provider_target("provider-helm", "{\"items\":[]}", TEST_LOCAL_REGISTRY) + .expect_err("expected no-existing-Provider error"); assert_no_existing_provider_error(err, "provider-helm"); } @@ -1245,10 +1254,7 @@ mod tests { // "already_patched" bucket. We must still find it instead of erroring. let json = provider_json(&[( "crossplane-contrib-provider-helm", - &format!( - "{}/hops-ops/provider-helm:dev-abc123", - TEST_LOCAL_REGISTRY - ), + &format!("{}/hops-ops/provider-helm:dev-abc123", TEST_LOCAL_REGISTRY), Some("crossplane-contrib-provider-helm-drc"), )]); @@ -1333,7 +1339,8 @@ mod tests { fn resolve_provider_target_returns_clear_error_on_invalid_json() { let err = resolve_provider_target("provider-helm", "not-json", TEST_LOCAL_REGISTRY) .expect_err("expected parse error"); - assert!(err.to_string().contains("failed to parse providers list JSON")); + assert!(err + .to_string() + .contains("failed to parse providers list JSON")); } } - diff --git a/src/commands/secrets/list.rs b/src/commands/secrets/list.rs index 4ae9878..218ca7f 100644 --- a/src/commands/secrets/list.rs +++ b/src/commands/secrets/list.rs @@ -41,7 +41,8 @@ pub fn run() -> Result<(), Box> { pushed_rows.push(PushedSecretRow { name: secret.name.clone(), owner: owner_label(&secret), - cluster: tag_value(&secret, "hops.ops.com.ai/cluster").unwrap_or("-".to_string()), + cluster: tag_value(&secret, "hops.ops.com.ai/cluster") + .unwrap_or("-".to_string()), namespace: tag_value(&secret, "hops.ops.com.ai/namespace") .unwrap_or("-".to_string()), kms_key: shorten_kms_key(&kms), diff --git a/src/commands/vars/mod.rs b/src/commands/vars/mod.rs index 306d7cb..d2b3370 100644 --- a/src/commands/vars/mod.rs +++ b/src/commands/vars/mod.rs @@ -132,10 +132,7 @@ pub(crate) fn require_command(program: &str) -> Result<(), Box> { } } -pub(crate) fn run_command_output( - program: &str, - args: &[&str], -) -> Result, Box> { +pub(crate) fn run_command_output(program: &str, args: &[&str]) -> Result, Box> { log::debug!("Running: {} {}", program, args.join(" ")); let output = Command::new(program).args(args).output()?; if !output.status.success() { diff --git a/src/commands/vars/sync.rs b/src/commands/vars/sync.rs index 2528330..9c3950d 100644 --- a/src/commands/vars/sync.rs +++ b/src/commands/vars/sync.rs @@ -243,11 +243,6 @@ fn set_github_variable( if !status.success() { return Err(format!("gh variable set exited with {}", status).into()); } - log::info!( - "Set GitHub variable '{}' in '{}/{}'", - var_name, - owner, - repo - ); + log::info!("Set GitHub variable '{}' in '{}/{}'", var_name, owner, repo); Ok(()) } diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs new file mode 100644 index 0000000..b2882cb --- /dev/null +++ b/tests/colima_smoke_workflow.rs @@ -0,0 +1,90 @@ +//! Structural checks for the shipped colima macOS smoke workflow. +//! +//! These tests drive the real file under `.github/workflows/` so a missing +//! or regressed smoke contract fails `cargo test` without needing a macOS GHA run. + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-colima-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected colima smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn colima_smoke_workflow_exists_and_pins_nested_virt_runner() { + let text = workflow_text(); + assert!( + text.contains("runs-on: macos-15-intel"), + "must pin nested-virt-capable Intel runner" + ); + assert!( + !text + .lines() + .any(|l| l.trim() == "runs-on: macos-latest"), + "must not use bare macos-latest as sole runs-on" + ); + assert!( + text.to_lowercase().contains("nested") && text.to_lowercase().contains("virt"), + "must comment on runner/virt constraints" + ); +} + +#[test] +fn colima_smoke_workflow_kind_parity_sequence() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend colima", + "local doctor", + "localhost:30500", + "registry.crossplane-system.svc.cluster.local:5000", + "--context colima", + "local stop", + "local destroy", + ] { + assert!( + text.contains(needle), + "colima smoke missing kind-parity fragment: {needle}" + ); + } + // stop/start resume: start without --backend after stop + assert!( + text.contains("local start\n") || text.lines().any(|l| l.trim() == "./target/debug/hops-cli local start"), + "must start again without --backend after stop" + ); + for tool in ["colima", "docker", "kubectl", "helm"] { + assert!( + text.contains(tool), + "prereq install must mention {tool}" + ); + } +} + +#[test] +fn colima_smoke_workflow_sizes_vm_for_gha_intel_runner() { + let text = workflow_text(); + // Defaults (8/16/60) exceed macos-15-intel; CI must pass explicit smaller sizes. + assert!( + text.contains("--cpus") && text.contains("--memory") && text.contains("--disk"), + "must pass --cpus/--memory/--disk so the VZ VM fits the runner" + ); + // 8Gi left CoreDNS thrashing; smoke needs headroom above that floor. + assert!( + text.contains("--memory 10") || text.contains("--memory 11") || text.contains("--memory 12"), + "must allocate at least 10Gi to the colima VM on GHA intel runners" + ); + assert!( + text.contains("ha.stderr.log"), + "failure dump must include lima hostagent stderr for nested-virt debug" + ); +}