diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml new file mode 100644 index 0000000..b1bfa70 --- /dev/null +++ b/.github/workflows/on-pr-colima-smoke.yaml @@ -0,0 +1,166 @@ +name: colima backend smoke + +# End-to-end check of `hops local` on the colima backend on macOS. +# Mirrors on-pr-kind-smoke.yaml with --backend colima and kubectl --context colima. +# +# Runner constraints: +# - Colima needs nested virtualization (Lima VM). Pin macos-15-intel — that +# image is known to support nested virt for Colima/Lima on GHA. +# - Do NOT use bare macos-latest alone (tracks arm images; arm64 GHA macOS +# historically fails with HV_UNSUPPORTED for nested virt). +# - Prefer localhost:30500 for registry traffic, not the Colima VM IP +# (macOS 15 Local Network Privacy can block non-root VM-IP access). +# +# Install: brew provides colima/docker/kubectl/helm only. Do not pre-start +# colima here — hops local start --backend colima owns cluster bring-up. +# +# Sizing: hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard +# macos-15-intel runners (~4 CPU / ~14 GiB). Pass explicit smaller sizes +# so the VZ VM can allocate; stop/start resume uses the persisted profile. + +on: + pull_request: + workflow_dispatch: + +jobs: + colima-smoke: + # Nested-virt-capable Intel pin (not bare macos-latest). + runs-on: macos-15-intel + # Nested virt + cold Crossplane pulls can take well over 30m. + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - uses: Swatinem/rust-cache@v2 + + - name: Install colima, docker, kubectl, helm + run: | + set -euxo pipefail + # Tools only — do not `colima start` here so the smoke exercises hops. + brew install colima docker kubectl helm + colima version + docker version --format '{{.Client.Version}}' || docker --version + kubectl version --client + helm version --short + + - name: Build hops + run: cargo build + + - name: Host resources (for sizing) + run: | + set -euxo pipefail + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + + - name: hops local start --backend colima + run: | + set -euxo pipefail + # Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ. + # Memory 10 (not 8): at 8Gi CoreDNS/metrics-server thrash and smoke pods + # sit in ContainerCreating without IPs even after images pull. Leave ~4Gi + # for host macOS + VZ. Disk 40: registry PVC requests 20Gi. + ./target/debug/hops-cli local start --backend colima \ + --cpus 3 --memory 10 --disk 40 + + - name: hops local doctor + run: ./target/debug/hops-cli local doctor + + - name: Wait for node + CoreDNS (nested virt settles) + run: | + set -euxo pipefail + # After cold start the control plane is still soft: probes time out and + # new pods stick in ContainerCreating. Wait for basics before registry. + kubectl --context colima wait --for=condition=Ready nodes --all --timeout=120s + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=kube-dns --timeout=180s || \ + kubectl --context colima -n kube-system wait --for=condition=Ready \ + pod -l k8s-app=coredns --timeout=180s || true + kubectl --context colima -n kube-system get pods -o wide || true + + - name: Registry round-trip through both pull names + run: | + set -euxo pipefail + docker pull public.ecr.aws/docker/library/busybox:stable + docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci + docker push localhost:30500/smoke/busybox:ci + + # Service-name pull: containerd resolves via its certs.d alias. + kubectl --context colima run smoke-svc-name \ + --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # localhost:30500 pull: what provider runtime pods reference. + kubectl --context colima run smoke-localhost \ + --image=localhost:30500/smoke/busybox:ci \ + --restart=Never --command -- sleep 300 + + # Nested virt: image pull can succeed while CNI/IP assignment lags. + if ! kubectl --context colima wait --for=condition=Ready \ + pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then + echo "==== smoke pods not Ready ====" + kubectl --context colima get pods smoke-svc-name smoke-localhost -o wide || true + kubectl --context colima describe pod smoke-svc-name smoke-localhost || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-svc-name || true + kubectl --context colima get events -A --field-selector involvedObject.name=smoke-localhost || true + exit 1 + fi + kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false + + - name: Stop/start resume path (uses persisted backend, no flag) + run: | + set -euxo pipefail + ./target/debug/hops-cli local stop + ./target/debug/hops-cli local start + # After colima VM stop/start, docker container IDs are gone and pods + # often sit in Error until kubelet recreates them. Wait generously and + # force a rollout restart if Available still stalls. + if ! kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s; then + echo "deployments not Available after resume; restarting..." + kubectl --context colima -n crossplane-system get pods -o wide || true + kubectl --context colima -n crossplane-system rollout restart \ + deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true + kubectl --context colima -n crossplane-system wait \ + --for=condition=Available deployment/crossplane deployment/registry \ + --timeout=420s + fi + ./target/debug/hops-cli local doctor + + # Capture live cluster state BEFORE destroy so failures are diagnosable. + - name: Debug dump on failure + if: failure() + run: | + set +e + echo "==== host ====" + sysctl -n hw.ncpu + sysctl hw.memsize + df -h / + echo "==== colima ====" + colima status || true + colima list || true + echo "==== cluster ====" + kubectl --context colima get nodes,pods -A -o wide || true + echo "==== smoke pods (default) ====" + kubectl --context colima describe pod smoke-svc-name smoke-localhost 2>/dev/null || true + echo "==== crossplane-system ====" + kubectl --context colima describe pods -n crossplane-system || true + kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -100 || true + echo "==== lima ha.stderr / serial ====" + for f in \ + "$HOME/.colima/_lima/colima/ha.stderr.log" \ + "$HOME/.colima/_lima/colima/ha.stdout.log" \ + "$HOME/.colima/_lima/colima/serial.log" \ + "$HOME/.colima/_lima/colima/serialv.log" + do + if [ -f "$f" ]; then + echo "----- $f -----" + tail -n 200 "$f" || true + fi + done + ./target/debug/hops-cli local doctor || true + + - name: hops local destroy + if: always() + run: ./target/debug/hops-cli local destroy || true diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs index 0db9db0..abb27ac 100644 --- a/src/commands/local/mod.rs +++ b/src/commands/local/mod.rs @@ -255,11 +255,16 @@ pub(crate) fn command_exists(program: &str) -> bool { /// Poll until the Kubernetes API server is reachable. pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { log::info!("Waiting for Kubernetes API..."); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["cluster-info"]); + // ~10 minutes — nested-virt apiserver can stay overloaded after package install. + for _ in 0..120 { + let result = run_cmd_output("kubectl", &["get", "--raw", "/readyz"]); if result.is_ok() { return Ok(()); } + // Fall back to a cheap list if /readyz is denied on some setups. + if run_cmd_output("kubectl", &["get", "ns", "default"]).is_ok() { + return Ok(()); + } std::thread::sleep(std::time::Duration::from_secs(5)); } Err("Timed out waiting for Kubernetes API".into()) @@ -267,24 +272,54 @@ pub(crate) fn wait_for_kubernetes() -> Result<(), Box> { /// Pipe a YAML string into `kubectl apply -f -`. /// Automatically injects `--context` when configured. +/// +/// Uses `--validate=false` so a slow/overloaded API server (common on nested +/// virt CI while Crossplane is warming) does not fail the apply solely because +/// OpenAPI schema download timed out. Retries a few times for transient +/// connection errors. pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box> { - let full = with_kube_context(&["apply", "-f", "-"]); - let mut child = Command::new("kubectl") - .args(&full) - .stdin(Stdio::piped()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn()?; + let full = with_kube_context(&["apply", "--validate=false", "-f", "-"]); + let mut last_status = None; + // Nested-virt CI (colima/GHA) can lose the apiserver for minutes after + // Crossplane/provider install (TLS handshake timeouts). Retry with backoff + // and re-probe the API between attempts. + const ATTEMPTS: u32 = 12; + + for attempt in 1..=ATTEMPTS { + let mut child = Command::new("kubectl") + .args(&full) + .stdin(Stdio::piped()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(yaml.as_bytes())?; + } - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(yaml.as_bytes())?; + let status = child.wait()?; + if status.success() { + return Ok(()); + } + last_status = Some(status); + log::warn!( + "kubectl apply failed (attempt {}/{}, status {}); waiting for API...", + attempt, + ATTEMPTS, + status + ); + // Best-effort API recovery before the next apply. + let _ = wait_for_kubernetes(); + std::thread::sleep(std::time::Duration::from_secs(10)); } - let status = child.wait()?; - if !status.success() { - return Err(format!("kubectl apply exited with {}", status).into()); - } - Ok(()) + Err(format!( + "kubectl apply exited with {} after retries", + last_status + .map(|s| s.to_string()) + .unwrap_or_else(|| "unknown".into()) + ) + .into()) } /// Apply a JSON merge patch with `kubectl patch --type merge`. diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs index 136d760..b151675 100644 --- a/src/commands/local/start.rs +++ b/src/commands/local/start.rs @@ -41,6 +41,20 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - for _ in 0..60 { + wait_for_deployment_attempts(namespace, name, 60) +} + +/// Longer wait used for Crossplane on cold nested-virt runners (~15 minutes). +fn wait_for_deployment_with_diagnostics( + namespace: &str, + name: &str, +) -> Result<(), Box> { + match wait_for_deployment_attempts(namespace, name, 180) { + Ok(()) => Ok(()), + Err(e) => { + dump_namespace_diagnostics(namespace); + Err(e) + } + } +} + +fn wait_for_deployment_attempts( + namespace: &str, + name: &str, + attempts: u32, +) -> Result<(), Box> { + for i in 0..attempts { let output = run_cmd_output( "kubectl", &[ @@ -133,20 +196,70 @@ fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box } } + // Periodic progress so CI logs show the wait is alive. + if i > 0 && i % 12 == 0 { + log::info!( + "Still waiting for deployment {}/{} ({}s elapsed)...", + namespace, + name, + i * 5 + ); + let _ = run_cmd( + "kubectl", + &["get", "pods", "-n", namespace, "-o", "wide"], + ); + } + thread::sleep(Duration::from_secs(5)); } Err(format!("Timed out waiting for deployment {}/{}", namespace, name).into()) } -/// Poll until a CRD exists in the cluster. +fn dump_namespace_diagnostics(namespace: &str) { + log::error!("Diagnostics for namespace {} after readiness timeout:", namespace); + let _ = run_cmd("kubectl", &["get", "pods", "-n", namespace, "-o", "wide"]); + let _ = run_cmd("kubectl", &["describe", "pods", "-n", namespace]); + let _ = run_cmd( + "kubectl", + &[ + "get", + "events", + "-n", + namespace, + "--sort-by=.lastTimestamp", + ], + ); + let _ = run_cmd("kubectl", &["get", "nodes", "-o", "wide"]); +} + +/// Poll until a CRD exists **and** is Established (API serves the kind). +/// +/// Merely creating the CRD object is not enough: under load the apiserver can +/// return the CRD while discovery still lacks the kind, causing +/// `no matches for kind "ProviderConfig"` on the next apply. fn wait_for_crd(crd: &str) -> Result<(), Box> { log::info!("Waiting for CRD {}...", crd); - for _ in 0..60 { - let result = run_cmd_output("kubectl", &["get", "crd", crd]); - if result.is_ok() { - return Ok(()); + for _ in 0..120 { + let exists = run_cmd_output("kubectl", &["get", "crd", crd]).is_ok(); + if exists { + let established = run_cmd_output( + "kubectl", + &[ + "get", + "crd", + crd, + "-o", + "jsonpath={.status.conditions[?(@.type==\"Established\")].status}", + ], + ) + .unwrap_or_default(); + if established.trim() == "True" { + // Brief settle so discovery caches pick up the new kind. + thread::sleep(Duration::from_secs(2)); + return Ok(()); + } } thread::sleep(Duration::from_secs(5)); } - Err(format!("Timed out waiting for CRD {}", crd).into()) + Err(format!("Timed out waiting for CRD {} to be Established", crd).into()) } diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs new file mode 100644 index 0000000..b2882cb --- /dev/null +++ b/tests/colima_smoke_workflow.rs @@ -0,0 +1,90 @@ +//! Structural checks for the shipped colima macOS smoke workflow. +//! +//! These tests drive the real file under `.github/workflows/` so a missing +//! or regressed smoke contract fails `cargo test` without needing a macOS GHA run. + +use std::fs; +use std::path::PathBuf; + +fn workflow_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-colima-smoke.yaml") +} + +fn workflow_text() -> String { + let path = workflow_path(); + assert!( + path.is_file(), + "expected colima smoke workflow at {}", + path.display() + ); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +#[test] +fn colima_smoke_workflow_exists_and_pins_nested_virt_runner() { + let text = workflow_text(); + assert!( + text.contains("runs-on: macos-15-intel"), + "must pin nested-virt-capable Intel runner" + ); + assert!( + !text + .lines() + .any(|l| l.trim() == "runs-on: macos-latest"), + "must not use bare macos-latest as sole runs-on" + ); + assert!( + text.to_lowercase().contains("nested") && text.to_lowercase().contains("virt"), + "must comment on runner/virt constraints" + ); +} + +#[test] +fn colima_smoke_workflow_kind_parity_sequence() { + let text = workflow_text(); + for needle in [ + "cargo build", + "start --backend colima", + "local doctor", + "localhost:30500", + "registry.crossplane-system.svc.cluster.local:5000", + "--context colima", + "local stop", + "local destroy", + ] { + assert!( + text.contains(needle), + "colima smoke missing kind-parity fragment: {needle}" + ); + } + // stop/start resume: start without --backend after stop + assert!( + text.contains("local start\n") || text.lines().any(|l| l.trim() == "./target/debug/hops-cli local start"), + "must start again without --backend after stop" + ); + for tool in ["colima", "docker", "kubectl", "helm"] { + assert!( + text.contains(tool), + "prereq install must mention {tool}" + ); + } +} + +#[test] +fn colima_smoke_workflow_sizes_vm_for_gha_intel_runner() { + let text = workflow_text(); + // Defaults (8/16/60) exceed macos-15-intel; CI must pass explicit smaller sizes. + assert!( + text.contains("--cpus") && text.contains("--memory") && text.contains("--disk"), + "must pass --cpus/--memory/--disk so the VZ VM fits the runner" + ); + // 8Gi left CoreDNS thrashing; smoke needs headroom above that floor. + assert!( + text.contains("--memory 10") || text.contains("--memory 11") || text.contains("--memory 12"), + "must allocate at least 10Gi to the colima VM on GHA intel runners" + ); + assert!( + text.contains("ha.stderr.log"), + "failure dump must include lima hostagent stderr for nested-virt debug" + ); +}