From e3f4e9269a4a2adadf9c9c886ac1ef5727c210fb Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 13:52:13 -0500
Subject: [PATCH 01/11] ci: add macOS colima backend smoke workflow

Mirror kind smoke on macos-15-intel with nested-virt pin,
--backend colima, and kubectl --context colima.

Implements [[tasks/hops-cli-colima-macos-smoke]] under
[[tasks/hops-cli-macos-backend-smoke]].
---
 .github/workflows/on-pr-colima-smoke.yaml | 93 +++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 .github/workflows/on-pr-colima-smoke.yaml

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
new file mode 100644
index 0000000..fd05b46
--- /dev/null
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -0,0 +1,93 @@
+name: colima backend smoke
+
+# End-to-end check of `hops local` on the colima backend on macOS.
+# Mirrors on-pr-kind-smoke.yaml with --backend colima and kubectl --context colima.
+#
+# Runner constraints:
+# - Colima needs nested virtualization (Lima VM). Pin macos-15-intel — that
+#   image is known to support nested virt for Colima/Lima on GHA.
+# - Do NOT use bare macos-latest alone (tracks arm images; arm64 GHA macOS
+#   historically fails with HV_UNSUPPORTED for nested virt).
+# - Prefer localhost:30500 for registry traffic, not the Colima VM IP
+#   (macOS 15 Local Network Privacy can block non-root VM-IP access).
+#
+# Install: brew provides colima/docker/kubectl/helm only. Do not pre-start
+# colima here — hops local start --backend colima owns cluster bring-up.
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  colima-smoke:
+    # Nested-virt-capable Intel pin (not bare macos-latest).
+    runs-on: macos-15-intel
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Install colima, docker, kubectl, helm
+        run: |
+          set -euxo pipefail
+          # Tools only — do not `colima start` here so the smoke exercises hops.
+          brew install colima docker kubectl helm
+          colima version
+          docker version --format '{{.Client.Version}}' || docker --version
+          kubectl version --client
+          helm version --short
+
+      - name: Build hops
+        run: cargo build
+
+      - name: hops local start --backend colima
+        run: ./target/debug/hops-cli local start --backend colima
+
+      - name: hops local doctor
+        run: ./target/debug/hops-cli local doctor
+
+      - name: Registry round-trip through both pull names
+        run: |
+          set -euxo pipefail
+          docker pull public.ecr.aws/docker/library/busybox:stable
+          docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci
+          docker push localhost:30500/smoke/busybox:ci
+
+          # Service-name pull: containerd resolves via its certs.d alias.
+          kubectl --context colima run smoke-svc-name \
+            --image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \
+            --restart=Never --command -- sleep 300
+
+          # localhost:30500 pull: what provider runtime pods reference.
+          kubectl --context colima run smoke-localhost \
+            --image=localhost:30500/smoke/busybox:ci \
+            --restart=Never --command -- sleep 300
+
+          kubectl --context colima wait --for=condition=Ready \
+            pod/smoke-svc-name pod/smoke-localhost --timeout=180s
+          kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false
+
+      - name: Stop/start resume path (uses persisted backend, no flag)
+        run: |
+          set -euxo pipefail
+          ./target/debug/hops-cli local stop
+          ./target/debug/hops-cli local start
+          kubectl --context colima -n crossplane-system wait \
+            --for=condition=Available deployment/crossplane deployment/registry --timeout=300s
+          ./target/debug/hops-cli local doctor
+
+      - name: hops local destroy
+        if: always()
+        run: ./target/debug/hops-cli local destroy
+
+      - name: Debug dump on failure
+        if: failure()
+        run: |
+          set +e
+          colima status || true
+          colima list || true
+          kubectl --context colima get nodes,pods -A || true
+          ./target/debug/hops-cli local doctor || true

From 0f89fa0af8105179e46b6386e125d1b3126962b0 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 13:53:38 -0500
Subject: [PATCH 02/11] test: structural checks for colima macOS smoke workflow

Assert the shipped on-pr-colima-smoke.yaml pins macos-15-intel and
mirrors kind smoke without a full GHA run.
---
 tests/colima_smoke_workflow.rs | 71 ++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 tests/colima_smoke_workflow.rs

diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs
new file mode 100644
index 0000000..fe40c7e
--- /dev/null
+++ b/tests/colima_smoke_workflow.rs
@@ -0,0 +1,71 @@
+//! Structural checks for the shipped colima macOS smoke workflow.
+//!
+//! These tests drive the real file under `.github/workflows/` so a missing
+//! or regressed smoke contract fails `cargo test` without needing a macOS GHA run.
+
+use std::fs;
+use std::path::PathBuf;
+
+fn workflow_path() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".github/workflows/on-pr-colima-smoke.yaml")
+}
+
+fn workflow_text() -> String {
+    let path = workflow_path();
+    assert!(
+        path.is_file(),
+        "expected colima smoke workflow at {}",
+        path.display()
+    );
+    fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))
+}
+
+#[test]
+fn colima_smoke_workflow_exists_and_pins_nested_virt_runner() {
+    let text = workflow_text();
+    assert!(
+        text.contains("runs-on: macos-15-intel"),
+        "must pin nested-virt-capable Intel runner"
+    );
+    assert!(
+        !text
+            .lines()
+            .any(|l| l.trim() == "runs-on: macos-latest"),
+        "must not use bare macos-latest as sole runs-on"
+    );
+    assert!(
+        text.to_lowercase().contains("nested") && text.to_lowercase().contains("virt"),
+        "must comment on runner/virt constraints"
+    );
+}
+
+#[test]
+fn colima_smoke_workflow_kind_parity_sequence() {
+    let text = workflow_text();
+    for needle in [
+        "cargo build",
+        "start --backend colima",
+        "local doctor",
+        "localhost:30500",
+        "registry.crossplane-system.svc.cluster.local:5000",
+        "--context colima",
+        "local stop",
+        "local destroy",
+    ] {
+        assert!(
+            text.contains(needle),
+            "colima smoke missing kind-parity fragment: {needle}"
+        );
+    }
+    // stop/start resume: start without --backend after stop
+    assert!(
+        text.contains("local start\n") || text.lines().any(|l| l.trim() == "./target/debug/hops-cli local start"),
+        "must start again without --backend after stop"
+    );
+    for tool in ["colima", "docker", "kubectl", "helm"] {
+        assert!(
+            text.contains(tool),
+            "prereq install must mention {tool}"
+        );
+    }
+}

From 2349ced23f04b7e133c43c22c02ba8f9b016fabd Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 14:04:12 -0500
Subject: [PATCH 03/11] ci: run colima smoke for all PR base branches

Drop pull_request.branches: [main] so this job runs on stacked
PRs into feat/cluster-backend-abstraction.
---
 .github/workflows/on-pr-colima-smoke.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index fd05b46..dcb58f1 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -16,8 +16,6 @@ name: colima backend smoke
 
 on:
   pull_request:
-    branches:
-      - main
   workflow_dispatch:
 
 jobs:

From ad46b410c4750fc60bfd0157059ea060c9134b7d Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 14:31:45 -0500
Subject: [PATCH 04/11] ci: size colima smoke VM for macos-15-intel runners

Hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard GHA intel
runners (~4 / ~14), so VZ hostagent exits during start. Pass explicit
smaller sizes and dump lima ha.stderr on failure.
---
 .github/workflows/on-pr-colima-smoke.yaml | 37 +++++++++++++++++++++--
 tests/colima_smoke_workflow.rs            | 14 +++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index dcb58f1..999686b 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -13,6 +13,10 @@ name: colima backend smoke
 #
 # Install: brew provides colima/docker/kubectl/helm only. Do not pre-start
 # colima here — hops local start --backend colima owns cluster bring-up.
+#
+# Sizing: hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard
+# macos-15-intel runners (~4 CPU / ~14 GiB). Pass explicit smaller sizes
+# so the VZ VM can allocate; stop/start resume uses the persisted profile.
 
 on:
   pull_request:
@@ -41,8 +45,19 @@ jobs:
       - name: Build hops
         run: cargo build
 
+      - name: Host resources (for sizing)
+        run: |
+          set -euxo pipefail
+          sysctl -n hw.ncpu
+          sysctl hw.memsize
+          df -h /
+
       - name: hops local start --backend colima
-        run: ./target/debug/hops-cli local start --backend colima
+        run: |
+          set -euxo pipefail
+          # Fit macos-15-intel (~4 CPU / ~14 GiB). Defaults (8/16/60) OOM the VZ VM.
+          ./target/debug/hops-cli local start --backend colima \
+            --cpus 2 --memory 4 --disk 20
 
       - name: hops local doctor
         run: ./target/debug/hops-cli local doctor
@@ -79,13 +94,31 @@ jobs:
 
       - name: hops local destroy
         if: always()
-        run: ./target/debug/hops-cli local destroy
+        run: ./target/debug/hops-cli local destroy || true
 
       - name: Debug dump on failure
         if: failure()
         run: |
           set +e
+          echo "==== host ===="
+          sysctl -n hw.ncpu
+          sysctl hw.memsize
+          df -h /
+          echo "==== colima ===="
           colima status || true
           colima list || true
+          echo "==== lima ha.stderr / serial ===="
+          for f in \
+            "$HOME/.colima/_lima/colima/ha.stderr.log" \
+            "$HOME/.colima/_lima/colima/ha.stdout.log" \
+            "$HOME/.colima/_lima/colima/serial.log" \
+            "$HOME/.colima/_lima/colima/serialv.log"
+          do
+            if [ -f "$f" ]; then
+              echo "----- $f -----"
+              tail -n 200 "$f" || true
+            fi
+          done
+          ls -la "$HOME/.colima/_lima/colima/" 2>/dev/null || true
           kubectl --context colima get nodes,pods -A || true
           ./target/debug/hops-cli local doctor || true
diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs
index fe40c7e..5ab340b 100644
--- a/tests/colima_smoke_workflow.rs
+++ b/tests/colima_smoke_workflow.rs
@@ -69,3 +69,17 @@ fn colima_smoke_workflow_kind_parity_sequence() {
         );
     }
 }
+
+#[test]
+fn colima_smoke_workflow_sizes_vm_for_gha_intel_runner() {
+    let text = workflow_text();
+    // Defaults (8/16/60) exceed macos-15-intel; CI must pass explicit smaller sizes.
+    assert!(
+        text.contains("--cpus") && text.contains("--memory") && text.contains("--disk"),
+        "must pass --cpus/--memory/--disk so the VZ VM fits the runner"
+    );
+    assert!(
+        text.contains("ha.stderr.log"),
+        "failure dump must include lima hostagent stderr for nested-virt debug"
+    );
+}

From 30ea3ca4a03457106887468b6d670b70b8f0f33e Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 14:49:54 -0500
Subject: [PATCH 05/11] ci: give colima smoke enough memory for Crossplane

2 CPU / 4 GiB booted Colima but helm --wait for Crossplane hit
5m with Available: 0/1. Use 3/8 on macos-15-intel (~14 GiB host)
and raise Crossplane helm --wait timeout to 10m for nested virt.
---
 .github/workflows/on-pr-colima-smoke.yaml | 4 +++-
 src/commands/local/start.rs               | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index 999686b..5eb9c9b 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -56,8 +56,10 @@ jobs:
         run: |
           set -euxo pipefail
           # Fit macos-15-intel (~4 CPU / ~14 GiB). Defaults (8/16/60) OOM the VZ VM.
+          # 2/4 was enough to boot but Crossplane stayed 0/1 within helm 5m;
+          # use 3/8 so k3s + Crossplane can schedule on nested virt.
           ./target/debug/hops-cli local start --backend colima \
-            --cpus 2 --memory 4 --disk 20
+            --cpus 3 --memory 8 --disk 20
 
       - name: hops local doctor
         run: ./target/debug/hops-cli local doctor
diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs
index 136d760..ada2fc7 100644
--- a/src/commands/local/start.rs
+++ b/src/commands/local/start.rs
@@ -69,8 +69,10 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
             "crossplane-system",
             "--create-namespace",
             "--wait",
+            // Nested-virt CI (colima on GHA) routinely needs >5m for image pull
+            // + schedule after the docker-daemon restart path.
             "--timeout",
-            "5m",
+            "10m",
         ],
     )?;
 

From 86f2355c874f3c0b320c01ce66a059d97d26b117 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 15:21:39 -0500
Subject: [PATCH 06/11] fix: wait longer for Crossplane on nested-virt colima

Helm --wait hit 10m with Available:0/1 on GHA macos-15-intel. Apply
the chart without --wait, poll deployments for ~15m with periodic
pod dumps, wait for nodes Ready after docker restart, and capture
cluster state before destroy on CI failure.
---
 .github/workflows/on-pr-colima-smoke.yaml | 22 +++---
 src/commands/local/start.rs               | 86 ++++++++++++++++++++---
 2 files changed, 91 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index 5eb9c9b..8d64bf7 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -26,7 +26,8 @@ jobs:
   colima-smoke:
     # Nested-virt-capable Intel pin (not bare macos-latest).
     runs-on: macos-15-intel
-    timeout-minutes: 60
+    # Nested virt + cold Crossplane pulls can take well over 30m.
+    timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4
 
@@ -56,8 +57,8 @@ jobs:
         run: |
           set -euxo pipefail
           # Fit macos-15-intel (~4 CPU / ~14 GiB). Defaults (8/16/60) OOM the VZ VM.
-          # 2/4 was enough to boot but Crossplane stayed 0/1 within helm 5m;
-          # use 3/8 so k3s + Crossplane can schedule on nested virt.
+          # 2/4 boots Colima but Crossplane stayed 0/1; 3/8 leaves headroom for
+          # k3s + Crossplane + registry on nested virt.
           ./target/debug/hops-cli local start --backend colima \
             --cpus 3 --memory 8 --disk 20
 
@@ -94,10 +95,7 @@ jobs:
             --for=condition=Available deployment/crossplane deployment/registry --timeout=300s
           ./target/debug/hops-cli local doctor
 
-      - name: hops local destroy
-        if: always()
-        run: ./target/debug/hops-cli local destroy || true
-
+      # Capture live cluster state BEFORE destroy so failures are diagnosable.
       - name: Debug dump on failure
         if: failure()
         run: |
@@ -109,6 +107,10 @@ jobs:
           echo "==== colima ===="
           colima status || true
           colima list || true
+          echo "==== cluster ===="
+          kubectl --context colima get nodes,pods -A -o wide || true
+          kubectl --context colima describe pods -n crossplane-system || true
+          kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -80 || true
           echo "==== lima ha.stderr / serial ===="
           for f in \
             "$HOME/.colima/_lima/colima/ha.stderr.log" \
@@ -121,6 +123,8 @@ jobs:
               tail -n 200 "$f" || true
             fi
           done
-          ls -la "$HOME/.colima/_lima/colima/" 2>/dev/null || true
-          kubectl --context colima get nodes,pods -A || true
           ./target/debug/hops-cli local doctor || true
+
+      - name: hops local destroy
+        if: always()
+        run: ./target/debug/hops-cli local destroy || true
diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs
index ada2fc7..4d141e4 100644
--- a/src/commands/local/start.rs
+++ b/src/commands/local/start.rs
@@ -41,6 +41,20 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
     // 3. Make the node runtime trust the cluster-internal registry over HTTP.
     backend.ensure_registry_trust()?;
 
+    // After a docker daemon restart (colima path), re-confirm the node is Ready
+    // so Crossplane pods are not stuck Pending on NotReady.
+    log::info!("Waiting for Kubernetes nodes to be Ready...");
+    run_cmd(
+        "kubectl",
+        &[
+            "wait",
+            "--for=condition=Ready",
+            "nodes",
+            "--all",
+            "--timeout=300s",
+        ],
+    )?;
+
     // 4. Add Crossplane Helm repo
     log::info!("Adding Crossplane Helm repo...");
     run_cmd(
@@ -57,6 +71,11 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
     run_cmd("helm", &["repo", "update", "crossplane-stable"])?;
 
     // 5. Install Crossplane
+    //
+    // Do not use helm --wait: on nested-virt CI (colima/GHA) image pulls and
+    // scheduling can exceed helm's single wait window, and a failed --wait
+    // leaves us without structured kubectl diagnostics. Apply the chart, then
+    // poll deployments ourselves with a longer budget + failure dumps.
     log::info!("Installing Crossplane...");
     run_cmd(
         "helm",
@@ -68,17 +87,15 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
             "-n",
             "crossplane-system",
             "--create-namespace",
-            "--wait",
-            // Nested-virt CI (colima on GHA) routinely needs >5m for image pull
-            // + schedule after the docker-daemon restart path.
             "--timeout",
-            "10m",
+            "5m",
         ],
     )?;
 
-    // 6. Wait for Crossplane deployment
+    // 6. Wait for Crossplane deployment (and rbac-manager, which shares the chart)
     log::info!("Waiting for Crossplane to be ready...");
-    wait_for_deployment("crossplane-system", "crossplane")?;
+    wait_for_deployment_with_diagnostics("crossplane-system", "crossplane")?;
+    wait_for_deployment_with_diagnostics("crossplane-system", "crossplane-rbac-manager")?;
 
     // 7. Deploy per-provider DRCs (each pins its own cluster-admin SA)
     log::info!("Applying DeploymentRuntimeConfigs (per-provider)...");
@@ -113,9 +130,31 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
     Ok(())
 }
 
-/// Poll until a deployment's Available condition is True.
+/// Poll until a deployment's Available condition is True (~5 minutes).
 fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box<dyn Error>> {
-    for _ in 0..60 {
+    wait_for_deployment_attempts(namespace, name, 60)
+}
+
+/// Longer wait used for Crossplane on cold nested-virt runners (~15 minutes).
+fn wait_for_deployment_with_diagnostics(
+    namespace: &str,
+    name: &str,
+) -> Result<(), Box<dyn Error>> {
+    match wait_for_deployment_attempts(namespace, name, 180) {
+        Ok(()) => Ok(()),
+        Err(e) => {
+            dump_namespace_diagnostics(namespace);
+            Err(e)
+        }
+    }
+}
+
+fn wait_for_deployment_attempts(
+    namespace: &str,
+    name: &str,
+    attempts: u32,
+) -> Result<(), Box<dyn Error>> {
+    for i in 0..attempts {
         let output = run_cmd_output(
             "kubectl",
             &[
@@ -135,11 +174,42 @@ fn wait_for_deployment(namespace: &str, name: &str) -> Result<(), Box<dyn Error>
             }
         }
 
+        // Periodic progress so CI logs show the wait is alive.
+        if i > 0 && i % 12 == 0 {
+            log::info!(
+                "Still waiting for deployment {}/{} ({}s elapsed)...",
+                namespace,
+                name,
+                i * 5
+            );
+            let _ = run_cmd(
+                "kubectl",
+                &["get", "pods", "-n", namespace, "-o", "wide"],
+            );
+        }
+
         thread::sleep(Duration::from_secs(5));
     }
     Err(format!("Timed out waiting for deployment {}/{}", namespace, name).into())
 }
 
+fn dump_namespace_diagnostics(namespace: &str) {
+    log::error!("Diagnostics for namespace {} after readiness timeout:", namespace);
+    let _ = run_cmd("kubectl", &["get", "pods", "-n", namespace, "-o", "wide"]);
+    let _ = run_cmd("kubectl", &["describe", "pods", "-n", namespace]);
+    let _ = run_cmd(
+        "kubectl",
+        &[
+            "get",
+            "events",
+            "-n",
+            namespace,
+            "--sort-by=.lastTimestamp",
+        ],
+    );
+    let _ = run_cmd("kubectl", &["get", "nodes", "-o", "wide"]);
+}
+
 /// Poll until a CRD exists in the cluster.
 fn wait_for_crd(crd: &str) -> Result<(), Box<dyn Error>> {
     log::info!("Waiting for CRD {}...", crd);

From a5c1ecf769a3ea728f7b6e5ce577ccfe601925b3 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 15:56:21 -0500
Subject: [PATCH 07/11] fix: harden kubectl apply under nested-virt API load

Crossplane came Ready on GHA colima but ProviderConfigs apply failed
with openapi schema download timeouts. Apply with --validate=false,
retry transient failures, and re-wait for the API after Crossplane
becomes leader. Soft-wait rbac-manager when core is healthy.
---
 src/commands/local/mod.rs   | 51 +++++++++++++++++++++++++++----------
 src/commands/local/start.rs | 18 +++++++++++--
 2 files changed, 53 insertions(+), 16 deletions(-)

diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs
index 0db9db0..bbb039b 100644
--- a/src/commands/local/mod.rs
+++ b/src/commands/local/mod.rs
@@ -267,24 +267,47 @@ pub(crate) fn wait_for_kubernetes() -> Result<(), Box<dyn Error>> {
 
 /// Pipe a YAML string into `kubectl apply -f -`.
 /// Automatically injects `--context` when configured.
+///
+/// Uses `--validate=false` so a slow/overloaded API server (common on nested
+/// virt CI while Crossplane is warming) does not fail the apply solely because
+/// OpenAPI schema download timed out. Retries a few times for transient
+/// connection errors.
 pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box<dyn Error>> {
-    let full = with_kube_context(&["apply", "-f", "-"]);
-    let mut child = Command::new("kubectl")
-        .args(&full)
-        .stdin(Stdio::piped())
-        .stdout(Stdio::inherit())
-        .stderr(Stdio::inherit())
-        .spawn()?;
+    let full = with_kube_context(&["apply", "--validate=false", "-f", "-"]);
+    let mut last_status = None;
+
+    for attempt in 1..=5 {
+        let mut child = Command::new("kubectl")
+            .args(&full)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::inherit())
+            .stderr(Stdio::inherit())
+            .spawn()?;
+
+        if let Some(ref mut stdin) = child.stdin {
+            stdin.write_all(yaml.as_bytes())?;
+        }
 
-    if let Some(ref mut stdin) = child.stdin {
-        stdin.write_all(yaml.as_bytes())?;
+        let status = child.wait()?;
+        if status.success() {
+            return Ok(());
+        }
+        last_status = Some(status);
+        log::warn!(
+            "kubectl apply failed (attempt {}/5, status {}); retrying...",
+            attempt,
+            status
+        );
+        std::thread::sleep(std::time::Duration::from_secs(5));
     }
 
-    let status = child.wait()?;
-    if !status.success() {
-        return Err(format!("kubectl apply exited with {}", status).into());
-    }
-    Ok(())
+    Err(format!(
+        "kubectl apply exited with {} after retries",
+        last_status
+            .map(|s| s.to_string())
+            .unwrap_or_else(|| "unknown".into())
+    )
+    .into())
 }
 
 /// Apply a JSON merge patch with `kubectl patch --type merge`.
diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs
index 4d141e4..1c9ace9 100644
--- a/src/commands/local/start.rs
+++ b/src/commands/local/start.rs
@@ -92,10 +92,20 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
         ],
     )?;
 
-    // 6. Wait for Crossplane deployment (and rbac-manager, which shares the chart)
+    // 6. Wait for Crossplane core deployment.
+    // rbac-manager can flap under nested-virt resource pressure; the core
+    // controller is what providers need. Best-effort wait for rbac-manager.
     log::info!("Waiting for Crossplane to be ready...");
     wait_for_deployment_with_diagnostics("crossplane-system", "crossplane")?;
-    wait_for_deployment_with_diagnostics("crossplane-system", "crossplane-rbac-manager")?;
+    if let Err(e) = wait_for_deployment_attempts("crossplane-system", "crossplane-rbac-manager", 36)
+    {
+        log::warn!(
+            "crossplane-rbac-manager not Available yet ({e}); continuing — core Crossplane is ready"
+        );
+    }
+
+    // API can be briefly overloaded right after Crossplane becomes leader.
+    wait_for_kubernetes()?;
 
     // 7. Deploy per-provider DRCs (each pins its own cluster-admin SA)
     log::info!("Applying DeploymentRuntimeConfigs (per-provider)...");
@@ -112,6 +122,10 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
     wait_for_crd("providerconfigs.helm.m.crossplane.io")?;
     wait_for_crd("providerconfigs.kubernetes.m.crossplane.io")?;
 
+    // Re-check API before ProviderConfigs (openapi validation can time out if
+    // the apiserver is still busy — apply also uses --validate=false).
+    wait_for_kubernetes()?;
+
     // 10. Apply ProviderConfigs
     log::info!("Applying ProviderConfigs...");
     kubectl_apply_stdin(PC_HELM)?;

From 30e5e524f87ad000eb44d43d39fe891c403c340e Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 16:28:43 -0500
Subject: [PATCH 08/11] fix: wait for Established CRDs and room for registry
 PVC
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ProviderConfig apply raced CRD discovery; wait for Established.
Registry PVC requests 20Gi — give colima smoke a 40Gi VM disk so
local-path can bind, and wait longer with diagnostics for registry.
---
 .github/workflows/on-pr-colima-smoke.yaml |  8 +++---
 src/commands/local/start.rs               | 35 ++++++++++++++++++-----
 2 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index 8d64bf7..ed5f6c8 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -56,11 +56,11 @@ jobs:
       - name: hops local start --backend colima
         run: |
           set -euxo pipefail
-          # Fit macos-15-intel (~4 CPU / ~14 GiB). Defaults (8/16/60) OOM the VZ VM.
-          # 2/4 boots Colima but Crossplane stayed 0/1; 3/8 leaves headroom for
-          # k3s + Crossplane + registry on nested virt.
+          # Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ.
+          # Disk 40: registry PVC requests 20Gi; a 20Gi VM disk is too tight once
+          # the OS + k3s + images are present (PVC Pending → registry never Ready).
           ./target/debug/hops-cli local start --backend colima \
-            --cpus 3 --memory 8 --disk 20
+            --cpus 3 --memory 8 --disk 40
 
       - name: hops local doctor
         run: ./target/debug/hops-cli local doctor
diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs
index 1c9ace9..fd5b71f 100644
--- a/src/commands/local/start.rs
+++ b/src/commands/local/start.rs
@@ -134,7 +134,9 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
     // 11. Deploy local OCI registry for Crossplane packages
     log::info!("Deploying local package registry...");
     kubectl_apply_stdin(REGISTRY)?;
-    wait_for_deployment("crossplane-system", "registry")?;
+    // Nested virt: image pull + schedule for the registry can exceed the
+    // default ~5m wait used for lighter resources.
+    wait_for_deployment_with_diagnostics("crossplane-system", "registry")?;
 
     // 12. Point the node at the registry Service's ClusterIP so pulls of the
     //     cluster-internal registry names resolve.
@@ -224,15 +226,34 @@ fn dump_namespace_diagnostics(namespace: &str) {
     let _ = run_cmd("kubectl", &["get", "nodes", "-o", "wide"]);
 }
 
-/// Poll until a CRD exists in the cluster.
+/// Poll until a CRD exists **and** is Established (API serves the kind).
+///
+/// Merely creating the CRD object is not enough: under load the apiserver can
+/// return the CRD while discovery still lacks the kind, causing
+/// `no matches for kind "ProviderConfig"` on the next apply.
 fn wait_for_crd(crd: &str) -> Result<(), Box<dyn Error>> {
     log::info!("Waiting for CRD {}...", crd);
-    for _ in 0..60 {
-        let result = run_cmd_output("kubectl", &["get", "crd", crd]);
-        if result.is_ok() {
-            return Ok(());
+    for _ in 0..120 {
+        let exists = run_cmd_output("kubectl", &["get", "crd", crd]).is_ok();
+        if exists {
+            let established = run_cmd_output(
+                "kubectl",
+                &[
+                    "get",
+                    "crd",
+                    crd,
+                    "-o",
+                    "jsonpath={.status.conditions[?(@.type==\"Established\")].status}",
+                ],
+            )
+            .unwrap_or_default();
+            if established.trim() == "True" {
+                // Brief settle so discovery caches pick up the new kind.
+                thread::sleep(Duration::from_secs(2));
+                return Ok(());
+            }
         }
         thread::sleep(Duration::from_secs(5));
     }
-    Err(format!("Timed out waiting for CRD {}", crd).into())
+    Err(format!("Timed out waiting for CRD {} to be Established", crd).into())
 }

From a7188bf1e8fa1ad3bb38c614de404a9cb4e052e1 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 17:31:44 -0500
Subject: [PATCH 09/11] ci: recover colima stop/start resume after VM container
 churn

After colima stop/start, pods Error on missing docker containers.
Wait longer for Available and rollout-restart crossplane/registry
if the first wait stalls.
---
 .github/workflows/on-pr-colima-smoke.yaml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index ed5f6c8..3804d66 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -91,8 +91,20 @@ jobs:
           set -euxo pipefail
           ./target/debug/hops-cli local stop
           ./target/debug/hops-cli local start
-          kubectl --context colima -n crossplane-system wait \
-            --for=condition=Available deployment/crossplane deployment/registry --timeout=300s
+          # After colima VM stop/start, docker container IDs are gone and pods
+          # often sit in Error until kubelet recreates them. Wait generously and
+          # force a rollout restart if Available still stalls.
+          if ! kubectl --context colima -n crossplane-system wait \
+              --for=condition=Available deployment/crossplane deployment/registry \
+              --timeout=420s; then
+            echo "deployments not Available after resume; restarting..."
+            kubectl --context colima -n crossplane-system get pods -o wide || true
+            kubectl --context colima -n crossplane-system rollout restart \
+              deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true
+            kubectl --context colima -n crossplane-system wait \
+              --for=condition=Available deployment/crossplane deployment/registry \
+              --timeout=420s
+          fi
           ./target/debug/hops-cli local doctor
 
       # Capture live cluster state BEFORE destroy so failures are diagnosable.

From d8345e96587c92ed30afcca6672c1072cc6c5943 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 17:51:11 -0500
Subject: [PATCH 10/11] fix: survive apiserver overload when applying registry

Nested-virt colima CI loses the apiserver (TLS timeouts) right after
Crossplane/provider install. Wait longer for /readyz, retry kubectl
apply with API re-probes, and pre-pull registry:2 before deploy.
---
 src/commands/local/mod.rs   | 22 +++++++++++++++++-----
 src/commands/local/start.rs |  6 ++++++
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/commands/local/mod.rs b/src/commands/local/mod.rs
index bbb039b..abb27ac 100644
--- a/src/commands/local/mod.rs
+++ b/src/commands/local/mod.rs
@@ -255,11 +255,16 @@ pub(crate) fn command_exists(program: &str) -> bool {
 /// Poll until the Kubernetes API server is reachable.
 pub(crate) fn wait_for_kubernetes() -> Result<(), Box<dyn Error>> {
     log::info!("Waiting for Kubernetes API...");
-    for _ in 0..60 {
-        let result = run_cmd_output("kubectl", &["cluster-info"]);
+    // ~10 minutes — nested-virt apiserver can stay overloaded after package install.
+    for _ in 0..120 {
+        let result = run_cmd_output("kubectl", &["get", "--raw", "/readyz"]);
         if result.is_ok() {
             return Ok(());
         }
+        // Fall back to a cheap list if /readyz is denied on some setups.
+        if run_cmd_output("kubectl", &["get", "ns", "default"]).is_ok() {
+            return Ok(());
+        }
         std::thread::sleep(std::time::Duration::from_secs(5));
     }
     Err("Timed out waiting for Kubernetes API".into())
@@ -275,8 +280,12 @@ pub(crate) fn wait_for_kubernetes() -> Result<(), Box<dyn Error>> {
 pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box<dyn Error>> {
     let full = with_kube_context(&["apply", "--validate=false", "-f", "-"]);
     let mut last_status = None;
+    // Nested-virt CI (colima/GHA) can lose the apiserver for minutes after
+    // Crossplane/provider install (TLS handshake timeouts). Retry with backoff
+    // and re-probe the API between attempts.
+    const ATTEMPTS: u32 = 12;
 
-    for attempt in 1..=5 {
+    for attempt in 1..=ATTEMPTS {
         let mut child = Command::new("kubectl")
             .args(&full)
             .stdin(Stdio::piped())
@@ -294,11 +303,14 @@ pub fn kubectl_apply_stdin(yaml: &str) -> Result<(), Box<dyn Error>> {
         }
         last_status = Some(status);
         log::warn!(
-            "kubectl apply failed (attempt {}/5, status {}); retrying...",
+            "kubectl apply failed (attempt {}/{}, status {}); waiting for API...",
             attempt,
+            ATTEMPTS,
             status
         );
-        std::thread::sleep(std::time::Duration::from_secs(5));
+        // Best-effort API recovery before the next apply.
+        let _ = wait_for_kubernetes();
+        std::thread::sleep(std::time::Duration::from_secs(10));
     }
 
     Err(format!(
diff --git a/src/commands/local/start.rs b/src/commands/local/start.rs
index fd5b71f..b151675 100644
--- a/src/commands/local/start.rs
+++ b/src/commands/local/start.rs
@@ -132,6 +132,12 @@ pub fn run(backend: backend::Backend, args: &StartArgs) -> Result<(), Box<dyn Er
     kubectl_apply_stdin(PC_K8S)?;
 
     // 11. Deploy local OCI registry for Crossplane packages
+    //
+    // Provider install can leave the apiserver briefly unresponsive; re-wait
+    // and best-effort pre-pull registry:2 so the pod is not cold-started.
+    wait_for_kubernetes()?;
+    log::info!("Pre-pulling registry:2 (best effort)...");
+    let _ = run_cmd("docker", &["pull", "registry:2"]);
     log::info!("Deploying local package registry...");
     kubectl_apply_stdin(REGISTRY)?;
     // Nested virt: image pull + schedule for the registry can exceed the

From 7c950e1e6a38621be5e3a542292c4ddfb90468e7 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Sat, 11 Jul 2026 18:38:37 -0500
Subject: [PATCH 11/11] fix(ci): give colima smoke more RAM and settle time for
 registry pulls

At 3/8/40, dual-name registry smoke pods sat in ContainerCreating without
IPs while CoreDNS and metrics-server thrashed. Bump VM memory to 10Gi,
wait for node/CoreDNS before the push, extend Ready timeout, and dump
smoke pod describe on failure.
---
 .github/workflows/on-pr-colima-smoke.yaml | 36 +++++++++++++++++++----
 tests/colima_smoke_workflow.rs            |  5 ++++
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-pr-colima-smoke.yaml b/.github/workflows/on-pr-colima-smoke.yaml
index 3804d66..b1bfa70 100644
--- a/.github/workflows/on-pr-colima-smoke.yaml
+++ b/.github/workflows/on-pr-colima-smoke.yaml
@@ -57,14 +57,27 @@ jobs:
         run: |
           set -euxo pipefail
           # Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ.
-          # Disk 40: registry PVC requests 20Gi; a 20Gi VM disk is too tight once
-          # the OS + k3s + images are present (PVC Pending → registry never Ready).
+          # Memory 10 (not 8): at 8Gi CoreDNS/metrics-server thrash and smoke pods
+          # sit in ContainerCreating without IPs even after images pull. Leave ~4Gi
+          # for host macOS + VZ. Disk 40: registry PVC requests 20Gi.
           ./target/debug/hops-cli local start --backend colima \
-            --cpus 3 --memory 8 --disk 40
+            --cpus 3 --memory 10 --disk 40
 
       - name: hops local doctor
         run: ./target/debug/hops-cli local doctor
 
+      - name: Wait for node + CoreDNS (nested virt settles)
+        run: |
+          set -euxo pipefail
+          # After cold start the control plane is still soft: probes time out and
+          # new pods stick in ContainerCreating. Wait for basics before registry.
+          kubectl --context colima wait --for=condition=Ready nodes --all --timeout=120s
+          kubectl --context colima -n kube-system wait --for=condition=Ready \
+            pod -l k8s-app=kube-dns --timeout=180s || \
+            kubectl --context colima -n kube-system wait --for=condition=Ready \
+              pod -l k8s-app=coredns --timeout=180s || true
+          kubectl --context colima -n kube-system get pods -o wide || true
+
       - name: Registry round-trip through both pull names
         run: |
           set -euxo pipefail
@@ -82,8 +95,16 @@ jobs:
             --image=localhost:30500/smoke/busybox:ci \
             --restart=Never --command -- sleep 300
 
-          kubectl --context colima wait --for=condition=Ready \
-            pod/smoke-svc-name pod/smoke-localhost --timeout=180s
+          # Nested virt: image pull can succeed while CNI/IP assignment lags.
+          if ! kubectl --context colima wait --for=condition=Ready \
+              pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then
+            echo "==== smoke pods not Ready ===="
+            kubectl --context colima get pods smoke-svc-name smoke-localhost -o wide || true
+            kubectl --context colima describe pod smoke-svc-name smoke-localhost || true
+            kubectl --context colima get events -A --field-selector involvedObject.name=smoke-svc-name || true
+            kubectl --context colima get events -A --field-selector involvedObject.name=smoke-localhost || true
+            exit 1
+          fi
           kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false
 
       - name: Stop/start resume path (uses persisted backend, no flag)
@@ -121,8 +142,11 @@ jobs:
           colima list || true
           echo "==== cluster ===="
           kubectl --context colima get nodes,pods -A -o wide || true
+          echo "==== smoke pods (default) ===="
+          kubectl --context colima describe pod smoke-svc-name smoke-localhost 2>/dev/null || true
+          echo "==== crossplane-system ===="
           kubectl --context colima describe pods -n crossplane-system || true
-          kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -80 || true
+          kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -100 || true
           echo "==== lima ha.stderr / serial ===="
           for f in \
             "$HOME/.colima/_lima/colima/ha.stderr.log" \
diff --git a/tests/colima_smoke_workflow.rs b/tests/colima_smoke_workflow.rs
index 5ab340b..b2882cb 100644
--- a/tests/colima_smoke_workflow.rs
+++ b/tests/colima_smoke_workflow.rs
@@ -78,6 +78,11 @@ fn colima_smoke_workflow_sizes_vm_for_gha_intel_runner() {
         text.contains("--cpus") && text.contains("--memory") && text.contains("--disk"),
         "must pass --cpus/--memory/--disk so the VZ VM fits the runner"
     );
+    // 8Gi left CoreDNS thrashing; smoke needs headroom above that floor.
+    assert!(
+        text.contains("--memory 10") || text.contains("--memory 11") || text.contains("--memory 12"),
+        "must allocate at least 10Gi to the colima VM on GHA intel runners"
+    );
     assert!(
         text.contains("ha.stderr.log"),
         "failure dump must include lima hostagent stderr for nested-virt debug"