Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions .github/workflows/on-pr-colima-smoke.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
name: colima backend smoke

# End-to-end check of `hops local` on the colima backend on macOS.
# Mirrors on-pr-kind-smoke.yaml with --backend colima and kubectl --context colima.
#
# Runner constraints:
# - Colima needs nested virtualization (Lima VM). Pin macos-15-intel — that
# image is known to support nested virt for Colima/Lima on GHA.
# - Do NOT use bare macos-latest alone (tracks arm images; arm64 GHA macOS
# historically fails with HV_UNSUPPORTED for nested virt).
# - Prefer localhost:30500 for registry traffic, not the Colima VM IP
# (macOS 15 Local Network Privacy can block non-root VM-IP access).
#
# Install: brew provides colima/docker/kubectl/helm only. Do not pre-start
# colima here — hops local start --backend colima owns cluster bring-up.
#
# Sizing: hops defaults (8 CPU / 16 GiB / 60 GiB) exceed standard
# macos-15-intel runners (~4 CPU / ~14 GiB). Pass explicit smaller sizes
# so the VZ VM can allocate; stop/start resume uses the persisted profile.

on:
pull_request:
workflow_dispatch:

jobs:
colima-smoke:
# Nested-virt-capable Intel pin (not bare macos-latest).
runs-on: macos-15-intel
# Nested virt + cold Crossplane pulls can take well over 30m.
timeout-minutes: 90
steps:
- uses: actions/checkout@v4

- uses: Swatinem/rust-cache@v2

- name: Install colima, docker, kubectl, helm
run: |
set -euxo pipefail
# Tools only — do not `colima start` here so the smoke exercises hops.
brew install colima docker kubectl helm
colima version
docker version --format '{{.Client.Version}}' || docker --version
kubectl version --client
helm version --short

- name: Build hops
run: cargo build

- name: Host resources (for sizing)
run: |
set -euxo pipefail
sysctl -n hw.ncpu
sysctl hw.memsize
df -h /

- name: hops local start --backend colima
run: |
set -euxo pipefail
# Fit macos-15-intel (~4 CPU / ~14 GiB RAM). Defaults (8/16/60) OOM VZ.
# Memory 10 (not 8): at 8Gi CoreDNS/metrics-server thrash and smoke pods
# sit in ContainerCreating without IPs even after images pull. Leave ~4Gi
# for host macOS + VZ. Disk 40: registry PVC requests 20Gi.
./target/debug/hops-cli local start --backend colima \
--cpus 3 --memory 10 --disk 40

- name: hops local doctor
run: ./target/debug/hops-cli local doctor

- name: Wait for node + CoreDNS (nested virt settles)
run: |
set -euxo pipefail
# After cold start the control plane is still soft: probes time out and
# new pods stick in ContainerCreating. Wait for basics before registry.
kubectl --context colima wait --for=condition=Ready nodes --all --timeout=120s
kubectl --context colima -n kube-system wait --for=condition=Ready \
pod -l k8s-app=kube-dns --timeout=180s || \
kubectl --context colima -n kube-system wait --for=condition=Ready \
pod -l k8s-app=coredns --timeout=180s || true
kubectl --context colima -n kube-system get pods -o wide || true

- name: Registry round-trip through both pull names
run: |
set -euxo pipefail
docker pull public.ecr.aws/docker/library/busybox:stable
docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci
docker push localhost:30500/smoke/busybox:ci

# Service-name pull: containerd resolves via its certs.d alias.
kubectl --context colima run smoke-svc-name \
--image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \
--restart=Never --command -- sleep 300

# localhost:30500 pull: what provider runtime pods reference.
kubectl --context colima run smoke-localhost \
--image=localhost:30500/smoke/busybox:ci \
--restart=Never --command -- sleep 300

# Nested virt: image pull can succeed while CNI/IP assignment lags.
if ! kubectl --context colima wait --for=condition=Ready \
pod/smoke-svc-name pod/smoke-localhost --timeout=420s; then
echo "==== smoke pods not Ready ===="
kubectl --context colima get pods smoke-svc-name smoke-localhost -o wide || true
kubectl --context colima describe pod smoke-svc-name smoke-localhost || true
kubectl --context colima get events -A --field-selector involvedObject.name=smoke-svc-name || true
kubectl --context colima get events -A --field-selector involvedObject.name=smoke-localhost || true
exit 1
fi
kubectl --context colima delete pod smoke-svc-name smoke-localhost --wait=false

- name: Stop/start resume path (uses persisted backend, no flag)
run: |
set -euxo pipefail
./target/debug/hops-cli local stop
./target/debug/hops-cli local start
# After colima VM stop/start, docker container IDs are gone and pods
# often sit in Error until kubelet recreates them. Wait generously and
# force a rollout restart if Available still stalls.
if ! kubectl --context colima -n crossplane-system wait \
--for=condition=Available deployment/crossplane deployment/registry \
--timeout=420s; then
echo "deployments not Available after resume; restarting..."
kubectl --context colima -n crossplane-system get pods -o wide || true
kubectl --context colima -n crossplane-system rollout restart \
deployment/crossplane deployment/crossplane-rbac-manager deployment/registry || true
kubectl --context colima -n crossplane-system wait \
--for=condition=Available deployment/crossplane deployment/registry \
--timeout=420s
fi
./target/debug/hops-cli local doctor

# Capture live cluster state BEFORE destroy so failures are diagnosable.
- name: Debug dump on failure
if: failure()
run: |
set +e
echo "==== host ===="
sysctl -n hw.ncpu
sysctl hw.memsize
df -h /
echo "==== colima ===="
colima status || true
colima list || true
echo "==== cluster ===="
kubectl --context colima get nodes,pods -A -o wide || true
echo "==== smoke pods (default) ===="
kubectl --context colima describe pod smoke-svc-name smoke-localhost 2>/dev/null || true
echo "==== crossplane-system ===="
kubectl --context colima describe pods -n crossplane-system || true
kubectl --context colima get events -A --sort-by=.lastTimestamp | tail -100 || true
echo "==== lima ha.stderr / serial ===="
for f in \
"$HOME/.colima/_lima/colima/ha.stderr.log" \
"$HOME/.colima/_lima/colima/ha.stdout.log" \
"$HOME/.colima/_lima/colima/serial.log" \
"$HOME/.colima/_lima/colima/serialv.log"
do
if [ -f "$f" ]; then
echo "----- $f -----"
tail -n 200 "$f" || true
fi
done
./target/debug/hops-cli local doctor || true

- name: hops local destroy
if: always()
run: ./target/debug/hops-cli local destroy || true
60 changes: 60 additions & 0 deletions .github/workflows/on-pr-kind-smoke.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
name: kind backend smoke

# End-to-end check of `hops local` on the kind backend, which is the CI
# path (ubuntu runners ship kind, docker, kubectl, and helm). Exercises the
# registry NodePort mapping and BOTH containerd certs.d trust names, plus the
# stop/start resume path.

on:
pull_request:

jobs:
kind-smoke:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4

- uses: Swatinem/rust-cache@v2

- name: Build hops
run: cargo build

- name: hops local start --backend kind
run: ./target/debug/hops-cli local start --backend kind

- name: hops local doctor
run: ./target/debug/hops-cli local doctor

- name: Registry round-trip through both pull names
run: |
set -euxo pipefail
docker pull public.ecr.aws/docker/library/busybox:stable
docker tag public.ecr.aws/docker/library/busybox:stable localhost:30500/smoke/busybox:ci
docker push localhost:30500/smoke/busybox:ci

# Service-name pull: containerd resolves via its certs.d alias.
kubectl --context kind-hops run smoke-svc-name \
--image=registry.crossplane-system.svc.cluster.local:5000/smoke/busybox:ci \
--restart=Never --command -- sleep 300

# localhost:30500 pull: what provider runtime pods reference.
kubectl --context kind-hops run smoke-localhost \
--image=localhost:30500/smoke/busybox:ci \
--restart=Never --command -- sleep 300

kubectl --context kind-hops wait --for=condition=Ready \
pod/smoke-svc-name pod/smoke-localhost --timeout=180s
kubectl --context kind-hops delete pod smoke-svc-name smoke-localhost --wait=false

- name: Stop/start resume path (uses persisted backend, no flag)
run: |
set -euxo pipefail
./target/debug/hops-cli local stop
./target/debug/hops-cli local start
kubectl --context kind-hops -n crossplane-system wait \
--for=condition=Available deployment/crossplane deployment/registry --timeout=300s
./target/debug/hops-cli local doctor

- name: hops local destroy
run: ./target/debug/hops-cli local destroy
67 changes: 62 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@

This tool supports three related workflows:

- Local cluster setup on Colima
- Local cluster setup on colima or kind
- Configuration package install/uninstall against the connected cluster
- XR observe/manage/adopt/orphan workflows for existing infrastructure

For local development, it can also:

- Install and manage Colima
- Install and manage a local cluster backend (colima or kind)
- Start a local k8s cluster with Crossplane installed via Helm
- Install the Kubernetes and Helm Crossplane providers
- Deploy an in-cluster OCI registry (`crossplane-system/registry`)
Expand Down Expand Up @@ -56,7 +56,7 @@ See "Releases" for available versions and changenotes.
- `up` (Upbound CLI, used by `up project build`)
- `aws` CLI v2 (used by `local aws` to export profile credentials)

Note: `hops-cli local install` installs `colima` through Homebrew.
Note: `hops-cli local install` installs the selected backend (`colima` or `kind`) through Homebrew.

## Build

Expand Down Expand Up @@ -87,7 +87,7 @@ hops service --help
`hops-cli` is organized into a few command groups:

- `local`
- Manage a local Colima-based control plane, install providers, and bootstrap AWS or GitHub provider auth.
- Manage a local control plane (colima or kind backend), install providers, and bootstrap AWS or GitHub provider auth.
- `config`
- Build, install, reload, and uninstall Crossplane configuration packages against the connected cluster.
- `secrets`
Expand Down Expand Up @@ -192,7 +192,8 @@ Examples:
## Create a Local Control Plane

```bash
# 1) Install Colima (via Homebrew)
# 1) Install the backend (via Homebrew). Defaults to colima on macOS;
# pass --backend kind to use kind on any docker daemon.
hops local install

# 2) Start local k8s + Crossplane + providers + local registry
Expand All @@ -211,6 +212,62 @@ hops local zitadel --source-context pat-local --domain auth.ops.com.ai
hops config install --repo hops-ops/aws-auto-eks-cluster --version v0.11.0
```

### Cluster backends

`hops local` supports three backends behind the same commands:

- **colima** — a VM running dockerd + k3s. macOS/Linux; supports `--cpus`,
`--memory`, `--disk`, and `hops local resize`.
- **kind** — cluster nodes as docker containers on any reachable docker
daemon: Docker Desktop, colima's dockerd, or CI runners. No VM of its own,
so sizing flags don't apply (size the docker daemon instead); requires
kind >= v0.27.
- **dory** — [dory](https://augani.github.io/dory)'s built-in k3s, driven
headlessly through the `dory` CLI (`dory k8s enable/disable/status`).
Requires the Dory app running (it provides the engine and forwards
published ports to localhost) and a `dory` CLI with headless k8s support.
hops writes `~/.dory/k8s/registries.yaml` (k3s' native registry trust) and
publishes the registry NodePort at cluster create. The VM is sized in the
Dory app, so hops sizing flags don't apply.

Select with the global `--backend` flag:

```bash
hops local start --backend kind
```

The chosen backend is persisted to `~/.hops/local/backend` on a successful
start, so later commands (`stop`, `destroy`, `doctor`, package installs)
target the same cluster without the flag. Resolution order: `--backend` flag >
persisted choice > existing cluster detection (colima wins) > platform
default (macOS: colima, otherwise kind).

Unless `--context` is given, kubectl commands automatically use the backend's
kubeconfig context (`colima`, `kind-hops`, or `dory`), regardless of your
current-context. For dory, hops also prepends `~/.kube/dory-config` (where
dory keeps its kubeconfig) to `KUBECONFIG` for its own kubectl/helm calls.

#### Using dory

With a `dory` CLI that supports `dory k8s enable` (headless Kubernetes),
use the native backend:

```bash
hops local start --backend dory
```

For `hops provider install` / `hops config install` builds, point your docker
CLI at dory's engine (`export DOCKER_HOST=unix://$HOME/.dory/engine.sock`) so
image builds/pushes land on the daemon that reaches the registry.

Without the headless CLI, dory still exposes a real docker socket, so the
kind backend works against it:

```bash
docker context use dory # or: export DOCKER_HOST=unix://$HOME/.dory/dory.sock
hops local start --backend kind
```

### Local provider setup and auth

`hops local aws`, `hops local github`, and `hops local zitadel` install the provider package and bootstrap auth into a local control plane. The exception is `--refresh`, which updates credentials only.
Expand Down
11 changes: 5 additions & 6 deletions src/commands/auth/bootstrap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ pub fn run(args: &BootstrapArgs) -> Result<(), Box<dyn Error>> {

fn write_secret(path: &PathBuf, value: &str, force: bool) -> Result<(), Box<dyn Error>> {
if path.exists() && !force {
log::info!(" skip {}: already present (use --force to overwrite)", path.display());
log::info!(
" skip {}: already present (use --force to overwrite)",
path.display()
);
return Ok(());
}
if let Some(parent) = path.parent() {
Expand Down Expand Up @@ -151,11 +154,7 @@ mod tests {
"no lowercase: {}",
pwd
);
assert!(
pwd.chars().any(|c| c.is_ascii_digit()),
"no digit: {}",
pwd
);
assert!(pwd.chars().any(|c| c.is_ascii_digit()), "no digit: {}", pwd);
assert!(
pwd.chars().any(|c| !c.is_ascii_alphanumeric()),
"no symbol: {}",
Expand Down
Loading
Loading