Reject unhealthy local k3s substrate states

docs/CONFIGURATION.md

@@ -22,8 +22,8 @@ edge config show
 
 ```yaml
 repos:
-  k3sNvidiaEdge: /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/k3s-nvidia-edge
-  llmObservabilityStack: /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/llm-observability-stack
+  k3sNvidiaEdge: /media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia/Project-Edge-Computing-LLM/k3s-nvidia-edge
+  llmObservabilityStack: /media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia/Project-Edge-Computing-LLM/llm-observability-stack
 
 cluster:
   kubeconfig: ""

docs/LIVE-VALIDATION-2026-07-19.md

@@ -0,0 +1,20 @@
+# Live validation: 2026-07-19
+
+All local Go gates passed on Ubuntu 24.04: module verification, formatting,
+unit tests, race tests, vet, build, and `govulncheck` (no reachable
+vulnerabilities).
+
+The live `edge validate infra --skip-cuda` audit exposed a false-positive in
+the earlier validator. A successful `kubectl get pods` command was reported as
+healthy even when the returned GPU Operator pods included CrashLoopBackOff and
+unready containers. The host had also rebooted onto a different network
+interface while k3s retained a stale InternalIP and Flannel interface.
+
+Validation now parses pod phase and container readiness, permits successfully
+Completed validator pods, and rejects every other unready pod. It also compares
+the advertised k3s InternalIP with the host's current IPv4 interfaces. These
+checks run before CUDA validation, so a stale network cannot be mistaken for a
+healthy GPU substrate merely because `nvidia.com/gpu` remains allocatable.
+
+No kubeconfig content, Secrets, host addresses, pod logs, prompts, responses,
+or model data are recorded here.

docs/LOCAL-DEPENDENCY-INVENTORY.md

@@ -1,8 +1,11 @@
 # Local dependency and repository inventory
 
-Validated on 2026-07-17 against
+Validated again on 2026-07-19 against
 `/media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia`.
-The scan found 58 local Git worktrees. This document distinguishes the platform's
+The scan found 64 local Git worktrees. See
+[LOCAL-REPOSITORY-AUDIT-2026-07-19.md](LOCAL-REPOSITORY-AUDIT-2026-07-19.md)
+for fetch, synchronization, test, release, and live-cluster results. This
+document distinguishes the platform's
 direct source/runtime dependencies from incidental links in vendored charts,
 lockfiles, generated package metadata, and documentation.
 

docs/LOCAL-REPOSITORY-AUDIT-2026-07-19.md

@@ -0,0 +1,111 @@
+# Local repository audit: 2026-07-19
+
+## Scope and method
+
+The audit recursively inspected all 64 Git worktrees under
+`Project-Linux-Kubernetes-Nvidia`, including remotes, tracking branches,
+working-tree state, latest commits, repository instructions, modules, charts,
+CI workflows, and direct Edge platform relationships. Every `origin` fetch
+completed successfully. All worktrees were clean at the baseline.
+
+The four `Edge-Computing-LLM` organization repositories were compared with the
+GitHub organization API, pull requests, branches, and Actions runs. Their prior
+dated branches had already been merged and had byte-identical trees to `main`,
+so the local checkouts were safely fast-forwarded to `origin/main` before this
+audit's new branches were created.
+
+## Collection status
+
+- 64 Git worktrees total
+- 4 organization-owned deployable/evidence repositories
+- 7 third-party dashboard template candidates
+- 53 other upstream/reference worktrees
+- 42 clean reference branches behind their freshly fetched upstream tip
+- 22 clean branches synchronized at the baseline
+- 0 locally divergent or ahead-only reference branches
+- 0 failed remote fetches
+
+Reference branches were not fast-forwarded because they are vendor research
+inputs, not organization-owned deployment sources. Their remote-tracking refs
+are current, which is sufficient for dependency and release comparison without
+rewriting the user's chosen checkout commits.
+
+## Reference branches behind upstream
+
+The values below are commit counts (`ahead/behind`) after fetching.
+
+| Area | Repository | Ahead/behind |
+|---|---|---:|
+| Apache | echarts | 0/172 |
+| Artifact Hub | hub | 0/2 |
+| Cloudflare | cloudflared | 0/13 |
+| CoreDNS | coredns | 0/82 |
+| Dashboard candidate | next-shadcn-admin-dashboard | 0/18 |
+| Dashboard candidate | next-shadcn-dashboard-starter | 0/86 |
+| Dashboard candidate | nextadmin-dashboard | 0/1 |
+| Go | go | 0/5 |
+| Helm | helm | 0/6 |
+| Kubernetes | kubernetes-client/python | 0/22 |
+| Kubernetes | dra-driver-nvidia-gpu (two worktrees) | 0/36 each |
+| Kubernetes | node-feature-discovery | 0/19 |
+| Kubernetes | client-go | 0/19 |
+| Kubernetes | kubectl | 0/20 |
+| NVIDIA | DCGM | 0/1 |
+| NVIDIA | dcgm-exporter | 0/1 |
+| NVIDIA | go-dcgm | 0/2 |
+| NVIDIA | gpu-operator | 0/85 |
+| NVIDIA | k8s-device-plugin | 0/43 |
+| NVIDIA | libnvidia-container | 0/4 |
+| NVIDIA | nvidia-container-toolkit | 0/71 |
+| Ollama | ollama | 0/35 |
+| OpenAI | codex | 0/11 |
+| OpenAI | openai-agents-python | 0/4 |
+| OpenTelemetry | collector | 0/25 |
+| OpenTelemetry | collector-contrib | 0/183 |
+| OpenTelemetry | Helm charts (primary) | 0/19 |
+| OpenTelemetry | operator | 0/22 |
+| OpenTelemetry | Python | 0/28 |
+| OpenTelemetry | Python contrib | 0/20 |
+| OpenTelemetry | Python GenAI | 0/71 |
+| OpenTelemetry | semantic-conventions-genai | 0/10 |
+| Prometheus | community Helm charts (primary) | 0/40 |
+| Prometheus | kube-prometheus | 0/2 |
+| Rancher/k3s | k3s | 0/32 |
+| Rancher | local-path-provisioner | 0/4 |
+| Traefik | traefik | 0/8 |
+| Zensical | zensical | 0/2 |
+| Duplicate worktree | Prometheus Community charts | 0/109 |
+| Duplicate worktree | Grafana charts | 0/4 |
+| Duplicate worktree | OpenTelemetry charts | 0/41 |
+
+## Edge repository gates
+
+All four repositories passed formatting, `go mod verify`, unit tests, race
+tests, vet, builds, and `govulncheck`; no reachable Go vulnerabilities were
+found. The Helm repositories additionally passed dependency resolution, lint,
+and default/CPU/local/GeForce/full-NVIDIA rendering. The current GitHub `main`
+workflow is green in all four repositories and Dependabot reports no open alert
+in the three repositories where alert access is enabled.
+
+Coverage remains the largest engineering-quality gap. Core package coverage
+ranges from roughly 17% to 56%, while orchestration and CLI entry packages are
+mostly untested. Code scanning has no analysis workflow, secret scanning is
+disabled, and Dependabot alerts are disabled for `llm-observability-stack`.
+
+## Live Ubuntu/k3s/NVIDIA result
+
+The host runs Ubuntu 24.04, k3s v1.36.2+k3s1, and GPU Operator v26.3.3 with a
+GeForce 940M advertising one allocatable GPU. Qwen is registered, resident,
+using GPU layers, and within the 850 MiB evidence ceiling.
+
+The cluster is not fully healthy after a network-adapter change. k3s advertises
+a stale InternalIP and Flannel interface that no longer exist on the host.
+Pod-to-API routing fails, which cascades into GPU Operator, Node Feature
+Discovery, metrics-server, kube-state-metrics, and node-exporter failures. The
+new validators reject this state instead of accepting GPU capacity as a proxy
+for substrate health.
+
+Repairing `/etc/rancher/k3s/config.yaml` and restarting k3s requires local sudo
+authentication and was therefore not performed automatically. No addresses,
+kubeconfig content, Secrets, prompts, responses, or model weights are recorded
+in this audit.

internal/kubernetes/kubernetes.go

@@ -2,6 +2,9 @@ package kubernetes
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
 
 	"github.com/Edge-Computing-LLM/edge-cli/internal/execx"
 )
@@ -18,10 +21,134 @@ func (c Client) Nodes(ctx context.Context) execx.Result {
 	return c.Runner.Check(ctx, "cluster nodes", execx.Command{Name: "kubectl", Args: []string{"get", "nodes", "-o", "wide"}})
 }
 
+func (c Client) LocalNodeAddress(ctx context.Context) execx.Result {
+	nodes, err := c.Runner.Output(ctx, execx.Command{Name: "kubectl", Args: []string{"get", "nodes", "-o", "json"}})
+	if err != nil {
+		return execx.Result{Label: "k3s node address", Output: nodes, Err: err}
+	}
+	addresses, err := c.Runner.Output(ctx, execx.Command{Name: "ip", Args: []string{"-j", "-4", "address", "show"}})
+	if err != nil {
+		return execx.Result{Label: "k3s node address", Output: addresses, Err: err}
+	}
+	if err := ValidateLocalNodeAddresses([]byte(nodes), []byte(addresses)); err != nil {
+		return execx.Result{Label: "k3s node address", Err: err}
+	}
+	return execx.Result{Label: "k3s node address", Output: "advertised InternalIP matches a current host interface"}
+}
+
 func (c Client) RuntimeClassNvidia(ctx context.Context) execx.Result {
 	return c.Runner.Check(ctx, "NVIDIA RuntimeClass", execx.Command{Name: "kubectl", Args: []string{"get", "runtimeclass", "nvidia"}})
 }
 
+func (c Client) PodsReady(ctx context.Context, namespace string) execx.Result {
+	output, err := c.Runner.Output(ctx, execx.Command{Name: "kubectl", Args: []string{"get", "pods", "-n", namespace, "-o", "json"}})
+	if err != nil {
+		return execx.Result{Label: namespace + " pod health", Output: output, Err: err}
+	}
+	summary, err := ValidatePodReadiness([]byte(output))
+	return execx.Result{Label: namespace + " pod health", Output: summary, Err: err}
+}
+
+type nodeList struct {
+	Items []struct {
+		Status struct {
+			Addresses []struct {
+				Type    string `json:"type"`
+				Address string `json:"address"`
+			} `json:"addresses"`
+		} `json:"status"`
+	} `json:"items"`
+}
+
+type hostInterface struct {
+	AddressInfo []struct {
+		Family string `json:"family"`
+		Local  string `json:"local"`
+	} `json:"addr_info"`
+}
+
+func ValidateLocalNodeAddresses(nodesJSON, addressesJSON []byte) error {
+	var nodes nodeList
+	if err := json.Unmarshal(nodesJSON, &nodes); err != nil {
+		return fmt.Errorf("decode Kubernetes nodes: %w", err)
+	}
+	var interfaces []hostInterface
+	if err := json.Unmarshal(addressesJSON, &interfaces); err != nil {
+		return fmt.Errorf("decode host interfaces: %w", err)
+	}
+	local := map[string]bool{}
+	for _, iface := range interfaces {
+		for _, address := range iface.AddressInfo {
+			if address.Family == "inet" {
+				local[address.Local] = true
+			}
+		}
+	}
+	if len(nodes.Items) == 0 {
+		return fmt.Errorf("no Kubernetes nodes found")
+	}
+	for _, node := range nodes.Items {
+		internalIP := ""
+		for _, address := range node.Status.Addresses {
+			if address.Type == "InternalIP" {
+				internalIP = address.Address
+				break
+			}
+		}
+		if internalIP == "" {
+			return fmt.Errorf("k3s node has no InternalIP")
+		}
+		if !local[internalIP] {
+			return fmt.Errorf("k3s node InternalIP is not assigned to a current host interface; check node-ip and flannel-iface in /etc/rancher/k3s/config.yaml")
+		}
+	}
+	return nil
+}
+
+type podList struct {
+	Items []struct {
+		Metadata struct {
+			Name string `json:"name"`
+		} `json:"metadata"`
+		Status struct {
+			Phase             string `json:"phase"`
+			ContainerStatuses []struct {
+				Ready bool `json:"ready"`
+			} `json:"containerStatuses"`
+		} `json:"status"`
+	} `json:"items"`
+}
+
+func ValidatePodReadiness(data []byte) (string, error) {
+	var pods podList
+	if err := json.Unmarshal(data, &pods); err != nil {
+		return "", fmt.Errorf("decode Kubernetes pods: %w", err)
+	}
+	if len(pods.Items) == 0 {
+		return "", fmt.Errorf("no pods found")
+	}
+	active, completed := 0, 0
+	var unhealthy []string
+	for _, pod := range pods.Items {
+		if pod.Status.Phase == "Succeeded" {
+			completed++
+			continue
+		}
+		active++
+		ready := pod.Status.Phase == "Running" && len(pod.Status.ContainerStatuses) > 0
+		for _, status := range pod.Status.ContainerStatuses {
+			ready = ready && status.Ready
+		}
+		if !ready {
+			unhealthy = append(unhealthy, pod.Metadata.Name+" ("+pod.Status.Phase+")")
+		}
+	}
+	if len(unhealthy) > 0 {
+		return fmt.Sprintf("%d active, %d completed", active, completed), fmt.Errorf("unhealthy pods: %s", strings.Join(unhealthy, ", "))
+	}
+	return fmt.Sprintf("%d active pods ready, %d completed", active, completed), nil
+}
+
 func (c Client) Namespace(ctx context.Context, ns string) execx.Result {
 	return c.Runner.Check(ctx, "namespace "+ns, execx.Command{Name: "kubectl", Args: []string{"get", "namespace", ns}})
 }

internal/kubernetes/kubernetes_test.go

@@ -0,0 +1,33 @@
+package kubernetes
+
+import "testing"
+
+func TestValidateLocalNodeAddresses(t *testing.T) {
+	nodes := []byte(`{"items":[{"status":{"addresses":[{"type":"InternalIP","address":"192.0.2.10"}]}}]}`)
+	current := []byte(`[{"addr_info":[{"family":"inet","local":"192.0.2.10"}]}]`)
+	stale := []byte(`[{"addr_info":[{"family":"inet","local":"192.0.2.20"}]}]`)
+	if err := ValidateLocalNodeAddresses(nodes, current); err != nil {
+		t.Fatalf("current address rejected: %v", err)
+	}
+	if err := ValidateLocalNodeAddresses(nodes, stale); err == nil {
+		t.Fatal("stale node address accepted")
+	}
+}
+
+func TestValidatePodReadiness(t *testing.T) {
+	healthy := []byte(`{"items":[
+		{"metadata":{"name":"operator"},"status":{"phase":"Running","containerStatuses":[{"ready":true}]}},
+		{"metadata":{"name":"validator"},"status":{"phase":"Succeeded","containerStatuses":[{"ready":false}]}}
+	]}`)
+	if _, err := ValidatePodReadiness(healthy); err != nil {
+		t.Fatalf("healthy pod set rejected: %v", err)
+	}
+
+	unhealthy := []byte(`{"items":[
+		{"metadata":{"name":"operator"},"status":{"phase":"Running","containerStatuses":[{"ready":false}]}},
+		{"metadata":{"name":"worker"},"status":{"phase":"Failed","containerStatuses":[{"ready":false}]}}
+	]}`)
+	if _, err := ValidatePodReadiness(unhealthy); err == nil {
+		t.Fatal("unhealthy pod set accepted")
+	}
+}

internal/modules/infra/infra.go

@@ -71,10 +71,11 @@ func Doctor(ctx context.Context, opts Options) error {
 	)
 	k := kubernetes.Client{Runner: r}
 	results = append(results, k.Nodes(ctx))
+	results = append(results, k.LocalNodeAddress(ctx))
 	if opts.NVIDIAEnabled {
 		results = append(results,
 			k.RuntimeClassNvidia(ctx),
-			r.Check(ctx, "GPU Operator pods", execx.Command{Name: "kubectl", Args: []string{"get", "pods", "-n", "gpu-operator", "-o", "wide"}}),
+			k.PodsReady(ctx, "gpu-operator"),
 			gpuCapacityResult(ctx, r),
 			r.Check(ctx, "CUDA validation support", execx.Command{Name: "kubectl", Args: []string{"get", "runtimeclass", "nvidia"}}),
 		)
@@ -147,11 +148,12 @@ func Validate(ctx context.Context, opts Options) error {
 	results := []execx.Result{
 		k.ClusterInfo(ctx),
 		k.Nodes(ctx),
+		k.LocalNodeAddress(ctx),
 	}
 	if opts.NVIDIAEnabled {
 		results = append(results,
 			k.RuntimeClassNvidia(ctx),
-			r.Check(ctx, "GPU Operator pods", execx.Command{Name: "kubectl", Args: []string{"get", "pods", "-n", "gpu-operator", "-o", "wide"}}),
+			k.PodsReady(ctx, "gpu-operator"),
 			gpuCapacityResult(ctx, r),
 		)
 	}