diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 3b04ee5..0e27d4b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -22,8 +22,8 @@ edge config show ```yaml repos: - k3sNvidiaEdge: /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/k3s-nvidia-edge - llmObservabilityStack: /media/waqasm86/External1/Waqas-Projects/Project-Edge-Computing-LLM/llm-observability-stack + k3sNvidiaEdge: /media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia/Project-Edge-Computing-LLM/k3s-nvidia-edge + llmObservabilityStack: /media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia/Project-Edge-Computing-LLM/llm-observability-stack cluster: kubeconfig: "" diff --git a/docs/LIVE-VALIDATION-2026-07-19.md b/docs/LIVE-VALIDATION-2026-07-19.md new file mode 100644 index 0000000..7dabf31 --- /dev/null +++ b/docs/LIVE-VALIDATION-2026-07-19.md @@ -0,0 +1,20 @@ +# Live validation: 2026-07-19 + +All local Go gates passed on Ubuntu 24.04: module verification, formatting, +unit tests, race tests, vet, build, and `govulncheck` (no reachable +vulnerabilities). + +The live `edge validate infra --skip-cuda` audit exposed a false-positive in +the earlier validator. A successful `kubectl get pods` command was reported as +healthy even when the returned GPU Operator pods included CrashLoopBackOff and +unready containers. The host had also rebooted onto a different network +interface while k3s retained a stale InternalIP and Flannel interface. + +Validation now parses pod phase and container readiness, permits successfully +Completed validator pods, and rejects every other unready pod. It also compares +the advertised k3s InternalIP with the host's current IPv4 interfaces. These +checks run before CUDA validation, so a stale network cannot be mistaken for a +healthy GPU substrate merely because `nvidia.com/gpu` remains allocatable. + +No kubeconfig content, Secrets, host addresses, pod logs, prompts, responses, +or model data are recorded here. diff --git a/docs/LOCAL-DEPENDENCY-INVENTORY.md b/docs/LOCAL-DEPENDENCY-INVENTORY.md index ab8fcd2..73ef442 100644 --- a/docs/LOCAL-DEPENDENCY-INVENTORY.md +++ b/docs/LOCAL-DEPENDENCY-INVENTORY.md @@ -1,8 +1,11 @@ # Local dependency and repository inventory -Validated on 2026-07-17 against +Validated again on 2026-07-19 against `/media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia`. -The scan found 58 local Git worktrees. This document distinguishes the platform's +The scan found 64 local Git worktrees. See +[LOCAL-REPOSITORY-AUDIT-2026-07-19.md](LOCAL-REPOSITORY-AUDIT-2026-07-19.md) +for fetch, synchronization, test, release, and live-cluster results. This +document distinguishes the platform's direct source/runtime dependencies from incidental links in vendored charts, lockfiles, generated package metadata, and documentation. diff --git a/docs/LOCAL-REPOSITORY-AUDIT-2026-07-19.md b/docs/LOCAL-REPOSITORY-AUDIT-2026-07-19.md new file mode 100644 index 0000000..5eff03f --- /dev/null +++ b/docs/LOCAL-REPOSITORY-AUDIT-2026-07-19.md @@ -0,0 +1,111 @@ +# Local repository audit: 2026-07-19 + +## Scope and method + +The audit recursively inspected all 64 Git worktrees under +`Project-Linux-Kubernetes-Nvidia`, including remotes, tracking branches, +working-tree state, latest commits, repository instructions, modules, charts, +CI workflows, and direct Edge platform relationships. Every `origin` fetch +completed successfully. All worktrees were clean at the baseline. + +The four `Edge-Computing-LLM` organization repositories were compared with the +GitHub organization API, pull requests, branches, and Actions runs. Their prior +dated branches had already been merged and had byte-identical trees to `main`, +so the local checkouts were safely fast-forwarded to `origin/main` before this +audit's new branches were created. + +## Collection status + +- 64 Git worktrees total +- 4 organization-owned deployable/evidence repositories +- 7 third-party dashboard template candidates +- 53 other upstream/reference worktrees +- 42 clean reference branches behind their freshly fetched upstream tip +- 22 clean branches synchronized at the baseline +- 0 locally divergent or ahead-only reference branches +- 0 failed remote fetches + +Reference branches were not fast-forwarded because they are vendor research +inputs, not organization-owned deployment sources. Their remote-tracking refs +are current, which is sufficient for dependency and release comparison without +rewriting the user's chosen checkout commits. + +## Reference branches behind upstream + +The values below are commit counts (`ahead/behind`) after fetching. + +| Area | Repository | Ahead/behind | +|---|---|---:| +| Apache | echarts | 0/172 | +| Artifact Hub | hub | 0/2 | +| Cloudflare | cloudflared | 0/13 | +| CoreDNS | coredns | 0/82 | +| Dashboard candidate | next-shadcn-admin-dashboard | 0/18 | +| Dashboard candidate | next-shadcn-dashboard-starter | 0/86 | +| Dashboard candidate | nextadmin-dashboard | 0/1 | +| Go | go | 0/5 | +| Helm | helm | 0/6 | +| Kubernetes | kubernetes-client/python | 0/22 | +| Kubernetes | dra-driver-nvidia-gpu (two worktrees) | 0/36 each | +| Kubernetes | node-feature-discovery | 0/19 | +| Kubernetes | client-go | 0/19 | +| Kubernetes | kubectl | 0/20 | +| NVIDIA | DCGM | 0/1 | +| NVIDIA | dcgm-exporter | 0/1 | +| NVIDIA | go-dcgm | 0/2 | +| NVIDIA | gpu-operator | 0/85 | +| NVIDIA | k8s-device-plugin | 0/43 | +| NVIDIA | libnvidia-container | 0/4 | +| NVIDIA | nvidia-container-toolkit | 0/71 | +| Ollama | ollama | 0/35 | +| OpenAI | codex | 0/11 | +| OpenAI | openai-agents-python | 0/4 | +| OpenTelemetry | collector | 0/25 | +| OpenTelemetry | collector-contrib | 0/183 | +| OpenTelemetry | Helm charts (primary) | 0/19 | +| OpenTelemetry | operator | 0/22 | +| OpenTelemetry | Python | 0/28 | +| OpenTelemetry | Python contrib | 0/20 | +| OpenTelemetry | Python GenAI | 0/71 | +| OpenTelemetry | semantic-conventions-genai | 0/10 | +| Prometheus | community Helm charts (primary) | 0/40 | +| Prometheus | kube-prometheus | 0/2 | +| Rancher/k3s | k3s | 0/32 | +| Rancher | local-path-provisioner | 0/4 | +| Traefik | traefik | 0/8 | +| Zensical | zensical | 0/2 | +| Duplicate worktree | Prometheus Community charts | 0/109 | +| Duplicate worktree | Grafana charts | 0/4 | +| Duplicate worktree | OpenTelemetry charts | 0/41 | + +## Edge repository gates + +All four repositories passed formatting, `go mod verify`, unit tests, race +tests, vet, builds, and `govulncheck`; no reachable Go vulnerabilities were +found. The Helm repositories additionally passed dependency resolution, lint, +and default/CPU/local/GeForce/full-NVIDIA rendering. The current GitHub `main` +workflow is green in all four repositories and Dependabot reports no open alert +in the three repositories where alert access is enabled. + +Coverage remains the largest engineering-quality gap. Core package coverage +ranges from roughly 17% to 56%, while orchestration and CLI entry packages are +mostly untested. Code scanning has no analysis workflow, secret scanning is +disabled, and Dependabot alerts are disabled for `llm-observability-stack`. + +## Live Ubuntu/k3s/NVIDIA result + +The host runs Ubuntu 24.04, k3s v1.36.2+k3s1, and GPU Operator v26.3.3 with a +GeForce 940M advertising one allocatable GPU. Qwen is registered, resident, +using GPU layers, and within the 850 MiB evidence ceiling. + +The cluster is not fully healthy after a network-adapter change. k3s advertises +a stale InternalIP and Flannel interface that no longer exist on the host. +Pod-to-API routing fails, which cascades into GPU Operator, Node Feature +Discovery, metrics-server, kube-state-metrics, and node-exporter failures. The +new validators reject this state instead of accepting GPU capacity as a proxy +for substrate health. + +Repairing `/etc/rancher/k3s/config.yaml` and restarting k3s requires local sudo +authentication and was therefore not performed automatically. No addresses, +kubeconfig content, Secrets, prompts, responses, or model weights are recorded +in this audit. diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index 0de2a0e..25124bc 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -2,6 +2,9 @@ package kubernetes import ( "context" + "encoding/json" + "fmt" + "strings" "github.com/Edge-Computing-LLM/edge-cli/internal/execx" ) @@ -18,10 +21,134 @@ func (c Client) Nodes(ctx context.Context) execx.Result { return c.Runner.Check(ctx, "cluster nodes", execx.Command{Name: "kubectl", Args: []string{"get", "nodes", "-o", "wide"}}) } +func (c Client) LocalNodeAddress(ctx context.Context) execx.Result { + nodes, err := c.Runner.Output(ctx, execx.Command{Name: "kubectl", Args: []string{"get", "nodes", "-o", "json"}}) + if err != nil { + return execx.Result{Label: "k3s node address", Output: nodes, Err: err} + } + addresses, err := c.Runner.Output(ctx, execx.Command{Name: "ip", Args: []string{"-j", "-4", "address", "show"}}) + if err != nil { + return execx.Result{Label: "k3s node address", Output: addresses, Err: err} + } + if err := ValidateLocalNodeAddresses([]byte(nodes), []byte(addresses)); err != nil { + return execx.Result{Label: "k3s node address", Err: err} + } + return execx.Result{Label: "k3s node address", Output: "advertised InternalIP matches a current host interface"} +} + func (c Client) RuntimeClassNvidia(ctx context.Context) execx.Result { return c.Runner.Check(ctx, "NVIDIA RuntimeClass", execx.Command{Name: "kubectl", Args: []string{"get", "runtimeclass", "nvidia"}}) } +func (c Client) PodsReady(ctx context.Context, namespace string) execx.Result { + output, err := c.Runner.Output(ctx, execx.Command{Name: "kubectl", Args: []string{"get", "pods", "-n", namespace, "-o", "json"}}) + if err != nil { + return execx.Result{Label: namespace + " pod health", Output: output, Err: err} + } + summary, err := ValidatePodReadiness([]byte(output)) + return execx.Result{Label: namespace + " pod health", Output: summary, Err: err} +} + +type nodeList struct { + Items []struct { + Status struct { + Addresses []struct { + Type string `json:"type"` + Address string `json:"address"` + } `json:"addresses"` + } `json:"status"` + } `json:"items"` +} + +type hostInterface struct { + AddressInfo []struct { + Family string `json:"family"` + Local string `json:"local"` + } `json:"addr_info"` +} + +func ValidateLocalNodeAddresses(nodesJSON, addressesJSON []byte) error { + var nodes nodeList + if err := json.Unmarshal(nodesJSON, &nodes); err != nil { + return fmt.Errorf("decode Kubernetes nodes: %w", err) + } + var interfaces []hostInterface + if err := json.Unmarshal(addressesJSON, &interfaces); err != nil { + return fmt.Errorf("decode host interfaces: %w", err) + } + local := map[string]bool{} + for _, iface := range interfaces { + for _, address := range iface.AddressInfo { + if address.Family == "inet" { + local[address.Local] = true + } + } + } + if len(nodes.Items) == 0 { + return fmt.Errorf("no Kubernetes nodes found") + } + for _, node := range nodes.Items { + internalIP := "" + for _, address := range node.Status.Addresses { + if address.Type == "InternalIP" { + internalIP = address.Address + break + } + } + if internalIP == "" { + return fmt.Errorf("k3s node has no InternalIP") + } + if !local[internalIP] { + return fmt.Errorf("k3s node InternalIP is not assigned to a current host interface; check node-ip and flannel-iface in /etc/rancher/k3s/config.yaml") + } + } + return nil +} + +type podList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Phase string `json:"phase"` + ContainerStatuses []struct { + Ready bool `json:"ready"` + } `json:"containerStatuses"` + } `json:"status"` + } `json:"items"` +} + +func ValidatePodReadiness(data []byte) (string, error) { + var pods podList + if err := json.Unmarshal(data, &pods); err != nil { + return "", fmt.Errorf("decode Kubernetes pods: %w", err) + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no pods found") + } + active, completed := 0, 0 + var unhealthy []string + for _, pod := range pods.Items { + if pod.Status.Phase == "Succeeded" { + completed++ + continue + } + active++ + ready := pod.Status.Phase == "Running" && len(pod.Status.ContainerStatuses) > 0 + for _, status := range pod.Status.ContainerStatuses { + ready = ready && status.Ready + } + if !ready { + unhealthy = append(unhealthy, pod.Metadata.Name+" ("+pod.Status.Phase+")") + } + } + if len(unhealthy) > 0 { + return fmt.Sprintf("%d active, %d completed", active, completed), fmt.Errorf("unhealthy pods: %s", strings.Join(unhealthy, ", ")) + } + return fmt.Sprintf("%d active pods ready, %d completed", active, completed), nil +} + func (c Client) Namespace(ctx context.Context, ns string) execx.Result { return c.Runner.Check(ctx, "namespace "+ns, execx.Command{Name: "kubectl", Args: []string{"get", "namespace", ns}}) } diff --git a/internal/kubernetes/kubernetes_test.go b/internal/kubernetes/kubernetes_test.go new file mode 100644 index 0000000..f7e8470 --- /dev/null +++ b/internal/kubernetes/kubernetes_test.go @@ -0,0 +1,33 @@ +package kubernetes + +import "testing" + +func TestValidateLocalNodeAddresses(t *testing.T) { + nodes := []byte(`{"items":[{"status":{"addresses":[{"type":"InternalIP","address":"192.0.2.10"}]}}]}`) + current := []byte(`[{"addr_info":[{"family":"inet","local":"192.0.2.10"}]}]`) + stale := []byte(`[{"addr_info":[{"family":"inet","local":"192.0.2.20"}]}]`) + if err := ValidateLocalNodeAddresses(nodes, current); err != nil { + t.Fatalf("current address rejected: %v", err) + } + if err := ValidateLocalNodeAddresses(nodes, stale); err == nil { + t.Fatal("stale node address accepted") + } +} + +func TestValidatePodReadiness(t *testing.T) { + healthy := []byte(`{"items":[ + {"metadata":{"name":"operator"},"status":{"phase":"Running","containerStatuses":[{"ready":true}]}}, + {"metadata":{"name":"validator"},"status":{"phase":"Succeeded","containerStatuses":[{"ready":false}]}} + ]}`) + if _, err := ValidatePodReadiness(healthy); err != nil { + t.Fatalf("healthy pod set rejected: %v", err) + } + + unhealthy := []byte(`{"items":[ + {"metadata":{"name":"operator"},"status":{"phase":"Running","containerStatuses":[{"ready":false}]}}, + {"metadata":{"name":"worker"},"status":{"phase":"Failed","containerStatuses":[{"ready":false}]}} + ]}`) + if _, err := ValidatePodReadiness(unhealthy); err == nil { + t.Fatal("unhealthy pod set accepted") + } +} diff --git a/internal/modules/infra/infra.go b/internal/modules/infra/infra.go index 64b5fec..2c6c819 100644 --- a/internal/modules/infra/infra.go +++ b/internal/modules/infra/infra.go @@ -71,10 +71,11 @@ func Doctor(ctx context.Context, opts Options) error { ) k := kubernetes.Client{Runner: r} results = append(results, k.Nodes(ctx)) + results = append(results, k.LocalNodeAddress(ctx)) if opts.NVIDIAEnabled { results = append(results, k.RuntimeClassNvidia(ctx), - r.Check(ctx, "GPU Operator pods", execx.Command{Name: "kubectl", Args: []string{"get", "pods", "-n", "gpu-operator", "-o", "wide"}}), + k.PodsReady(ctx, "gpu-operator"), gpuCapacityResult(ctx, r), r.Check(ctx, "CUDA validation support", execx.Command{Name: "kubectl", Args: []string{"get", "runtimeclass", "nvidia"}}), ) @@ -147,11 +148,12 @@ func Validate(ctx context.Context, opts Options) error { results := []execx.Result{ k.ClusterInfo(ctx), k.Nodes(ctx), + k.LocalNodeAddress(ctx), } if opts.NVIDIAEnabled { results = append(results, k.RuntimeClassNvidia(ctx), - r.Check(ctx, "GPU Operator pods", execx.Command{Name: "kubectl", Args: []string{"get", "pods", "-n", "gpu-operator", "-o", "wide"}}), + k.PodsReady(ctx, "gpu-operator"), gpuCapacityResult(ctx, r), ) }