feat(cli): telemetry data-schema+ post-create git init

docs/telemetry-enhancements.md

@@ -0,0 +1,127 @@
+# CLI Telemetry — Enhancements & Roadmap
+
+Possible improvements to the ChaiBuilder CLI telemetry (CLI events, the
+`/in/cli/track` ingestion endpoint, and PostHog dashboards).
+
+**Priority:** P0 (go-live) · P1 (high) · P2 (nice) · P3 (later).
+**Effort:** S (hours) · M (a day) · L (multi-day).
+
+---
+
+## Current state
+
+- CLI emits `cmd:step:action[:result]` events for `create`, `new`, `switch`
+  (e.g. `create:db_migration:finish:error`).
+- Collected: `os`, `cli_version`, per-run anonymous `distinct_id`/`run_id`,
+  event properties (`template`, `error_type`, redacted `error`).
+- First-run notice + opt-out (`DO_NOT_TRACK`, `CHAIBUILDER_TELEMETRY_DISABLED`).
+- Endpoint `POST /in/cli/track` (zod-validated, body-size cap) forwards to PostHog
+  after the response. Never blocks or crashes the CLI.
+- **Not live yet:** endpoint PR unmerged; prod PostHog key not set.
+
+---
+
+## 0. Go-live checklist (P0)
+
+| Item | Effort |
+|---|---|
+| Merge the endpoint PR (`cbpl` #104) and deploy | S |
+| Set `POSTHOG_API_KEY` + `POSTHOG_HOST` in prod Vercel (cbpl) | S |
+| Merge CLI telemetry PR (`cli` #8), publish the CLI | S |
+| Merge `new`/`switch` PR (`cli` #9) after #8 | S |
+| Delete/exclude seed test events in PostHog (`seed-*` distinct_ids) | S |
+
+---
+
+## 1. Event data & schema
+
+**Status:** 1.1–1.5 ✅ implemented (see "Implemented — 1.1–1.5" below). 1.6–1.7 pending.
+
+| # | Enhancement | Why | P / Effort |
+|---|---|---|---|
+| 1.1 ✅ | **Structured error `code`** on `*:finish:error` (e.g. `MIGRATION_FAILED`, `TEMPLATE_DOWNLOAD_FAILED`) | Break down failures by a clean enum instead of the redacted free-text `error`; no PII risk | P1 / S |
+| 1.2 ✅ | **Persistent anonymous id** (write a UUID once to `~/.config/chaibuilder/id`, reuse) | Distinguish unique machines from repeat runs; today `distinct_id` is per-run, so "users" == "runs" | P1 / S |
+| 1.3 ✅ | **CI detection** (`is_ci` from `process.env.CI`) | Segment automated/CI runs from real developers | P2 / S |
+| 1.4 ✅ | **Command-outcome event** (`create:done` / `create:failed`) | One event for success-rate queries without a multi-step funnel | P2 / S |
+| 1.5 ✅ | **Step durations** | The redesign dropped `duration_ms`. Either rely on funnel "time between steps", or re-add a `duration_ms` on finish events | P2 / S |
+| 1.6 | **Optional safe dimensions** (`packageManager`, `db: local\|remote`, `node`, `arch`) | Richer segmentation — currently intentionally OS-only; revisit with Suraj | P2 / S |
+| 1.7 | **Homepage source** (`remote` vs bundled `fallback`) on seed | Know how often the remote homepage fetch fails | P3 / S |
+
+### Implemented — 1.1–1.5
+
+- **1.1 error codes** — every `*:finish:error` now carries a stable `code` (`errorCode(step)`: `MIGRATION_FAILED`, `TEMPLATE_DOWNLOAD_FAILED`, else `<STEP>_FAILED`). `check-env` keeps its own `INVALID_ENV` / `MISSING_ENV_FILE`.
+- **1.2 persistent id** — `distinct_id` is now a UUID persisted once to `~/.config/chaibuilder/id` (best-effort, never throws); `run_id` stays per-run. Skipped entirely when the user has opted out.
+- **1.3 CI detection** — `is_ci` rides in `properties` on every event (`CI`, `GITHUB_ACTIONS`, `GITLAB_CI`, `CIRCLECI`, `TRAVIS`, `BUILDKITE`, `TF_BUILD`, `JENKINS_URL`).
+- **1.4 command outcome** — one terminal event per run: `create|new|switch|check_env` × `:done` / `:failed`. Suppressed on user cancel. `switch:done` also carries `changed: true|false`.
+- **1.5 durations** — `duration_ms` on every step `*:finish:*` and on the outcome event.
+
+**No endpoint change required:** `distinct_id`/`run_id` are existing top-level fields (values only changed); `is_ci`, `code`, `duration_ms` ride inside `properties`, which the endpoint forwards to PostHog wholesale.
+
+---
+
+## 2. Privacy & consent
+
+| # | Enhancement | Why | P / Effort |
+|---|---|---|---|
+| 2.1 | **`telemetry` subcommand** (`status` / `enable` / `disable`) | Let users manage telemetry without env vars; persist the choice | P1 / S |
+| 2.2 | **Persist opt-out** to the config file | Today opt-out is env-only; persist a disabled flag so it sticks | P1 / S |
+| 2.3 | **Notify-only first run** (don't collect on the very first run) | Stricter consent — the run where the notice first appears sends nothing | P2 / S |
+| 2.4 | **Public "what we collect" page** linked from the notice | Transparency; standard for CLI telemetry | P2 / S |
+| 2.5 | **Redaction hardening + fuzz tests** | Expand patterns (short tokens, connection strings), add property-based tests | P2 / M |
+
+---
+
+## 3. Endpoint & infrastructure
+
+| # | Enhancement | Why | P / Effort |
+|---|---|---|---|
+| 3.1 | **Batch events per run** (buffer, send once via PostHog `/batch`) | ~13 POSTs/run → 1; less overhead and fewer dropped events at exit | P1 / M |
+| 3.2 | **Point CLI at `www.chaibuilder.com`** (or add a non-www route) | Avoid the `chaibuilder.com` → `www` 308 hop on every event | P2 / S |
+| 3.3 | **Basic rate limiting / abuse guard** on the endpoint | It's public + unauthenticated; cap per-IP | P2 / M |
+| 3.4 | **Geo/country** (forward client `$ip`) | Where are users? — privacy trade-off, gate behind the notice | P2 / S |
+| 3.5 | **`posthog-node` SDK** instead of raw fetch | Built-in batching/retries/flush | P3 / S |
+| 3.6 | **Retry/queue failed sends** (persist, retry next run) | Recover events lost to transient network failures | P3 / M |
+
+---
+
+## 4. Dashboards & analysis (PostHog)
+
+| # | Enhancement | Why | P / Effort |
+|---|---|---|---|
+| 4.1 | **Core tiles** — success funnel, create step funnel, failures-by-step, runs/day, OS, CLI version, new/switch usage | The base dashboard (in progress) | P0 / S |
+| 4.2 | **Exclude seed/test distinct_ids** from all tiles | Keep metrics clean once real data flows | P0 / S |
+| 4.3 | **Alerts** — success-rate drop / failure spike → Slack/email | Catch regressions in the wild fast | P1 / S |
+| 4.4 | **Failure-reason table** (once 1.1 lands) — breakdown by `code` | Actionable "why installs/migrations fail" | P1 / S |
+| 4.5 | **CLI-version adoption cohort** | Track migration off broken/old versions | P2 / S |
+| 4.6 | **Weekly digest / subscription** | Passive visibility for the team | P2 / S |
+
+---
+
+## 5. Testing & CI
+
+| # | Enhancement | Why | P / Effort |
+|---|---|---|---|
+| 5.1 | **`cli-e2e` telemetry assertions** (run against a mock endpoint, assert emitted events) | Guard the event contract end-to-end | P1 / M |
+| 5.2 | **`cli-e2e` in CI** (GitHub Actions on PRs) | Auto-run create/new/switch on every change | P1 / M |
+| 5.3 | **Error + remote-DB scenarios** in `cli-e2e` | Cover failure paths + libSQL/Turso, not just the happy path | P2 / M |
+| 5.4 | **Cross-platform e2e** (Windows/Linux runners) | Catch path/PM issues per-OS | P3 / L |
+
+---
+
+## 6. CLI UX / observability
+
+| # | Enhancement | Why | P / Effort |
+|---|---|---|---|
+| 6.1 | **`CHAIBUILDER_TELEMETRY_DEBUG`** — print events to stderr | Debug what's emitted without a server | P2 / S |
+| 6.2 | **Telemetry on remaining commands** (`check-env`, future ones) | Full coverage as the CLI grows | P2 / S |
+| 6.3 | **Event sampling** (if volume ever gets large) | Cost/volume control | P3 / S |
+
+---
+
+## Suggested first slice (post go-live)
+
+1. **1.1 structured error codes** + **4.4 failure table** — turns the dashboard from "how many fail" into "why they fail."
+2. **1.2 persistent id** — real unique-user counts.
+3. **2.1 telemetry subcommand** + **2.2 persist opt-out** — proper user control.
+4. **5.1 / 5.2 e2e telemetry + CI** — lock the contract.
+5. **3.1 batching** — efficiency once volume is real.

src/commands/check-env-run.ts

@@ -7,12 +7,15 @@ import { errorDetails, stepEvent } from '../lib/telemetry.js'
 const CMD = 'check_env'
 
 export async function runCheckEnv(ports: Ports): Promise<{ ok: boolean; exitCode: number }> {
+  const startedAt = Date.now()
+  const dur = () => ({ duration_ms: Date.now() - startedAt })
   ports.telemetry.capture(stepEvent(CMD, 'run', 'start'))
   try {
     const envPath = ports.fs.join(ports.fs.cwd(), '.env')
     if (!(await ports.fs.exists(envPath))) {
       ports.log.error('Missing .env file')
-      ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'error'), { code: 'MISSING_ENV_FILE' })
+      ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'error'), { code: 'MISSING_ENV_FILE', ...dur() })
+      ports.telemetry.capture(`${CMD}:failed`, { code: 'MISSING_ENV_FILE', ...dur() })
       return { ok: false, exitCode: 1 }
     }
 
@@ -32,18 +35,18 @@ export async function runCheckEnv(ports: Ports): Promise<{ ok: boolean; exitCode
 
     if (result.ok) {
       ports.log.success('Required env vars look good')
-      ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'success'))
+      ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'success'), dur())
+      ports.telemetry.capture(`${CMD}:done`, dur())
     } else {
-      ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'error'), {
-        code: 'INVALID_ENV',
-        missing_count: result.missing.length,
-        empty_count: result.empty.length,
-      })
+      const props = { code: 'INVALID_ENV', missing_count: result.missing.length, empty_count: result.empty.length }
+      ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'error'), { ...props, ...dur() })
+      ports.telemetry.capture(`${CMD}:failed`, { ...props, ...dur() })
     }
 
     return { ok: result.ok, exitCode: result.ok ? 0 : 1 }
   } catch (err) {
-    ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'error'), errorDetails(err))
+    ports.telemetry.capture(stepEvent(CMD, 'run', 'finish', 'error'), { ...errorDetails(err), ...dur() })
+    ports.telemetry.capture(`${CMD}:failed`, { ...errorDetails(err), ...dur() })
     throw err
   } finally {
     await ports.telemetry.flush()

src/commands/create-app-run.ts

@@ -4,12 +4,13 @@ import type { Step, WizardState } from '../core/context.js'
 import { CliError } from '../core/errors.js'
 import { runSteps } from '../core/run-steps.js'
 import { TEMPLATE_SOURCE } from '../constants.js'
-import { errorDetails, stepEvent } from '../lib/telemetry.js'
+import { errorCode, errorDetails, stepEvent, tagCode } from '../lib/telemetry.js'
 import { checkUpdate } from '../steps/check-update.js'
 import { chooseDatabase } from '../steps/choose-database.js'
 import { detectPm } from '../steps/detect-package-manager.js'
 import { downloadTemplate } from '../steps/download-template.js'
 import { finalizeLocalDb } from '../steps/finalize-local-db.js'
+import { initGit } from '../steps/init-git.js'
 import { installDeps } from '../steps/install-deps.js'
 import { outro } from '../steps/outro.js'
 import { promptAppName } from '../steps/prompt-app-name.js'
@@ -29,22 +30,26 @@ function isCancellation(ports: Ports, err: unknown): boolean {
 
 function tracked(step: string, fn: Step, propsFn?: PropsFn): Step {
   return async (ctx, ports) => {
+    const startedAt = Date.now()
     ports.telemetry.capture(stepEvent(CMD, step, 'start'), propsFn?.(ctx))
     try {
       const patch = await fn(ctx, ports)
       if (ports.prompts.isCancel(patch)) return patch
-      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'success'), propsFn?.({ ...ctx, ...patch }))
+      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'success'), { ...propsFn?.({ ...ctx, ...patch }), duration_ms: Date.now() - startedAt })
       return patch
     } catch (err) {
       if (isCancellation(ports, err)) throw err
-      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'error'), { ...propsFn?.(ctx), ...errorDetails(err) })
+      const code = errorCode(step)
+      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'error'), { ...propsFn?.(ctx), ...errorDetails(err), code, duration_ms: Date.now() - startedAt })
+      tagCode(err, code)
       throw err
     }
   }
 }
 
 function trackedSequence(step: string, fns: Step[], propsFn?: PropsFn): Step {
   return async (ctx, ports) => {
+    const startedAt = Date.now()
     ports.telemetry.capture(stepEvent(CMD, step, 'start'), propsFn?.(ctx))
     let state = ctx
     try {
@@ -53,11 +58,13 @@ function trackedSequence(step: string, fns: Step[], propsFn?: PropsFn): Step {
         if (ports.prompts.isCancel(patch)) return patch
         state = { ...state, ...patch }
       }
-      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'success'), propsFn?.(state))
+      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'success'), { ...propsFn?.(state), duration_ms: Date.now() - startedAt })
       return state
     } catch (err) {
       if (isCancellation(ports, err)) throw err
-      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'error'), { ...propsFn?.(ctx), ...errorDetails(err) })
+      const code = errorCode(step)
+      ports.telemetry.capture(stepEvent(CMD, step, 'finish', 'error'), { ...propsFn?.(ctx), ...errorDetails(err), code, duration_ms: Date.now() - startedAt })
+      tagCode(err, code)
       throw err
     }
   }
@@ -80,6 +87,7 @@ export const createAppSteps = [
   tracked('db_migration', runMigrations),
   trackedSequence('create_admin_user', [phaseHeader('[2/3] Create admin user'), setupAppAndUser, seedDb]),
   finalizeLocalDb,
+  initGit,
   outro,
 ]
 
@@ -97,7 +105,9 @@ export async function runCreateApp(
     cwd: ports.fs.cwd(),
     ...initial,
   }
-  const final = await runSteps(createAppSteps, state, ports)
+  const startedAt = Date.now()
+  const final = await runSteps(createAppSteps, state, ports, { cmd: CMD, startedAt })
+  ports.telemetry.capture(`${CMD}:done`, { duration_ms: Date.now() - startedAt })
   await ports.telemetry.flush()
   return final
 }

src/commands/new-app-run.ts

@@ -5,7 +5,7 @@ import { parseEnv, upsertEnv } from '../lib/env-file.js'
 import { createAppRecord, findUserIdByEmail, openDb } from '../lib/app-record.js'
 import { runAdminOp } from '../lib/admin-runner.js'
 import { resolvePackageManagerForDir } from '../lib/package-manager.js'
-import { trackStep } from '../lib/telemetry.js'
+import { errorDetails, trackStep } from '../lib/telemetry.js'
 
 
 const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/
@@ -18,6 +18,7 @@ const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/
  * CHAIBUILDER_APP_KEY at it.
  */
 export async function runNewApp(ports: Ports): Promise<void> {
+  const startedAt = Date.now()
   ports.log.intro('Create a new ChaiBuilder app')
   try {
     const appDir = ports.fs.cwd()
@@ -97,7 +98,11 @@ export async function runNewApp(ports: Ports): Promise<void> {
     ports.log.message(pc.bold('Next steps'))
     ports.log.note(`Active app is now ${pc.cyan(appName)} (${appId}).`)
     ports.log.note('Restart your dev server to load it.')
+    ports.telemetry.capture('new:done', { duration_ms: Date.now() - startedAt })
     ports.log.outro('Done')
+  } catch (err) {
+    ports.telemetry.capture('new:failed', { ...errorDetails(err), duration_ms: Date.now() - startedAt })
+    throw err
   } finally {
     await ports.telemetry.flush()
   }

src/commands/switch-app-run.ts

@@ -3,10 +3,11 @@ import type { Ports } from '../core/adapters.js'
 import { CliError } from '../core/errors.js'
 import { parseEnv, upsertEnv } from '../lib/env-file.js'
 import { listApps, openDb, type AppSummary } from '../lib/app-record.js'
-import { trackStep } from '../lib/telemetry.js'
+import { errorDetails, trackStep } from '../lib/telemetry.js'
 
 
 export async function runSwitchApp(ports: Ports): Promise<void> {
+  const startedAt = Date.now()
   ports.log.intro('Switch ChaiBuilder app')
   try {
     const appDir = ports.fs.cwd()
@@ -57,7 +58,10 @@ export async function runSwitchApp(ports: Ports): Promise<void> {
     })
     if (ports.prompts.isCancel(selected)) return void ports.log.outro('Cancelled')
 
-    if (selected === current) return void ports.log.outro('No change — already the active app.')
+    if (selected === current) {
+      ports.telemetry.capture('switch:done', { duration_ms: Date.now() - startedAt, changed: false })
+      return void ports.log.outro('No change — already the active app.')
+    }
 
     await trackStep(ports.telemetry, 'switch', 'switch_app', async () => {
       const content = await ports.fs.readFile(envPath)
@@ -67,7 +71,11 @@ export async function runSwitchApp(ports: Ports): Promise<void> {
     ports.log.message(pc.bold('Switched'))
     ports.log.note(`Active app: ${pc.cyan(picked?.name ?? String(selected))}`)
     ports.log.note('Restart your dev server to load it.')
+    ports.telemetry.capture('switch:done', { duration_ms: Date.now() - startedAt, changed: true })
     ports.log.outro('Done')
+  } catch (err) {
+    ports.telemetry.capture('switch:failed', { ...errorDetails(err), duration_ms: Date.now() - startedAt })
+    throw err
   } finally {
     await ports.telemetry.flush()
   }

src/core/real-adapters.ts

@@ -8,7 +8,7 @@ import { createRequire } from 'node:module'
 import path from 'node:path'
 import { execa } from 'execa'
 import { downloadTemplate } from 'giget'
-import { createTelemetry } from '../lib/telemetry.js'
+import { createTelemetry, resolveDistinctId, telemetryDisabled } from '../lib/telemetry.js'
 import type {
   ExecPort,
   FsPort,
@@ -233,7 +233,10 @@ export function createRealPorts(): Ports {
     exec: createExec(),
     template: createTemplate(),
     log: createLog(),
-    telemetry: createTelemetry({ cliVersion: cliVersion() }),
+    telemetry: createTelemetry({
+      cliVersion: cliVersion(),
+      distinctId: telemetryDisabled() ? undefined : resolveDistinctId(),
+    }),
     randomBytes: (size) => nodeRandomBytes(size),
   }
 }

src/core/run-steps.ts

@@ -1,12 +1,14 @@
 import type { Ports } from './adapters.js'
 import type { Step, WizardState } from './context.js'
 import { CliError } from './errors.js'
+import { errorDetails, readCode } from '../lib/telemetry.js'
 import pc from 'picocolors'
 
 export async function runSteps(
   steps: Step[],
   initial: WizardState,
   ports: Ports,
+  opts?: { cmd?: string; startedAt?: number },
 ): Promise<WizardState> {
   let state = { ...initial }
 
@@ -20,13 +22,20 @@ export async function runSteps(
       }
       state = { ...state, ...patch }
     } catch (err: any) {
-      if (ports.prompts.isCancel(err)) {
+      const cancelled = ports.prompts.isCancel(err) || (err instanceof CliError && err.message === 'Cancelled')
+      if (cancelled) {
         await ports.telemetry.flush()
         console.error(pc.red('Cancelled'))
         process.exit(1)
       }
-      // Step-level telemetry (finish/error) is emitted by the tracked() wrappers
-      // in create-app-run before the throw reaches here.
+      // terminal command-outcome event (step-level finish/error already emitted upstream)
+      if (opts?.cmd) {
+        ports.telemetry.capture(`${opts.cmd}:failed`, {
+          ...errorDetails(err),
+          code: readCode(err) ?? `${opts.cmd.toUpperCase()}_FAILED`,
+          ...(opts.startedAt != null ? { duration_ms: Date.now() - opts.startedAt } : {}),
+        })
+      }
       await ports.telemetry.flush()
       console.error(pc.red(err.message || String(err)))
       process.exit(1)