feat: emit onboarding, demo, starter and first-action telemetry (#297)

Emits the four new product events the API now ingests: an onboarding step per
screen, a demo experiment (curated conversation or real run), a starter-prompt
click, and the first action taken on the demo or on a new project.

Three of the four originate in the UI, which previously had no telemetry path
beyond consent, so this adds POST /api/telemetry/event. Every field is matched
against a fixed allowlist before it reaches capture(), so the local endpoint
cannot emit arbitrary telemetry.

first_action is claimed once per surface and persisted, so relaunching cannot
promote a later action into the first slot. The claim happens only when
telemetry is enabled — claiming first would burn the slot for a later opt-in.

Starter clicks report slot position because the prompts are model-generated;
the accepted range carries headroom so a fifth box cannot 400 a whole batch.

Requires the matching API contract to be deployed first.

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

src/commands/create_experiment.rs

@@ -34,5 +34,11 @@ pub async fn run(mut args: crate::CreateExperimentArgs) -> Result<()> {
         })
         .await?;
     crate::telemetry::capture_experiment_started("create", true, None);
+    let surface = if args.project_id == crate::local::demo::PROJECT_ID {
+        "demo"
+    } else {
+        "project"
+    };
+    crate::telemetry::capture_first_action(surface, "create_experiment");
     Ok(())
 }

src/commands/up.rs

@@ -554,6 +554,7 @@ fn router(state: AppState, remote_auth: Option<RemoteAuth>) -> Router {
             "/api/settings/telemetry",
             get(telemetry_settings).post(set_telemetry_settings),
         )
+        .route("/api/telemetry/event", post(record_ui_event))
         .route(
             "/api/settings/profile",
             get(profile_settings).post(set_profile_settings),
@@ -4932,6 +4933,55 @@ async fn telemetry_settings() -> ApiResult {
         .map_err(|e| ApiError::from(anyhow!("telemetry task failed: {e}")))?
 }
 
+/// A product event raised by the UI rather than by a command. Every field is
+/// matched against a fixed allowlist in `telemetry`, so this local endpoint
+/// cannot emit arbitrary telemetry.
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct UiEventReq {
+    name: String,
+    #[serde(default)]
+    step: Option<String>,
+    #[serde(default)]
+    kind: Option<String>,
+    #[serde(default)]
+    experiment: Option<String>,
+    #[serde(default)]
+    slot: Option<u8>,
+    #[serde(default)]
+    surface: Option<String>,
+    #[serde(default)]
+    action: Option<String>,
+}
+
+async fn record_ui_event(Json(req): Json<UiEventReq>) -> ApiResult {
+    match req.name.as_str() {
+        "onboarding_step_viewed" => {
+            if let Some(step) = req.step.as_deref() {
+                crate::telemetry::capture_onboarding_step_viewed(step);
+            }
+        }
+        "demo_experiment_started" => {
+            if let (Some(kind), Some(experiment)) = (req.kind.as_deref(), req.experiment.as_deref())
+            {
+                crate::telemetry::capture_demo_experiment_started(kind, experiment);
+            }
+        }
+        "project_starter_clicked" => {
+            if let Some(slot) = req.slot {
+                crate::telemetry::capture_project_starter_clicked(slot);
+            }
+        }
+        "first_action" => {
+            if let (Some(surface), Some(action)) = (req.surface.as_deref(), req.action.as_deref()) {
+                crate::telemetry::capture_first_action(surface, action);
+            }
+        }
+        _ => return Ok(Json(json!({ "ok": false }))),
+    }
+    Ok(Json(json!({ "ok": true })))
+}
+
 #[derive(Deserialize)]
 struct SetTelemetryReq {
     enabled: bool,

src/plane/local_plane.rs

@@ -315,6 +315,12 @@ impl LocalPlane {
         if result.is_ok() {
             let target = backend_label.as_deref().unwrap_or("unknown");
             crate::telemetry::capture_experiment_started("run", true, Some(target));
+            // A launch out of the bundled demo is the clearest signal the demo
+            // converted into real work, so it is counted separately.
+            if self.id == crate::local::demo::PROJECT_ID {
+                crate::telemetry::capture_demo_experiment_started("run", target);
+                crate::telemetry::capture_first_action("demo", "run_experiment");
+            }
         }
         result
     }

src/telemetry.rs

@@ -163,6 +163,10 @@ pub(crate) struct Settings {
     /// place; only the silent apply stops.
     #[serde(default)]
     pub auto_update: Option<bool>,
+    /// Surfaces whose `first_action` event has already been sent, so a restart
+    /// cannot re-report a later action as the user's first one.
+    #[serde(default)]
+    pub first_action_reported: Vec<String>,
 }
 
 /// A paper the user linked to their researcher profile.
@@ -1013,6 +1017,86 @@ impl TelemetrySession {
 ///
 /// Non-blocking: the send is spawned and registered for the exit-time flush, so
 /// this returns immediately and never delays the command's own success output.
+/// Onboarding screens, in order; must match the API's
+/// `CLI_ANALYTICS_ONBOARDING_STEPS` or the event is rejected at ingest.
+pub(crate) const ONBOARDING_STEPS: [&str; 3] = ["welcome", "environment", "profile"];
+pub(crate) const DEMO_EXPERIMENT_KINDS: [&str; 2] = ["curated", "run"];
+/// Starter prompts are model-generated, so only the slot position is stable.
+/// The upper bound is headroom — the UI renders whatever the model returns.
+pub(crate) const STARTER_SLOTS: std::ops::RangeInclusive<u8> = 1..=8;
+pub(crate) const FIRST_ACTION_SURFACES: [&str; 2] = ["demo", "project"];
+pub(crate) const FIRST_ACTIONS: [&str; 7] = [
+    "starter_click",
+    "typed_prompt",
+    "open_experiment",
+    "open_file",
+    "run_experiment",
+    "create_experiment",
+    "open_settings",
+];
+
+pub(crate) fn capture_onboarding_step_viewed(step: &str) {
+    if !ONBOARDING_STEPS.contains(&step) {
+        return;
+    }
+    capture("onboarding_step_viewed", json!({ "step": step }));
+}
+
+/// `curated` replays a recorded demo conversation; `run` launches real compute.
+pub(crate) fn capture_demo_experiment_started(kind: &str, experiment: &str) {
+    if !DEMO_EXPERIMENT_KINDS.contains(&kind) || experiment.is_empty() {
+        return;
+    }
+    capture(
+        "demo_experiment_started",
+        json!({ "kind": kind, "experiment": experiment }),
+    );
+}
+
+pub(crate) fn capture_project_starter_clicked(slot: u8) {
+    if !STARTER_SLOTS.contains(&slot) {
+        return;
+    }
+    capture("project_starter_clicked", json!({ "slot": slot }));
+}
+
+/// First action on a surface, sent at most once per surface for the life of the
+/// install, so quitting and relaunching cannot promote a later action.
+pub(crate) fn capture_first_action(surface: &str, action: &str) {
+    if !FIRST_ACTION_SURFACES.contains(&surface) || !FIRST_ACTIONS.contains(&action) {
+        return;
+    }
+    // Check enablement before claiming: `capture` drops the event when telemetry
+    // is off, and claiming first would burn the slot for a later opt-in.
+    if !is_enabled(flag()) {
+        return;
+    }
+    if !claim_first_action_surface(surface) {
+        return;
+    }
+    capture(
+        "first_action",
+        json!({ "surface": surface, "action": action }),
+    );
+}
+
+/// Claim `surface`'s single first-action slot; true only for the winning caller.
+fn claim_first_action_surface(surface: &str) -> bool {
+    let mut claimed = false;
+    let _ = mutate_settings(|settings| {
+        if settings
+            .first_action_reported
+            .iter()
+            .any(|seen| seen == surface)
+        {
+            return;
+        }
+        settings.first_action_reported.push(surface.to_string());
+        claimed = true;
+    });
+    claimed
+}
+
 pub(crate) fn capture_experiment_started(kind: &str, local: bool, target: Option<&str>) {
     let mut extra = json!({ "kind": kind, "local": local });
     if let (Some(obj), Some(t)) = (extra.as_object_mut(), target) {
@@ -1515,6 +1599,62 @@ mod tests {
         let _ = std::fs::remove_dir_all(&dir);
     }
 
+    #[test]
+    fn first_action_claims_each_surface_once() {
+        let _g = EnvGuard::new(OPT_VARS);
+        let dir = std::env::temp_dir().join(format!("orx-tel-first-{}", uuid::Uuid::new_v4()));
+        std::env::set_var("XDG_CONFIG_HOME", &dir);
+
+        assert!(claim_first_action_surface("project"));
+        assert!(!claim_first_action_surface("project"));
+        assert!(claim_first_action_surface("demo"));
+        assert_eq!(
+            load_settings()
+                .map(|s| s.first_action_reported)
+                .unwrap_or_default(),
+            vec!["project".to_string(), "demo".to_string()],
+        );
+
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
+    #[test]
+    fn a_disabled_first_action_does_not_burn_the_surface() {
+        let _g = EnvGuard::new(OPT_VARS);
+        let dir = std::env::temp_dir().join(format!("orx-tel-burn-{}", uuid::Uuid::new_v4()));
+        std::env::set_var("XDG_CONFIG_HOME", &dir);
+        std::env::set_var("ORX_NO_TELEMETRY", "1");
+
+        capture_first_action("project", "typed_prompt");
+        assert!(
+            load_settings()
+                .map(|s| s.first_action_reported.is_empty())
+                .unwrap_or(true),
+            "an opt-out must leave the slot free for a later opt-in"
+        );
+
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
+    #[test]
+    fn dimension_events_reject_values_outside_the_allowlists() {
+        let _g = EnvGuard::new(OPT_VARS);
+        let dir = std::env::temp_dir().join(format!("orx-tel-vocab-{}", uuid::Uuid::new_v4()));
+        std::env::set_var("XDG_CONFIG_HOME", &dir);
+
+        capture_first_action("website", "typed_prompt");
+        capture_first_action("project", "danced");
+        assert!(
+            load_settings()
+                .map(|s| s.first_action_reported.is_empty())
+                .unwrap_or(true),
+            "an unknown surface or action must never claim a slot"
+        );
+        assert!(!STARTER_SLOTS.contains(&0) && !STARTER_SLOTS.contains(&9));
+
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
     #[test]
     fn every_event_name_is_cli_prefixed() {
         let _g = EnvGuard::new(OPT_VARS);
@@ -1538,6 +1678,10 @@ mod tests {
             ("skill_invoked", "cli_skill_invoked"),
             ("experiment_started", "cli_experiment_started"),
             ("telemetry_consent", "cli_telemetry_consent"),
+            ("onboarding_step_viewed", "cli_onboarding_step_viewed"),
+            ("demo_experiment_started", "cli_demo_experiment_started"),
+            ("project_starter_clicked", "cli_project_starter_clicked"),
+            ("first_action", "cli_first_action"),
         ] {
             let p = build_payload(bare, "did", json!({}));
             assert_eq!(

ui/src/App.tsx

@@ -81,7 +81,9 @@ import {
   DEMO_MAIN_SESSION_ID,
   DEMO_OVERVIEW_ARTIFACT,
   DEMO_RUN_EXPERIMENT_PROMPT,
+  captureUiEvent,
   isDemoProjectId,
+  type FirstAction,
   openProject,
   updateUiState,
   type AgentSelection,
@@ -707,11 +709,29 @@ export default function App({ runtime, projectId, pane }: { runtime: RuntimeInfo
   const projectIdRef = useRef(projectId);
   projectIdRef.current = projectId;
 
+  // The CLI keeps only the earliest report per surface, so these are safe to
+  // fire on every open.
+  const reportFirstAction = useCallback(
+    (action: FirstAction) => {
+      if (!projectId) return;
+      captureUiEvent({
+        name: "first_action",
+        surface: isDemoProjectId(projectId) ? "demo" : "project",
+        action,
+      });
+    },
+    [projectId],
+  );
+  useEffect(() => {
+    if (mainView !== "chat" && mainView !== "skills") reportFirstAction("open_settings");
+  }, [mainView, reportFirstAction]);
+
   const openExperimentsTab = useCallback((replace = false) => {
     if (replace && (!navigationRef.current.isTask || navigationRef.current.pane)) return;
+    reportFirstAction("open_experiment");
     setExperimentsTabOpen(true);
     navigatePane({ kind: "home", view: "experiments" }, replace);
-  }, [navigatePane]);
+  }, [navigatePane, reportFirstAction]);
 
   const loadInitialState = () => {
     void projectsQuery.refetch();
@@ -867,6 +887,7 @@ export default function App({ runtime, projectId, pane }: { runtime: RuntimeInfo
     intent: TabOpenIntent = "preview",
     runId?: string,
   ) => {
+    reportFirstAction("open_experiment");
     const tab = { id, view };
     setExpTabs((prev) => (prev.some((t) => sameExpTab(t, tab)) ? prev : [...prev, tab]));
     openRightTab(tab, intent, runId);

ui/src/api.ts

@@ -16,6 +16,13 @@ export const isDemoProjectId = (id: string) => id.startsWith("demo_");
 export const DEMO_MAIN_SESSION_ID = "chat_demo_nanochat_v1";
 export const DEMO_FIGURE_SESSION_ID = "chat_demo_nanochat_figures_v1";
 export const DEMO_LITERATURE_SESSION_ID = "chat_demo_nanochat_literature_v1";
+
+/** Stable analytics labels for the bundled demo's recorded conversations. */
+export const DEMO_EXPERIMENT_LABELS: Record<string, string> = {
+  [DEMO_MAIN_SESSION_ID]: "cpu_end_to_end",
+  [DEMO_FIGURE_SESSION_ID]: "figures",
+  [DEMO_LITERATURE_SESSION_ID]: "literature",
+};
 export const DEMO_OVERVIEW_ARTIFACT = "cpu-apple-silicon-pipeline-results.md";
 export const DEMO_RUN_EXPERIMENT_PROMPT =
   "Run the Muon matrix LR 2× probe experiment. When it finishes, compare its step-100 and step-200 val_bpb against the baseline and tell me whether doubling the matrix learning rate helps early training.";
@@ -1422,6 +1429,29 @@ export const getTelemetry = (signal?: AbortSignal) => get<TelemetrySettings>("/a
 export const setTelemetry = (enabled: boolean) =>
   post<TelemetrySettings>("/api/settings/telemetry", { enabled });
 
+export type OnboardingStep = "welcome" | "environment" | "profile";
+export type FirstActionSurface = "demo" | "project";
+export type FirstAction =
+  | "starter_click"
+  | "typed_prompt"
+  | "open_experiment"
+  | "open_file"
+  | "run_experiment"
+  | "create_experiment"
+  | "open_settings";
+
+type UiEvent =
+  | { name: "onboarding_step_viewed"; step: OnboardingStep }
+  | { name: "demo_experiment_started"; kind: "curated" | "run"; experiment: string }
+  | { name: "project_starter_clicked"; slot: number }
+  | { name: "first_action"; surface: FirstActionSurface; action: FirstAction };
+
+/** Product events raised by the UI. Fire-and-forget: analytics must never
+ * surface an error or block the interaction that triggered it. */
+export const captureUiEvent = (event: UiEvent): void => {
+  void post<{ ok: boolean }>("/api/telemetry/event", event).catch(() => {});
+};
+
 export type HarnessId = "claude-code" | "codex" | "opencode";
 
 export interface HarnessModel {

ui/src/components/ChatPanel.tsx

@@ -75,6 +75,8 @@ import {
   deleteChatSession,
   DEMO_FIGURE_SESSION_ID,
   DEMO_LITERATURE_SESSION_ID,
+  captureUiEvent,
+  DEMO_EXPERIMENT_LABELS,
   DEMO_PROJECT_ID,
   forkChatTurn,
   fmtNumber,
@@ -93,6 +95,7 @@ import {
   setChatSessionArchived,
   setChatSessionPermissionMode,
   setChatSessionPlanMode,
+  type FirstActionSurface,
   type ChatImageAttachment,
   type ChatMessage,
   type ChatPart,
@@ -4996,6 +4999,9 @@ export function ChatPanel({
   });
   const starterPrompts = starterQuery.data?.prompts ?? null;
   const starterLoading = starterHarness !== null && starterQuery.isPending;
+  // The demo project is the only surface that isn't a user-created project.
+  const telemetrySurface: FirstActionSurface =
+    projectId === DEMO_PROJECT_ID ? "demo" : "project";
   const applyStarterPrompt = (prompt: string) => {
     setDraft(prompt);
     setSkillMenuDismissed(false);
@@ -5046,6 +5052,11 @@ export function ChatPanel({
 
   /** `queue` (the ⌘/Ctrl+Enter chord) parks the message even on a harness that steers. */
   async function send({ queue = false }: { queue?: boolean } = {}) {
+    captureUiEvent({
+      name: "first_action",
+      surface: telemetrySurface,
+      action: "typed_prompt",
+    });
     if (preparingSend.current) return;
     // Slash tokens stay in the wire form: the server resolves every selected
     // skill and supplies this exact message as their shared request context.
@@ -5759,7 +5770,22 @@ export function ChatPanel({
             revealTitle={titleReveals.get(s.id)}
             onOpen={() => {
               onActiveSessionChange(s.id);
-              if (projectId === DEMO_PROJECT_ID) markDemoSessionRead(s.id);
+              if (projectId === DEMO_PROJECT_ID) {
+                markDemoSessionRead(s.id);
+                const experiment = DEMO_EXPERIMENT_LABELS[s.id];
+                if (experiment) {
+                  captureUiEvent({
+                    name: "demo_experiment_started",
+                    kind: "curated",
+                    experiment,
+                  });
+                  captureUiEvent({
+                    name: "first_action",
+                    surface: "demo",
+                    action: "open_experiment",
+                  });
+                }
+              }
               setUnreadSessionIds((current) => {
                 if (!current.has(s.id)) return current;
                 const next = new Set(current);
@@ -5928,7 +5954,15 @@ export function ChatPanel({
                       key={index}
                       type="button"
                       className={`flex min-h-22 w-full min-w-0 cursor-pointer flex-col items-start justify-center gap-1.5 rounded-xl border bg-background px-5 py-4 text-start font-sans transition-colors duration-120 ease-standard hover:bg-surface ${tone.box}`}
-                      onClick={() => applyStarterPrompt(item.prompt)}
+                      onClick={() => {
+                        captureUiEvent({ name: "project_starter_clicked", slot: index + 1 });
+                        captureUiEvent({
+                          name: "first_action",
+                          surface: telemetrySurface,
+                          action: "starter_click",
+                        });
+                        applyStarterPrompt(item.prompt);
+                      }}
                     >
                       <span className="flex items-center gap-2.5 text-base font-medium text-text">
                         <Icon size={17} className={tone.icon} />

ui/src/components/Onboarding.tsx

@@ -13,12 +13,14 @@ import { ArrowLeft, ArrowRight, RefreshCw, X } from "lucide-react";
 import { Wordmark } from "./Wordmark";
 import { useEffect, useRef, useState } from "react";
 import {
+  captureUiEvent,
   harnessModelLabel,
   completeOnboarding,
   reasoningFor,
   type AgentSelection,
   type Harness,
   type HarnessId,
+  type OnboardingStep,
   type LinkedPaper,
   type PaperHit,
   type Project,
@@ -69,6 +71,10 @@ const RESEARCH_AREAS = [
  * installation. The data-dir choice lives in
  * Settings → Storage (which can also *move* existing data); usage analytics is
  * opt-out via Settings or `orx telemetry off`. */
+/** In order — the API funnel keys on these names for all time, so renumbering
+ * the screens must not redefine a historical step. */
+const ONBOARDING_STEP_NAMES: readonly OnboardingStep[] = ["welcome", "environment", "profile"];
+
 export function Onboarding({
   onDone,
   preferredAgent,
@@ -79,6 +85,10 @@ export function Onboarding({
   const completeOnboardingMutation = useMutation({ mutationFn: (args: Parameters<typeof completeOnboarding>) => completeOnboarding(...args) });
 
   const [step, setStep] = useState<0 | 1 | 2>(0);
+  useEffect(() => {
+    const name = ONBOARDING_STEP_NAMES[step];
+    if (name) captureUiEvent({ name: "onboarding_step_viewed", step: name });
+  }, [step]);
   const harnessQuery = useQuery(getHarnessesQuery());
   const pathQuery = useQuery(getProjectPathStatusQuery());
   const harnesses = harnessQuery.data ?? null;