diff --git a/website/src/data/benchmarks/luna-android.json b/website/src/data/benchmarks/luna-android.json index 1195aafa..9d1014c1 100644 --- a/website/src/data/benchmarks/luna-android.json +++ b/website/src/data/benchmarks/luna-android.json @@ -5,7 +5,7 @@ "suite": "readiness", "platform": "android", "protocolVersion": 4, - "recordedOn": "2026-09-04", + "recordedOn": "2026-09-06", "primaryMetric": "Dispatch to validated Settings screenshot", "pricing": { "model": "gpt-5.6-luna", @@ -18,7 +18,7 @@ "environment": { "machine": { "model": "Mac mini", - "chip": "Apple M4 (10-core)", + "chip": "Apple M4", "memory": "16 GB" }, "macos": "macOS 26.5.2 (25F84)", @@ -35,212 +35,275 @@ "arm": "stim", "valid": true, "invalidReasons": [], - "settingsReadySeconds": 160.72, - "appAliveSeconds": 111.744, + "settingsReadySeconds": 200.226, + "appAliveSeconds": 142.349, "diagnosisSeconds": null, "diagnosisCommandCount": null, "launchCrashAudit": null, "diagnosisUsage": null, "estimatedDiagnosisCostUsd": null, - "totalSeconds": 198.525, - "commandCount": 19, + "totalSeconds": 256.821, + "commandCount": 25, "usage": { - "input_tokens": 529542, - "cached_input_tokens": 467200, - "output_tokens": 5451, - "reasoning_output_tokens": 1709 + "input_tokens": 700029, + "cached_input_tokens": 650752, + "output_tokens": 8459, + "reasoning_output_tokens": 3626 }, - "estimatedTokenCostUsd": 0.028353600000000003, - "summary": "Created an isolated worktree, worked on the JavaScript change, and ran Stim's Android workflow. It reached Settings and captured valid agent-device proof. The record includes 1 failed command attempt before completion.", + "estimatedTokenCostUsd": 0.03302124000000001, + "summary": "Created an isolated worktree, worked on the JavaScript change, and ran Stim's Android workflow. It reached Settings and captured valid agent-device proof.", "messages": [ { "id": "item_0", - "atSeconds": 5.372, - "text": "I’m using the Stim skill because this task requires an isolated worktree, Metro, and an Android launch. I’ll create the requested worktree first, then make the subtitle change and verify it on-device with the exact agent-device session commands." + "atSeconds": 5.793, + "text": "I’m using the Stim skill for the isolated React Native run and the agent-device skill for the required Android interaction and proof capture. I’ll create the requested worktree first, then perform the edit, launch, navigation, and all eight proof commands in order." }, { "id": "item_4", - "atSeconds": 21.304, - "text": "The Stim workflow guide confirms the required sequence: create the isolated worktree, run `stim start`, then the Android launch, and trust the reported device serial. I’m creating the worktree now and will use only the printed absolute path afterward." + "atSeconds": 16.197, + "text": "The requested worktree was created successfully. I’m now operating exclusively from that checkout and loading the installed Stim workflow guide before starting the app." + }, + { + "id": "item_6", + "atSeconds": 79.684, + "text": "`stim worktree warm` is still active without emitting errors or progress; I’m allowing it to finish before touching the checkout." }, { "id": "item_11", - "atSeconds": 66.005, - "text": "The requested worktree is `worktree/bench+javascript-stim`, and the subtitle is updated there. I’m starting its isolated Metro server now, followed by the requested Android build/launch." + "atSeconds": 102.747, + "text": "The target subtitle is in `app/(tabs)/settings.tsx`; the edit is JavaScript-only, so Metro Fast Refresh will apply it after launch." }, { - "id": "item_14", - "atSeconds": 126.791, - "text": "Android launched successfully on the exact run emulator serial ``; Metro is on port 8082 and the bundle is loaded. I’m now switching to the required agent-device skill to open the isolated session, start run-scoped video, handle any onboarding, and reach Settings." + "id": "item_15", + "atSeconds": 113.479, + "text": "Metro is running on port 8082, and Stim is launching the requested Android 36 Play Store emulator. I’ll use the exact serial reported by this launch (``) for the isolated agent-device session." }, { - "id": "item_19", - "atSeconds": 151.755, - "text": "The isolated agent-device session opened on ``, recording started at the requested path, and the Settings tab is visible. The exact new subtitle is already present in the settled UI snapshot; I’m running the required wait and evidence commands individually now." + "id": "item_16", + "atSeconds": 150.174, + "text": "The app launched successfully on `` with Metro on 8082. I’m beginning the mandated proof protocol now; each shell invocation will be exactly the displayed command, in order." }, { - "id": "item_26", - "atSeconds": 189.845, - "text": "The exact subtitle wait succeeded, the Settings screenshot was saved and copied, recording stop reported the saved MP4, and the isolated agent-device session is closed. Metro remains running; I’m handing off the worktree and proof paths now." + "id": "item_22", + "atSeconds": 196.51, + "text": "The wait succeeded and found the exact subtitle. The screenshot invocation hit a shell working-directory typo before executing; no proof command ran, so I’m retrying the same mandated step from the correct run-worktree path." }, { "id": "item_27", - "atSeconds": 198.344, - "text": "Completed.\n\n- Worktree: `worktree/bench+javascript-stim`\n- Screenshot: [settings.png](results/luna-android/javascript-stim/proof/settings.png)\n- Recording: [session.mp4](results/luna-android/javascript-stim/proof/session.mp4)\n\nMetro remains running." + "atSeconds": 227.531, + "text": "The recording stopped and produced the video, and the video was copied. The final close invocation again failed before shell execution due the tool’s working-directory resolution; I’m checking the known run-worktree path and will retry only that same mandated close command." + }, + { + "id": "item_31", + "atSeconds": 245.408, + "text": "All eight mandated proof commands have now succeeded in order, including the exact-text wait, screenshot copy, recording stop, video copy, and isolated session close. I’m doing a final read-only verification of the changed line and both proof artifacts; Metro/app remain untouched." + }, + { + "id": "item_35", + "atSeconds": 256.637, + "text": "Completed in the requested run worktree:\n\n- [Run worktree](./worktrees/bench-javascript-stim)\n- [Settings screenshot](results/luna-android/javascript-stim/proof/settings.png)\n- [Session recording](results/luna-android/javascript-stim/proof/session.mp4)\n\nAll eight proof commands succeeded in order." } ], "commands": [ { - "id": "item_2", - "startSeconds": 12.091, - "endSeconds": 12.091, + "id": "item_1", + "startSeconds": 9.332, + "endSeconds": 9.332, "command": "sed -n '1,240p' results/luna-android/javascript-stim/runner-home/.codex/skills/stim/SKILL.md", - "output": "---\nname: stim\ndescription: The React Native / Expo CLI for AI agents. Use when an agent needs an isolated Metro server and simulator or emulator, must build and launch an app, inspect build or runtime errors, create a parallel worktree, or identify the correct device for UI interaction.\n---\n\n# Stim\n\nBefore using Stim, load the workflow guide from the installed CLI:\n\n```bash\nstim guide agent\n```\n\nFollow the version-matched instructions it prints.\n", + "output": "---\nname: stim\ndescription: The React Native / Expo CLI for AI agents. Use when running or building a React Native or Expo app on a simulator, emulator, or device, including expo run:ios, expo run:android, react-native run-ios, react-native run-android, expo start, and Metro; when reading build, launch, redbox, or runtime errors from the app logs; when working in parallel worktrees; or when identifying the correct device for UI interaction.\n---\n\n# Stim\n\nBefore using Stim, load the workflow guide from the installed CLI:\n\n```bash\nstim guide agent\n```\n\nFollow the version-matched instructions it prints.\n", "exitCode": 0 }, { - "id": "item_3", - "startSeconds": 16.873, - "endSeconds": 17.053, - "command": "stim guide agent", - "output": "Commands use `stim`. If it is not installed globally, replace `stim` with `npx stim-cli`.\n\nAGENT WORKFLOW\n\nUse Stim to run React Native and Expo apps without sharing a Metro port or\ndevice with another workspace. Prefer plain output: it streams each phase and\nends with the facts the next step needs. Use --json only when a script must\nparse a stable payload.\n\nNORMAL WORKFLOW\n\nWork in the current checkout by default. When the task needs another branch or\nan isolated environment, create a worktree and carry its dependencies and\nnative outputs.\n\nBefore native worktree work, run doctor for the platform in scope. It checks\nthe main checkout from a linked worktree. Fix relevant findings and inspect the\nupstream gap.\n\n stim doctor --platform ios # or: --platform android\n\n # In the main checkout, seed the shared build caches when more native\n # worktrees are coming. Skip this for one-off or JavaScript-only work.\n stim start\n stim ios # or: stim android\n stim stop\n\n # Branches from HEAD. --carry-ignored carries installed dependencies and\n # native output. Pass it on the first creation, never on a retry. The command\n # prints the new absolute path.\n stim worktree create --carry-ignored\n cd \n\n stim start\n stim ios # or: stim android\n stim logs --errors\n\n # JavaScript and TypeScript edits use Fast Refresh.\n stim logs --since 30s --level error\n\n stim stop\n stim worktree remove\n\nRULES DURING THE LOOP\n\n- Run start before a debug ios or android build. If it returns STIM_NO_METRO,\n run stim start and retry.\n- Run ios or android again after a native input changes. A JavaScript-only\n change does not need one.\n- A cold native build can outlive a shell timeout. Run the same command again:\n the second call joins the active build or returns its result.\n- ios and android install the app, launch it, and check readiness. Trust the\n exact device, app, Metro, and launch facts in the final summary. Use the full\n reported device ID. Never assume a simulator named booted belongs to this\n workspace.\n- After each ios or android run, give the user one compact result: exact device,\n app id, launch state, cache result, total duration, and whether stim logs\n --errors passed. Include a remedy only when action remains. Do not repeat the\n phase transcript.\n- An OK summary with no launch qualifier proves the launch. \"bundle requested,\n still building\" means Metro has not finished; wait and query the logs. For\n launch UNVERIFIED, follow the printed remedy before claiming success. JSON\n reports these as true, \"bundling\", and \"unverified\" in launched.\n- Exit code 0 from logs --errors is the pass condition. Human output can show\n \"No matching log records\" on stderr. JSON mode prints zero bytes when no\n records match. Do not read the NDJSON files directly.\n- Use stim status when resuming a workspace or recovering missing device,\n port, server, or build facts. A normal start and platform run already print\n them. Use stim doctor when a build is unexpectedly slow or the environment\n looks incomplete.\n\nOWNERSHIP AND DELETION\n\nStim creates, boots, and deletes only devices it created. Owned simulators use\nthe stim-