{"account": {"accounted_usd_after": 0.04811556, "stop_usd": 0.2}, "baseline": [{"detail": "Answered 200 in 0.6s", "label": "The page loads", "status": "passed"}, {"detail": "167 characters of text. A sign-in wall or an error message would also pass this.", "label": "The page shows words", "status": "passed"}, {"detail": "0 found", "label": "No script errors on load", "status": "passed"}], "browser": "Playwright Chromium", "counts": {"claimed": 0, "confirmed": 0, "gets_stuck": 3, "gets_through": 0, "same_every_try": 3, "screen_confirmed": 0, "sessions": 3, "steps": 16, "unclear": 0, "visitors": 3}, "elapsed_ms": 17885, "findings": [{"count": 3, "evidence": [{"goal": "Read the getting started guide", "name": "First-time visitor", "said": "stuck", "screenshot": "v-newcomer-t1-06.png", "step": 6, "try": 1, "visitor": "newcomer"}, {"goal": "Read the getting started guide", "name": "In a hurry, on a phone", "said": "stuck on broken notes page", "screenshot": "v-phone-t1-04.png", "step": 4, "try": 1, "visitor": "phone"}, {"goal": "Read the getting started guide", "name": "Careful visitor", "said": "no way to save notes", "screenshot": "v-buyer-t1-06.png", "step": 6, "try": 1, "visitor": "buyer"}], "hits": 1, "kind": "gave_up_here", "page": "http://127.0.0.1:4102/demo/notes-broken", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "said": "stuck", "title": "add back button to guide page", "tries": 1, "what_happened": "Stopped on this page without finding a way forward."}], "finished_at": "2026-10-17T22:04:02.084142+00:00", "fixes": [{"count": 3, "evidence": [{"goal": "Read the getting started guide", "name": "First-time visitor", "said": "stuck", "screenshot": "v-newcomer-t1-06.png", "step": 6, "try": 1, "visitor": "newcomer"}, {"goal": "Read the getting started guide", "name": "In a hurry, on a phone", "said": "stuck on broken notes page", "screenshot": "v-phone-t1-04.png", "step": 4, "try": 1, "visitor": "phone"}, {"goal": "Read the getting started guide", "name": "Careful visitor", "said": "no way to save notes", "screenshot": "v-buyer-t1-06.png", "step": 6, "try": 1, "visitor": "buyer"}], "hits": 1, "kind": "gave_up_here", "page": "http://127.0.0.1:4102/demo/notes-broken", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "said": "stuck", "title": "add back button to guide page", "tries": 1, "what_happened": "Stopped on this page without finding a way forward."}], "fixes_note": "", "handed_back_test": true, "headline": "0 of 3 test visitors got the job done in one try. 3 got stuck.", "held_back": 0, "id": "20261018-000344-dd00ad", "job": {"label": "Task line set by you", "needs_write": false, "source": "you", "task_contract": {"intent": "read", "required_moves": [], "task": "Read the getting started guide"}, "text": "Read the getting started guide"}, "judge_model": "nvidia/Nemotron-3.5-Super-VL", "limitations": ["Test visitors are an NVIDIA model reading and clicking your pages. They are not people, and nothing here is a quote from a real user.", "They only read. Nothing was typed, no account was used, and anything behind sign-in was left untested.", "Three states are kept apart. Claimed: the visitor said the job was done. Screen confirmed: a second call, given the steps and the text of that screen, agreed and pointed at words on it. Job completed: the steps also show every part the task names. A task that needs sign-in or a save is never counted as completed. The model calls can be wrong."], "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "model_calls": 32, "model_cost_usd": 0.00082176, "provider": "Nebius Token Factory", "rounds": [{"elapsed_ms": 16192, "try": 1}], "sceptic": null, "seen_once": 0, "set_aside": [], "spend": {"calls": [{"accounted_usd": 1.614e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-01-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 640, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, {"accounted_usd": 1.626e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-01-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 665, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, {"accounted_usd": 1.674e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-01-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 847, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 223}}, {"accounted_usd": 3.414e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-01", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 715, "status": "usage_accounted", "usage": {"completion_tokens": 41, "prompt_tokens": 405}}, {"accounted_usd": 3.57e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-01", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 628, "status": "usage_accounted", "usage": {"completion_tokens": 47, "prompt_tokens": 407}}, {"accounted_usd": 3.642e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-01", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 706, "status": "usage_accounted", "usage": {"completion_tokens": 48, "prompt_tokens": 415}}, {"accounted_usd": 1.632e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-02-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 535, "status": "usage_accounted", "usage": {"completion_tokens": 20, "prompt_tokens": 192}}, {"accounted_usd": 1.644e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-02-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 543, "status": "usage_accounted", "usage": {"completion_tokens": 20, "prompt_tokens": 194}}, {"accounted_usd": 1.692e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-02-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 549, "status": "usage_accounted", "usage": {"completion_tokens": 20, "prompt_tokens": 202}}, {"accounted_usd": 3.252e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-02", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 488, "status": "usage_accounted", "usage": {"completion_tokens": 29, "prompt_tokens": 426}}, {"accounted_usd": 3.336e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-02", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 431, "status": "usage_accounted", "usage": {"completion_tokens": 32, "prompt_tokens": 428}}, {"accounted_usd": 3.432e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-02", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 559, "status": "usage_accounted", "usage": {"completion_tokens": 34, "prompt_tokens": 436}}, {"accounted_usd": 1.614e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-03-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 286, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, {"accounted_usd": 1.626e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-03-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 473, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, {"accounted_usd": 1.674e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-03-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 445, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 223}}, {"accounted_usd": 3.474e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-03", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 380, "status": "usage_accounted", "usage": {"completion_tokens": 39, "prompt_tokens": 423}}, {"accounted_usd": 3.414e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-03", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 522, "status": "usage_accounted", "usage": {"completion_tokens": 36, "prompt_tokens": 425}}, {"accounted_usd": 3.51e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-03", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 533, "status": "usage_accounted", "usage": {"completion_tokens": 38, "prompt_tokens": 433}}, {"accounted_usd": 1.614e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-04-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 572, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, {"accounted_usd": 1.626e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-04-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 639, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, {"accounted_usd": 1.674e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-04-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 634, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 223}}, {"accounted_usd": 3.552e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-04", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 559, "status": "usage_accounted", "usage": {"completion_tokens": 37, "prompt_tokens": 444}}, {"accounted_usd": 3.45e-05, "call": "studio-20261018-000344-dd00ad-t1-phone-04", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 356, "status": "usage_accounted", "usage": {"completion_tokens": 28, "prompt_tokens": 463}}, {"accounted_usd": 3.9e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-04", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 524, "status": "usage_accounted", "usage": {"completion_tokens": 51, "prompt_tokens": 446}}, {"accounted_usd": 1.614e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-05-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 671, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, {"accounted_usd": 1.722e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-05-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 478, "status": "usage_accounted", "usage": {"completion_tokens": 18, "prompt_tokens": 215}}, {"accounted_usd": 3.438e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-05", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 497, "status": "usage_accounted", "usage": {"completion_tokens": 31, "prompt_tokens": 449}}, {"accounted_usd": 3.642e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-05", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 622, "status": "usage_accounted", "usage": {"completion_tokens": 39, "prompt_tokens": 451}}, {"accounted_usd": 1.614e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-06-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 1181, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, {"accounted_usd": 1.626e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-06-look", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 803, "status": "usage_accounted", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, {"accounted_usd": 3.426e-05, "call": "studio-20261018-000344-dd00ad-t1-buyer-06", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 816, "status": "usage_accounted", "usage": {"completion_tokens": 26, "prompt_tokens": 467}}, {"accounted_usd": 3.438e-05, "call": "studio-20261018-000344-dd00ad-t1-newcomer-06", "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "ms": 927, "status": "usage_accounted", "usage": {"completion_tokens": 26, "prompt_tokens": 469}}], "calls_without_usage": 0, "completion_tokens": 828, "held_reservation_usd": 0.0, "listed_rate_estimate_usd": 0.00082176, "max_calls": 48, "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "model_calls": 32, "models": ["nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B"], "price_source": "https://nebius.com/services/token-factory/models/nvidia-nemotron-models-inference", "prompt_tokens": 10384, "provider": "Nebius Token Factory", "provider_invoice_usd": null, "run_cap_usd": 0.2}, "status": "complete", "tries": [[{"blocked": [], "claimed": false, "colour": "#1f6f5c", "confirmed": false, "end_ms": 16192, "goal": "Read the getting started guide", "id": "newcomer", "max_steps": 6, "name": "First-time visitor", "outcome": "stuck", "phone": false, "sceptic": false, "screen_confirmed": false, "start_ms": 1, "status": "complete", "steps": [{"action": "click", "choices_seen": 1, "elapsed_ms": 3002, "hit": {"kind": "error_page", "label": "Read the guide", "page_text": "404 This page does not exist.", "screenshot": "v-newcomer-t1-51.png", "url": "http://127.0.0.1:4102/demo/notes-broken/guide", "what_happened": "Clicked 'Read the guide' and the page answered with error 404."}, "http_status": 404, "look": {"found": false, "model_ms": 665, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, "model_ms": 628, "model_reply": "{\n  \"say\": \"Read the guide to learn how to organise my notes\",\n  \"action\": \"click\",\n  \"target\": 1,\n  \"wish\": \"guide should explain how to save notes\"\n}", "needs_account": [], "order": 1, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Read the guide to learn how to organise my notes", "screenshot": "v-newcomer-t1-01.png", "start_ms": 1647, "status": "error_page", "target_text": "Read the guide", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 47, "prompt_tokens": 407}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 1927, "http_status": 404, "look": {"found": false, "model_ms": 543, "on_page": true, "quote": "This page does not exist.", "usage": {"completion_tokens": 20, "prompt_tokens": 194}}, "model_ms": 431, "model_reply": "{\"say\": \"error page blocked the guide\", \"action\": \"back\", \"target\": 1, \"wish\": \"fix broken link\"}", "needs_account": [], "order": 2, "page_evidence_text": "404This page does not exist.", "page_text": "404 This page does not exist.", "say": "error page blocked the guide", "screenshot": "v-newcomer-t1-02.png", "start_ms": 4650, "status": "ok", "title": "Not found", "url": "http://127.0.0.1:4102/demo/notes-broken/guide", "usage": {"completion_tokens": 32, "prompt_tokens": 428}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 1821, "http_status": 200, "look": {"found": false, "model_ms": 473, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, "model_ms": 522, "model_reply": "{\"say\": \"Read the guide leads to error page\", \"action\": \"back\", \"target\": 2, \"wish\": \"guide should open without error\"}", "needs_account": [], "order": 3, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Read the guide leads to error page", "screenshot": "v-newcomer-t1-03.png", "start_ms": 6580, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 36, "prompt_tokens": 425}}, {"action": "scroll", "choices_seen": 0, "elapsed_ms": 2307, "http_status": 200, "look": {"found": false, "model_ms": 639, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, "model_ms": 524, "model_reply": "{\n  \"say\": \"I need to go back to the collection to see what's there\",\n  \"action\": \"scroll\",\n  \"target\": 0,\n  \"wish\": \"add a back button to the collection page\"\n}", "needs_account": [], "order": 4, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "I need to go back to the collection to see what's there", "screenshot": "v-newcomer-t1-04.png", "start_ms": 8402, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 51, "prompt_tokens": 446}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 2414, "http_status": 200, "look": {"found": true, "model_ms": 478, "on_page": false, "quote": "Read the guide", "usage": {"completion_tokens": 18, "prompt_tokens": 215}}, "model_ms": 622, "model_reply": "{\n  \"say\": \"no more pages to read\",\n  \"action\": \"back\",\n  \"target\": 2,\n  \"wish\": \"add next step link\"\n}", "needs_account": [], "order": 5, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "no more pages to read", "screenshot": "v-newcomer-t1-05.png", "start_ms": 10711, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 39, "prompt_tokens": 451}}, {"action": "stuck", "choices_seen": 0, "elapsed_ms": 3051, "http_status": 200, "look": {"found": false, "model_ms": 803, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 215}}, "model_ms": 927, "model_reply": "{\"say\":\"stuck\",\"action\":\"stuck\",\"target\":0,\"wish\":\"add back button to guide page\"}", "needs_account": [], "order": 6, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "stuck", "screenshot": "v-newcomer-t1-06.png", "start_ms": 13128, "status": "stuck", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 26, "prompt_tokens": 469}, "wish": "add back button to guide page"}], "time_to_goal_ms": null}, {"blocked": [], "claimed": false, "colour": "#b4532a", "confirmed": false, "end_ms": 10232, "goal": "Read the getting started guide", "id": "phone", "max_steps": 4, "name": "In a hurry, on a phone", "outcome": "stuck", "phone": true, "sceptic": false, "screen_confirmed": false, "start_ms": 3, "status": "complete", "steps": [{"action": "click", "choices_seen": 1, "elapsed_ms": 3101, "hit": {"kind": "error_page", "label": "Read the guide", "page_text": "404 This page does not exist.", "screenshot": "v-phone-t1-51.png", "url": "http://127.0.0.1:4102/demo/notes-broken/guide", "what_happened": "Clicked 'Read the guide' and the page answered with error 404."}, "http_status": 404, "look": {"found": false, "model_ms": 847, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 223}}, "model_ms": 706, "model_reply": "{\n  \"say\": \"Read the guide to learn how to organise my notes\",\n  \"action\": \"click\",\n  \"target\": 1,\n  \"wish\": \"guide should open directly not as separate page\"\n}", "needs_account": [], "order": 1, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Read the guide to learn how to organise my notes", "screenshot": "v-phone-t1-01.png", "start_ms": 1586, "status": "error_page", "target_text": "Read the guide", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 48, "prompt_tokens": 415}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 1908, "http_status": 404, "look": {"found": false, "model_ms": 549, "on_page": true, "quote": "This page does not exist.", "usage": {"completion_tokens": 20, "prompt_tokens": 202}}, "model_ms": 559, "model_reply": "{\"say\": \"error page, need to go back\", \"action\": \"back\", \"target\": 0, \"wish\": \"fix broken link\"}", "needs_account": [], "order": 2, "page_evidence_text": "404This page does not exist.", "page_text": "404 This page does not exist.", "say": "error page, need to go back", "screenshot": "v-phone-t1-02.png", "start_ms": 4691, "status": "ok", "title": "Not found", "url": "http://127.0.0.1:4102/demo/notes-broken/guide", "usage": {"completion_tokens": 34, "prompt_tokens": 436}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 1858, "http_status": 200, "look": {"found": false, "model_ms": 445, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 223}}, "model_ms": 533, "model_reply": "{\n  \"say\": \"Back to the collection\",\n  \"action\": \"back\",\n  \"target\": 2,\n  \"wish\": \"guide link should work\"\n}", "needs_account": [], "order": 3, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Back to the collection", "screenshot": "v-phone-t1-03.png", "start_ms": 6602, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 38, "prompt_tokens": 433}}, {"action": "stuck", "choices_seen": 0, "elapsed_ms": 1754, "http_status": 200, "look": {"found": false, "model_ms": 634, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 223}}, "model_ms": 356, "model_reply": "{\"say\":\"stuck on broken notes page\",\"action\":\"stuck\",\"target\":0,\"wish\":\"fix broken notes page\"}", "needs_account": [], "order": 4, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "stuck on broken notes page", "screenshot": "v-phone-t1-04.png", "start_ms": 8471, "status": "stuck", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 28, "prompt_tokens": 463}, "wish": "fix broken notes page"}], "time_to_goal_ms": null}, {"blocked": [], "claimed": false, "colour": "#8a3d6b", "confirmed": false, "end_ms": 15704, "goal": "Read the getting started guide", "id": "buyer", "max_steps": 6, "name": "Careful visitor", "outcome": "stuck", "phone": false, "sceptic": false, "screen_confirmed": false, "start_ms": 11, "status": "complete", "steps": [{"action": "click", "choices_seen": 1, "elapsed_ms": 2992, "hit": {"kind": "error_page", "label": "Read the guide", "page_text": "404 This page does not exist.", "screenshot": "v-buyer-t1-51.png", "url": "http://127.0.0.1:4102/demo/notes-broken/guide", "what_happened": "Clicked 'Read the guide' and the page answered with error 404."}, "http_status": 404, "look": {"found": false, "model_ms": 640, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, "model_ms": 715, "model_reply": "{\"say\": \"Read the guide to learn how to organise my notes.\", \"action\": \"click\", \"target\": 1, \"wish\": \"guide should explain how to add notes\"}", "needs_account": [], "order": 1, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Read the guide to learn how to organise my notes.", "screenshot": "v-buyer-t1-01.png", "start_ms": 1620, "status": "error_page", "target_text": "Read the guide", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 41, "prompt_tokens": 405}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 1859, "http_status": 404, "look": {"found": false, "model_ms": 535, "on_page": true, "quote": "This page does not exist.", "usage": {"completion_tokens": 20, "prompt_tokens": 192}}, "model_ms": 488, "model_reply": "{\"say\": \"back\", \"action\": \"back\", \"target\": 0, \"wish\": \"fix the broken link\"}", "needs_account": [], "order": 2, "page_evidence_text": "404This page does not exist.", "page_text": "404 This page does not exist.", "say": "back", "screenshot": "v-buyer-t1-02.png", "start_ms": 4613, "status": "ok", "title": "Not found", "url": "http://127.0.0.1:4102/demo/notes-broken/guide", "usage": {"completion_tokens": 29, "prompt_tokens": 426}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 1546, "http_status": 200, "look": {"found": false, "model_ms": 286, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, "model_ms": 380, "model_reply": "{\"say\": \"Read the guide caused error, need to go back\", \"action\": \"back\", \"target\": 2, \"wish\": \"guide should open without error\"}", "needs_account": [], "order": 3, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Read the guide caused error, need to go back", "screenshot": "v-buyer-t1-03.png", "start_ms": 6475, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 39, "prompt_tokens": 423}}, {"action": "scroll", "choices_seen": 0, "elapsed_ms": 2209, "http_status": 200, "look": {"found": false, "model_ms": 572, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, "model_ms": 559, "model_reply": "{\"say\": \"Read the guide leads to error page\", \"action\": \"scroll\", \"target\": 0, \"wish\": \"guide page should load without error\"}", "needs_account": [], "order": 4, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "Read the guide leads to error page", "screenshot": "v-buyer-t1-04.png", "start_ms": 8025, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 37, "prompt_tokens": 444}}, {"action": "back", "choices_seen": 0, "elapsed_ms": 2325, "http_status": 200, "look": {"found": false, "model_ms": 671, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, "model_ms": 497, "model_reply": "{\"say\": \"no more to read\", \"action\": \"back\", \"target\": 2, \"wish\": \"add more content\"}", "needs_account": [], "order": 5, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "no more to read", "screenshot": "v-buyer-t1-05.png", "start_ms": 10241, "status": "no_effect", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 31, "prompt_tokens": 449}}, {"action": "stuck", "choices_seen": 0, "elapsed_ms": 3131, "http_status": 200, "look": {"found": false, "model_ms": 1181, "on_page": false, "quote": "", "usage": {"completion_tokens": 14, "prompt_tokens": 213}}, "model_ms": 816, "model_reply": "{\"say\":\"no way to save notes\",\"action\":\"stuck\",\"target\":0,\"wish\":\"add save button\"}", "needs_account": [], "order": 6, "page_evidence_text": "Notes collectionOpen our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing.", "page_text": "Notes collection Open our guide to learn how to organise your notes. This demo app is part of the simulator and saves nothing. Read the guide Back to the collection", "say": "no way to save notes", "screenshot": "v-buyer-t1-06.png", "start_ms": 12570, "status": "stuck", "title": "Notes", "url": "http://127.0.0.1:4102/demo/notes-broken", "usage": {"completion_tokens": 26, "prompt_tokens": 467}, "wish": "add save button"}], "time_to_goal_ms": null}]], "tries_planned": 1, "url": "http://127.0.0.1:4102/demo/notes-broken", "verdict": "0 of 3 test visitors got the job done in one try. 3 got stuck.", "visitors": [{"agreement": 1, "claimed": 0, "colour": "#1f6f5c", "confirmed": 0, "goal": "Read the getting started guide", "id": "newcomer", "max_steps": 6, "name": "First-time visitor", "outcomes": ["stuck"], "phone": false, "sceptic": false, "screen_confirmed": 0, "verdict": "gets_stuck"}, {"agreement": 1, "claimed": 0, "colour": "#b4532a", "confirmed": 0, "goal": "Read the getting started guide", "id": "phone", "max_steps": 4, "name": "In a hurry, on a phone", "outcomes": ["stuck"], "phone": true, "sceptic": false, "screen_confirmed": 0, "verdict": "gets_stuck"}, {"agreement": 1, "claimed": 0, "colour": "#8a3d6b", "confirmed": 0, "goal": "Read the getting started guide", "id": "buyer", "max_steps": 6, "name": "Careful visitor", "outcomes": ["stuck"], "phone": false, "sceptic": false, "screen_confirmed": 0, "verdict": "gets_stuck"}], "visitors_planned": [{"colour": "#1f6f5c", "goal": "Read the getting started guide", "id": "newcomer", "max_steps": 6, "name": "First-time visitor"}, {"colour": "#b4532a", "goal": "Read the getting started guide", "id": "phone", "max_steps": 4, "name": "In a hurry, on a phone"}, {"colour": "#8a3d6b", "goal": "Read the getting started guide", "id": "buyer", "max_steps": 6, "name": "Careful visitor"}], "replay": {"name": "broken-demo-0003", "label": "Three visitors, broken demo app, Super VL judge, 18 Oct 00:03", "source": "swarm-product-2350/store/runs/20261018-000344-dd00ad/result.json", "sha256": "960eae1daca71d7e88ea0e2801cd4b7266f8c855bdf1dde45aff41416d873717"}}