{ "site": { "measured": "2026-07-03", "run": "run 07-03", "scope": "110 models · 42 integrations", "host": "benchmarks.speko.ai", "tagline": "Independent, reproducible voice-AI benchmarks: STT, TTS, LLM, S2S, and true cost-per-solve across the whole call.", "blurb": "Independent, reproducible voice-AI benchmarks: STT, TTS, LLM, S2S, turn-taking, and true cost-per-solve across the whole call. Published numbers are measured, never marketed." }, "boards": [ { "id": "stt", "navLabel": "Speech-to-Text", "identityLabel": "Provider / Model", "title": "Speech-to-Text", "subtitle": "Word error rate, latency and cost for every speech-to-text model we measure.", "live": true, "meta": "English · read speech (FLEURS, n=50) batch 2026-07-03 / streaming 2026-10-01 (FLEURS test+dev+train, n=1541) · spontaneous conversation, 60 speakers (EdAcc, n=283, two passes, real-time delivery) 2026-10-01 · latency vendor-direct US East 2026-09-08", "measured": "2026-07-03", "headline": [ "endpoint", "wer", "cost" ], "scatter": { "x": "endpoint", "y": "wer", "cornerLabel": "fast + accurate" }, "columns": [ { "id": "wer", "path": "batch", "label": "WER · batch", "dir": "lower", "info": "Word Error Rate on FLEURS read English (n=50), batch API. Read speech, not studio-clean audio. Lower is better." }, { "id": "werStream", "path": "stream", "label": "WER · stream", "partOf": "WER · streaming", "partLabel": "Read speech", "dir": "lower", "info": "WER on 1,541 FLEURS read-English clips through the live socket at real-time pace. Lower is better." }, { "id": "werConv", "path": "stream", "label": "WER · conversational", "partOf": "WER · streaming", "partLabel": "Conversation", "dir": "lower", "info": "WER on 283 clips of spontaneous accented conversation (EdAcc, 60 speakers), two passes at real-time pace. Lower is better." }, { "id": "endpoint", "path": "stream", "label": "Finalize", "dir": "lower", "info": "End-of-speech to final transcript, p50, vendor-direct from US East. Whisker spans p50→p90." }, { "id": "ttft", "path": "stream", "label": "Time to first token", "dir": "lower", "info": "First partial word, p50, vendor-direct from US East. Whisker spans p50→p90." }, { "id": "cost", "label": "Cost / min", "dir": "lower", "info": "$ per minute of audio, vendor list, streaming path. Pay-as-you-go where offered, otherwise the cheapest paid plan. Lower = cheaper; \"—\" where a vendor publishes no per-minute rate." } ], "rows": [ { "provider": "Speechmatics", "model": "Standard", "id": "speechmatics:standard", "measured": "2026-09-22", "measuredStream": "2026-10-01", "routable": false, "meta": "Vendor-direct benchmark", "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "18.08%", "s": 18.078, "tone": "warn", "ci": "15.9–20.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "5.0%", "s": 5, "ci": "95% CI 3.43–6.65 · n=50 · vendor-direct batch · FLEURS", "note": "Measured 2026-09-22" }, "werStream": { "v": "5.88%", "s": 5.88, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "4.57s", "s": 4569, "lo": 4569, "hi": 4968, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±3%", "tone": "bad" }, "ttft": { "v": "1.46s", "s": 1458, "lo": 1458, "hi": 1868, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±0%", "tone": "good" }, "cost": { "v": "$0.0040", "s": 0.004, "note": "Speechmatics public Pro list rate, streaming, checked 2026-09-22. https://www.speechmatics.com/pricing" } } }, { "provider": "Speechmatics", "model": "Enhanced", "id": "speechmatics:enhanced", "measured": "2026-09-22", "measuredStream": "2026-10-01", "routable": false, "meta": "Vendor-direct benchmark", "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "15.09%", "s": 15.0893, "tone": "warn", "ci": "13.2–17.6% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "3.9%", "s": 3.9, "ci": "95% CI 2.63–5.31 · n=50 · vendor-direct batch · FLEURS", "note": "Measured 2026-09-22" }, "werStream": { "v": "5.04%", "s": 5.04, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "4.49s", "s": 4490, "lo": 4490, "hi": 4792, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±2%", "tone": "bad" }, "ttft": { "v": "1.48s", "s": 1482, "lo": 1482, "hi": 1904, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±0%", "tone": "good" }, "cost": { "v": "$0.0072", "s": 0.007166666666666667, "note": "Speechmatics public Pro list rate, streaming, checked 2026-09-22. https://www.speechmatics.com/pricing" } } }, { "provider": "Speechmatics", "model": "Melia 1", "id": "speechmatics:melia-1", "measured": "2026-09-22", "routable": false, "batchOnly": true, "meta": "Vendor-direct benchmark", "rec": [ { "label": "Batch" } ], "cells": { "wer": { "v": "3.8%", "s": 3.8, "ci": "95% CI 2.38–5.25 · n=50 · vendor-direct batch · FLEURS", "note": "Measured 2026-09-22" }, "werStream": { "v": "—", "s": null, "dim": true, "note": "Batch-only" }, "werConv": { "v": "—", "s": null, "dim": true, "note": "Batch-only" }, "endpoint": { "v": "—", "s": null, "dim": true, "note": "Batch-only" }, "ttft": { "v": "—", "s": null, "dim": true, "note": "Batch-only" }, "cost": { "v": "—", "s": null, "dim": true, "note": "No streaming API. Public batch rate: $0.00215/min ($0.129/hour), checked 2026-09-22. https://www.speechmatics.com/pricing" } } }, { "provider": "Speechmatics", "model": "Linden 1", "id": "speechmatics:linden-1", "measured": "2026-09-22", "measuredStream": "2026-10-01", "routable": false, "streamOnly": true, "meta": "Vendor-direct benchmark", "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "16.33%", "s": 16.3346, "tone": "warn", "ci": "14.2–18.9% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "—", "s": null, "dim": true, "note": "Agent STT has no batch API" }, "werStream": { "v": "6.17%", "s": 6.17, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "395ms", "s": 395, "lo": 395, "hi": 607, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±12%", "tone": "warn" }, "ttft": { "v": "1.49s", "s": 1486, "lo": 1486, "hi": 1902, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±1%", "tone": "good" }, "cost": { "v": "$0.0027", "s": 0.0026666666666666666, "note": "Speechmatics public Pro list rate, streaming, checked 2026-09-22. https://www.speechmatics.com/pricing" } } }, { "provider": "Hopper", "model": "nemotron-asr", "id": "hopper:nemotron-asr", "measured": "2026-10-08", "routable": false, "status": { "label": "FAIR", "tone": "warn" }, "meta": "NVIDIA Nemotron 0.6B, served from Oregon", "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "23.05%", "s": 23.0505, "tone": "bad", "ci": "20.0–26.1% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-08", "measured": "2026-10-08", "note": "6 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "9.13%", "s": 9.13, "tone": "warn", "ci": "95% CI 7.0–11.4 · n=50 · vendor-direct batch · FLEURS", "note": "The scorer counts spelled-out numbers as errors. On the 40 clips without numbers it reads 7.6%." }, "werStream": { "v": "9.83%", "s": 9.83, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-08", "note": "Scored with the same normalizer as batch, which counts spelled-out numbers as errors." }, "endpoint": { "v": "656ms", "s": 656, "lo": 656, "hi": 896, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-10-08 · ±8%", "tone": "bad", "note": "Marker-limited: 68 of 200 readings land before the measured speech end, 28 by more than 0.5s (Modulate on the same clips: 27 and 12). Hopper serves from Oregon, so US readings include a cross-country round trip." }, "ttft": { "v": "1.52s", "s": 1520, "lo": 1520, "hi": 2060, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-10-08 · ±7%", "tone": "good" }, "cost": { "v": "$0.00083", "s": 0.0008333333333333334, "note": "Vendor list, $0.05 per audio hour." } }, "measuredStream": "2026-10-08" }, { "provider": "Smallest AI", "model": "Pulse", "id": "smallest:pulse", "status": { "label": "FAIR", "tone": "warn" }, "rec": [ { "label": "Streaming" }, { "label": "Real-time phone" } ], "cells": { "werConv": { "v": "14.46%", "s": 14.4624, "tone": "good", "ci": "12.4–16.7% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "2 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "5.1%", "s": 5.1, "tone": "warn" }, "werStream": { "v": "5.93%", "s": 5.93, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "839ms", "s": 839, "lo": 839, "hi": 1082, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "bad" }, "ttft": { "v": "2.27s", "s": 2272, "lo": 2272, "hi": 2323, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±1%", "tone": "warn" }, "cost": { "v": "$0.004", "s": 0.004, "note": "Streaming Pulse, the vendor's published per-model rate (smallest.ai/pricing/models; pre-recorded Pulse is $0.003/min). The ~$0.009/min on the main pricing page is the STT slice of a blended voice-agent stack, not this API's list rate." } }, "measuredStream": "2026-10-01" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "status": { "label": "FAIR", "tone": "warn" }, "rec": [ { "label": "Real-time phone" }, { "label": "Streaming" } ], "cells": { "werConv": { "v": "16.95%", "s": 16.9529, "tone": "warn", "ci": "14.4–19.9% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "1 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "11.96%", "s": 11.96, "tone": "bad", "ci": "n=50 · FLEURS clean read-English", "note": "Corrected 2026-09-02 from a published 9.8% that has no reproducible source. 11.96% is the mean per-clip WER of the committed batch artifact results/anchor-nova3-20260808.json (run of 2026-08-08), re-confirmed by rescoring that artifact with the current text normalizer — 11.96% as stored and 11.96% rescored." }, "werStream": { "v": "10.8%", "s": 10.8, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "189ms", "s": 189, "lo": 189, "hi": 770, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±9%", "tone": "good" }, "ttft": { "v": "976ms", "s": 976, "lo": 976, "hi": 1978, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±0%", "tone": "good" }, "cost": { "v": "$0.0048", "s": 0.0048, "note": "Streaming pay-as-you-go. Vendor flags it as a limited-time promotional rate; pre-recorded is $0.0043." } }, "measuredStream": "2026-10-01" }, { "provider": "Deepgram", "model": "Flux", "streamOnly": true, "id": "deepgram:flux-general-en", "measured": "2026-08-04", "status": { "label": "GOOD", "tone": "good" }, "rec": [ { "label": "Real-time phone" }, { "label": "Streaming" } ], "cells": { "werConv": { "v": "19.68%", "s": 19.684, "tone": "bad", "ci": "16.7–22.6% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "10 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "7.2%", "s": 7.2, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "712ms", "s": 712, "lo": 712, "hi": 1312, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±15%", "tone": "bad" }, "ttft": { "v": "811ms", "s": 811, "lo": 811, "hi": 1491, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "good" }, "cost": { "v": "$0.0065", "s": 0.0065, "note": "Flux English streaming pay-as-you-go. Vendor flags it as a limited-time promotional rate; flux-general-multi is $0.0078." } }, "measuredStream": "2026-10-01" }, { "provider": "Deepgram", "model": "flux-general-multi", "streamOnly": true, "id": "deepgram:flux-general-multi", "measured": "2026-09-28", "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "21.44%", "s": 21.4445, "tone": "bad", "ci": "18.0–24.6% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "8 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "8.73%", "s": 8.73, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "854ms", "s": 854, "lo": 854, "hi": 1613, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±10%", "tone": "bad" }, "ttft": { "v": "1.01s", "s": 1014, "lo": 1014, "hi": 1494, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±8%", "tone": "good" }, "cost": { "v": "$0.0078", "s": 0.0078, "note": "Flux Multilingual streaming pay-as-you-go, no promotional rate. Checked 2026-09-28." } }, "measuredStream": "2026-10-01" }, { "provider": "Deepgram", "model": "Nova-3 Medical", "id": "deepgram:nova-3-medical", "routable": false, "measured": "2026-09-28", "rec": [], "cells": { "werConv": { "v": "19.47%", "s": 19.4693, "tone": "warn", "ci": "16.8–22.3% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "4 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "13.12%", "s": 13.12, "tone": "bad", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "190ms", "s": 190, "lo": 190, "hi": 742, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±8%", "tone": "good" }, "ttft": { "v": "989ms", "s": 989, "lo": 989, "hi": 2004, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±2%", "tone": "good" }, "cost": { "v": "—", "s": null, "note": "Not on Deepgram’s pay-as-you-go price list (checked 2026-09-28)." } }, "measuredStream": "2026-10-01" }, { "provider": "Deepgram", "model": "Base Phonecall", "id": "deepgram:base-phonecall", "routable": false, "measured": "2026-09-28", "rec": [], "cells": { "werConv": { "v": "34.51%", "s": 34.507, "tone": "bad", "ci": "29.3–39.9% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "4 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "27.03%", "s": 27.03, "tone": "bad", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "171ms", "s": 171, "lo": 171, "hi": 572, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±14%", "tone": "good" }, "ttft": { "v": "1.01s", "s": 1008, "lo": 1008, "hi": 1989, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±46%", "tone": "good" }, "cost": { "v": "—", "s": null, "note": "Not on Deepgram’s pay-as-you-go price list (checked 2026-09-28)." } }, "measuredStream": "2026-10-01" }, { "provider": "Cartesia", "model": "Ink-2", "id": "cartesia:ink-2", "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "werConv": { "v": "16.68%", "s": 16.6781, "tone": "warn", "ci": "14.6–19.1% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "11.0%", "s": 11, "tone": "bad" }, "werStream": { "v": "10.12%", "s": 10.12, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "587ms", "s": 587, "lo": 587, "hi": 989, "ci": "p50–p90 · US East · n=200 · vendor-direct · pre-close (99/100 flush-forced) · ±34%", "tone": "bad" }, "ttft": { "v": "1.78s", "s": 1785, "lo": 1785, "hi": 2396, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±10%", "tone": "good" }, "cost": { "v": "$0.0090", "s": 0.009, "note": "Ink-2 bills 3 credits/sec on Pro, the cheapest paid plan; $0.0071 at Startup, $0.0067 at Scale. The 1 credit/sec rate is Ink-Whisper, a different model." } }, "measuredStream": "2026-10-01" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Multilingual" } ], "cells": { "werConv": { "v": "-", "s": null }, "wer": { "v": "3.9%", "s": 3.9, "tone": "good", "measured": "2026-09-24", "note": "Measured 2026-09-24. The earlier 2.8% did not reproduce on the same clips." }, "werStream": { "v": "-", "s": null }, "endpoint": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$0.0021", "s": 0.0021, "note": "File/sync SKU, Singapore. The realtime socket bills $0.0054." } } }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "id": "alibaba:qwen3-asr-flash-realtime", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Multilingual" }, { "label": "Streaming" } ], "cells": { "werConv": { "v": "12.2%", "s": 12.2, "tone": "good", "measured": "2026-08-28", "ci": "EdAcc n=97 · earlier set", "note": "Earlier 98-clip set: the session hung in the 2026-10-01 re-measure. UNUSABLE FOR A CALL: median 311s of wall clock per 7s clip, up to 1123s." }, "wer": { "v": "-", "s": null }, "werStream": { "v": "4.0%", "s": 4, "tone": "good", "measured": "2026-07-21", "ci": "n=50 · FLEURS · earlier set", "note": "Earlier 50-clip set: the session hung in the 2026-10-01 re-measure." }, "endpoint": { "v": "889ms", "s": 889, "lo": 889, "hi": 1206, "ci": "p50–p90 · US East · n=189 · vendor-direct · ±6%", "tone": "bad" }, "ttft": { "v": "6.81s", "s": 6809, "lo": 6809, "hi": 7499, "ci": "p50–p90 · US East · n=198 · vendor-direct · ±1%", "tone": "bad" }, "cost": { "v": "$0.0054", "s": 0.0054, "note": "qwen3-asr-flash-realtime, Singapore. The file/sync SKU is $0.0021." } } }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "status": { "label": "GOOD", "tone": "good" }, "rec": [ { "label": "Multilingual" }, { "label": "Global" } ], "cells": { "werConv": { "v": "18.43%", "s": 18.4301, "tone": "warn", "ci": "15.2–21.8% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "3.9%", "s": 3.9, "tone": "good" }, "werStream": { "v": "5.89%", "s": 5.89, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$0.0160", "s": 0.016, "tone": "bad", "note": "Speech-to-Text V2 Standard, first tier — one rate for streaming, sync and batch. Dynamic Batch Recognition is $0.003." } }, "measuredStream": "2026-10-01" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "streamOnly": true, "id": "elevenlabs:scribe_v2_realtime", "slug": "elevenlabs-scribe-v2", "measured": "2026-07-21", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Balanced" }, { "label": "Streaming" } ], "cells": { "werConv": { "v": "12.54%", "s": 12.5386, "tone": "good", "ci": "10.9–15.0% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "-", "s": null }, "werStream": { "v": "4.32%", "s": 4.32, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "1.75s", "s": 1745, "lo": 1745, "hi": 1975, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±4%", "tone": "bad" }, "ttft": { "v": "2.22s", "s": 2218, "lo": 2218, "hi": 2359, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±0%", "tone": "warn" }, "cost": { "v": "$0.0065", "s": 0.0065, "note": "Scribe v2 Realtime at $0.39/hr, flat across every tier. Batch Scribe v2 is $0.0037." } }, "measuredStream": "2026-10-01" }, { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "slug": "elevenlabs-scribe-v2-batch", "measured": "2026-06-03", "meta": "Historical batch measurement / FLEURS / n=50 / -16 LUFS / 2026-06-03", "batchOnly": true, "rec": [], "cells": { "wer": { "v": "2.9%", "s": 2.9, "tone": "good" }, "werStream": { "v": "-", "s": null }, "endpoint": { "v": "-", "s": null }, "ttft": { "v": "-", "s": null }, "cost": { "v": "$0.0037", "s": 0.0037 } } }, { "provider": "Fish Audio", "model": "transcribe-1-pro", "id": "fishaudio:transcribe-1-pro", "slug": "fish-audio-transcribe-1-pro", "batchOnly": true, "measured": "2026-09-22", "meta": "Vendor-direct batch / FLEURS n=50 + EdAcc n=82 / 2026-09-22", "rec": [], "cells": { "wer": { "v": "3.4%", "s": 3.4, "tone": "good", "ci": "2.1–4.9% · 95% clip bootstrap · FLEURS n=50", "note": "Vendor-direct batch; explicit speaker/event annotations removed, quoted overlap speech retained." }, "werConv": { "v": "-", "s": null }, "werStream": { "v": "-", "s": null }, "endpoint": { "v": "-", "s": null }, "ttft": { "v": "-", "s": null }, "cost": { "v": "$0.006", "s": 0.006, "note": "Batch ASR: $0.36/audio hour; billed duration rounded up to whole seconds. Fish Audio list price checked 2026-09-22." } } }, { "provider": "Fish Audio", "model": "transcribe-1", "id": "fishaudio:transcribe-1", "slug": "fish-audio-transcribe-1", "batchOnly": true, "measured": "2026-09-28", "meta": "Vendor-direct batch / FLEURS n=50 + EdAcc n=82 / 2026-09-28", "rec": [], "cells": { "wer": { "v": "3.2%", "s": 3.2, "tone": "good", "ci": "1.9–4.7% · 95% clip bootstrap · FLEURS n=50", "note": "Vendor-direct batch, same clips and scorer as transcribe-1-pro." }, "werConv": { "v": "-", "s": null }, "werStream": { "v": "-", "s": null }, "endpoint": { "v": "-", "s": null }, "ttft": { "v": "-", "s": null }, "cost": { "v": "$0.006", "s": 0.006, "note": "Batch ASR: $0.36/audio hour, same rate as transcribe-1-pro. Fish Audio list price checked 2026-09-22." } } }, { "provider": "Microsoft", "model": "MAI-Transcribe-2", "id": "azure:MAI-Transcribe-2", "batchOnly": true, "measured": "2026-09-03", "meta": "Vendor-direct Azure Speech fast transcription (enhancedMode), eastus / FLEURS n=50 + EdAcc n=82 / 2026-09-03", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Accuracy" }, { "label": "Value" } ], "cells": { "werConv": { "v": "-", "s": null }, "wer": { "v": "2.3%", "s": 2.3, "tone": "good", "note": "Same pass reproduced the Nova-3 anchor at its published 11.96%." }, "werStream": { "v": "-", "s": null }, "endpoint": { "v": "-", "s": null }, "ttft": { "v": "-", "s": null }, "cost": { "v": "$0.0017", "s": 0.001667, "note": "$0.10 per hour of audio, a launch price through 2026-12-31." } } }, { "provider": "Alibaba", "model": "Qwen-Audio-3.1-ASR-Flash", "id": "alibaba:qwen-audio-3.1-asr-flash", "routable": false, "batchOnly": true, "measured": "2026-09-24", "meta": "Vendor-direct DashScope batch, Singapore / FLEURS n=50 + EdAcc n=82 / 2026-09-24", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Accuracy" }, { "label": "Value" } ], "cells": { "werConv": { "v": "-", "s": null }, "wer": { "v": "2.0%", "s": 2, "tone": "good", "note": "Measured 2026-09-24. The Nova-3 anchor re-scored at its published 11.96% the same day." }, "werStream": { "v": "-", "s": null }, "endpoint": { "v": "-", "s": null }, "ttft": { "v": "-", "s": null }, "cost": { "v": "$0.00024", "s": 0.00024, "note": "Billed tokens at $0.15/M input and $0.47/M output, Singapore." } } }, { "provider": "Inworld", "model": "Realtime STT-1", "id": "inworld:inworld-stt-1", "routable": false, "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Streaming" }, { "label": "Value" } ], "cells": { "werConv": { "v": "12.14%", "s": 12.1436, "tone": "good", "ci": "10.4–14.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "3.3%", "s": 3.3, "tone": "good" }, "werStream": { "v": "27.92%", "s": 27.92, "tone": "bad", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01", "note": "188 of 1541 clips came back empty, scored as all-deletion." }, "endpoint": { "v": "670ms", "s": 670, "lo": 670, "hi": 1162, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±17%", "tone": "bad" }, "ttft": { "v": "1.32s", "s": 1320, "lo": 1320, "hi": 1919, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±8%", "tone": "good" }, "cost": { "v": "$0.0025", "s": 0.0025, "tone": "good", "note": "On-demand $0.15/hr. The $0.10/hr headline needs a paid Creator subscription." } }, "measuredStream": "2026-10-01" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Streaming" }, { "label": "Accuracy" } ], "cells": { "werConv": { "v": "14.54%", "s": 14.5397, "tone": "good", "ci": "12.4–16.9% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "Deletions carry most of it: 7.1% dropped words against 5.8% substitutions." }, "wer": { "v": "2.0%", "s": 2, "tone": "good" }, "werStream": { "v": "3.73%", "s": 3.73, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-09-30" }, "endpoint": { "v": "408ms", "s": 408, "lo": 408, "hi": 908, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±12%", "tone": "warn" }, "ttft": { "v": "1.01s", "s": 1007, "lo": 1007, "hi": 1507, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±10%", "tone": "good" }, "cost": { "v": "$0.0075", "s": 0.0075, "note": "Universal-3.5 Pro Realtime at $0.45/hr. Async pre-recorded is $0.0035. Diarization and other add-ons bill on top." } } }, { "provider": "AssemblyAI", "model": "Universal-3.6 Pro", "id": "assemblyai:universal-3-6-pro", "measured": "2026-09-30", "rec": [], "cells": { "werConv": { "v": "13.75%", "s": 13.7496, "tone": "good", "ci": "11.7–16.1% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "0.79pt better than 3.5 Pro on the same clips (95% CI 0.03–1.67)." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "3.07%", "s": 3.07, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-09-30", "note": "0.66pt better than 3.5 Pro on the same clips (95% CI 0.49–0.84)." }, "endpoint": { "v": "505ms", "s": 505, "lo": 505, "hi": 948, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-30 · ±10%", "tone": "warn" }, "ttft": { "v": "1.11s", "s": 1106, "lo": 1106, "hi": 1707, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-30 · ±5%", "tone": "good" }, "cost": { "v": "$0.0075", "s": 0.0075, "note": "Universal-3.6 Pro Realtime at $0.45/hr, unchanged from 3.5 Pro. Checked 2026-10-01." } } }, { "provider": "AssemblyAI", "model": "Universal-Streaming English", "streamOnly": true, "id": "assemblyai:universal-streaming-english", "measured": "2026-09-28", "rec": [ { "label": "Streaming" }, { "label": "Value" } ], "cells": { "werConv": { "v": "26.19%", "s": 26.1937, "tone": "bad", "ci": "23.0–29.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "14 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "7.36%", "s": 7.36, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01", "note": "5 of 1541 clips came back empty, scored as all-deletion." }, "endpoint": { "v": "452ms", "s": 452, "lo": 452, "hi": 694, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±12%", "tone": "warn" }, "ttft": { "v": "1.74s", "s": 1740, "lo": 1740, "hi": 2456, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "good" }, "cost": { "v": "$0.0025", "s": 0.0025, "note": "$0.15/hr, billed on WebSocket session duration, idle time included. Checked 2026-09-28." } }, "measuredStream": "2026-10-01" }, { "provider": "AssemblyAI", "model": "Universal-Streaming Multilingual", "streamOnly": true, "id": "assemblyai:universal-streaming-multilingual", "measured": "2026-09-28", "rec": [], "cells": { "werConv": { "v": "35.19%", "s": 35.1941, "tone": "bad", "ci": "30.0–40.6% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "34 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "—", "s": null }, "werStream": { "v": "15.37%", "s": 15.37, "tone": "bad", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01", "note": "77 of 1541 clips came back empty, scored as all-deletion." }, "endpoint": { "v": "495ms", "s": 495, "lo": 495, "hi": 765, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±9%", "tone": "warn" }, "ttft": { "v": "1.73s", "s": 1729, "lo": 1729, "hi": 2363, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "good" }, "cost": { "v": "$0.0025", "s": 0.0025, "note": "$0.15/hr, billed on WebSocket session duration, idle time included. Checked 2026-09-28." } }, "measuredStream": "2026-10-01" }, { "provider": "Modulate", "model": "Velma 2", "id": "modulate:velma-2-stt-streaming-english-v2", "measured": "2026-08-08", "status": { "label": "GOOD", "tone": "good" }, "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "14.41%", "s": 14.4109, "tone": "good", "ci": "12.0–17.3% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01", "note": "2 of 566 readings came back empty, scored as all-deletion." }, "wer": { "v": "4.4%", "s": 4.4, "tone": "good", "ci": "95% CI 2.8–6.2 · n=50" }, "werStream": { "v": "5.35%", "s": 5.35, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "1.21s", "s": 1205, "lo": 1205, "hi": 1408, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±4%", "tone": "bad" }, "ttft": { "v": "1.48s", "s": 1476, "lo": 1476, "hi": 1486, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±0%", "tone": "good" }, "cost": { "v": "$0.00083", "s": 0.000833, "note": "Published streaming rate for English Fast, $0.05/hr. Batch is $0.025/hr. The $0.06/hr this row used to carry is the MULTILINGUAL row, a different product from the one this id names. Not a measurement." } }, "measuredStream": "2026-10-01" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "status": { "label": "FAIR", "tone": "warn" }, "rec": [ { "label": "Streaming" }, { "label": "Multilingual" } ], "cells": { "werConv": { "v": "16.64%", "s": 16.6438, "tone": "warn", "ci": "14.4–19.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "7.5%", "s": 7.5, "tone": "warn", "ci": "95% CI 4.8–10.7 · n=50" }, "werStream": { "v": "6.31%", "s": 6.31, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "340ms", "s": 340, "lo": 340, "hi": 1329, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±16%", "tone": "warn" }, "ttft": { "v": "1.53s", "s": 1526, "lo": 1526, "hi": 2042, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±4%", "tone": "good" }, "cost": { "v": "$0.002", "s": 0.002 } }, "measuredStream": "2026-10-01" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Accuracy" }, { "label": "Healthcare", "caution": true } ], "cells": { "werConv": { "v": "32.91%", "s": 32.9097, "tone": "bad", "ci": "28.4–36.9% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "2.3%", "s": 2.3, "tone": "good" }, "werStream": { "v": "5.64%", "s": 5.64, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "848ms", "s": 848, "lo": 848, "hi": 1181, "ci": "p50–p90 · US East · n=199 · vendor-direct · ±10%", "tone": "bad" }, "ttft": { "v": "6.40s", "s": 6400, "lo": 6400, "hi": 10807, "ci": "p50–p90 · US East · n=199 · vendor-direct · ±10%", "tone": "bad" }, "cost": { "v": "$0.006", "s": 0.006 } }, "measuredStream": "2026-10-01" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "slug": "openai-gpt-transcribe", "measured": "2026-07-03", "measuredStream": "2026-10-01", "rec": [], "cells": { "werConv": { "v": "16.82%", "s": 16.8241, "tone": "warn", "ci": "14.3–19.5% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "2.5%", "s": 2.5, "tone": "good" }, "werStream": { "v": "5.85%", "s": 5.85, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "817ms", "s": 817, "lo": 817, "hi": 1169, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-15 · ±7%", "tone": "bad", "note": "Marker-limited: 48 of 200 readings land before the measured speech end, 17 by more than 0.5s. The model emits nothing until its VAD closes a segment, so first token is the wait a caller hears." }, "ttft": { "v": "6.28s", "s": 6277, "lo": 6277, "hi": 10853, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-15 · ±10%", "tone": "bad" }, "cost": { "v": "$0.0045", "s": 0.0045, "note": "Billed per minute, not per token." } } }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "streamOnly": true, "id": "openai:gpt-live-transcribe", "slug": "openai-gpt-transcribe-gpt-live-transcribe", "measured": "2026-07-21", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Streaming" }, { "label": "Real-time phone" } ], "cells": { "werConv": { "v": "13.53%", "s": 13.5263, "tone": "good", "ci": "11.3–16.0% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "-", "s": null }, "werStream": { "v": "4.89%", "s": 4.89, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "620ms", "s": 620, "lo": 620, "hi": 770, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-15 · ±3%", "tone": "bad" }, "ttft": { "v": "1.72s", "s": 1724, "lo": 1724, "hi": 2244, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-15 · ±4%", "tone": "good" }, "cost": { "v": "$0.017", "s": 0.017, "tone": "bad", "note": "gpt-live-transcribe, the streaming path this row is recommended for." } }, "measuredStream": "2026-10-01" }, { "provider": "OpenAI", "model": "GPT Realtime Whisper", "streamOnly": true, "id": "openai:gpt-realtime-whisper", "measured": "2026-09-21", "rec": [], "cells": { "werConv": { "v": "16.82%", "s": 16.8155, "tone": "warn", "ci": "14.0–19.9% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "-", "s": null }, "werStream": { "v": "5.02%", "s": 5.02, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "1.14s", "s": 1141, "lo": 1141, "hi": 2264, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±8%", "tone": "bad", "note": "Commit-to-final: the probe commits at speech end, so the vendor never decides the turn. The two US sweeps disagreed beyond the artifact’s bound; Singapore reads 1.34s." }, "ttft": { "v": "1.81s", "s": 1815, "lo": 1815, "hi": 2299, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "warn" }, "cost": { "v": "$0.017", "s": 0.017, "tone": "bad", "note": "Billed by audio duration at $0.017/minute, not by token (developers.openai.com model page, read 2026-09-21). Same rate as gpt-live-transcribe." } }, "measuredStream": "2026-10-01" }, { "provider": "Nari", "model": "Qwen3-ASR Fast", "id": "nari:qwen3-asr-fast", "measured": "2026-09-13", "rec": [], "cells": { "werConv": { "v": "11.99%", "s": 11.989, "tone": "good", "ci": "9.9–14.3% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "3.3%", "s": 3.3, "tone": "good", "ci": "2.0–4.8%" }, "werStream": { "v": "3.83%", "s": 3.83, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "33ms", "s": 33, "lo": 33, "hi": 45, "ci": "p50–p90 · US East · n=60 · vendor-direct · pre-close (30/30 flush-forced) · re-measured 2026-09-22 · ±9%", "tone": "good", "note": "Commit-to-final: turn detection is off, so the probe commits at the measured speech end. Singapore reads 196ms." }, "ttft": { "v": "1.74s", "s": 1735, "lo": 1735, "hi": 1744, "ci": "p50–p90 · US East · n=60 · vendor-direct · re-measured 2026-09-22 · ±0%", "tone": "good" }, "cost": { "v": "$0.002", "s": 0.002, "tone": "good", "note": "Vendor list $0.12 per input audio hour, Partner tier (docs.narilabs.com/models-and-pricing, read 2026-09-13). A free tier of the same model is listed at $0." } }, "measuredStream": "2026-10-01" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "measured": "2026-09-26", "rec": [], "cells": { "werConv": { "v": "12.02%", "s": 12.0234, "tone": "good", "ci": "10.0–14.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "3.4%", "s": 3.4, "tone": "good", "ci": "2.1–4.9%" }, "werStream": { "v": "3.88%", "s": 3.88, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "25ms", "s": 25, "lo": 25, "hi": 43, "ci": "p50–p90 · US East · n=60 · vendor-direct · pre-close (30/30 flush-forced) · re-measured 2026-09-26 · ±22%", "tone": "good", "note": "Commit-to-final: turn detection is off, so the probe commits at the measured speech end. Singapore reads 254ms." }, "ttft": { "v": "1.75s", "s": 1750, "lo": 1750, "hi": 1782, "ci": "p50–p90 · US East · n=60 · vendor-direct · re-measured 2026-09-26 · ±0%", "tone": "good" }, "cost": { "v": "$0.001", "s": 0.001, "tone": "good", "note": "Vendor list $0.06 per input audio hour (docs.narilabs.com/models-and-pricing, read 2026-09-26)." } }, "measuredStream": "2026-10-01" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Value" } ], "cells": { "werConv": { "v": "20.60%", "s": 20.6029, "tone": "bad", "ci": "17.1–24.3% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "2.7%", "s": 2.7, "tone": "good" }, "werStream": { "v": "5.69%", "s": 5.69, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "822ms", "s": 822, "lo": 822, "hi": 1090, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±9%", "tone": "bad" }, "ttft": { "v": "6.33s", "s": 6335, "lo": 6335, "hi": 10557, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±10%", "tone": "bad" }, "cost": { "v": "$0.003", "s": 0.003 } }, "measuredStream": "2026-10-01" }, { "provider": "xAI", "model": "Grok Voice Transcribe 1.0", "id": "xai:stt", "status": { "label": "FAIR", "tone": "warn" }, "measured": "2026-09-22", "measuredStream": "2026-10-01", "rec": [], "cells": { "werConv": { "v": "13.73%", "s": 13.7324, "tone": "good", "ci": "11.4–16.3% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "2.8%", "s": 2.8, "ci": "2.0–3.6% · 95% CI · n=100", "tone": "good", "note": "Measured 2026-09-22. FLEURS English at -16 LUFS, REST, pinned to grok-voice-transcribe-1.0: 100/100 clips, 0 empty, 0 retried. 2.0 vs 1.0 on the same clips: 13 better, 18 worse, 69 tied; paired difference -0.34 to +1.21 points (95% CI)." }, "werStream": { "v": "37.22%", "s": 37.22, "tone": "bad", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01", "note": "238 of 1541 clips came back empty, scored as all-deletion." }, "endpoint": { "v": "1.13s", "s": 1128, "lo": 1128, "hi": 1343, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±4%", "tone": "bad" }, "ttft": { "v": "1.03s", "s": 1031, "lo": 1031, "hi": 2006, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±0%", "tone": "good" }, "cost": { "v": "$0.0033", "s": 0.0033, "note": "Streaming at $0.20/hr. REST is $0.0017. Unchanged across 1.0 and 2.0." } } }, { "provider": "xAI", "model": "Grok Voice Transcribe 2.0", "id": "xai:grok-voice-transcribe-2.0", "status": { "label": "FAIR", "tone": "warn" }, "measured": "2026-09-22", "measuredStream": "2026-10-01", "rec": [], "cells": { "werConv": { "v": "14.76%", "s": 14.763, "tone": "warn", "ci": "12.8–17.2% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "3.2%", "s": 3.2, "ci": "2.3–4.1% · 95% CI · n=100", "tone": "good", "note": "Measured 2026-09-22. FLEURS English at -16 LUFS, REST, pinned to grok-voice-transcribe-2.0: 100/100 clips, 0 empty, 0 retried. 2.0 vs 1.0 on the same clips: 13 better, 18 worse, 69 tied; paired difference -0.34 to +1.21 points (95% CI)." }, "werStream": { "v": "34.46%", "s": 34.46, "tone": "bad", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01", "note": "276 of 1541 clips came back empty, scored as all-deletion." }, "endpoint": { "v": "1.06s", "s": 1059, "lo": 1059, "hi": 1262, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±5%", "tone": "bad" }, "ttft": { "v": "1.14s", "s": 1141, "lo": 1141, "hi": 2144, "ci": "p50–p90 · US East · n=200 · vendor-direct · re-measured 2026-09-23 · ±1%", "tone": "good", "note": "Bimodal: readings cluster near 1.1s and 2.1s." }, "cost": { "v": "$0.0033", "s": 0.0033, "note": "Streaming at $0.20/hr. REST is $0.0017. Unchanged across 1.0 and 2.0." } } }, { "provider": "Gradium", "model": "Gradium ASR", "id": "gradium:default", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "werConv": { "v": "24.23%", "s": 24.2271, "tone": "bad", "ci": "21.0–27.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "8.4%", "s": 8.4, "tone": "warn" }, "werStream": { "v": "11.64%", "s": 11.64, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "1.13s", "s": 1131, "lo": 1131, "hi": 1344, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "bad" }, "ttft": { "v": "2.44s", "s": 2438, "lo": 2438, "hi": 2949, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±4%", "tone": "warn" }, "cost": { "v": "$0.0104", "s": 0.0104, "note": "3 credits/sec on the XS tier, the cheapest paid plan; $0.0068 at the L tier." } }, "measuredStream": "2026-10-01" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "measured": "2026-08-07", "meta": "Gladia’s higher-accuracy Solaria-3 has no live endpoint — pre-recorded only.", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "werConv": { "v": "21.15%", "s": 21.1525, "tone": "bad", "ci": "18.0–24.4% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "5.0%", "s": 5, "tone": "warn" }, "werStream": { "v": "11.21%", "s": 11.21, "tone": "warn", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01", "note": "2 of 1541 clips came back empty, scored as all-deletion." }, "endpoint": { "v": "910ms", "s": 910, "lo": 910, "hi": 3100, "ci": "p50–p90 · US East · n=199 · vendor-direct · ±6%", "tone": "bad", "note": "Vendor-default endpointing (0.05s of silence). Gladia often splits one sentence into several finals; this times the last." }, "ttft": { "v": "1.61s", "s": 1609, "lo": 1609, "hi": 3655, "ci": "p50–p90 · US East · n=199 · vendor-direct · ±8%", "tone": "good", "note": "The probe turns on partials, which Gladia disables by default. Without them this would time the first final." }, "cost": { "v": "$0.0125", "s": 0.0125, "note": "Starter pay-as-you-go real-time ($0.75/hr). Pre-recorded is $0.0102/min, and committed Growth pricing goes to $0.0042." } }, "measuredStream": "2026-10-01" }, { "provider": "Meta", "model": "Muse Voice Transcribe", "id": "meta:muse-voice-transcribe-1.0", "measured": "2026-09-02", "status": { "label": "GOOD", "tone": "good" }, "rec": [ { "label": "Streaming" }, { "label": "Value" } ], "cells": { "werConv": { "v": "14.24%", "s": 14.2391, "tone": "good", "ci": "12.4–16.5% · EdAcc n=283 · 60 speakers, 2 passes, 2026-10-01", "measured": "2026-10-01" }, "wer": { "v": "—", "s": null, "dim": true, "note": "Not measured. Meta ships a prerecorded endpoint; no batch sweep has been run on it." }, "werStream": { "v": "4.94%", "s": 4.94, "tone": "good", "ci": "n=1541 · FLEURS test+dev+train · vendor-direct", "measured": "2026-10-01" }, "endpoint": { "v": "—", "s": null, "dim": true, "note": "Measured but withheld: on 9 of 30 clips the speech-end marker lands up to 2.4s after Meta committed the turn, while the transcript is complete -- so the marker, not the model, sets the number. Muse finalizes faster than this corpus can resolve." }, "ttft": { "v": "1.62s", "s": 1623, "lo": 1623, "hi": 2225, "ci": "p50–p90 · US East · n=199 · vendor-direct · ±3%", "tone": "good" }, "cost": { "v": "$0.003", "s": 0.003, "note": "Vendor list price: $3.00 per 1,000 minutes, equivalently $0.18/hr. Streaming and batch are priced identically -- the table does not differentiate -- and there is a single tier, no plan or volume rates." } }, "measuredStream": "2026-10-01" }, { "provider": "Google", "model": "Gemini 3.5 Transcribe Live", "id": "gemini:gemini-3-5-transcribe-live", "status": { "label": "FAIR", "tone": "warn" }, "rec": [ { "label": "Streaming" } ], "cells": { "werConv": { "v": "13.1%", "s": 13.1, "tone": "good", "ci": "EdAcc n=98 · earlier set", "note": "Earlier 98-clip set. The 2026-10-01 re-measure read 31% with 26 of 99 clips empty, so this cell is not on the new set." }, "endpoint": { "v": "1.28s", "s": 1282, "lo": 1282, "hi": 1542, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±6%", "tone": "bad" }, "ttft": { "v": "1.80s", "s": 1796, "lo": 1796, "hi": 2299, "ci": "p50–p90 · US East · n=200 · vendor-direct · ±5%", "tone": "good" }, "cost": { "v": "~$0.009", "s": 0.009, "note": "ESTIMATE, not a published per-minute rate. Google prices this model per token -- $3.50/1M audio in, $21.00/1M text out -- and the ~$0.005/min + ~$0.004/min figures on the pricing page are Google's own estimate at 25 audio tokens/sec input and 175 text tokens/min output, for a blended ~$0.009/min. Speech density moves it, so read it as an order of magnitude rather than a rate card. Every other price on this column is the vendor's published per-minute rate." } } } ], "reading": "" }, { "id": "stt-codeswitch", "navLabel": "Code-switching", "identityLabel": "Provider / Model", "title": "Code-switching", "subtitle": "Two languages in one utterance — error rate and whether the second language survives, per pair, over the live stream.", "live": true, "meta": "", "rankCol": "pairs", "headline": [ "pairs", "zh", "es" ], "columns": [ { "id": "pairs", "label": "Pairs handled", "dir": "higher", "info": "Language pairs where the second language survives (both-language coverage over 50%), of 4 tested." }, { "id": "zh", "label": "EN ↔ ZH", "dir": "lower", "info": "Mixed-token error rate on Mandarin-English code-switch (ASCEND, streaming); caption = both-language coverage. Lower is better." }, { "id": "es", "label": "EN ↔ ES", "dir": "lower", "info": "Word error rate on English-Spanish code-switch (FLEURS concat, streaming); caption = coverage. Lower is better." }, { "id": "de", "label": "EN ↔ DE", "dir": "lower", "info": "Word error rate on English-German code-switch (FLEURS concat, streaming); caption = coverage. Lower is better." }, { "id": "fr", "label": "EN ↔ FR", "dir": "lower", "info": "Word error rate on English-French code-switch (FLEURS concat, streaming); caption = coverage. Lower is better." } ], "rows": [ { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "measured": "2026-07-21", "rec": [], "cells": { "pairs": { "v": "4 / 4", "s": 4, "tone": "good" }, "zh": { "v": "11.8%", "s": 11.8, "tone": "good", "ci": "88% both-lang" }, "es": { "v": "4.9%", "s": 4.9, "tone": "good", "ci": "97% both-lang" }, "de": { "v": "5.1%", "s": 5.1, "tone": "good", "ci": "100% both-lang" }, "fr": { "v": "8.8%", "s": 8.8, "tone": "good", "ci": "89% both-lang" } } }, { "provider": "AssemblyAI", "model": "universal-3-5-pro", "id": "assemblyai:universal-3-5-pro", "measured": "2026-07-31", "rec": [], "cells": { "pairs": { "v": "4 / 4", "s": 4, "tone": "good" }, "zh": { "v": "9.8%", "s": 9.8, "tone": "good", "ci": "100% both-lang" }, "es": { "v": "3.5%", "s": 3.5, "tone": "good", "ci": "100% both-lang" }, "de": { "v": "3.7%", "s": 3.7, "tone": "good", "ci": "98% both-lang" }, "fr": { "v": "2.6%", "s": 2.6, "tone": "good", "ci": "93% both-lang" } } }, { "provider": "ElevenLabs", "model": "scribe_v2_realtime", "id": "elevenlabs:scribe_v2_realtime", "measured": "2026-07-21", "rec": [], "cells": { "pairs": { "v": "3 / 4", "s": 3, "tone": "warn" }, "zh": { "v": "75.0%", "s": 75, "tone": "bad", "ci": "34% both-lang" }, "es": { "v": "3.8%", "s": 3.8, "tone": "good", "ci": "100% both-lang" }, "de": { "v": "3.1%", "s": 3.1, "tone": "good", "ci": "98% both-lang" }, "fr": { "v": "7.8%", "s": 7.8, "tone": "good", "ci": "89% both-lang" } } } ], "reading": "" }, { "id": "tts", "navLabel": "Text-to-Speech", "identityLabel": "System / Model", "title": "Text-to-Speech", "subtitle": "Naturalness, drift, latency, robustness and cost for every text-to-speech model we measure.", "live": true, "meta": "", "measured": "2026-07-03", "headline": [ "natural", "synth", "cost" ], "scatter": { "x": "synth", "y": "natural", "cornerLabel": "fast + natural", "note": "naturalness = arena Elo from our own blind human A/B study on phone-agent lines; field mean 1500" }, "columns": [ { "id": "natural", "label": "Naturalness", "dir": "higher", "info": "Arena Elo from blind human A/B votes on phone-agent lines; field mean 1500, higher = preferred." }, { "id": "ci", "label": "Win % · 95% CI", "dir": "higher", "info": "Head-to-head win rate against the rest of the field, with its 95% confidence interval. Overlapping intervals = statistical tie — read the interval, not just the point. Higher = preferred." }, { "id": "drift", "label": "Drift", "dir": "lower", "info": "Voice steadiness: mean pairwise embedding distance over 20 repeats. Lower = steadier. \"det\" = deterministic output. \"—\" = not yet run for this model." }, { "id": "synth", "label": "Synth p50", "dir": "lower", "info": "Time to first audio, p50 (n=30, US East, sequential). Lower is better." }, { "id": "robustness", "label": "Robustness", "dir": "higher", "info": "Does it say the right WORDS for a number, date, currency amount or operator — a different question from drift, which is whether the voice sounds like itself. Scored 0–1 by fining each defect on its severity (fatal / serious / minor) and on how often a caller hits that input class. Higher = fewer defects. \"us-only\" / \"wide-us\" rows ran fewer fixtures and are optimistic against full-basis rows, so a small gap across bases is not meaningful." }, { "id": "cost", "label": "Cost / 1M chars", "dir": "lower", "info": "$ per 1M characters, vendor list. Pay-as-you-go where offered, otherwise the cheapest paid plan. Lower = cheaper." } ], "rows": [ { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "rec": [], "cells": { "natural": { "v": "1590", "s": 1590, "tone": "good" }, "ci": { "v": "66%", "s": 66, "tone": "good", "ci": "60–72%" }, "drift": { "v": "20", "s": 20 }, "synth": { "v": "266ms", "s": 266, "lo": 265, "hi": 268, "ci": "265–268 two-sweep p50" }, "cost": { "v": "$50.0", "s": 50, "tone": "good", "note": "v3 Conversational is its own $0.05/1k product on elevenlabs.io/pricing/api, half the plain v3 rate. The FAQ on that same page collapses the two at $0.10/1k; the itemized card and the per-plan table both say $0.05, so they win." }, "robustness": { "v": "0.30", "s": 30, "tone": "bad", "tie": 7, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “A refund of -$0.99 was applied.”\n→ it said “A refund of 99 cents was applied.”\n• fatal · large numbers — never said “three point four million” (37 of 43 transcripts)\n→ we sent “The clip hit 3.4M views and 1.2k shares.”\n→ it said “The clip hit three point four M views and one point two k shares.”\n• fatal · math operators — never said “greater than or equal to” (47 of 49 transcripts)\n→ we sent “Keep x ≥ 10 at all times.”\n→ it said “Keep X ten at all times.”\n+ 11 more fined" } } }, { "provider": "ElevenLabs", "model": "eleven_v4", "id": "elevenlabs:eleven_v4", "rec": [], "measured": "2026-09-28", "cells": { "natural": { "v": "~1708", "s": 1708, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "—", "s": null, "dim": true }, "synth": { "v": "844ms", "s": 844, "lo": 798, "hi": 864, "ci": "798–864 three-sweep p50" }, "cost": { "v": "$80.0", "s": 80, "note": "elevenlabs.io/pricing/api lists $0.08/1k, discounted 72% to $0.022/1k until 2026-10-12." }, "robustness": { "v": "0.90", "s": 90, "tone": "good", "tie": 0, "note": "Fined:\n• minor · formatting · promptable — said “slash” out loud (48 of 49 transcripts)\n→ we sent “Would you like the bar and/or dining menu?”\n→ it said “Would you like the bar and slash or dining menu?”\n• minor · times · promptable — said “slash” out loud (41 of 49 transcripts)\n→ we sent “Meet at 8:45 a.m. PT / 11:45 a.m. ET.”\n→ it said “Meet at eight forty-five AM Pacific Time slash eleven forty-five AM Eastern time.”" } } }, { "provider": "ElevenLabs", "model": "eleven_v4_turbo", "id": "elevenlabs:eleven_v4_turbo", "rec": [], "measured": "2026-09-28", "cells": { "natural": { "v": "~1670", "s": 1670, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "—", "s": null, "dim": true }, "synth": { "v": "205ms", "s": 205, "lo": 203, "hi": 213, "ci": "203–213 three-sweep p50", "tone": "good" }, "cost": { "v": "$40.0", "s": 40, "tone": "good", "note": "elevenlabs.io/pricing/api lists $0.04/1k, discounted 72% to $0.011/1k until 2026-10-12." }, "robustness": { "v": "0.90", "s": 90, "tone": "good", "tie": 0, "note": "Fined:\n• minor · formatting · promptable — said “slash” out loud (45 of 49 transcripts)\n→ we sent “Would you like the bar and/or dining menu?”\n→ it said “Would you like the bar and slash or dining menu?”\n• minor · times · promptable — said “slash” out loud (42 of 49 transcripts)\n→ we sent “Meet at 8:45 a.m. PT / 11:45 a.m. ET.”\n→ it said “Meet at eight forty-five a.m. Pacific time slash eleven forty-five a.m. Eastern time.”" } } }, { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "rec": [], "cells": { "natural": { "v": "1591", "s": 1591, "tone": "good" }, "ci": { "v": "66%", "s": 66, "tone": "good", "ci": "59–72%" }, "drift": { "v": "—", "s": null, "dim": true }, "synth": { "v": "978ms", "s": 978, "lo": 978, "hi": 1235, "ci": "978–1235 p50–p90" }, "cost": { "v": "~$33.3", "s": 33.3, "note": "Token-billed: $1/1M text in, $20/1M audio out; Google states 25 audio tokens/sec (ai.google.dev/gemini-api/docs/pricing). At 900 chars/min of speech that is ~$33 per 1M chars; text-in adds ~$0.25, negligible." }, "robustness": { "v": "0.69", "s": 69, "tone": "warn", "tie": 6, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “A refund of -$0.99 was applied.”\n→ it said “A refund of $0.99 was applied.”\n• minor · dates · promptable — kept the value but never said “july” (7 of 23 transcripts)\n→ we sent “Your appointment is on 07/15/2026.”\n→ it said “Your appointment is on seven fifteen, twenty twenty six.”\n• minor · formatting · promptable — said “agent id” out loud (18 of 37 transcripts)\n→ we sent “Transferring you now. ”\n→ it said “Transferring you now. Call: Transfer Agent ID: Agent 7.”\n+ 3 more fined" } } }, { "provider": "Gemini", "model": "gemini-3.8-flash-tts", "id": "google-tts:gemini-3.8-flash-tts", "rec": [], "measured": "2026-09-24", "cells": { "natural": { "v": "~1642", "s": 1642, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "—", "s": null, "dim": true }, "synth": { "v": "1031ms", "s": 1031, "lo": 1015, "hi": 1046, "ci": "1015–1046 two-sweep p50" }, "cost": { "v": "~$15.0", "s": 15, "note": "Token-billed: $0.50/1M text in, $9/1M audio out until 2026-12-31, then $1 / $18 (ai.google.dev/gemini-api/docs/pricing). At 25 audio tokens/sec and 900 chars/min of speech that is ~$15 per 1M chars, ~$30 from 2027-01-01." }, "robustness": { "v": "0.70", "s": 70, "tone": "good", "tie": 4, "note": "Fined:\n• fatal · currency — never said “four hundred fifty thousand” (45 of 49 transcripts)\n→ we sent “The budget is $450K for the year.”\n→ it said “The budget is four hundred fifty K for the year.”\n• minor · dates · promptable — kept the value but never said “march” (7 of 7 transcripts)\n→ we sent “Your slot is 03/09 this year.”\n→ it said “Your slot is o-three-o-nine this year.”\n• minor · times · promptable — said “slash” out loud (12 of 31 transcripts)\n→ we sent “Meet at 8:45 a.m. PT / 11:45 a.m. ET.”\n→ it said “Meet at eight forty-five a.m. PT slash eleven forty-five a.m. ET.”\n+ 2 more fined" } } }, { "provider": "Gemini", "model": "gemini-3.8-flash-lite-tts", "id": "google-tts:gemini-3.8-flash-lite-tts", "rec": [], "measured": "2026-09-24", "cells": { "natural": { "v": "~1639", "s": 1639, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "—", "s": null, "dim": true }, "synth": { "v": "497ms", "s": 497, "lo": 483, "hi": 511, "ci": "483–511 two-sweep p50" }, "cost": { "v": "~$10.0", "s": 10, "note": "Token-billed: $0.50/1M text in, $6/1M audio out until 2026-12-31, then $1 / $12 (ai.google.dev/gemini-api/docs/pricing). At 25 audio tokens/sec and 900 chars/min of speech that is ~$10 per 1M chars, ~$20 from 2027-01-01." }, "robustness": { "v": "0.73", "s": 73, "tone": "good", "tie": 5, "note": "Fined:\n• fatal · currency — dropped the minus sign (4 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is twelve dollars and fifty cents.”\n• minor · dates · promptable — kept the value but never said “january” (7 of 7 transcripts)\n→ we sent “Renewal falls on 1/1/2027.”\n→ it said “Renewal falls on one one twenty twenty-seven.”\n• minor · dates · promptable (international, ×0.4) — kept the value but never said “march” (7 of 7 transcripts)\n→ we sent “The order shipped on 2026-03-09.”\n→ it said “The order shipped on twenty twenty six zero three zero nine.”\n+ 3 more fined" } } }, { "provider": "Deepgram", "model": "aura-2", "id": "deepgram:aura-2", "rec": [], "cells": { "natural": { "v": "1584", "s": 1584, "tone": "good" }, "ci": { "v": "65%", "s": 65, "tone": "good", "ci": "58–70%" }, "drift": { "v": "9", "s": 9, "tone": "good" }, "synth": { "v": "125ms", "s": 125, "lo": 125, "hi": 138, "ci": "125–138 p50–p90", "tone": "good" }, "cost": { "v": "$30.0", "s": 30, "tone": "good" }, "robustness": { "v": "0.20", "s": 20, "tone": "bad", "tie": 6, "note": "Fined:\n• fatal · currency — dropped the minus sign (35 of 35 transcripts)\n→ we sent “Your account shows −$85.40 after the refund.”\n→ it said “Your account shows 85.4 after the refund.”\n• fatal · dates — never said “friday” or “sunday” (21 of 35 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Fri Sun, October ninth eleven.”\n• fatal · large numbers — never said “zero zero” (5 of 5 transcripts)\n→ we sent “Only 007 seats remain.”\n→ it said “Only seven seats remain.”\n+ 22 more fined" } } }, { "provider": "Deepgram", "model": "flux", "id": "deepgram:flux", "rec": [], "cells": { "natural": { "v": "~1550", "s": 1550 }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "25", "s": 25, "tone": "warn", "note": "variance analyzer, 20 repeats, run 2026-09-02T10-00-22-058Z, measured 2026-09-02. Single draw; not comparable with other rows in this column." }, "synth": { "v": "106ms", "s": 106, "lo": 106, "hi": 148, "ci": "106–148 p50–p90", "tone": "good", "note": "Mean of two 2026-08-12 sweeps (104/108 p50), n=30 each, concurrency 1, one keep-alive connection, 0 rejected. Anchors in the same sweeps came in 11–26% off their published cells, so the column needs a full re-baseline before these are read against each other." }, "cost": { "v": "$45.0", "s": 45, "tone": "warn", "note": "Deepgram published list, Pay As You Go: $0.0450 per 1,000 characters = $45/1M. Same unit that puts Aura-2 at $0.030/1k = the $30 cell above, so the two are directly comparable — Flux TTS costs 50% more than Aura-2. Growth tier is $0.0405/1k ($40.5/1M). Free to build against through 2026-09-12 (45 concurrent streams globally, 5 EU/AU); billing starts 09-13." }, "robustness": { "v": "0.20", "s": 20, "tone": "bad", "tie": 4, "note": "Fined:\n• fatal · currency — dropped the minus sign (34 of 35 transcripts)\n→ we sent “Your account shows −$85.40 after the refund.”\n→ it said “Your account shows SMITE 5.4 after the refund.”\n• fatal · dates — never said “monday” (19 of 35 transcripts)\n→ we sent “Payments resume Mon, Feb 2.”\n→ it said “Pason's resume mon. February 2nd.”\n• fatal · large numbers — never said “zero zero” (5 of 5 transcripts)\n→ we sent “Only 007 seats remain.”\n→ it said “Only seven seats remain.”\n+ 25 more fined" } } }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "rec": [], "cells": { "natural": { "v": "1574", "s": 1574, "tone": "good" }, "ci": { "v": "65%", "s": 65, "tone": "good", "ci": "58–71%" }, "drift": { "v": "13", "s": 13, "tone": "good" }, "synth": { "v": "121ms", "s": 121, "lo": 121, "hi": 236, "ci": "121–236 p50–p90", "tone": "good" }, "cost": { "v": "$50.0", "s": 50, "note": "1 credit/char on Pro, the cheapest paid plan; $39.2 at Startup, $37.4 at Scale. Cloned voices bill 1.5x." }, "robustness": { "v": "0.18", "s": 18, "tone": "bad", "tie": 5, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is $12.50.”\n• fatal · dates — never said “friday” or “sunday” (41 of 49 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Fryson, October ninth, eleven.”\n• fatal · large numbers — never said “three point four million” (21 of 49 transcripts)\n→ we sent “The clip hit 3.4M views and 1.2k shares.”\n→ it said “The clip hit three point four M views and one point two K shares.”\n+ 22 more fined" } } }, { "provider": "Cartesia", "model": "sonic-3.6", "id": "cartesia:sonic-3.6", "rec": [], "measured": "2026-08-28", "cells": { "natural": { "v": "~1578", "s": 1578, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "10", "s": 10, "tone": "good" }, "synth": { "v": "120ms", "s": 120, "lo": 120, "hi": 135, "ci": "120–135 p50–p90 · US East · vendor-direct · n=400", "tone": "good", "note": "Vendor-direct from US East, n=400 (2 sweeps x 200), concurrency 1, one keep-alive socket." }, "cost": { "v": "$50.0", "s": 50, "note": "Derived, not published: Cartesia documents \"approximately 1 credit per character\" and Pro at $5/100K credits. No per-character USD rate exists. Cloned voices bill 1.5x." }, "robustness": { "v": "0.21", "s": 21, "tone": "bad", "tie": 2, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is twelve dollars and fifty cents.”\n• fatal · dates — never said “monday” (33 of 49 transcripts)\n→ we sent “Payments resume Mon, Feb 2.”\n→ it said “Payments resume on February 2nd.”\n• fatal · large numbers — never said “three point four million” (24 of 49 transcripts)\n→ we sent “The clip hit 3.4M views and 1.2k shares.”\n→ it said “The clip hit three point four M views and one point two K shares.”\n+ 22 more fined" } } }, { "provider": "Soniox", "model": "tts-rt-v1", "id": "soniox:tts-rt-v1", "routable": false, "rec": [], "cells": { "natural": { "v": "1569", "s": 1569, "tone": "good" }, "ci": { "v": "63%", "s": 63, "tone": "good", "ci": "56–69%" }, "drift": { "v": "24", "s": 24, "tone": "warn" }, "synth": { "v": "362ms", "s": 362, "lo": 362, "hi": 388, "ci": "362–388 p50–p90", "tone": "good" }, "cost": { "v": "~$13.0", "s": 13, "tone": "good", "note": "Token-billed: $4/1M text in, $21.50/1M audio out; the vendor states ~$0.70 per hour of generated speech (soniox.com/pricing). At 900 chars/min (54,000 chars/hour) that is ~$13 per 1M chars." }, "robustness": { "v": "0.35", "s": 35, "tone": "bad", "tie": 3, "note": "Fined:\n• fatal · currency — dropped the minus sign (4 of 7 transcripts)\n→ we sent “A refund of -$0.99 was applied.”\n→ it said “A refund of $9.99 was applied.”\n• fatal · dates — never said “sunday” (40 of 43 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Friday to Sun, October ninth to eleventh.”\n• fatal · large numbers — never said “three trillion” (48 of 49 transcripts)\n→ we sent “National debt passed 3,000,000,000,000 dollars.”\n→ it said “National debt passed three billion dollars.”\n+ 7 more fined" } } }, { "provider": "Soniox", "model": "tts-rt-v2", "id": "soniox:tts-rt-v2", "measured": "2026-08-13", "rec": [], "cells": { "natural": { "v": "~1606", "s": 1606, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "18", "s": 18 }, "synth": { "v": "381ms", "s": 381, "lo": 381, "hi": 417, "ci": "381–417 p50–p90", "tone": "good" }, "cost": { "v": "~$13.0", "s": 13, "tone": "good", "note": "Token-billed: $4/1M text in, $21.50/1M audio out; vendor-stated ~$0.70 per hour of speech (soniox.com/pricing). At 900 chars/min (54,000 chars/hour) that is ~$13 per 1M chars. Same rates as v1, which the pricing page no longer lists." }, "robustness": { "v": "0.49", "s": 49, "tone": "warn", "tie": 4, "note": "Fined:\n• fatal · dates — never said “sunday” (44 of 49 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Friday to Sun, October ninth to eleventh.”\n• fatal · large numbers — never said “three trillion” (48 of 49 transcripts)\n→ we sent “National debt passed 3,000,000,000,000 dollars.”\n→ it said “National debt passed $3 billion.”\n• minor · currency · promptable — kept the value but never said “ninety nine cent” (49 of 49 transcripts)\n→ we sent “Prices start at 99¢ each.”\n→ it said “Prices start at ninety nine three each.”\n+ 7 more fined" } } }, { "provider": "Speechify", "model": "simba-3.2", "id": "speechify:simba-3.2", "rec": [], "cells": { "natural": { "v": "1573", "s": 1573, "tone": "good" }, "ci": { "v": "63%", "s": 63, "tone": "good", "ci": "57–69%" }, "drift": { "v": "7", "s": 7, "tone": "good" }, "synth": { "v": "85ms", "s": 85, "lo": 85, "hi": 103, "ci": "85–103 p50–p90", "tone": "good", "note": "Re-measured 2026-09-21: two n=30 sweeps at concurrency 1 from US East, p50 82 and 87, p90 102 and 104. The pair missed the ≥90%-within-10% publish gate at 3/4 — the one failure is the co-swept deepgram:aura-2 anchor (26%), not this row, which agreed to 5.9% here and 0.0% on an earlier pair. Eight sweeps across two regions span 82–90ms, and a vendor-direct probe reads 56ms from the same region." }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "note": "Vendor list (Starter tier); $6–8 per 1M at Pro/Scale volume." }, "robustness": { "v": "0.21", "s": 21, "tone": "bad", "tie": 1, "note": "Fined:\n• fatal · currency — never said “four hundred fifty thousand” (49 of 49 transcripts)\n→ we sent “The budget is $450K for the year.”\n→ it said “The budget is four hundred and fifty k for the year.”\n• fatal · decimals — dropped the minus sign (7 of 7 transcripts)\n→ we sent “Growth slowed to -0.5% this month.”\n→ it said “Growth slowed to 0.5% this month.”\n• fatal · large numbers — never said “one million one” (49 of 49 transcripts)\n→ we sent “We logged 1,000,001 events overnight.”\n→ it said “We logged a thousand thousand one events overnight.”\n+ 28 more fined" } } }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "rec": [], "cells": { "natural": { "v": "1561", "s": 1561, "tone": "good" }, "ci": { "v": "63%", "s": 63, "tone": "good", "ci": "57–69%" }, "drift": { "v": "10", "s": 10, "tone": "good" }, "synth": { "v": "116ms", "s": 116, "lo": 116, "hi": 125, "ci": "116–125 p50–p90", "tone": "good" }, "cost": { "v": "$25.0", "s": 25, "tone": "good", "note": "On-demand. Drops to $20 on Creator and $12.50 on Growth." }, "robustness": { "v": "0.61", "s": 61, "tone": "warn", "tie": 3, "note": "Fined:\n• fatal · units — never said “one and a half” (48 of 49 transcripts)\n→ we sent “Add 1 1/2 cups of flour.”\n→ it said “Add one one-half cups of flour.”\n• fatal · ordinals (×0.5) — never said “one thousandth” (49 of 49 transcripts)\n→ we sent “This is our 1,000th order.”\n→ it said “This is our one zeroth order”\n• minor · dates · promptable — said “slash” out loud (4 of 7 transcripts)\n→ we sent “The deadline moved to 10/02.”\n→ it said “The deadline moved to 10. Slash zero two.”\n+ 5 more fined" } } }, { "provider": "Inworld", "model": "inworld-tts-2-flash", "id": "inworld:inworld-tts-2-flash", "measured": "2026-09-03", "rec": [], "cells": { "natural": { "v": "~1501", "s": 1501, "dim": true }, "drift": { "v": "13", "s": 13, "tone": "good" }, "synth": { "v": "82ms", "s": 82, "lo": 82, "hi": 89, "ci": "82–89 p50–p90", "tone": "good" }, "cost": { "v": "$15.0", "s": 15, "tone": "good", "note": "On-demand." }, "robustness": { "v": "0.58", "s": 57.99999999999999, "tone": "warn", "tie": 4, "note": "Fined:\n• fatal · units — never said “one and a half” (49 of 49 transcripts)\n→ we sent “Add 1 1/2 cups of flour.”\n→ it said “Add one one-half cups of flour.”\n• fatal · ordinals (×0.5) — never said “one thousandth” (49 of 49 transcripts)\n→ we sent “This is our 1,000th order.”\n→ it said “This is our one zeroth order.”\n• minor · currency · promptable — kept the value but never said “ninety nine cent” (49 of 49 transcripts)\n→ we sent “Prices start at 99¢ each.”\n→ it said “Prices start at ninety-nine ed-ee each.”\n+ 6 more fined" } } }, { "provider": "Fish Audio", "model": "s2.1-pro", "id": "fishaudio:s2.1-pro", "rec": [], "cells": { "natural": { "v": "~1566", "s": 1566, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "15", "s": 15 }, "synth": { "v": "185ms", "s": 185, "lo": 185, "hi": 235, "ci": "185–235 p50–p90", "tone": "good" }, "cost": { "v": "$15.0", "s": 15, "tone": "good", "note": "Billed per UTF-8 byte, so this holds for Latin text only — CJK runs ~3x. The s2.1-pro-free tier is $0 through 2026-08-31." }, "robustness": { "v": "0.17", "s": 17, "tone": "bad", "tie": 6, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is $12.50.”\n• fatal · dates — never said “friday” or “sunday” (35 of 49 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Fryson, October ninth eleven.”\n• fatal · decimals — never said “plus or minus” or “zero point zero five” (5 of 7 transcripts)\n→ we sent “Tolerance is ±0.05 mm on that part.”\n→ it said “Tolerance is smaanemems yamunt, zero five millimeter, on that part.”\n+ 13 more fined" } } }, { "provider": "Smallest", "model": "lightning_v3.1", "id": "smallest:lightning_v3.1", "rec": [], "cells": { "natural": { "v": "1544", "s": 1544, "tone": "good" }, "ci": { "v": "58%", "s": 58, "tone": "good", "ci": "51–64%" }, "drift": { "v": "det", "s": 0.27, "tone": "good" }, "synth": { "v": "173ms", "s": 173, "lo": 173, "hi": 193, "ci": "173–193 p50–p90", "tone": "good" }, "cost": { "v": "$17.5", "s": 17.5, "tone": "good", "note": "Pay-as-you-go; $0.175/10k chars (smallest.ai/pricing/models)." }, "robustness": { "v": "0.25", "s": 25, "tone": "bad", "tie": 3, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is $12.50.”\n• fatal · math operators — never said “divided” (6 of 7 transcripts)\n→ we sent “That is 100÷4=25 per person.”\n→ it said “That is $134 equals 25 per person.”\n• fatal · currency (international, ×0.4) — dropped the minus sign (7 of 7 transcripts)\n→ we sent “Balance: -€45,90 outstanding.”\n→ it said “Balance: 45 euros and 90 cents outstanding.”\n+ 6 more fined\nNot re-measured: v1.3 only (125 fixtures), score held at its last published value. Smallest refuses English from our account (LANGUAGE_NOT_ENABLED_IN_REGION), so the v1.4/v1.5 fixtures could not run, and re-scoring the old fine on the new curve (0.47) would rank an untested row above re-tested ones." } } }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "rec": [], "cells": { "natural": { "v": "1495", "s": 1495 }, "ci": { "v": "48%", "s": 48, "ci": "42–55%" }, "drift": { "v": "15", "s": 15 }, "synth": { "v": "272ms", "s": 272, "lo": 272, "hi": 325, "ci": "272–325 p50–p90", "tone": "good" }, "cost": { "v": "$15.0", "s": 15, "tone": "good" }, "robustness": { "v": "0.45", "s": 45, "tone": "warn", "tie": 5, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “Your balance is -$1,204.50 as of today.”\n→ it said “Your balance is $1,204.50 as of today.”\n• fatal · dates — never said “friday” or “sunday” (20 of 43 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Fri to Sun, Oct ninth to eleventh.”\n• fatal · decimals — dropped the minus sign (1 of 7 transcripts)\n→ we sent “Growth slowed to -0.5% this month.”\n→ it said “Growth slowed to 0.5% this month.”\n+ 3 more fined" } } }, { "provider": "Gradium", "model": "Gradium TTS", "slug": "gradium-default", "id": "gradium:default", "rec": [], "measured": "2026-10-01", "cells": { "natural": { "v": "~1592", "s": 1592, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "14", "s": 14 }, "synth": { "v": "105ms", "s": 105, "tone": "good", "note": "Measured 2026-10-01 on voice Brooklyn: three n=30 sweeps at p50 107, 103 and 105. The anchor rows matched their published cells in only 6 of 9 readings, so the formal two-sweep gate did not pass." }, "cost": { "v": "$57.8", "s": 57.8, "note": "1 credit/char on the XS tier, the cheapest paid plan; $35.9 at the L tier." }, "robustness": { "v": "0.27", "s": 27, "tone": "bad", "tie": 4, "note": "Fined:\n• fatal · decimals — never said “plus or minus” or “zero point zero five” (7 of 7 transcripts)\n→ we sent “Tolerance is ±0.05 mm on that part.”\n→ it said “Tolerance is prime oh five millimeters on that part.”\n• fatal · large numbers — never said “three trillion” (48 of 49 transcripts)\n→ we sent “National debt passed 3,000,000,000,000 dollars.”\n→ it said “National debt passed three billion boundaries.”\n• fatal · math operators — never said “divided” (7 of 7 transcripts)\n→ we sent “That is 100÷4=25 per person.”\n→ it said “That is 174 equals 25 per person.”\n+ 25 more fined" } } }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "rec": [], "cells": { "natural": { "v": "1424", "s": 1424 }, "ci": { "v": "37%", "s": 37, "ci": "31–43%" }, "drift": { "v": "22", "s": 22, "tone": "warn" }, "synth": { "v": "691ms", "s": 691, "lo": 691, "hi": 1171, "ci": "691–1171 p50–p90" }, "cost": { "v": "~$20.0", "s": 20, "tone": "good", "note": "Token-billed: $0.60/1M text in, $12/1M audio out; OpenAI publishes no per-character or per-minute rate. At ~25 audio tokens/sec (not OpenAI-published) and 900 chars/min, ~$20 per 1M chars. Legacy tts-1 is $15, tts-1-hd $30 per 1M chars." }, "robustness": { "v": "0.44", "s": 44, "tone": "bad", "tie": 6, "note": "Fined:\n• fatal · dates — never said “friday” or “sunday” (20 of 35 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs. Fry to sun, October 9, to eleventh.”\n• fatal · large numbers — never said “one million one” (9 of 31 transcripts)\n→ we sent “We logged 1,000,001 events overnight.”\n→ it said “We logged one thousand one hundred and one events overnight.”\n• fatal · math operators — never said “greater than or equal to” (20 of 23 transcripts)\n→ we sent “Keep x ≥ 10 at all times.”\n→ it said “Keep x equals 10 at all times.”\n+ 6 more fined" } } }, { "provider": "Rime", "model": "arcanav3", "id": "rime:arcanav3", "rec": [], "cells": { "natural": { "v": "1429", "s": 1429 }, "ci": { "v": "36%", "s": 36, "ci": "31–42%" }, "drift": { "v": "21", "s": 21 }, "synth": { "v": "238ms", "s": 238, "lo": 238, "hi": 248, "ci": "238–248 p50–p90", "tone": "good" }, "cost": { "v": "$40.0", "s": 40, "note": "Arcana on Starter; $30 on Growth. The vendor pricing page contradicts itself — its FAQ quotes $0.05/1k and its card quotes the cheaper Mist rate." }, "robustness": { "v": "0.35", "s": 35, "tone": "bad", "tie": 7, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is twelve dollars fifty cents.”\n• fatal · large numbers — never said “three trillion” (49 of 49 transcripts)\n→ we sent “National debt passed 3,000,000,000,000 dollars.”\n→ it said “National debt passed three billion, zero zero zero dollars.”\n• fatal · units — never said “one twenty over eighty” (44 of 49 transcripts)\n→ we sent “Blood pressure was 120/80 this morning.”\n→ it said “Blood pressure was one hundred twenty eightyths this morning.”\n+ 14 more fined" } } }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "rec": [], "cells": { "natural": { "v": "1431", "s": 1431 }, "ci": { "v": "36%", "s": 36, "ci": "30–43%" }, "drift": { "v": "12", "s": 12, "tone": "good" }, "synth": { "v": "294ms", "s": 294, "lo": 294, "hi": 354, "ci": "294–354 p50–p90", "tone": "good" }, "cost": { "v": "$100.0", "s": 100, "tone": "warn", "note": "HD tier; speech-2.8-turbo ≈ $60/1M." }, "robustness": { "v": "0.15", "s": 15, "tone": "bad", "tie": 2, "note": "Fined:\n• fatal · currency — dropped the minus sign (49 of 49 transcripts)\n→ we sent “Your account shows −$85.40 after the refund.”\n→ it said “Your account shows $85.40 after the refund.”\n• fatal · decimals — never said “point seven five” (49 of 49 transcripts)\n→ we sent “Give .75 of the usual dose.”\n→ it said “Give seventy-five of the usual dose.”\n• fatal · formatting — never said “pick either option” (49 of 49 transcripts)\n→ we sent “You can pick *either* option today.”\n→ it said “You can pick times either times option today.”\n+ 11 more fined" } } }, { "provider": "Hume", "model": "octave-2", "id": "hume:octave-2", "rec": [], "cells": { "natural": { "v": "1377", "s": 1377, "tone": "warn" }, "ci": { "v": "29%", "s": 29, "tone": "warn", "ci": "24–34%" }, "drift": { "v": "25", "s": 25, "tone": "warn" }, "synth": { "v": "448ms", "s": 448, "lo": 448, "hi": 497, "ci": "448–497 p50–p90" }, "cost": { "v": "$100.0", "s": 100, "tone": "warn", "note": "Starter tier, no pay-as-you-go rate published. Falls to $50 on Business; entry overage is $150." }, "robustness": { "v": "0.24", "s": 24, "tone": "bad", "tie": 2, "note": "Fined:\n• fatal · currency — dropped the minus sign (6 of 7 transcripts)\n→ we sent “A refund of -$0.99 was applied.”\n→ it said “A refund of $0.99 was applied.”\n• fatal · decimals — never said “one point zero five” (21 of 23 transcripts)\n→ we sent “The fee rose from 1.05% to 1.5%.”\n→ it said “The fee rose from one point five percent to one point five percent.”\n• fatal · large numbers — never said “four thousand ninety six” (5 of 18 transcripts)\n→ we sent “The file is 4096 bytes.”\n→ it said “The file is four nine hundred and ninety six bytes.”\n+ 14 more fined" } } }, { "provider": "Qwen", "model": "qwen3-tts-flash", "id": "alibaba:qwen3-tts-flash", "rec": [], "cells": { "natural": { "v": "1310", "s": 1310, "tone": "bad" }, "ci": { "v": "21%", "s": 21, "tone": "bad", "ci": "16–27%" }, "drift": { "v": "21", "s": 21 }, "synth": { "v": "472ms", "s": 472, "lo": 472, "hi": 504, "ci": "472–504 p50–p90" }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "note": "International (Singapore) list: $0.10 per 10,000 input characters, output audio not billed (alibabacloud.com/help/en/model-studio/model-pricing). A true per-character rate, no conversion. Mainland-China scope is ~$0.115/10k." }, "robustness": { "v": "0.15", "s": 15, "tone": "bad", "tie": 6, "note": "Fined:\n• fatal · currency — dropped the minus sign (3 of 5 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is twelve point five dollars.”\n• fatal · dates — never said “nineteen nineties” (31 of 35 transcripts)\n→ we sent “This design dates to the 1990s.”\n→ it said “This design dates to the one thousand nine hundred and nineties.”\n• fatal · decimals — never said “zero point zero zero zero zero zero one” (24 of 31 transcripts)\n→ we sent “The drift is 0.000001 per cycle.”\n→ it said “The drift is oh point oh oh oh oh one per cycle.”\n+ 12 more fined" } } }, { "provider": "Palabra", "model": "palabra-tts-v1", "id": "palabra:palabra-tts-v1", "rec": [], "cells": { "natural": { "v": "~1520", "s": 1520, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "11", "s": 11, "tone": "good" }, "synth": { "v": "72ms", "s": 72, "lo": 72, "hi": 79, "ci": "72–79 p50–p90", "tone": "good" }, "cost": { "v": "$12.0", "s": 12, "tone": "good" }, "robustness": { "v": "0.16", "s": 16, "tone": "bad", "tie": 8, "note": "Fined:\n• fatal · currency — dropped the minus sign (7 of 7 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is $12.50.”\n• fatal · dates — never said “friday” or “sunday” (10 of 29 transcripts)\n→ we sent “The fair runs Fri–Sun, Oct 9–11.”\n→ it said “The fair runs Fri-Sun, October ninth-eleven.”\n• fatal · decimals — never said “plus or minus” or “zero point zero five” or “millimetre” (6 of 7 transcripts)\n→ we sent “Tolerance is ±0.05 mm on that part.”\n→ it said “Tolerance is not you too zero five mon that part.”\n+ 19 more fined" } } }, { "provider": "Bland", "model": "bland-speech", "id": "bland:bland-speech", "rec": [], "measured": "2026-08-20", "cells": { "natural": { "v": "~1569", "s": 1569, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "12", "s": 12, "tone": "good" }, "synth": { "v": "303ms", "s": 303, "lo": 303, "hi": 386, "ci": "303–386 p50–p90" }, "cost": { "v": "$40.0", "s": 40, "tone": "warn", "note": "Pay-as-you-go, $0.04/1,000 chars on bland.ai/speech, which states the same rate as \"1 million characters total $40\". The per-minute figures on bland.ai/pricing are the voice-agent bundle, not this API." }, "robustness": { "v": "0.22", "s": 22, "tone": "bad", "tie": 25, "note": "Fined:\n• fatal · currency — dropped the minus sign (5 of 5 transcripts)\n→ we sent “The balance is -$12.50.”\n→ it said “The balance is twelve dollars and fifty cents.”\n• fatal · decimals — dropped the minus sign (6 of 7 transcripts)\n→ we sent “Growth slowed to -0.5% this month.”\n→ it said “Growth slowed to 0.5% this month.”\n• fatal · large numbers — never said “eight point one billion” (7 of 14 transcripts)\n→ we sent “Population reached 8,100,000,000 in 2023.”\n→ it said “Population reached eight thousand one hundred thousand twenty twenty three.”\n+ 15 more fined" } } }, { "provider": "Nari", "model": "Qwen3-TTS Fast", "id": "nari:qwen3-tts-fast", "rec": [], "measured": "2026-09-12", "cells": { "natural": { "v": "~1536", "s": 1536, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "det", "s": null, "dim": true, "note": "Deterministic: identical audio every run. The endpoint pins seed 0 (x-nari-seed) on a sampling model, so this describes the deployment, not the voice. A response cache was ruled out." }, "synth": { "measured": "2026-09-26", "v": "64ms", "s": 64, "lo": 64, "hi": 65, "ci": "64–65 p50–p90", "tone": "good", "note": "Four n=30 sweeps from US East on 2026-09-26 read p50 62, 65, 64 and 63. Both sweep pairs missed the publish gate on unstable anchors (cartesia:sonic-3.5 and deepgram:aura-2), and smallest:lightning_v3.1 failed over on every call. This row stayed within 5%." }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "note": "Vendor list, Partner tier (docs.narilabs.com/models-and-pricing, read 2026-09-13). Nari also lists a `qwen3-tts-fast:free` tier at $0; this row measured the PAID `qwen3-tts-fast` id, so the paid rate is the one published. The standard (non-fast) qwen3-tts is $5/1M." }, "robustness": { "v": "0.30", "s": 30, "tone": "bad", "tie": 3, "note": "Fined:\n• fatal · currency — never said “three point two million” (48 of 49 transcripts)\n→ we sent “We raised $3.2M in the seed round.”\n→ it said “We raised three dollars point two millions in the seed round.”\n• fatal · large numbers — never said “four thousand ninety six” (7 of 7 transcripts)\n→ we sent “The file is 4096 bytes.”\n→ it said “The file is four hundred ninety-six bytes.”\n• fatal · math operators — never said “three quarters” (6 of 7 transcripts)\n→ we sent “Give me 3/4 of the batch.”\n→ it said “Give me March fourth of the batch.”\n+ 13 more fined" } } }, { "provider": "Nari", "model": "Qwen3-TTS", "id": "nari:qwen3-tts", "rec": [], "measured": "2026-09-26", "cells": { "natural": { "v": "~1510", "s": 1510, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "det", "s": null, "dim": true, "note": "Deterministic: identical audio every run. The endpoint pins seed 0 (x-nari-seed) on a sampling model, so this describes the deployment, not the voice. A response cache was ruled out." }, "synth": { "v": "75ms", "s": 75, "lo": 75, "hi": 79, "ci": "75–79 p50–p90", "tone": "good", "note": "Four n=30 sweeps from US East on 2026-09-26 read p50 74, 76, 76 and 75, in the same runs as qwen3-tts-fast and with the same anchor misses. This row stayed within 3%." }, "cost": { "v": "$5.0", "s": 5, "tone": "good", "note": "Vendor list $5 per 1M input characters (docs.narilabs.com/models-and-pricing, read 2026-09-26)." }, "robustness": { "v": "0.32", "s": 32, "tone": "bad", "tie": 4, "note": "Fined:\n• fatal · currency — never said “three point two million” (48 of 49 transcripts)\n→ we sent “We raised $3.2M in the seed round.”\n→ it said “We raised three dollars lot twenty-one in the seed round.”\n• fatal · large numbers — never said “three trillion” (49 of 49 transcripts)\n→ we sent “National debt passed 3,000,000,000,000 dollars.”\n→ it said “National debt passed $3 billion.”\n• fatal · math operators — never said “one thousand twenty four” (43 of 49 transcripts)\n→ we sent “Since 2^10=1024, it fits in one block.”\n→ it said “Since two to the power of ten equals w times ten twenty or twenty-four, it fits in one blo”\n+ 13 more fined" } } }, { "provider": "Hopper", "model": "Qwen3-TTS", "id": "hopper:qwen3-tts", "routable": false, "meta": "served from Oregon", "rec": [], "measured": "2026-10-08", "cells": { "natural": { "v": "~1406", "s": 1406, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "20", "s": 20, "note": "Measured 2026-10-08. All 20 takes of the same text differ." }, "synth": { "v": "—", "s": null, "dim": true }, "cost": { "v": "~$5.67", "s": 5.67, "note": "Billed at $0.30 per audio hour. At the 14.7 characters per second measured on voice hudson, that is about $5.67 per 1M characters." }, "robustness": { "v": "0.29", "s": 28.999999999999996, "tone": "bad", "tie": 7, "note": "Fined:\n• fatal · currency — dropped the minus sign (4 of 7 transcripts)\n→ we sent “Your balance is -$1,204.50 as of today.”\n→ it said “Your balance is nineus tweepounds and four thousand fifty dollars as of today.”\n• fatal · large numbers — never said “eight point one billion” (6 of 7 transcripts)\n→ we sent “Population reached 8,100,000,000 in 2023.”\n→ it said “Population reached eight million one hundred million in twenty twenty-three.”\n• fatal · units — never said “one and a half” (49 of 49 transcripts)\n→ we sent “Add 1 1/2 cups of flour.”\n→ it said “Add one one-half cups of flour.”\n+ 27 more fined" } } }, { "provider": "Paxa", "model": "paxa-tts-flash-v1", "id": "paxa:paxa-tts-flash-v1", "rec": [], "measured": "2026-09-28", "cells": { "natural": { "v": "~1325", "s": 1325, "dim": true, "measured": "2026-10-05" }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "0", "s": 0, "note": "Measured 2026-09-28. All 20 takes differ, but barely: mean pairwise distance 0.0087, against 0.0959 for Deepgram aura-2." }, "synth": { "v": "389ms", "s": 389, "lo": 389, "hi": 445, "ci": "389–445 p50–p90", "tone": "good", "measured": "2026-10-10", "note": "Two n=100 sweeps from US East on 2026-10-10 read p50 372 and 405. Cartesia reproduced its cell in the same runs; Deepgram ran 9–12% slow and Inworld 10% fast." }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "note": "Paxa bills speech at 10 credits per 1,000 characters, $10 per one million characters (paxalabs.com/docs/credits, read 2026-10-06). A credit costs $0.001 on refill (10,000 for $10), and live calls on 2026-10-06 were charged 0.01 credit per character. This row read $15 from the 2026-09-12 pricing page until then. Billing is on input characters." }, "robustness": { "v": "0.25", "s": 25, "tone": "bad", "tie": 3, "note": "Fined:\n• fatal · currency — never said “three point two million” (48 of 49 transcripts)\n→ we sent “We raised $3.2M in the seed round.”\n→ it said “We raised some toong dalaam in the seed round.”\n• fatal · large numbers — never said “three point four million” (35 of 49 transcripts)\n→ we sent “The clip hit 3.4M views and 1.2k shares.”\n→ it said “The clip hit three point four M views and one point two K shares.”\n• fatal · math operators — never said “to the power of ten” (32 of 49 transcripts)\n→ we sent “Since 2^10=1024, it fits in one block.”\n→ it said “Since two ten equals one thousand twenty-four it fits in one block.”\n+ 28 more fined" } } }, { "provider": "Maya", "model": "Maya 2 Native", "id": "maya:Maya 2 Native", "rec": [], "measured": "2026-08-14", "cells": { "natural": { "v": "—", "s": null, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "drift": { "v": "—", "s": null, "dim": true }, "synth": { "v": "100ms", "s": 100, "lo": 100, "hi": 114, "ci": "100–114 p50–p90", "tone": "good" }, "cost": { "v": "~$4.0", "s": 4, "tone": "good", "note": "Quoted by Maya directly. They publish no rate card — no pricing page and nothing in docs.mayaresearch.ai — so unlike every other row this rate cannot be checked against a vendor page." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not measured: the NSW fixtures are en-US and Maya ships no en-US voice — its `en` is Indian English, with no en-US/en-GB variant." } } }, { "provider": "ElevenLabs", "model": "eleven_flash_v2_5", "id": "elevenlabs:eleven_flash_v2_5", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1494", "s": 1494, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$50.0", "s": 50, "tone": "good", "note": "API rate card (elevenlabs.io/pricing/api, 2026-09-08): Flash and Turbo $0.05 per 1K characters; v3 and Multilingual v2 $0.10." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "ElevenLabs", "model": "eleven_turbo_v2_5", "id": "elevenlabs:eleven_turbo_v2_5", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1466", "s": 1466, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$50.0", "s": 50, "tone": "good", "note": "API rate card (elevenlabs.io/pricing/api, 2026-09-08): Flash and Turbo $0.05 per 1K characters; v3 and Multilingual v2 $0.10." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "ElevenLabs", "model": "eleven_multilingual_v2", "id": "elevenlabs:eleven_multilingual_v2", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1472", "s": 1472, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$100", "s": 100, "tone": "warn", "note": "API rate card (elevenlabs.io/pricing/api, 2026-09-08): Multilingual v2 $0.10 per 1K characters, the v3 rate." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "ElevenLabs", "model": "eleven_flash_v2", "id": "elevenlabs:eleven_flash_v2", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1507", "s": 1507, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$50.0", "s": 50, "tone": "good", "note": "API rate card (elevenlabs.io/pricing/api, 2026-09-08): Flash and Turbo $0.05 per 1K characters." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "ElevenLabs", "model": "eleven_turbo_v2", "id": "elevenlabs:eleven_turbo_v2", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1492", "s": 1492, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$50.0", "s": 50, "tone": "good", "note": "API rate card (elevenlabs.io/pricing/api, 2026-09-08): Flash and Turbo $0.05 per 1K characters." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "Cartesia", "model": "sonic-3", "id": "cartesia:sonic-3", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1551", "s": 1551, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$50.0", "s": 50, "note": "Derived, not published: Cartesia documents \"approximately 1 credit per character\" and Pro at $5/100K credits — the same basis as the sonic-3.5 and sonic-3.6 rows." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "Rime", "model": "coda", "id": "rime:coda", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1535", "s": 1535, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$50.0", "s": 50, "note": "Starter tier, rime.ai/pricing 2026-09-08: Coda $0.05 / 1K characters." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "Rime", "model": "mistv3", "id": "rime:mistv3", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1428", "s": 1428, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$30.0", "s": 30, "tone": "good", "note": "Starter tier, rime.ai/pricing 2026-09-08: Mist v3 $0.03 / 1K characters." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "Hume", "model": "octave-1", "id": "hume:octave-1", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1455", "s": 1455, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$100.0", "s": 100, "tone": "warn", "note": "Hume prices Octave per character with no per-version rate; Starter tier, as on the octave-2 row." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "MiniMax", "model": "speech-2.6-hd", "id": "minimax:speech-2.6-hd", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1450", "s": 1450, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "~$100.0", "s": 100, "tone": "warn", "note": "Estimate. The vendor pricing page lists point plans, not per-model rates (2026-09-08); HD taken at the tier the speech-2.8-hd row cites." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "MiniMax", "model": "speech-2.6-turbo", "id": "minimax:speech-2.6-turbo", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1408", "s": 1408, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "~$60.0", "s": 60, "note": "Estimate. The vendor pricing page lists point plans, not per-model rates (2026-09-08); Turbo taken at the ≈$60 the speech-2.8-hd row cites for its turbo sibling." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "Smallest", "model": "lightning_v3.1_pro", "id": "smallest:lightning_v3.1_pro", "rec": [], "measured": "2026-09-08", "cells": { "natural": { "v": "~1447", "s": 1447, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$19.5", "s": 19.5, "note": "Pay-as-you-go; $0.195/10k chars (smallest.ai/pricing/models, corroborated by the model card)." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-08 with an auto-MOS naturalness estimate). Pending, not exempt." } } }, { "provider": "MiniMax", "model": "speech-2.8-turbo", "id": "minimax:speech-2.8-turbo", "rec": [], "measured": "2026-09-29", "cells": { "natural": { "v": "~1385", "s": 1385, "dim": true }, "ci": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$60.0", "s": 60, "note": "MiniMax's pay-as-you-go rate for speech-2.8-turbo, the price the router bills." }, "robustness": { "v": "—", "s": null, "dim": true, "note": "Not yet measured: the en-US NSW robustness battery has not been run on this model (row added 2026-09-29 with an auto-MOS naturalness estimate). Pending, not exempt." } } } ], "reading": "" }, { "id": "llm", "navLabel": "LLM", "identityLabel": "Provider / Model", "title": "LLM", "subtitle": "First-token latency, task completion, fabrication and cost for voice-agent LLMs.", "live": true, "meta": "English · corpus v3-16d3e8496362, measured 2026-10-04, vendor-direct, streamed (19 scenarios × 5 iters = n=95 task runs; 10 fabrication probes × 3 = n=30; real calls 60 scenarios × 3 = n=180) · TTFT/cost from US East", "measured": "2026-10-04", "headline": [ "ttft", "task", "cost" ], "scatter": { "x": "ttft", "y": "task", "cornerLabel": "fast + gets the job done" }, "columns": [ { "id": "score", "label": "Score", "dir": "higher", "info": "Retired. The board publishes no composite — each parameter is ranked on its own card." }, { "id": "task", "label": "Task done", "dir": "higher", "info": "Mean per-run score over 19 scripted scenarios: 1 completed, 0 acted dangerously, part-credit if it stopped early but safely." }, { "id": "refusal", "label": "Refusal quality", "dir": "higher", "info": "When it does not know, whether it names the gap and offers a way forward. Higher is better." }, { "id": "fabric", "label": "Fabrication", "dir": "lower", "info": "Invented a policy or fact it was never given, % of out-of-policy questions." }, { "id": "deadair", "label": "Dead-air", "dir": "lower", "info": "Turns where the caller hears silence (no spoken words), % of turns." }, { "id": "toolsilence", "label": "Tool silence", "dir": "lower", "info": "Tool-call turns with no spoken lead-in, % — the caller waits in silence. Not in the score." }, { "id": "stall", "label": "Stalled", "dir": "lower", "info": "Ended a turn mid-task with no tool call — the caller waits and has to re-ask, % of runs." }, { "id": "ttft", "label": "TTFT p50", "dir": "lower", "info": "Time to first output on a real agent turn (system prompt + tools), p50; whisker to p90." }, { "id": "cost", "label": "Cost / 1M tok", "dir": "lower", "info": "$ per 1M output tokens (host / vendor list pricing)." } ], "rows": [ { "provider": "Baseten", "model": "GLM-4.7", "id": "baseten:zai-org/GLM-4.7", "measured": "2026-10-04", "status": { "label": "RETIRED", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "94", "s": 94, "tone": "good" }, "stall": { "v": "27.8%", "s": 27.8, "tone": "warn", "ci": "25 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "6.0%", "s": 6, "tone": "good", "ci": "16 of 268 turns" }, "toolsilence": { "v": "9.5%", "s": 9.5, "tone": "good", "ci": "16 of 168 tool-call turns" }, "task": { "v": "80.88%", "s": 80.88, "tone": "good", "ci": "67–92% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "80.0%", "s": 80, "tone": "good", "ci": "names the gap 67% · offers a route 83% · 10 probes x 3 iterations" }, "ttft": { "v": "148ms", "s": 148, "lo": 148, "hi": 161, "ci": "148–161 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$2.20", "s": 2.2 }, "consistency": { "v": "58%", "s": 58, "tone": "warn", "ci": "11 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "77%", "s": 77, "tone": "warn", "ci": "139 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "78%", "s": 78 }, "realcall:orders": { "v": "89%", "s": 89 }, "realcall:intake": { "v": "61%", "s": 61 }, "realcall:payments": { "v": "92%", "s": 92 }, "realcall:frontdesk": { "v": "67%", "s": 67 } }, "lead": true }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "id": "anthropic:claude-haiku-4-5", "status": { "label": "STRONG", "tone": "good" }, "rec": [ { "label": "Live agent" } ], "cells": { "score": { "v": "89", "s": 89, "tone": "good" }, "stall": { "v": "24.4%", "s": 24.4, "tone": "warn", "ci": "22 of 90 runs" }, "fabric": { "v": "3%", "s": 3, "tone": "good", "ci": "1 of 30 probe runs" }, "deadair": { "v": "2.0%", "s": 2, "tone": "good", "ci": "5 of 252 turns" }, "toolsilence": { "v": "3.3%", "s": 3.3, "tone": "good", "ci": "5 of 152 tool-call turns" }, "task": { "v": "81.67%", "s": 81.67, "tone": "good", "ci": "65–94% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "84.4%", "s": 84.4, "tone": "good", "ci": "names the gap 77% · offers a route 80% · 10 probes x 3 iterations" }, "toolarg": { "v": "77%", "s": 77, "tone": "bad" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "391ms", "s": 391, "lo": 391, "hi": 574, "ci": "391–574 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$5.00", "s": 5 }, "consistency": { "v": "63%", "s": 63, "tone": "warn", "ci": "12 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "74%", "s": 74, "tone": "warn", "ci": "134 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "75%", "s": 75 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "64%", "s": 64 }, "realcall:payments": { "v": "78%", "s": 78 }, "realcall:frontdesk": { "v": "64%", "s": 64 } }, "lead": false }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "id": "together:deepseek-ai/DeepSeek-V4.1-Flash", "measured": "2026-10-04", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "89", "s": 89, "tone": "good" }, "stall": { "v": "14.4%", "s": 14.4, "tone": "warn", "ci": "13 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "34.3%", "s": 34.3, "tone": "bad", "ci": "109 of 318 turns" }, "toolsilence": { "v": "49.5%", "s": 49.5, "tone": "bad", "ci": "109 of 220 tool-call turns" }, "task": { "v": "84.21%", "s": 84.21, "tone": "good", "ci": "69–97% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "68.9%", "s": 68.9, "tone": "warn", "ci": "names the gap 70% · offers a route 37% · 10 probes x 3 iterations" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "157ms", "s": 157, "lo": 157, "hi": 793, "ci": "157–793 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.20", "s": 1.2, "note": "Vendor list, serverless, per 1M output tokens ($0.30 in)." }, "consistency": { "v": "68%", "s": 68, "tone": "warn", "ci": "13 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "86%", "s": 86, "tone": "good", "ci": "155 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "92%", "s": 92 }, "realcall:orders": { "v": "89%", "s": 89 }, "realcall:intake": { "v": "81%", "s": 81 }, "realcall:payments": { "v": "94%", "s": 94 }, "realcall:frontdesk": { "v": "75%", "s": 75 } }, "lead": false }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "86", "s": 86, "tone": "good" }, "stall": { "v": "7.8%", "s": 7.8, "tone": "good", "ci": "7 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "43.4%", "s": 43.4, "tone": "bad", "ci": "139 of 320 turns" }, "toolsilence": { "v": "62.3%", "s": 62.3, "tone": "bad", "ci": "139 of 223 tool-call turns" }, "task": { "v": "86.14%", "s": 86.14, "tone": "good", "ci": "72–97% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "80.0%", "s": 80, "tone": "good", "ci": "names the gap 70% · offers a route 83% · 10 probes x 3 iterations · 2 silent turns scored zero" }, "ttft": { "v": "246ms", "s": 246, "lo": 246, "hi": 592, "ci": "246–592 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.49", "s": 1.49, "note": "Cerebras list pricing for this model, $0.99/1M input and $1.49/1M output, from the vendor’s own model page (inference-docs.cerebras.ai/models/qwen-3.8-27b, read 2026-09-06). The same weights cost $2.55/1M output on the Alibaba row." }, "consistency": { "v": "68%", "s": 68, "tone": "warn", "ci": "13 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "86%", "s": 86, "tone": "good", "ci": "154 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "1.1%", "s": 1.11, "tone": "warn", "ci": "2 of 180 calls" }, "realcall:scheduling": { "v": "83%", "s": 83 }, "realcall:orders": { "v": "78%", "s": 78 }, "realcall:intake": { "v": "81%", "s": 81 }, "realcall:payments": { "v": "94%", "s": 94 }, "realcall:frontdesk": { "v": "92%", "s": 92 } }, "lead": false }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "id": "baseten:deepseek-ai/DeepSeek-V4-Flash-0731", "measured": "2026-10-04", "status": { "label": "STALLS", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "83", "s": 83, "tone": "good" }, "stall": { "v": "3.3%", "s": 3.3, "tone": "good", "ci": "3 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "36.8%", "s": 36.8, "tone": "bad", "ci": "116 of 315 turns" }, "toolsilence": { "v": "53.2%", "s": 53.2, "tone": "bad", "ci": "116 of 218 tool-call turns" }, "task": { "v": "93.16%", "s": 93.16, "tone": "good", "ci": "83–100% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "72.2%", "s": 72.2, "tone": "warn", "ci": "names the gap 57% · offers a route 67% · 10 probes x 3 iterations" }, "ttft": { "v": "558ms", "s": 558, "lo": 558, "hi": 1026, "ci": "558–1026 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$0.26", "s": 0.26 }, "consistency": { "v": "89%", "s": 89, "tone": "good", "ci": "17 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "88%", "s": 88, "tone": "good", "ci": "159 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "81%", "s": 81 }, "realcall:orders": { "v": "94%", "s": 94 }, "realcall:intake": { "v": "89%", "s": 89 }, "realcall:payments": { "v": "92%", "s": 92 }, "realcall:frontdesk": { "v": "86%", "s": 86 } }, "lead": false }, { "provider": "Hopper", "model": "gemma-4-31b", "id": "hopper:gemma-4-31b", "routable": false, "meta": "non-thinking, served from Oregon", "measured": "2026-10-08", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "83", "s": 83, "tone": "good" }, "stall": { "v": "10.0%", "s": 10, "tone": "good", "ci": "9 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "63.7%", "s": 63.7, "tone": "bad", "ci": "174 of 273 turns" }, "toolsilence": { "v": "91.0%", "s": 91, "tone": "bad", "ci": "162 of 178 tool-call turns" }, "task": { "v": "85.26%", "s": 85.26, "tone": "good", "ci": "68–100% 95% CI bootstrap over items · n=95 · measured 2026-10-08" }, "refusal": { "v": "96.7%", "s": 96.7, "tone": "good", "ci": "names the gap 97% · offers a route 93% · 10 probes x 3 iterations" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "150ms", "s": 150, "lo": 150, "hi": 158, "ci": "150–158 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.20", "s": 1.2, "note": "Vendor list, per 1M output tokens ($0.40 in, $0.20 cached)." }, "consistency": { "v": "84%", "s": 84, "tone": "good", "ci": "16 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "87%", "s": 87, "tone": "good", "ci": "156 of 180 calls clean · measured 2026-10-08" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "83%", "s": 83 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "97%", "s": 97 }, "realcall:payments": { "v": "78%", "s": 78 }, "realcall:frontdesk": { "v": "83%", "s": 83 } }, "lead": false }, { "provider": "Cerebras", "model": "gpt-oss-120b", "id": "cerebras:gpt-oss-120b", "status": { "label": "DEAD AIR", "tone": "warn" }, "rec": [ { "label": "Needs bridging", "caution": true } ], "cells": { "score": { "v": "81", "s": 81, "tone": "good" }, "stall": { "v": "12.2%", "s": 12.2, "tone": "warn", "ci": "11 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "70.5%", "s": 70.5, "tone": "bad", "ci": "227 of 322 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (227 tool-call turns) · ≥82%" }, "task": { "v": "82.02%", "s": 82.02, "tone": "good", "ci": "70–93% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "83.3%", "s": 83.3, "tone": "good", "ci": "names the gap 77% · offers a route 83% · 10 probes x 3 iterations · 3 silent turns scored zero" }, "toolarg": { "v": "50%", "s": 50, "tone": "bad" }, "placeholder": { "v": "13%", "s": 13, "tone": "bad" }, "ttft": { "v": "139ms", "s": 139, "lo": 139, "hi": 261, "ci": "139–261 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$0.75", "s": 0.75, "tone": "good" }, "consistency": { "v": "58%", "s": 58, "tone": "warn", "ci": "11 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "53%", "s": 53, "tone": "bad", "ci": "95 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "31.7%", "s": 31.67, "tone": "bad", "ci": "57 of 180 calls" }, "realcall:scheduling": { "v": "61%", "s": 61 }, "realcall:orders": { "v": "78%", "s": 78 }, "realcall:intake": { "v": "8%", "s": 8 }, "realcall:payments": { "v": "64%", "s": 64 }, "realcall:frontdesk": { "v": "53%", "s": 53 } }, "lead": false }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "id": "baseten:deepseek-ai/DeepSeek-V4.1-Flash", "measured": "2026-10-04", "status": { "label": "STALLS", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "81", "s": 81, "tone": "good" }, "stall": { "v": "16.7%", "s": 16.7, "tone": "warn", "ci": "15 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "36.6%", "s": 36.6, "tone": "bad", "ci": "117 of 320 turns" }, "toolsilence": { "v": "52.7%", "s": 52.7, "tone": "bad", "ci": "117 of 222 tool-call turns" }, "task": { "v": "82.19%", "s": 82.19, "tone": "good", "ci": "69–94% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "74.4%", "s": 74.4, "tone": "warn", "ci": "names the gap 70% · offers a route 60% · 10 probes x 3 iterations · 1 silent turn scored zero" }, "ttft": { "v": "458ms", "s": 458, "lo": 458, "hi": 1062, "ci": "458–1062 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.20", "s": 1.2 }, "consistency": { "v": "58%", "s": 58, "tone": "warn", "ci": "11 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "88%", "s": 88, "tone": "good", "ci": "158 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "81%", "s": 81 }, "realcall:orders": { "v": "94%", "s": 94 }, "realcall:intake": { "v": "92%", "s": 92 }, "realcall:payments": { "v": "94%", "s": 94 }, "realcall:frontdesk": { "v": "78%", "s": 78 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "id": "openai:gpt-5.4-mini", "measured": "2026-10-04", "status": { "label": "DEAD AIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "81", "s": 81, "tone": "good" }, "stall": { "v": "12.2%", "s": 12.2, "tone": "warn", "ci": "11 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "62.2%", "s": 62.2, "tone": "bad", "ci": "166 of 267 turns" }, "toolsilence": { "v": "99.4%", "s": 99.4, "tone": "bad", "ci": "166 of 167 tool-call turns" }, "task": { "v": "91.75%", "s": 91.75, "tone": "good", "ci": "84–98% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "82.2%", "s": 82.2, "tone": "good", "ci": "names the gap 80% · offers a route 83% · 10 probes x 3 iterations · 5 silent turns scored zero" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "390ms", "s": 390, "lo": 390, "hi": 477, "ci": "390–477 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$4.50", "s": 4.5, "note": "Vendor list, standard tier, per 1M output tokens ($0.75 in)." }, "consistency": { "v": "68%", "s": 68, "tone": "warn", "ci": "13 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "80%", "s": 80, "tone": "good", "ci": "144 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "86%", "s": 86 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "67%", "s": 67 }, "realcall:payments": { "v": "78%", "s": 78 }, "realcall:frontdesk": { "v": "78%", "s": 78 } }, "lead": false }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "id": "baseten:nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B", "measured": "2026-10-04", "status": { "label": "STALLS", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "80", "s": 80, "tone": "good" }, "stall": { "v": "16.7%", "s": 16.7, "tone": "warn", "ci": "15 of 90 runs" }, "fabric": { "v": "10%", "s": 10, "tone": "good", "ci": "3 of 30 probe runs" }, "deadair": { "v": "64.1%", "s": 64.1, "tone": "bad", "ci": "195 of 304 turns" }, "toolsilence": { "v": "95.6%", "s": 95.6, "tone": "bad", "ci": "195 of 204 tool-call turns" }, "task": { "v": "86.84%", "s": 86.84, "tone": "good", "ci": "74–97% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "75.6%", "s": 75.6, "tone": "warn", "ci": "names the gap 83% · offers a route 60% · 10 probes x 3 iterations · 1 silent turn scored zero" }, "ttft": { "v": "245ms", "s": 245, "lo": 245, "hi": 299, "ci": "245–299 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$2.40", "s": 2.4 }, "consistency": { "v": "79%", "s": 79, "tone": "good", "ci": "15 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "78%", "s": 78, "tone": "warn", "ci": "141 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "81%", "s": 81 }, "realcall:orders": { "v": "81%", "s": 81 }, "realcall:intake": { "v": "56%", "s": 56 }, "realcall:payments": { "v": "94%", "s": 94 }, "realcall:frontdesk": { "v": "81%", "s": 81 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "id": "openai:gpt-4.1-mini", "status": { "label": "FABRICATES", "tone": "warn" }, "rec": [ { "label": "Fast + capable", "caution": true } ], "cells": { "score": { "v": "79", "s": 79, "tone": "warn" }, "stall": { "v": "2.2%", "s": 2.2, "tone": "good", "ci": "2 of 90 runs" }, "fabric": { "v": "3%", "s": 3, "tone": "good", "ci": "1 of 30 probe runs" }, "deadair": { "v": "61.4%", "s": 61.4, "tone": "bad", "ci": "167 of 272 turns" }, "toolsilence": { "v": "97.1%", "s": 97.1, "tone": "bad", "ci": "167 of 172 tool-call turns" }, "task": { "v": "92.63%", "s": 92.63, "tone": "good", "ci": "80–100% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "83.3%", "s": 83.3, "tone": "good", "ci": "names the gap 80% · offers a route 80% · 10 probes x 3 iterations · 3 silent turns scored zero" }, "toolarg": { "v": "27%", "s": 27, "tone": "bad" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "480ms", "s": 480, "lo": 480, "hi": 755, "ci": "480–755 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.60", "s": 1.6, "tone": "good" }, "consistency": { "v": "89%", "s": 89, "tone": "good", "ci": "17 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "69%", "s": 69, "tone": "warn", "ci": "125 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "67%", "s": 67 }, "realcall:orders": { "v": "81%", "s": 81 }, "realcall:intake": { "v": "61%", "s": 61 }, "realcall:payments": { "v": "78%", "s": 78 }, "realcall:frontdesk": { "v": "61%", "s": 61 } }, "lead": false }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "id": "gemini:gemini-3.5-flash-lite", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "78", "s": 78, "tone": "warn" }, "stall": { "v": "5.6%", "s": 5.6, "tone": "good", "ci": "5 of 90 runs" }, "fabric": { "v": "10%", "s": 10, "tone": "good", "ci": "3 of 30 probe runs" }, "deadair": { "v": "63.1%", "s": 63.1, "tone": "bad", "ci": "173 of 274 turns" }, "toolsilence": { "v": "99.4%", "s": 99.4, "tone": "bad", "ci": "173 of 174 tool-call turns" }, "task": { "v": "94.74%", "s": 94.74, "tone": "good", "ci": "79–100% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "70.0%", "s": 70, "tone": "warn", "ci": "names the gap 63% · offers a route 70% · 10 probes x 3 iterations · 6 silent turns scored zero" }, "toolarg": { "v": "100%", "s": 100, "tone": "bad" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "500ms", "s": 500, "lo": 500, "hi": 621, "ci": "500–621 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$2.50", "s": 2.5 }, "consistency": { "v": "95%", "s": 95, "tone": "good", "ci": "18 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "71%", "s": 71, "tone": "warn", "ci": "127 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "69%", "s": 69 }, "realcall:orders": { "v": "89%", "s": 89 }, "realcall:intake": { "v": "25%", "s": 25 }, "realcall:payments": { "v": "92%", "s": 92 }, "realcall:frontdesk": { "v": "78%", "s": 78 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-4.1", "id": "openai:gpt-4.1", "status": { "label": "STRONG", "tone": "good" }, "rec": [ { "label": "Live agent" } ], "cells": { "score": { "v": "77", "s": 77, "tone": "warn" }, "stall": { "v": "8.9%", "s": 8.9, "tone": "good", "ci": "8 of 90 runs" }, "fabric": { "v": "10%", "s": 10, "tone": "good", "ci": "3 of 30 probe runs" }, "deadair": { "v": "61.5%", "s": 61.5, "tone": "bad", "ci": "166 of 270 turns" }, "toolsilence": { "v": "97.6%", "s": 97.6, "tone": "bad", "ci": "166 of 170 tool-call turns" }, "task": { "v": "89.12%", "s": 89.12, "tone": "good", "ci": "75–100% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "78.9%", "s": 78.9, "tone": "warn", "ci": "names the gap 70% · offers a route 80% · 10 probes x 3 iterations · 3 silent turns scored zero" }, "toolarg": { "v": "0%", "s": 0, "tone": "good" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "462ms", "s": 462, "lo": 462, "hi": 605, "ci": "462–605 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$8.00", "s": 8, "tone": "warn" }, "consistency": { "v": "84%", "s": 84, "tone": "good", "ci": "16 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "78%", "s": 78, "tone": "warn", "ci": "140 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "78%", "s": 78 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "69%", "s": 69 }, "realcall:payments": { "v": "86%", "s": 86 }, "realcall:frontdesk": { "v": "64%", "s": 64 } }, "lead": false }, { "provider": "Anthropic", "model": "Claude Haiku 5.5", "id": "anthropic:claude-haiku-5-5", "routable": false, "measured": "2026-10-08", "status": { "label": "DEAD AIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "73", "s": 73, "tone": "warn" }, "stall": { "v": "16.7%", "s": 16.7, "tone": "warn", "ci": "15 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "59.2%", "s": 59.2, "tone": "bad", "ci": "177 of 299 turns" }, "toolsilence": { "v": "88.9%", "s": 88.9, "tone": "bad", "ci": "177 of 199 tool-call turns" }, "task": { "v": "82.46%", "s": 82.46, "tone": "good", "ci": "65–96% 95% CI bootstrap over items · n=95 · measured 2026-10-08" }, "refusal": { "v": "93.3%", "s": 93.3, "tone": "good", "ci": "names the gap 90% · offers a route 90% · 10 probes x 3 iterations" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "579ms", "s": 579, "lo": 579, "hi": 1157, "ci": "579–1157 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$0.50", "s": 0.5, "note": "Vendor list, per 1M output tokens ($0.10 in); prompts over 100K tokens bill $2.50 out." }, "consistency": { "v": "74%", "s": 74, "tone": "good", "ci": "14 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "86%", "s": 86, "tone": "good", "ci": "155 of 180 calls clean · measured 2026-10-08" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "92%", "s": 92 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "92%", "s": 92 }, "realcall:payments": { "v": "89%", "s": 89 }, "realcall:frontdesk": { "v": "67%", "s": 67 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "id": "openai:gpt-5.6-terra", "meta": "reasoning off (required with tools)", "measured": "2026-10-04", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "73", "s": 73, "tone": "warn" }, "stall": { "v": "21.1%", "s": 21.1, "tone": "warn", "ci": "19 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "64.0%", "s": 64, "tone": "bad", "ci": "181 of 283 turns" }, "toolsilence": { "v": "98.9%", "s": 98.9, "tone": "bad", "ci": "181 of 183 tool-call turns" }, "task": { "v": "83.51%", "s": 83.51, "tone": "good", "ci": "68–96% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "88.9%", "s": 88.9, "tone": "good", "ci": "names the gap 90% · offers a route 87% · 10 probes x 3 iterations · 1 silent turn scored zero" }, "toolarg": { "v": "7%", "s": 7, "tone": "warn" }, "placeholder": { "v": "100%", "s": 100, "tone": "bad" }, "ttft": { "v": "580ms", "s": 580, "lo": 580, "hi": 731, "ci": "580–731 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$12.00", "s": 12, "tone": "warn" }, "consistency": { "v": "74%", "s": 74, "tone": "good", "ci": "14 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "85%", "s": 85, "tone": "good", "ci": "153 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "83%", "s": 83 }, "realcall:orders": { "v": "89%", "s": 89 }, "realcall:intake": { "v": "78%", "s": 78 }, "realcall:payments": { "v": "92%", "s": 92 }, "realcall:frontdesk": { "v": "83%", "s": 83 } }, "lead": false }, { "provider": "Inception", "model": "Mercury Voice", "id": "inception:mercury-voice", "meta": "diffusion LLM", "measured": "2026-10-04", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "73", "s": 73, "tone": "warn" }, "stall": { "v": "16.7%", "s": 16.7, "tone": "warn", "ci": "15 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "72.8%", "s": 72.8, "tone": "bad", "ci": "244 of 335 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (244 tool-call turns) · ≥82%" }, "task": { "v": "80.53%", "s": 80.53, "tone": "good", "ci": "68–91% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "75.6%", "s": 75.6, "tone": "warn", "ci": "names the gap 67% · offers a route 70% · 10 probes x 3 iterations · 3 silent turns scored zero" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "443ms", "s": 443, "lo": 443, "hi": 1286, "ci": "443–1286 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$0.75", "s": 0.75, "note": "Vendor list (promotional), per 1M output tokens ($0.20 in)." }, "consistency": { "v": "42%", "s": 42, "tone": "warn", "ci": "8 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "88%", "s": 88, "tone": "good", "ci": "159 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "92%", "s": 92 }, "realcall:orders": { "v": "97%", "s": 97 }, "realcall:intake": { "v": "86%", "s": 86 }, "realcall:payments": { "v": "86%", "s": 86 }, "realcall:frontdesk": { "v": "81%", "s": 81 } }, "lead": false }, { "provider": "Baseten", "model": "gpt-oss-120b", "id": "baseten:openai/gpt-oss-120b", "measured": "2026-10-04", "status": { "label": "DEAD AIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "72", "s": 72, "tone": "warn" }, "stall": { "v": "12.2%", "s": 12.2, "tone": "warn", "ci": "11 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "70.2%", "s": 70.2, "tone": "bad", "ci": "224 of 319 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (224 tool-call turns) · ≥82%" }, "task": { "v": "80.53%", "s": 80.53, "tone": "good", "ci": "66–94% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "80.0%", "s": 80, "tone": "good", "ci": "names the gap 67% · offers a route 83% · 10 probes x 3 iterations · 3 silent turns scored zero" }, "ttft": { "v": "504ms", "s": 504, "lo": 504, "hi": 1215, "ci": "504–1215 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$0.50", "s": 0.5 }, "consistency": { "v": "68%", "s": 68, "tone": "warn", "ci": "13 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "73%", "s": 73, "tone": "warn", "ci": "132 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "8.3%", "s": 8.33, "tone": "bad", "ci": "15 of 180 calls" }, "realcall:scheduling": { "v": "72%", "s": 72 }, "realcall:orders": { "v": "75%", "s": 75 }, "realcall:intake": { "v": "72%", "s": 72 }, "realcall:payments": { "v": "86%", "s": 86 }, "realcall:frontdesk": { "v": "61%", "s": 61 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "id": "openai:gpt-5.6-luna", "meta": "reasoning off (required with tools)", "measured": "2026-10-04", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "70", "s": 70, "tone": "warn" }, "stall": { "v": "25.6%", "s": 25.6, "tone": "warn", "ci": "23 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "68.1%", "s": 68.1, "tone": "bad", "ci": "216 of 317 turns" }, "toolsilence": { "v": "99.5%", "s": 99.5, "tone": "bad", "ci": "216 of 217 tool-call turns" }, "task": { "v": "75.61%", "s": 75.61, "tone": "warn", "ci": "58–91% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "95.6%", "s": 95.6, "tone": "good", "ci": "names the gap 97% · offers a route 93% · 10 probes x 3 iterations" }, "toolarg": { "v": "100%", "s": 100, "tone": "bad" }, "placeholder": { "v": "93%", "s": 93, "tone": "bad" }, "ttft": { "v": "546ms", "s": 546, "lo": 546, "hi": 683, "ci": "546–683 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.20", "s": 1.2, "tone": "good" }, "consistency": { "v": "53%", "s": 53, "tone": "warn", "ci": "10 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "78%", "s": 78, "tone": "warn", "ci": "140 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "83%", "s": 83 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "64%", "s": 64 }, "realcall:payments": { "v": "81%", "s": 81 }, "realcall:frontdesk": { "v": "69%", "s": 69 } }, "lead": false }, { "provider": "Together", "model": "Llama-3.3-70B", "id": "together:meta-llama/Llama-3.3-70B-Instruct-Turbo", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "66", "s": 66, "tone": "warn" }, "stall": { "v": "72.2%", "s": 72.2, "tone": "bad", "ci": "65 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "48.5%", "s": 48.5, "tone": "bad", "ci": "94 of 194 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (94 tool-call turns) · ≥82%" }, "task": { "v": "54.30%", "s": 54.3, "tone": "warn", "ci": "39–70% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "20.0%", "s": 20, "tone": "bad", "ci": "names the gap 33% · offers a route 13% · 10 probes x 3 iterations · 19 silent turns scored zero" }, "toolarg": { "v": "100%", "s": 100, "tone": "bad" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "525ms", "s": 525, "lo": 525, "hi": 725, "ci": "525–725 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$1.04", "s": 1.04, "tone": "good" }, "consistency": { "v": "32%", "s": 32, "tone": "bad", "ci": "6 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "57%", "s": 57, "tone": "bad", "ci": "103 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "5.0%", "s": 5, "tone": "bad", "ci": "9 of 180 calls" }, "realcall:scheduling": { "v": "58%", "s": 58 }, "realcall:orders": { "v": "58%", "s": 58 }, "realcall:intake": { "v": "50%", "s": 50 }, "realcall:payments": { "v": "64%", "s": 64 }, "realcall:frontdesk": { "v": "56%", "s": 56 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5.2", "id": "openai:gpt-5.2", "measured": "2026-10-04", "status": { "label": "DEAD AIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "65", "s": 65, "tone": "warn" }, "stall": { "v": "28.9%", "s": 28.9, "tone": "warn", "ci": "26 of 90 runs" }, "fabric": { "v": "10%", "s": 10, "tone": "good", "ci": "3 of 30 probe runs" }, "deadair": { "v": "71.0%", "s": 71, "tone": "bad", "ci": "245 of 345 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (245 tool-call turns) · ≥82%" }, "task": { "v": "75.61%", "s": 75.61, "tone": "warn", "ci": "59–91% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "91.1%", "s": 91.1, "tone": "good", "ci": "names the gap 90% · offers a route 87% · 10 probes x 3 iterations" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "664ms", "s": 664, "lo": 664, "hi": 825, "ci": "664–825 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$14.00", "s": 14, "note": "Vendor list, standard tier, per 1M output tokens ($1.75 in)." }, "consistency": { "v": "58%", "s": 58, "tone": "warn", "ci": "11 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "87%", "s": 87, "tone": "good", "ci": "157 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "83%", "s": 83 }, "realcall:orders": { "v": "92%", "s": 92 }, "realcall:intake": { "v": "86%", "s": 86 }, "realcall:payments": { "v": "89%", "s": 89 }, "realcall:frontdesk": { "v": "86%", "s": 86 } }, "lead": false }, { "provider": "Gemini", "model": "gemini-3.8-flash", "id": "gemini:gemini-3.8-flash", "measured": "2026-10-04", "status": { "label": "UNSAFE ACTIONS", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "60", "s": 60, "tone": "warn" }, "stall": { "v": "13.3%", "s": 13.3, "tone": "warn", "ci": "12 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "65.5%", "s": 65.5, "tone": "bad", "ci": "190 of 290 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (190 tool-call turns) · ≥82%" }, "task": { "v": "85.79%", "s": 85.79, "tone": "good", "ci": "71–98% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "96.7%", "s": 96.7, "tone": "good", "ci": "names the gap 97% · offers a route 97% · 10 probes x 3 iterations" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "1.21s", "s": 1214, "lo": 1214, "hi": 2905, "ci": "1.21–2.90s p50–p90 · n=38", "tone": "warn" }, "cost": { "v": "$3.75", "s": 3.75, "note": "Vendor list, paid tier, per 1M output tokens; rises to $7.50 on 2027-01-01." }, "consistency": { "v": "79%", "s": 79, "tone": "good", "ci": "15 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "87%", "s": 87, "tone": "good", "ci": "157 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "81%", "s": 81 }, "realcall:orders": { "v": "94%", "s": 94 }, "realcall:intake": { "v": "81%", "s": 81 }, "realcall:payments": { "v": "89%", "s": 89 }, "realcall:frontdesk": { "v": "92%", "s": 92 } }, "lead": false }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "id": "anthropic:claude-sonnet-5", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "59", "s": 59, "tone": "bad" }, "stall": { "v": "18.9%", "s": 18.9, "tone": "warn", "ci": "17 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "34.6%", "s": 34.6, "tone": "bad", "ci": "101 of 292 turns" }, "toolsilence": { "v": "52.6%", "s": 52.6, "tone": "bad", "ci": "101 of 192 tool-call turns" }, "task": { "v": "83.33%", "s": 83.33, "tone": "good", "ci": "68–95% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "90.0%", "s": 90, "tone": "good", "ci": "names the gap 77% · offers a route 93% · 10 probes x 3 iterations" }, "toolarg": { "v": "0%", "s": 0, "tone": "good" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "1.47s", "s": 1473, "lo": 1473, "hi": 2744, "ci": "1.47–2.74s p50–p90 · n=38", "tone": "warn" }, "cost": { "v": "$10.00", "s": 10, "tone": "warn", "note": "Introductory rate through 2026-08-31; $15.00 from 2026-09-01." }, "consistency": { "v": "53%", "s": 53, "tone": "warn", "ci": "10 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "89%", "s": 89, "tone": "good", "ci": "161 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "86%", "s": 86 }, "realcall:orders": { "v": "94%", "s": 94 }, "realcall:intake": { "v": "97%", "s": 97 }, "realcall:payments": { "v": "97%", "s": 97 }, "realcall:frontdesk": { "v": "72%", "s": 72 } }, "lead": false }, { "provider": "Gemini", "model": "gemini-3.7-flash", "id": "gemini:gemini-3.7-flash", "measured": "2026-10-04", "status": { "label": "UNSAFE ACTIONS", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "56", "s": 56, "tone": "bad" }, "stall": { "v": "11.1%", "s": 11.1, "tone": "warn", "ci": "10 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "64.5%", "s": 64.5, "tone": "bad", "ci": "182 of 282 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (182 tool-call turns) · ≥82%" }, "task": { "v": "86.84%", "s": 86.84, "tone": "good", "ci": "71–100% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "97.8%", "s": 97.8, "tone": "good", "ci": "names the gap 100% · offers a route 97% · 10 probes x 3 iterations" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "1.38s", "s": 1381, "lo": 1381, "hi": 2648, "ci": "1.38–2.65s p50–p90 · n=38", "tone": "warn" }, "cost": { "v": "$3.75", "s": 3.75, "note": "Vendor list, paid tier, per 1M output tokens ($0.75 in); rises to $7.50 on 2027-01-01." }, "consistency": { "v": "84%", "s": 84, "tone": "good", "ci": "16 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "85%", "s": 85, "tone": "good", "ci": "153 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "75%", "s": 75 }, "realcall:orders": { "v": "89%", "s": 89 }, "realcall:intake": { "v": "81%", "s": 81 }, "realcall:payments": { "v": "92%", "s": 92 }, "realcall:frontdesk": { "v": "89%", "s": 89 } }, "lead": false }, { "provider": "Alibaba", "model": "qwen-turbo", "id": "alibaba:qwen-turbo", "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "score": { "v": "52", "s": 52, "tone": "bad" }, "stall": { "v": "72.2%", "s": 72.2, "tone": "bad", "ci": "65 of 90 runs" }, "fabric": { "v": "90%", "s": 90, "tone": "bad", "ci": "27 of 30 probe runs" }, "deadair": { "v": "32.9%", "s": 32.9, "tone": "bad", "ci": "49 of 149 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 10 items (49 tool-call turns) · ≥69%" }, "task": { "v": "30.61%", "s": 30.61, "tone": "bad", "ci": "12–52% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "11.1%", "s": 11.1, "tone": "bad", "ci": "names the gap 0% · offers a route 10% · 10 probes x 3 iterations · 3 silent turns scored zero" }, "toolarg": { "v": "0%", "s": 0, "tone": "good" }, "placeholder": { "v": "73%", "s": 73, "tone": "bad" }, "ttft": { "v": "430ms", "s": 430, "lo": 430, "hi": 544, "ci": "430–544 p50–p90 · n=38", "tone": "good" }, "cost": { "v": "$0.20", "s": 0.2, "tone": "good" }, "consistency": { "v": "26%", "s": 26, "tone": "bad", "ci": "5 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "33%", "s": 33, "tone": "bad", "ci": "60 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.6%", "s": 0.56, "tone": "warn", "ci": "1 of 180 calls" }, "realcall:scheduling": { "v": "22%", "s": 22 }, "realcall:orders": { "v": "56%", "s": 56 }, "realcall:intake": { "v": "3%", "s": 3 }, "realcall:payments": { "v": "58%", "s": 58 }, "realcall:frontdesk": { "v": "28%", "s": 28 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5-mini", "id": "openai:gpt-5-mini", "meta": "reasoning", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "45", "s": 45, "tone": "bad" }, "stall": { "v": "14.4%", "s": 14.4, "tone": "warn", "ci": "13 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "71.9%", "s": 71.9, "tone": "bad", "ci": "238 of 331 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (238 tool-call turns) · ≥82%" }, "task": { "v": "81.93%", "s": 81.93, "tone": "good", "ci": "69–94% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "83.3%", "s": 83.3, "tone": "good", "ci": "names the gap 83% · offers a route 83% · 10 probes x 3 iterations · 5 silent turns scored zero" }, "toolarg": { "v": "100%", "s": 100, "tone": "bad" }, "placeholder": { "v": "7%", "s": 7, "tone": "warn" }, "ttft": { "v": "1.73s", "s": 1729, "lo": 1729, "hi": 5944, "ci": "1.73–5.94s p50–p90 · n=38", "tone": "warn" }, "cost": { "v": "$2.00", "s": 2 }, "consistency": { "v": "63%", "s": 63, "tone": "warn", "ci": "12 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "77%", "s": 77, "tone": "warn", "ci": "139 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "1.7%", "s": 1.67, "tone": "warn", "ci": "3 of 180 calls" }, "realcall:scheduling": { "v": "78%", "s": 78 }, "realcall:orders": { "v": "72%", "s": 72 }, "realcall:intake": { "v": "72%", "s": 72 }, "realcall:payments": { "v": "81%", "s": 81 }, "realcall:frontdesk": { "v": "83%", "s": 83 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5", "id": "openai:gpt-5", "meta": "reasoning", "status": { "label": "SOLID", "tone": "good" }, "rec": [], "cells": { "score": { "v": "40", "s": 40, "tone": "bad" }, "stall": { "v": "24.4%", "s": 24.4, "tone": "warn", "ci": "22 of 90 runs" }, "fabric": { "v": "7%", "s": 7, "tone": "good", "ci": "2 of 30 probe runs" }, "deadair": { "v": "69.5%", "s": 69.5, "tone": "bad", "ci": "223 of 321 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (223 tool-call turns) · ≥82%" }, "task": { "v": "75.61%", "s": 75.61, "tone": "warn", "ci": "58–91% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "84.4%", "s": 84.4, "tone": "good", "ci": "names the gap 77% · offers a route 77% · 10 probes x 3 iterations" }, "toolarg": { "v": "100%", "s": 100, "tone": "bad" }, "placeholder": { "v": "0%", "s": 0, "tone": "good" }, "ttft": { "v": "1.86s", "s": 1859, "lo": 1859, "hi": 8352, "ci": "1.86–8.35s p50–p90 · n=38", "tone": "warn" }, "cost": { "v": "$10.00", "s": 10, "tone": "warn" }, "consistency": { "v": "58%", "s": 58, "tone": "warn", "ci": "11 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "81%", "s": 81, "tone": "good", "ci": "146 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "4.4%", "s": 4.44, "tone": "warn", "ci": "8 of 180 calls" }, "realcall:scheduling": { "v": "83%", "s": 83 }, "realcall:orders": { "v": "94%", "s": 94 }, "realcall:intake": { "v": "61%", "s": 61 }, "realcall:payments": { "v": "89%", "s": 89 }, "realcall:frontdesk": { "v": "78%", "s": 78 } }, "lead": false }, { "provider": "xAI", "model": "Grok-4.3", "id": "xai:grok-4.3", "status": { "label": "SLOW", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "38", "s": 38, "tone": "bad" }, "stall": { "v": "5.6%", "s": 5.6, "tone": "good", "ci": "5 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "56.9%", "s": 56.9, "tone": "bad", "ci": "165 of 290 turns" }, "toolsilence": { "v": "86.8%", "s": 86.8, "tone": "bad", "ci": "165 of 190 tool-call turns" }, "task": { "v": "90.88%", "s": 90.88, "tone": "good", "ci": "78–100% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "96.7%", "s": 96.7, "tone": "good", "ci": "names the gap 97% · offers a route 97% · 10 probes x 3 iterations · 1 silent turn scored zero" }, "toolarg": { "v": "0%", "s": 0, "tone": "good" }, "placeholder": { "v": "20%", "s": 20, "tone": "bad" }, "ttft": { "v": "2.38s", "s": 2381, "lo": 2381, "hi": 5377, "ci": "2.38–5.38s p50–p90 · n=38", "tone": "bad" }, "cost": { "v": "$2.50", "s": 2.5 }, "consistency": { "v": "84%", "s": 84, "tone": "good", "ci": "16 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "75%", "s": 75, "tone": "warn", "ci": "135 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "0.0%", "s": 0, "tone": "good", "ci": "0 of 180 calls" }, "realcall:scheduling": { "v": "72%", "s": 72 }, "realcall:orders": { "v": "86%", "s": 86 }, "realcall:intake": { "v": "56%", "s": 56 }, "realcall:payments": { "v": "78%", "s": 78 }, "realcall:frontdesk": { "v": "83%", "s": 83 } }, "lead": false }, { "provider": "OpenAI", "model": "gpt-5-nano", "id": "openai:gpt-5-nano", "meta": "reasoning", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "28", "s": 28, "tone": "bad" }, "stall": { "v": "24.4%", "s": 24.4, "tone": "warn", "ci": "22 of 90 runs" }, "fabric": { "v": "0%", "s": 0, "tone": "good", "ci": "0 of 10 probes (30 probe runs) · ≤31%" }, "deadair": { "v": "73.2%", "s": 73.2, "tone": "bad", "ci": "235 of 321 turns" }, "toolsilence": { "v": "100.0%", "s": 100, "tone": "bad", "ci": "all of 19 items (228 tool-call turns) · ≥82%" }, "task": { "v": "77.02%", "s": 77.02, "tone": "warn", "ci": "63–90% 95% CI bootstrap over items · n=95 · measured 2026-10-04" }, "refusal": { "v": "84.4%", "s": 84.4, "tone": "good", "ci": "names the gap 77% · offers a route 77% · 10 probes x 3 iterations" }, "toolarg": { "v": "100%", "s": 100, "tone": "bad" }, "placeholder": { "v": "23%", "s": 23, "tone": "bad" }, "ttft": { "v": "2.41s", "s": 2414, "lo": 2414, "hi": 6873, "ci": "2.41–6.87s p50–p90 · n=37", "tone": "bad" }, "cost": { "v": "$0.40", "s": 0.4, "tone": "good" }, "consistency": { "v": "53%", "s": 53, "tone": "warn", "ci": "10 of 19 scenarios passed all 5 runs" }, "realcall": { "v": "76%", "s": 76, "tone": "warn", "ci": "136 of 180 calls clean · measured 2026-10-04" }, "leak": { "v": "1.1%", "s": 1.11, "tone": "warn", "ci": "2 of 180 calls" }, "realcall:scheduling": { "v": "67%", "s": 67 }, "realcall:orders": { "v": "86%", "s": 86 }, "realcall:intake": { "v": "78%", "s": 78 }, "realcall:payments": { "v": "81%", "s": 81 }, "realcall:frontdesk": { "v": "67%", "s": 67 } }, "lead": false }, { "provider": "Together", "model": "DeepSeek-V4-Pro", "id": "together:deepseek-ai/DeepSeek-V4-Pro", "status": { "label": "NOT REAL-TIME", "tone": "warn" }, "rec": [ { "label": "Quality ceiling", "caution": true } ], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "—", "s": null, "dim": true, "note": "Not measured on the current probe set — this row has no probe replies to score." }, "toolarg": { "v": "0%", "s": 0, "tone": "good" }, "placeholder": { "v": "3%", "s": 3, "tone": "warn" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$3.96", "s": 3.96, "note": "Together's only V4 Pro SKU, DeepSeek-V4-Pro-0813, at $3.96/1M output ($1.32 in). The $3.48 this row used to carry is BASETEN's rate for the same model; the bare `DeepSeek-V4-Pro` id on together.ai is unreleased and carries no price." }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false }, { "provider": "LiveKit", "model": "kimi-k2.5", "id": "livekit:moonshotai/kimi-k2.5", "meta": "worker transport", "status": { "label": "RETIRED", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "—", "s": null, "dim": true, "note": "Not measured on the current probe set — this row has no probe replies to score." }, "toolarg": { "v": "0%", "s": 0, "tone": "good" }, "placeholder": { "v": "7%", "s": 7, "tone": "warn" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "—", "s": null, "dim": true, "note": "LiveKit sunset kimi-k2.5 on 2026-07-23 and publishes no rate for it. Replacement Kimi K2.6 is $4.00 / 1M output." }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false }, { "provider": "Cerebras", "model": "gemma-4-31b", "id": "cerebras:gemma-4-31b", "status": { "label": "RETIRED", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "—", "s": null, "dim": true, "note": "Not measured on the current probe set — this row has no probe replies to score." }, "toolarg": { "v": "33%", "s": 33, "tone": "bad" }, "placeholder": { "v": "100%", "s": 100, "tone": "bad" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$1.49", "s": 1.49 }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false }, { "provider": "Baseten", "model": "inkling-small", "id": "baseten:thinkingmachines/inkling-small", "meta": "reasoning off (provider default)", "measured": "2026-08-17", "status": { "label": "RETIRED", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "—", "s": null, "dim": true, "note": "Not measured on the current probe set — this row has no probe replies to score." }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$1.20", "s": 1.2 }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false }, { "provider": "Baseten", "model": "inkling", "id": "baseten:thinkingmachines/inkling", "meta": "reasoning off (provider default)", "measured": "2026-08-17", "status": { "label": "RETIRED", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "—", "s": null, "dim": true, "note": "Not measured on the current probe set — this row has no probe replies to score." }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$4.05", "s": 4.05 }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false }, { "provider": "LiveKit", "model": "gemma-4-31b-it", "id": "livekit:google/gemma-4-31b-it", "meta": "worker transport", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "—", "s": null, "dim": true, "note": "Not measured on the current probe set — this row has no probe replies to score." }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "Three independent n=30 runs gave 33%, 3% and 20% — unstable run to run, so no rate is published. Its Cerebras twin held 30/33/37% on the same weights." }, "placeholder": { "v": "100%", "s": 100, "tone": "bad" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$1.20", "s": 1.2, "tone": "good" }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false }, { "provider": "Crusoe", "model": "gemma-4-31b-it", "id": "crusoe:google/gemma-4-31b-it", "meta": "non-thinking, vendor-direct", "measured": "2026-09-28", "status": { "label": "UNSAFE ACTIONS", "tone": "warn" }, "rec": [], "cells": { "score": { "v": "—", "s": null, "dim": true, "note": "No real-time first-token measurement, so a speed-weighted score is not computed — unranked rather than judged on partial data." }, "stall": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "fabric": { "v": "—", "s": null, "dim": true, "note": "fabrication probes not run for this model" }, "deadair": { "v": "—", "s": null, "dim": true, "note": "not measured on the current corpus" }, "toolsilence": { "v": "—", "s": null, "dim": true, "note": "no tool-call turns to measure" }, "task": { "v": "—", "s": null, "tone": "warn", "ci": "no measurement on the current corpus — transport unavailable" }, "refusal": { "v": "90.0%", "s": 90, "tone": "good", "ci": "names the gap 87% · offers a route 90% · 10 probes x 3 iterations · 2 silent turns scored zero" }, "toolarg": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "placeholder": { "v": "—", "s": null, "dim": true, "note": "not measured for this model" }, "ttft": { "v": "—", "s": null, "dim": true, "ci": "not measured on the vendor-direct regional sweep" }, "cost": { "v": "$0.40", "s": 0.4, "note": "Vendor list, serverless, per 1M output tokens ($0.14 in)." }, "consistency": { "v": "—", "s": null, "dim": true }, "realcall": { "v": "—", "s": null, "dim": true }, "leak": { "v": "—", "s": null, "dim": true }, "realcall:scheduling": { "v": "—", "s": null, "dim": true }, "realcall:orders": { "v": "—", "s": null, "dim": true }, "realcall:intake": { "v": "—", "s": null, "dim": true }, "realcall:payments": { "v": "—", "s": null, "dim": true }, "realcall:frontdesk": { "v": "—", "s": null, "dim": true } }, "lead": false } ], "reading": "" }, { "id": "s2s", "navLabel": "Speech-to-Speech", "identityLabel": "Provider / Model", "title": "S2S", "subtitle": "Native voice-to-voice, measured across six concierge scenarios.", "live": true, "meta": "6 scenarios · n=3 · clean audio · latency US East · 2026-07-19 (2.0 row 07-31 · gpt-live-1 row 09-11, provider-direct, backend given the date · gemini-3.8 rows 09-16, provider-direct, no clock on any arm · grok-voice-3.0 row 10-15, provider-direct, anchored to the 2.0 row) · scorer v2 rescore 09-11", "measured": "2026-07-19", "headline": [ "action", "reliab", "lat" ], "scatter": { "x": "lat", "y": "score", "cornerLabel": "fast + capable" }, "columns": [ { "id": "score", "label": "Overall", "dir": "higher", "info": "Completion × capability (action weighted), mean over 6 concierge scenarios, n=3." }, { "id": "action", "label": "Action", "dir": "higher", "info": "Fired the write tool after the caller confirmed — not just said it would. 0–1." }, { "id": "reliab", "label": "Call completion", "dir": "higher", "info": "Share of the scripted call completed before going silent or dropping." }, { "id": "cap", "label": "Task capability", "dir": "higher", "info": "Mean of 6 transcript checks: tool choice, no false calls, honest recovery, no invented prices, latest-value recall, brevity." }, { "id": "lat", "label": "First sound", "dir": "lower", "info": "End of speech to the first sound back, p50, US East. On a duplex model that sound can be an acknowledgement, not the answer." }, { "id": "answer", "label": "Answer", "dir": "lower", "info": "End of speech to the first sound that carries the answer, p50. Filler does not count." }, { "id": "price", "label": "Audio price", "info": "Vendor pricing in each vendor's own denomination — not directly comparable." } ], "rows": [ { "provider": "xAI", "model": "grok-voice-3.0", "id": "xai:grok-voice-3.0", "rec": [], "measured": "2026-10-15", "cells": { "score": { "v": "0.86", "s": 86 }, "action": { "v": "0.67", "s": 67 }, "reliab": { "v": "100%", "s": 100 }, "cap": { "v": "0.94", "s": 94 }, "lat": { "v": "782ms", "s": 782, "tone": "good", "ci": "1084·1754", "note": "Published 2.0 latency minus the same-run 3.0 gap (38ms). Range is this run’s own p90·p95, US East, provider-direct." }, "answer": { "v": "1598ms", "s": 1598 }, "price": { "v": "$0.08 · min", "s": null } } }, { "provider": "xAI", "model": "grok-voice-think-fast-2.0", "id": "xai:grok-voice-think-fast-2.0", "rec": [], "cells": { "score": { "v": "0.85", "s": 85 }, "action": { "v": "0.67", "s": 67 }, "reliab": { "v": "100%", "s": 100 }, "cap": { "v": "0.92", "s": 92 }, "lat": { "v": "820ms", "s": 820, "tone": "good", "ci": "1141·3419" }, "price": { "v": "$0.08 · min", "s": null } } }, { "provider": "Gemini", "model": "gemini-3.1-flash-live", "id": "google:gemini-3.1-flash-live-preview", "slug": "google-gemini-3-1-flash-live", "rec": [], "cells": { "score": { "v": "0.77", "s": 77 }, "action": { "v": "0.78", "s": 78 }, "reliab": { "v": "92%", "s": 92 }, "cap": { "v": "0.86", "s": 86 }, "lat": { "v": "1108ms", "s": 1108, "tone": "warn", "ci": "1209·1285" }, "price": { "v": "$3.00 / $12.00 · 1M tok", "s": null, "note": "Google publishes both bases: $0.005/min in, $0.018/min out." } } }, { "provider": "OpenAI", "model": "gpt-realtime", "id": "openai:gpt-realtime", "rec": [], "cells": { "score": { "v": "0.79", "s": 79 }, "action": { "v": "0.84", "s": 84 }, "reliab": { "v": "88%", "s": 88 }, "cap": { "v": "0.92", "s": 92 }, "lat": { "v": "494ms", "s": 494, "tone": "good", "ci": "768·870" }, "price": { "v": "$32 / $64 · 1M tok", "s": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-2.1-mini", "id": "openai:gpt-realtime-2.1-mini", "rec": [], "cells": { "score": { "v": "0.76", "s": 76 }, "action": { "v": "0.83", "s": 83 }, "reliab": { "v": "93%", "s": 93 }, "cap": { "v": "0.80", "s": 80 }, "lat": { "v": "1011ms", "s": 1011, "tone": "warn", "ci": "1360·1542" }, "price": { "v": "$10 / $20 · 1M tok", "s": null } } }, { "provider": "Gemini", "model": "gemini-3.8-live", "id": "google:gemini-3.8-live", "rec": [], "measured": "2026-09-16", "cells": { "score": { "v": "0.87", "s": 87 }, "action": { "v": "0.83", "s": 83 }, "reliab": { "v": "98%", "s": 98 }, "cap": { "v": "0.92", "s": 92 }, "lat": { "v": "1211ms", "s": 1211, "tone": "warn", "ci": "2086·—" }, "answer": { "v": "2013ms", "s": 2013 }, "price": { "v": "$0.005 · min", "s": null } } }, { "provider": "Gemini", "model": "gemini-3.8-live-extended-thinking", "id": "google:gemini-3.8-live-extended-thinking", "rec": [], "measured": "2026-09-16", "cells": { "score": { "v": "0.83", "s": 83 }, "action": { "v": "0.83", "s": 83 }, "reliab": { "v": "96%", "s": 96 }, "cap": { "v": "0.88", "s": 88 }, "lat": { "v": "844ms", "s": 844, "ci": "1203·—" }, "answer": { "v": "7522ms", "s": 7522 }, "price": { "v": "$0.005 · min", "s": null } } }, { "provider": "OpenAI", "model": "gpt-live-1", "id": "openai:gpt-live-1", "rec": [], "measured": "2026-09-11", "cells": { "score": { "v": "0.86", "s": 86 }, "action": { "v": "0.78", "s": 78 }, "reliab": { "v": "100%", "s": 100 }, "cap": { "v": "0.90", "s": 90 }, "lat": { "v": "1217ms", "s": 1217, "tone": "warn", "ci": "1793·—", "note": "US East, provider-direct, n=5, 2026-09-11. A gpt-realtime-2.1 control in the same run measured 972ms. First voiced audio; the spoken answer after a tool call arrives later than on gpt-realtime." }, "answer": { "v": "4187ms", "s": 4187 }, "price": { "v": "$0.05 · min", "s": null, "note": "Plus backend tokens (gpt-5.6-luna), billed separately." } } }, { "provider": "OpenAI", "model": "gpt-realtime-2", "id": "openai:gpt-realtime-2", "rec": [], "cells": { "score": { "v": "0.67", "s": 67 }, "action": { "v": "0.87", "s": 87 }, "reliab": { "v": "81%", "s": 81 }, "cap": { "v": "0.83", "s": 83 }, "lat": { "v": "1103ms", "s": 1103, "tone": "warn", "ci": "1468·1626" }, "price": { "v": "$32 / $64 · 1M tok", "s": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-2.1", "id": "openai:gpt-realtime-2.1", "rec": [], "cells": { "score": { "v": "0.63", "s": 63 }, "action": { "v": "0.56", "s": 56 }, "reliab": { "v": "90%", "s": 90 }, "cap": { "v": "0.76", "s": 76 }, "lat": { "v": "1104ms", "s": 1104, "tone": "warn", "ci": "1514·2146" }, "price": { "v": "$32 / $64 · 1M tok", "s": null } } }, { "provider": "OpenAI", "model": "gpt-realtime-mini", "id": "openai:gpt-realtime-mini", "rec": [], "cells": { "score": { "v": "0.65", "s": 65 }, "action": { "v": "0.77", "s": 77 }, "reliab": { "v": "74%", "s": 74, "tone": "warn" }, "cap": { "v": "0.93", "s": 93 }, "lat": { "v": "614ms", "s": 614, "tone": "good", "ci": "812·847" }, "price": { "v": "$10 / $20 · 1M tok", "s": null } } } ], "reading": "" }, { "id": "stacks", "navLabel": "Cost / solve", "identityLabel": "Stack · STT / LLM / TTS", "title": "True cost-per-solve · whole stack", "subtitle": "Cost per grounded solve across a full STT+LLM+TTS stack.", "live": true, "meta": "dental-booking task · n=3/stack · grounded (tool fired) · measured 2026-07-03", "measured": "2026-07-03", "rankCol": "solve", "headline": [ "solve", "grounded", "call" ], "columns": [ { "id": "grounded", "label": "Grounded", "dir": "higher", "info": "% of runs the required tool actually fired. Higher is better." }, { "id": "turns", "label": "Turns", "info": "Mean turns to resolution." }, { "id": "call", "label": "$/call", "dir": "lower", "info": "Cost per call: real consumption × billed rates." }, { "id": "solve", "label": "$/solve", "dir": "lower", "info": "Cost per grounded solve. Lower = cheaper." } ], "rows": [ { "provider": "", "model": "CHEAPEST", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "cartesia:sonic-3.5" ], "lead": true, "status": { "label": "GROUNDED", "tone": "good" }, "rec": [], "cells": { "grounded": { "v": "100%", "s": 100, "tone": "good" }, "turns": { "v": "5.3", "s": 5.3, "dim": true }, "call": { "v": "$0.058", "s": 0.058 }, "solve": { "v": "$0.058", "s": 0.058, "tone": "good" } } }, { "provider": "", "model": "DEEPGRAM-NATIVE", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "deepgram:aura-2" ], "status": { "label": "GROUNDED", "tone": "good" }, "rec": [], "cells": { "grounded": { "v": "100%", "s": 100, "tone": "good" }, "turns": { "v": "5.0", "s": 5, "dim": true }, "call": { "v": "$0.123", "s": 0.123 }, "solve": { "v": "$0.123", "s": 0.123, "tone": "good" } } }, { "provider": "", "model": "SINGLE-VENDOR", "stack": [ "cartesia:ink-2", "cerebras:gpt-oss-120b", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "6.0", "s": 6, "dim": true }, "call": { "v": "$0.087", "s": 0.087 }, "solve": { "v": "$0.131", "s": 0.131, "tone": "good" } } }, { "provider": "", "model": "EXPRESSIVE VOICE", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "minimax:speech-2.6-hd" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "4.3", "s": 4.3, "dim": true }, "call": { "v": "$0.092", "s": 0.092 }, "solve": { "v": "$0.137", "s": 0.137, "tone": "good" } } }, { "provider": "", "model": "PREMIUM VOICE / CHEAP BRAIN", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "elevenlabs:eleven_flash_v2_5" ], "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "grounded": { "v": "33%", "s": 33, "tone": "bad" }, "turns": { "v": "1.7", "s": 1.7, "dim": true }, "call": { "v": "$0.055", "s": 0.055 }, "solve": { "v": "$0.166", "s": 0.166, "tone": "warn" } } }, { "provider": "", "model": "PREMIUM BRAIN / CHEAP VOICE", "stack": [ "deepgram:nova-3", "openai:gpt-4.1", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "4.3", "s": 4.3, "dim": true }, "call": { "v": "$0.111", "s": 0.111 }, "solve": { "v": "$0.166", "s": 0.166, "tone": "warn" } } }, { "provider": "", "model": "CHEAP-OPENAI", "stack": [ "deepgram:nova-3", "openai:gpt-5-nano", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "10.0", "s": 10, "dim": true }, "call": { "v": "$0.145", "s": 0.145 }, "solve": { "v": "$0.218", "s": 0.218, "tone": "warn" } } }, { "provider": "", "model": "ACCURACY", "stack": [ "elevenlabs:scribe_v2_realtime", "openai:gpt-4.1", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "5.7", "s": 5.7, "dim": true }, "call": { "v": "$0.147", "s": 0.147 }, "solve": { "v": "$0.221", "s": 0.221, "tone": "warn" } } }, { "provider": "", "model": "PREMIUM (ALL-BRAND)", "stack": [ "elevenlabs:scribe_v2_realtime", "openai:gpt-4.1", "elevenlabs:eleven_flash_v2_5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "7.3", "s": 7.3, "dim": true }, "call": { "v": "$0.469", "s": 0.469 }, "solve": { "v": "$0.703", "s": 0.703, "tone": "bad" } } }, { "provider": "", "model": "BALANCED", "stack": [ "deepgram:nova-3", "openai:gpt-4.1-mini", "cartesia:sonic-3.5" ], "status": { "label": "NO SOLVE", "tone": "bad" }, "rec": [], "cells": { "grounded": { "v": "0%", "s": 0, "tone": "bad" }, "turns": { "v": "3.3", "s": 3.3, "dim": true }, "call": { "v": "$0.079", "s": 0.079 }, "solve": { "v": "never booked", "s": null, "tone": "bad" } } } ], "reading": "" }, { "id": "turntaking", "navLabel": "Turn-taking", "identityLabel": "Model", "title": "Turn-taking · end-of-turn detection", "subtitle": "End-of-turn detection, ranked on 200 real human clips.", "live": true, "meta": "English · 200 real human clips · pipecat smart-turn-v3.1-test · transcripts via Deepgram nova-3", "rankCol": "acc", "headline": [ "acc", "cut", "lat" ], "columns": [ { "id": "acc", "label": "EOT accuracy", "dir": "higher", "info": "Correct END-vs-WAIT decisions across 200 real clips, at each model’s best operating threshold (min false-cutoff s.t. END-recall ≥ 85%)." }, { "id": "recall", "label": "END-recall", "dir": "higher", "info": "Completed turns correctly ended — the agent didn’t leave the caller hanging." }, { "id": "cut", "label": "False-cutoff", "dir": "lower", "info": "WAIT clips wrongly ended = talked over the caller mid-thought. The costly live-call error." }, { "id": "lat", "label": "Inference", "dir": "lower", "info": "Model forward pass, warm CPU. Text models additionally wait on the STT transcript; audio runs in parallel with STT." } ], "rows": [ { "provider": "Smart Turn v3.2", "model": "Pipecat · Whisper-tiny", "meta": "audio · prosody", "rec": [], "lead": true, "cells": { "acc": { "v": "94.0%", "s": 94, "tone": "good" }, "recall": { "v": "89.0%", "s": 89, "tone": "good" }, "cut": { "v": "1.0%", "s": 1, "tone": "good", "note": "One false cutoff in 100 incomplete clips." }, "lat": { "v": "32ms", "s": 32, "tone": "good", "ci": "49 p90", "note": "Runs in parallel with STT — no transcript wait." } } }, { "provider": "LiveKit Intl", "model": "turn-detector v0.4.1", "meta": "text · transcript", "rec": [], "cells": { "acc": { "v": "87.0%", "s": 87, "tone": "good" }, "recall": { "v": "86.0%", "s": 86, "tone": "good" }, "cut": { "v": "12.0%", "s": 12, "tone": "warn" }, "lat": { "v": "29ms", "s": 29, "tone": "good", "ci": "57 p90", "note": "Waits for the STT transcript (~0–125ms) before running." } } }, { "provider": "Turnsense", "model": "SmolLM2-135M", "meta": "text · transcript", "rec": [], "cells": { "acc": { "v": "86.0%", "s": 86, "tone": "good" }, "recall": { "v": "88.0%", "s": 88, "tone": "good" }, "cut": { "v": "16.0%", "s": 16, "tone": "warn" }, "lat": { "v": "127ms", "s": 127, "tone": "warn", "ci": "193 p90", "note": "Pads every input to 256 tokens regardless of length — slowest as-shipped." } } }, { "provider": "LiveKit EN", "model": "turn-detector v1.2.2", "meta": "text · transcript", "rec": [], "cells": { "acc": { "v": "82.0%", "s": 82, "tone": "warn" }, "recall": { "v": "87.0%", "s": 87, "tone": "good" }, "cut": { "v": "23.0%", "s": 23, "tone": "warn" }, "lat": { "v": "5ms", "s": 5, "tone": "good", "ci": "24 p90" } } }, { "provider": "VAD + silence timer", "model": "baseline", "meta": "audio · energy", "rec": [], "cells": { "acc": { "v": "46.9%", "s": 46.9, "tone": "bad", "dim": true }, "recall": { "v": "100%", "s": 100, "tone": "neutral", "dim": true }, "cut": { "v": "100%", "s": 100, "tone": "bad", "dim": true, "note": "A silence timer ends every turn — the floor semantic detection has to beat." }, "lat": { "v": "~1ms", "s": 1, "tone": "neutral", "dim": true } } } ], "reading": "" } ], "boardsByLanguage": { "stt": { "es": { "label": "Spanish", "metric": "WER", "meta": "FLEURS es_419 · n=50 · corpus WER · batch 2026-07-21 · streaming 2026-10-01 · n=908 streaming (full test split)", "path": "batch", "rows": [ { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 3.8, "ci": [ 2.4, 5.4 ], "errStream": 3.11, "ciStream": [ 2.6, 3.62 ], "clipsStream": 908, "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-4o-transcribe-es-fleurs-ab-es-test-p1.json", "runDateStream": "2026-10-01", "tone": "good" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "err": 4, "ci": [ 2.6, 5.6 ], "errStream": 2.8, "ciStream": [ 2.36, 3.27 ], "provenanceStream": "apps/benchmark-runner/results/fleurs-es-u35-u36.json", "clipsStream": 908, "runDateStream": "2026-09-30", "tone": "good", "runDate": "2026-08-17", "provenance": "apps/benchmark-runner/results/es-assemblyai-sync-fleurs50.json" }, { "provider": "AssemblyAI", "model": "Universal-3.6 Pro", "id": "assemblyai:universal-3-6-pro", "errStream": 2.53, "ciStream": [ 2.12, 2.93 ], "clipsStream": 908, "runDateStream": "2026-09-30", "provenanceStream": "apps/benchmark-runner/results/fleurs-es-u35-u36.json", "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 4.4, "ci": [ 2.9, 6.1 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 4.6, "ci": [ 2.9, 6.5 ], "errStream": 4.14, "ciStream": [ 3.5, 4.87 ], "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-4o-mini-transcribe-es-fleurs-ab-es-test-p1.json", "clipsStream": 908, "runDateStream": "2026-10-01", "tone": "good" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 3.12, "ciStream": [ 2.67, 3.59 ], "clipsStream": 908, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/nari-qwen3-asr-es-fleurs-ab-es-test-p1.json", "tone": "good" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 5.4, "ci": [ 3.8, 7 ], "errStream": 3.56, "ciStream": [ 3.08, 4.06 ], "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/google-chirp-3-es-fleurs-ab-es-test-p1.json", "clipsStream": 908, "runDateStream": "2026-10-01", "tone": "warn" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "errStream": 4.46, "ciStream": [ 3.92, 5.05 ], "clipsStream": 908, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/modulate-velma-2-stt-streaming-es-fleurs-ab-es-test-p1.json", "tone": "good" }, { "provider": "Gradium", "model": "Gradium ASR", "id": "gradium:default", "err": 6.7, "ci": [ 4.5, 9.2 ], "errStream": 8.5, "ciStream": [ 6.8, 10.5 ], "finalsPerClip": 19.6, "clipsStream": 49, "provenanceStream": "apps/benchmark-runner/results/es-stream-wer-50.json", "runDateStream": "2026-08-21", "tone": "warn", "runDate": "2026-07-31", "provenance": "apps/benchmark-runner/results/gradium-ml-es-NEW.json" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "errStream": 4.94, "ciStream": [ 4.13, 5.82 ], "clipsStream": 908, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/meta-muse-voice-transcribe-1.0-es-fleurs-ab-es-test-p1.json", "tone": "good" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 7.3, "ci": [ 5.4, 9.5 ], "errStream": 7.68, "ciStream": [ 7.06, 8.27 ], "clipsStream": 908, "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/cartesia-ink-whisper-es-fleurs-ab-es-test-p1.json", "runDateStream": "2026-10-01", "tone": "warn", "perMinUsd": 0.0021667 }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 7.3, "ci": [ 4, 10.5 ], "errStream": 3.62, "ciStream": [ 3.17, 4.13 ], "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/soniox-stt-rt-v5-es-fleurs-ab-es-test-p1.json", "clipsStream": 908, "runDateStream": "2026-10-01", "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 8.4, "ci": [ 6.1, 11 ], "errStream": 5.24, "ciStream": [ 4.59, 5.85 ], "clipsStream": 908, "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/deepgram-nova-3-es-fleurs-ab-es-test-p1.json", "runDateStream": "2026-10-01", "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 8.4, "ci": [ 6.1, 11.1 ], "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-2", "id": "deepgram:nova-2", "err": 11.3, "ci": [ 7.8, 15.7 ], "errStream": 9.06, "ciStream": [ 8.07, 10.06 ], "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/deepgram-nova-2-es-fleurs-ab-es-test-p1.json", "clipsStream": 908, "runDateStream": "2026-10-01", "tone": "bad", "perMinUsd": 0.0058333 }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 3.02, "ciStream": [ 2.62, 3.42 ], "clipsStream": 908, "runDateStream": "2026-10-01", "tone": "good", "runDate": "2026-08-21", "provenance": "apps/benchmark-runner/results/es-stream-wer-50.json", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/elevenlabs-scribe-v2-realtime-es-fleurs-ab-es-test-p1.json" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 6.6, "ciStream": [ 3.4, 11.6 ], "finalsPerClip": 1.1, "runDateStream": "2026-08-21", "tone": "warn", "runDate": "2026-08-21", "provenance": "apps/benchmark-runner/results/es-stream-wer-50.json", "provenanceStream": "apps/benchmark-runner/results/es-stream-wer-50.json" }, { "provider": "Smallest AI", "model": "Pulse", "id": "smallest:pulse", "errStream": 5.35, "ciStream": [ 4.68, 6.05 ], "clipsStream": 908, "runDateStream": "2026-10-01", "tone": "bad", "runDate": "2026-08-21", "provenance": "apps/benchmark-runner/results/es-stream-wer-50.json", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/smallest-pulse-es-fleurs-ab-es-test-p1.json" } ] }, "de": { "label": "German", "metric": "WER", "meta": "FLEURS de_de · n=50 · corpus WER · batch 2026-07-21 · streaming 2026-10-01 · n=862 streaming (full test split)", "path": "batch", "rows": [ { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 2.1, "ci": [ 1.2, 3 ], "errStream": 4.5, "ciStream": [ 3.94, 5.12 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-4o-transcribe-de-fleurs-ab-de-test-p1.json", "tone": "good" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "err": 2.6, "ci": [ 1.5, 3.9 ], "errStream": 3.61, "ciStream": [ 3.13, 4.11 ], "clipsStream": 862, "runDateStream": "2026-09-30", "provenanceStream": "apps/benchmark-runner/results/fleurs-de-u35-u36.json", "tone": "good", "runDate": "2026-08-17", "provenance": "apps/benchmark-runner/results/de-assemblyai-sync-fleurs50.json" }, { "provider": "AssemblyAI", "model": "Universal-3.6 Pro", "id": "assemblyai:universal-3-6-pro", "errStream": 3.11, "ciStream": [ 2.68, 3.57 ], "clipsStream": 862, "runDateStream": "2026-09-30", "provenanceStream": "apps/benchmark-runner/results/fleurs-de-u35-u36.json", "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 3, "ci": [ 1.9, 4.3 ], "tone": "good" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "errStream": 4.61, "ciStream": [ 3.99, 5.23 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/meta-muse-voice-transcribe-1.0-de-fleurs-ab-de-test-p1.json", "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 4.1, "ci": [ 2.5, 5.9 ], "errStream": 6.31, "ciStream": [ 5.64, 7.04 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-4o-mini-transcribe-de-fleurs-ab-de-test-p1.json", "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 4.07, "ciStream": [ 3.54, 4.63 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/elevenlabs-scribe-v2-realtime-de-fleurs-ab-de-test-p1.json", "tone": "good" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 5, "ci": [ 3.5, 6.6 ], "errStream": 5.27, "ciStream": [ 4.68, 5.89 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/google-chirp-3-de-fleurs-ab-de-test-p1.json", "tone": "warn" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "errStream": 4.49, "ciStream": [ 3.97, 5.08 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/modulate-velma-2-stt-streaming-de-fleurs-ab-de-test-p1.json", "tone": "good" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 4.58, "ciStream": [ 4.05, 5.17 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/nari-qwen3-asr-de-fleurs-ab-de-test-p1.json", "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 5.5, "ciStream": [ 3.2, 8.1 ], "finalsPerClip": 1.3, "runDateStream": "2026-07-21", "tone": "warn" }, { "provider": "Gradium", "model": "Gradium ASR", "id": "gradium:default", "err": 5.7, "ci": [ 4, 7.4 ], "errStream": 9.3, "ciStream": [ 7.6, 11.3 ], "finalsPerClip": 17.8, "runDateStream": "2026-07-21", "tone": "warn", "runDate": "2026-07-31", "provenance": "apps/benchmark-runner/results/gradium-ml-de-NEW.json" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 7.4, "ci": [ 5.4, 9.4 ], "errStream": 11.03, "ciStream": [ 10.32, 11.73 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/cartesia-ink-whisper-de-fleurs-ab-de-test-p1.json", "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse", "id": "smallest:pulse", "errStream": 9.73, "ciStream": [ 8.96, 10.58 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/smallest-pulse-de-fleurs-ab-de-test-p1.json", "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 9.2, "ci": [ 6.7, 11.6 ], "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 11.2, "ci": [ 8.6, 14 ], "errStream": 10.22, "ciStream": [ 9.34, 11.18 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/deepgram-nova-3-de-fleurs-ab-de-test-p1.json", "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-2", "id": "deepgram:nova-2", "err": 12, "ci": [ 9.4, 14.8 ], "errStream": 10.97, "ciStream": [ 10, 11.98 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/deepgram-nova-2-de-fleurs-ab-de-test-p1.json", "tone": "bad" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 13.4, "ci": [ 6.7, 21.1 ], "errStream": 4.99, "ciStream": [ 4.42, 5.61 ], "clipsStream": 862, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/soniox-stt-rt-v5-de-fleurs-ab-de-test-p1.json", "tone": "bad" } ] }, "fr": { "label": "French", "metric": "WER", "meta": "FLEURS fr_fr · n=50 · corpus WER · batch 2026-07-21 · streaming 2026-10-01 · n=676 streaming (full test split)", "path": "batch", "rows": [ { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "err": 3.3, "ci": [ 1.9, 4.9 ], "errStream": 3.88, "ciStream": [ 3.29, 4.45 ], "clipsStream": 676, "runDateStream": "2026-09-30", "tone": "good", "runDate": "2026-08-17", "provenance": "apps/benchmark-runner/results/fr-assemblyai-sync-fleurs50.json", "provenanceStream": "apps/benchmark-runner/results/fleurs-fr-u35-u36.json" }, { "provider": "AssemblyAI", "model": "Universal-3.6 Pro", "id": "assemblyai:universal-3-6-pro", "errStream": 3.55, "ciStream": [ 3.05, 4.11 ], "clipsStream": 676, "runDateStream": "2026-09-30", "provenanceStream": "apps/benchmark-runner/results/fleurs-fr-u35-u36.json", "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 4, "ci": [ 2.7, 5.4 ], "errStream": 6.1, "ciStream": [ 5.32, 6.91 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "good", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-4o-transcribe-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 4.1, "ciStream": [ 2.3, 6.3 ], "finalsPerClip": 0.9, "clipsStream": 47, "runDateStream": "2026-08-21", "tone": "good", "provenanceStream": "apps/benchmark-runner/results/fr-stream-wer-50.json" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 4.4, "ci": [ 2.7, 6.5 ], "tone": "good" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 4.84, "ciStream": [ 4.21, 5.44 ], "clipsStream": 676, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/nari-qwen3-asr-fr-fleurs-ab-fr-test-p1.json", "tone": "good" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "errStream": 5.86, "ciStream": [ 5.19, 6.59 ], "clipsStream": 676, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/modulate-velma-2-stt-streaming-fr-fleurs-ab-fr-test-p1.json", "tone": "good" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "errStream": 6.79, "ciStream": [ 6.05, 7.53 ], "clipsStream": 676, "runDateStream": "2026-10-01", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/meta-muse-voice-transcribe-1.0-fr-fleurs-ab-fr-test-p1.json", "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 5.14, "ciStream": [ 4.59, 5.72 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "warn", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/elevenlabs-scribe-v2-realtime-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 6.8, "ci": [ 4.6, 9.5 ], "errStream": 8.46, "ciStream": [ 7.45, 9.54 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "warn", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-4o-mini-transcribe-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "id": "openai:gpt-live-transcribe", "errStream": 6.85, "ciStream": [ 6.05, 7.67 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "warn", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/openai-gpt-live-transcribe-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "errStream": 7.45, "ciStream": [ 6.6, 8.37 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "warn", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/google-chirp-3-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 8.6, "ci": [ 5.6, 12.3 ], "errStream": 6.28, "ciStream": [ 5.61, 7.08 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "warn", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/soniox-stt-rt-v5-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 9.9, "ci": [ 7.9, 12.1 ], "errStream": 10.49, "ciStream": [ 9.64, 11.34 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "warn", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/deepgram-nova-3-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Gradium", "model": "Gradium ASR", "id": "gradium:default", "err": 10, "ci": [ 7.3, 12.7 ], "errStream": 11.8, "ciStream": [ 9.6, 14.7 ], "finalsPerClip": 16.4, "runDateStream": "2026-08-21", "tone": "warn", "runDate": "2026-07-31", "provenanceStream": "apps/benchmark-runner/results/fr-stream-wer-50.json", "provenance": "apps/benchmark-runner/results/gradium-ml-fr-NEW.json" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 10.4, "ci": [ 7.2, 14.2 ], "errStream": 16.69, "ciStream": [ 15.27, 18.18 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "bad", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/cartesia-ink-whisper-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Smallest AI", "model": "Pulse", "id": "smallest:pulse", "errStream": 11.28, "ciStream": [ 10.3, 12.3 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "bad", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/smallest-pulse-fr-fleurs-ab-fr-test-p1.json" }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 11.8, "ci": [ 8.5, 15.8 ], "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-2", "id": "deepgram:nova-2", "err": 12.9, "ci": [ 10.1, 15.6 ], "errStream": 13.18, "ciStream": [ 12.19, 14.13 ], "clipsStream": 676, "runDateStream": "2026-10-01", "tone": "bad", "provenanceStream": "gs://speko-bench-stt-ab/ab1/out/jobs/deepgram-nova-2-fr-fleurs-ab-fr-test-p1.json" } ] }, "ar": { "label": "Arabic", "metric": "CER", "meta": "FLEURS ar · n=50 batch, n=47 streaming (paired across rows) · corpus CER (diacritic-normalized) · batch 2026-07-21 · streaming 2026-08-21", "path": "batch", "rows": [ { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 2.7, "ci": [ 2.3, 3.2 ], "errStream": 3.3, "ciStream": [ 2.4, 4.6 ], "finalsPerClip": 1.3, "tone": "good" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "err": 3.3, "ci": [ 2.5, 4.2 ], "errStream": 3.5, "ciStream": [ 2.4, 4.9 ], "finalsPerClip": 1, "tone": "good", "runDate": "2026-08-17", "provenance": "apps/benchmark-runner/results/ar-assemblyai-sync-fleurs50.json" }, { "provider": "Munsit", "model": "munsit", "id": "munsit:munsit", "err": 3.5, "ci": [ 1.8, 5.7 ], "errStream": 2.7, "ciStream": [ 1.2, 4.8 ], "clipsStream": 47, "tone": "good", "runDate": "2026-10-08", "runDateStream": "2026-10-08", "provenance": "apps/benchmark-runner/results/ar-hamsa-munsit-20261008/stt-ar-50.json", "provenanceStream": "apps/benchmark-runner/results/ar-hamsa-munsit-20261008/stt-ar-50.json", "perMinUsd": 0.005 }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 3.5, "ci": [ 2.7, 4.5 ], "tone": "warn" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "errStream": 3.6, "ciStream": [ 2.1, 5.8 ], "clipsStream": 47, "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/multilingual-direct/2026-09-29-fr-ar-es-de/ml-stt-meta-nari-ar-20260929.json", "tone": "good" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 3.9, "ciStream": [ 3.1, 4.9 ], "clipsStream": 47, "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/multilingual-direct/2026-09-29-fr-ar-es-de/ml-stt-meta-nari-ar-20260929.json", "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 4.1, "ciStream": [ 3.1, 5.4 ], "finalsPerClip": 1.1, "tone": "warn", "runDate": "2026-08-21", "provenance": "apps/benchmark-runner/results/ar-stream-cer-50.json" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 4.2, "ciStream": [ 3.3, 5.2 ], "finalsPerClip": 1, "tone": "warn", "runDate": "2026-08-21", "provenance": "apps/benchmark-runner/results/ar-stream-cer-50.json" }, { "provider": "xAI", "model": "Grok STT", "id": "xai:stt", "errStream": 4.7, "ciStream": [ 2.5, 8.1 ], "finalsPerClip": 1.1, "tone": "warn", "runDate": "2026-08-21", "provenance": "apps/benchmark-runner/results/ar-stream-cer-50.json" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 4.7, "ci": [ 3.1, 6.8 ], "errStream": 4.9, "ciStream": [ 3.4, 7 ], "finalsPerClip": 1.3, "tone": "warn" }, { "provider": "Hamsa", "model": "S3", "id": "hamsa:s3", "err": 5, "ci": [ 3.5, 6.9 ], "tone": "warn", "runDate": "2026-08-16", "provenance": "apps/benchmark-runner/results/hamsa-ml-ar.json", "turnBased": true, "finalizeMs": 390, "finalizeBasis": { "surface": "speko-gateway", "vantage": "developer machine (~100ms of client RTT above the US East convention)", "measured": "2026-08-16", "n": 20, "note": "Warm-socket per-turn finalization p50 (p95 534ms), Arabic, run speko-stt-hamsa-s3-ar-2026-08-16. NOT the streaming endpoint finalize the vendor-direct sweep publishes, and the difference is the quantity rather than the route: Hamsa is turn-based whole-utterance, so the worker VAD-segments the call and hands over a complete turn, and this interval EXCLUDES the endpoint-detection wait every streaming row's finalize includes. The vendor-direct sweep refuses this vendor class for that reason, so no swept reading will replace this one." }, "perMinUsd": 0.15 }, { "provider": "Munsit", "model": "munsit-en-ar", "id": "munsit:munsit-en-ar", "err": 5.4, "ci": [ 3.6, 7.6 ], "errStream": 4.9, "ciStream": [ 3.3, 7 ], "clipsStream": 47, "tone": "warn", "runDate": "2026-10-08", "runDateStream": "2026-10-08", "provenance": "apps/benchmark-runner/results/ar-hamsa-munsit-20261008/stt-ar-50.json", "provenanceStream": "apps/benchmark-runner/results/ar-hamsa-munsit-20261008/stt-ar-50.json", "perMinUsd": 0.005 }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "errStream": 5, "ciStream": [ 2.8, 8.1 ], "clipsStream": 47, "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/multilingual-direct/2026-09-29-fr-ar-es-de/ml-stt-velma-soniox-ar-20260929.json", "tone": "good" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 5.1, "ci": [ 3.2, 8.3 ], "errStream": 3.6, "ciStream": [ 2.9, 4.4 ], "finalsPerClip": 1, "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 7.1, "ci": [ 6, 8.4 ], "errStream": 7.6, "ciStream": [ 6.1, 9.6 ], "finalsPerClip": 3.1, "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 10.2, "ci": [ 8.4, 12 ], "errStream": 10.1, "ciStream": [ 8.9, 11.5 ], "finalsPerClip": 3.1, "tone": "bad" } ] }, "fil": { "label": "Filipino", "metric": "WER", "meta": "FLEURS fil_ph · n=50 · corpus WER · 2026-08-21", "path": "batch", "rows": [ { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 6.1, "ci": [ 4.6, 8 ], "errStream": 12.3, "ciStream": [ 9.3, 16.3 ], "finalsPerClip": 3.9, "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 7.2, "ci": [ 5.7, 8.7 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 9.5, "ci": [ 6.6, 12.8 ], "errStream": 13.4, "ciStream": [ 10.2, 17 ], "finalsPerClip": 4.2, "tone": "good" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "errStream": 9.7, "ciStream": [ 6.9, 12.7 ], "clipsStream": 50, "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/meta-direct/2026-09-29-meta-muse-th-fil/ml-stt-meta-fil-20260929.json", "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 10.8, "ci": [ 8.6, 13.5 ], "errStream": 17.5, "ciStream": [ 13.6, 22.4 ], "finalsPerClip": 4.3, "tone": "warn" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 11.7, "ciStream": [ 9, 15.2 ], "finalsPerClip": 1, "tone": "warn" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "id": "openai:gpt-live-transcribe", "errStream": 12.5, "ciStream": [ 9.3, 16.5 ], "finalsPerClip": 1, "tone": "warn" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "err": 12.5, "ci": [ 8.8, 16.5 ], "errStream": 6.2, "ciStream": [ 4.7, 8 ], "finalsPerClip": 1.2, "tone": "warn" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 18.8, "ci": [ 15, 23.3 ], "tone": "warn" }, { "provider": "Alibaba", "model": "Qwen-ASR", "id": "alibaba:qwen-asr", "err": 18.8, "ci": [ 15, 23.3 ], "tone": "warn" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 20.7, "ci": [ 16.3, 26.2 ], "errStream": 17.2, "ciStream": [ 14.3, 20.7 ], "finalsPerClip": 3.4, "tone": "bad" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 21.9, "ci": [ 15.4, 28.5 ], "errStream": 8.9, "ciStream": [ 6.3, 12.2 ], "finalsPerClip": 1, "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 25.6, "ci": [ 21.9, 29.5 ], "errStream": 24.5, "ciStream": [ 21.1, 28.5 ], "finalsPerClip": 5.5, "tone": "bad" }, { "provider": "Nari", "model": "Qwen3-ASR Fast", "id": "nari:qwen3-asr-fast", "errStream": 26.4, "ciStream": [ 22.3, 30.8 ], "tone": "bad" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 26.7, "ciStream": [ 22.1, 31 ], "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/nari-direct/2026-09-29-nari-qwen3-asr-th-fil/ml-stt-nari-fil-20260929.json", "tone": "bad" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 27.3, "ci": [ 20.2, 35.4 ], "errStream": 26.6, "ciStream": [ 19.4, 34.3 ], "finalsPerClip": 4.8, "tone": "bad" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "id": "alibaba:qwen3-asr-flash-realtime", "errStream": 29.7, "ciStream": [ 23.9, 35.8 ], "finalsPerClip": 2.1, "latencyKey": "alibaba:qwen3-asr-flash-realtime", "tone": "warn" } ] }, "nb": { "label": "Norwegian", "metric": "WER", "meta": "FLEURS nb_no · n=50 · corpus WER · batch 2026-08-21 · streaming 2026-07-21", "path": "batch", "rows": [ { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 5.6, "ci": [ 3.8, 7.5 ], "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v1", "id": "elevenlabs:scribe_v1", "err": 5.7, "ci": [ 3.8, 7.7 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 6.4, "ci": [ 4.6, 8.3 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 6.8, "ci": [ 5, 8.6 ], "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 8.3, "ciStream": [ 6.3, 10.5 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "err": 8.4, "ci": [ 6.3, 10.7 ], "errStream": 13, "ciStream": [ 11, 14.8 ], "finalsPerClip": 1, "runDateStream": "2026-08-22", "provenanceStream": "apps/benchmark-runner/results/nb-stream-wer-modulate50.json", "finalizeMs": 577, "finalizeBasis": { "surface": "speko-gateway", "vantage": "US East, 11 simultaneous sockets", "measured": "2026-08-21", "n": 30, "p90Ms": 1237, "note": "Norwegian finalize p50 from the 2026-08-21 nb endpointing job, standing in for the whole model — the nb reading rather than the Filipino 402ms because fil is this board's one VAD-marker column, so not comparable to a vendor-direct sweep row. Read it against the per-language cells, which carry each language's OWN finalize (fil 402, ja 1428, nb 577): this model-level number is the nb one, and Japanese is 2.5x slower than it. Sweepable, unlike Hamsa — the english-v2 sibling is already in the sweep at 1207ms, so adding this pin retires the exception." }, "perMinUsd": 0.001, "tone": "good" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 8.6, "ci": [ 5.7, 12.6 ], "errStream": 8, "ciStream": [ 6.2, 10 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "err": 10.1, "ci": [ 8, 12.3 ], "errStream": 12.3, "ciStream": [ 9.1, 16 ], "finalsPerClip": 1, "tone": "warn", "runDate": "2026-08-21", "provenanceStream": "apps/benchmark-runner/results/nb-assemblyai-u35pro-stream50.json", "runDateStream": "2026-08-17", "provenance": "apps/benchmark-runner/results/nb-batch-fleurs50.json" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 11.2, "ci": [ 8.8, 13.8 ], "tone": "warn", "provenance": "apps/benchmark-runner/results/nb-batch-chirp3-fleurs50.json" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 12, "ci": [ 9.6, 14.6 ], "tone": "warn" }, { "provider": "Alibaba", "model": "Qwen-ASR", "id": "alibaba:qwen-asr", "err": 12, "ci": [ 9.6, 14.6 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 14.4, "ci": [ 10.5, 18.3 ], "tone": "bad" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 14.5, "ciStream": [ 11.8, 17.7 ], "finalsPerClip": 1, "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 14.5, "ci": [ 12.2, 17 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/nb-batch-cartesia-ink-whisper-fleurs50.json" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 14.6, "ci": [ 12.2, 17.2 ], "errStream": 14.9, "ciStream": [ 12.3, 17.5 ], "clipsStream": 45, "finalsPerClip": 3.2, "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-2", "id": "deepgram:nova-2", "err": 19.1, "ci": [ 16.4, 22.1 ], "errStream": 18.6, "ciStream": [ 15.4, 22.1 ], "clipsStream": 48, "finalsPerClip": 3.1, "tone": "bad" } ] }, "hi": { "label": "Hindi", "metric": "WER", "meta": "FLEURS hi_in · n=50 · corpus WER · 2026-08-04", "path": "batch", "rows": [ { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 6.1, "ci": [ 4.4, 8.1 ], "tone": "good", "provenance": "apps/benchmark-runner/results/hi-fleurs50-chirp-eu.json", "runDate": "2026-08-04" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 9.8, "ci": [ 7, 12.9 ], "tone": "good" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 12.3, "ci": [ 8, 17.5 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 12.7, "ci": [ 10, 15.9 ], "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 13.4, "ci": [ 10.5, 16.5 ], "tone": "warn" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "id": "assemblyai:universal-3-5-pro", "err": 17.2, "ci": [ 14.6, 20.3 ], "tone": "warn", "runDate": "2026-08-17", "provenance": "apps/benchmark-runner/results/hi-assemblyai-sync-fleurs50.json" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 20.3, "ci": [ 17.1, 23.8 ], "tone": "bad" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 21, "ci": [ 13, 30.9 ], "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-2", "id": "deepgram:nova-2", "err": 25.1, "ci": [ 21.2, 28.9 ], "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 41.9, "ci": [ 37.3, 46.9 ], "tone": "bad" } ] }, "ta": { "label": "Tamil", "metric": "CER", "meta": "FLEURS ta_in | target n=50 | 1 invalid clip excluded | corpus CER | 2026-08-04", "path": "batch", "rows": [ { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 7.4, "ci": [ 4.3, 13 ], "tone": "warn", "clips": 48, "comparison": { "metric": "WER", "err": 24.3 }, "provenance": "apps/benchmark-runner/results/ta-fleurs50-chirp-eu.json", "runDate": "2026-08-04" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 10.1, "ci": [ 8.1, 12.7 ], "tone": "bad", "clips": 49, "comparison": { "metric": "WER", "err": 33.2 } }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 12.1, "ci": [ 6.2, 19.3 ], "tone": "bad", "clips": 48, "comparison": { "metric": "WER", "err": 26.4 } }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 14.2, "ci": [ 9.9, 19.8 ], "tone": "bad", "clips": 49, "comparison": { "metric": "WER", "err": 36.9 } }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 14.6, "ci": [ 8.7, 20.2 ], "tone": "bad", "clips": 49, "comparison": { "metric": "WER", "err": 31.3 } }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 16.1, "ci": [ 12.1, 21.7 ], "tone": "bad", "clips": 49, "comparison": { "metric": "WER", "err": 40.3 } }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 30.6, "ci": [ 24.7, 37.4 ], "tone": "bad", "clips": 49, "comparison": { "metric": "WER", "err": 78.8 } } ] }, "te": { "label": "Telugu", "metric": "CER", "meta": "FLEURS te_in · n=50 · corpus CER · 2026-08-04", "path": "batch", "rows": [ { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 3.2, "ci": [ 2, 4.8 ], "tone": "good", "provenance": "apps/benchmark-runner/results/te-fleurs50-chirp-eu.json", "runDate": "2026-08-04" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 7.5, "ci": [ 5.6, 9.7 ], "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 8.5, "ci": [ 5.2, 13.6 ], "tone": "warn" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 9.8, "ci": [ 4.6, 15.9 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 10.2, "ci": [ 6.9, 14.9 ], "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 13.4, "ci": [ 9.4, 18.4 ], "tone": "bad" } ] }, "ko": { "label": "Korean", "metric": "CER", "meta": "FLEURS ko_kr · live streaming sockets · n=50, CER paired over 48 · Finalize + TTFT from US East · 2026-08-24", "path": "stream", "rows": [ { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "errStream": 4.2, "ciStream": [ 2.4, 6.2 ], "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 4.5, "ciStream": [ 2.6, 6.7 ], "tone": "good" }, { "provider": "xAI", "model": "Grok STT", "latencyKey": "xai:stt", "errStream": 4.7, "ciStream": [ 2.7, 7 ], "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 4.7, "ciStream": [ 2.5, 7.4 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "errStream": 4.9, "ciStream": [ 2.8, 7.2 ], "tone": "good" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "errStream": 5.9, "ciStream": [ 3.8, 8.2 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "errStream": 6.4, "ciStream": [ 3.9, 9.6 ], "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "errStream": 11.6, "ciStream": [ 8.5, 15.2 ], "tone": "bad" } ] }, "zh": { "label": "Chinese (Mandarin)", "metric": "CER", "meta": "FLEURS cmn_hans_cn (validation split) · live streaming sockets · n=50, CER paired over 47 · Finalize + TTFT from US East · 2026-08-24", "path": "stream", "rows": [ { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "errStream": 5.5, "ciStream": [ 3.2, 8 ], "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "latencyKey": "alibaba:qwen3-asr-flash-realtime", "errStream": 6, "ciStream": [ 3.5, 8.8 ], "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 6.2, "ciStream": [ 3.9, 8.8 ], "tone": "good" }, { "provider": "xAI", "model": "Grok STT", "latencyKey": "xai:stt", "errStream": 6.7, "ciStream": [ 4.1, 9.5 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "errStream": 8.3, "ciStream": [ 5.5, 11.1 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "errStream": 9.1, "ciStream": [ 6.2, 12.3 ], "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "errStream": 9.3, "ciStream": [ 6.8, 12.1 ], "tone": "warn" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "errStream": 10.3, "ciStream": [ 7.5, 13.3 ], "tone": "warn" } ] }, "ja": { "label": "Japanese", "metric": "CER", "meta": "FLEURS ja_jp · n=50 · pooled CER over hiragana yomi · batch + streaming · Finalize + TTFT from US East (Virginia) · 2026-09-02", "path": "batch", "rows": [ { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 1.6, "ci": [ 0.9, 2.5 ], "errStream": 5.2, "ciStream": [ 2.7, 8.7 ], "finalsPerClip": 2.4, "clipsStream": 49, "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 2, "ci": [ 1, 2.9 ], "tone": "good" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 2, "ci": [ 1.1, 2.8 ], "errStream": 1.8, "ciStream": [ 1.1, 2.5 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v1", "id": "elevenlabs:scribe_v1", "err": 2.1, "ci": [ 1.1, 3 ], "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 2.1, "ci": [ 1.1, 3.3 ], "errStream": 3.7, "ciStream": [ 2.1, 5.8 ], "finalsPerClip": 1.4, "clipsStream": 49, "tone": "good" }, { "provider": "Alibaba", "model": "Qwen-ASR", "id": "alibaba:qwen-asr", "err": 2.1, "ci": [ 1.1, 3.3 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 2.8, "ci": [ 1.7, 4 ], "errStream": 3.7, "ciStream": [ 2.4, 5.3 ], "finalsPerClip": 2.6, "clipsStream": 47, "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 2.8, "ci": [ 1.7, 4 ], "errStream": 4.5, "ciStream": [ 3.1, 6 ], "finalsPerClip": 2.5, "clipsStream": 46, "tone": "warn" }, { "provider": "AssemblyAI", "model": "Universal-3.5 Pro", "latencyKey": "assemblyai:universal-3-5-pro", "errStream": 3.4, "ciStream": [ 2.1, 4.9 ], "finalsPerClip": 1.3, "tone": "warn" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 4.4, "ciStream": [ 2.6, 7.1 ], "finalsPerClip": 1.1, "tone": "warn" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "err": 4.8, "ci": [ 3.4, 6.4 ], "errStream": 4.8, "ciStream": [ 3.5, 6.5 ], "finalsPerClip": 1.1, "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse Pro", "id": "smallest:pulse-pro", "err": 5.5, "ci": [ 3.7, 7.8 ], "tone": "warn" }, { "provider": "Smallest AI", "model": "Pulse", "latencyKey": "smallest:pulse", "errStream": 6.1, "ciStream": [ 4.1, 8.4 ], "finalsPerClip": 1.6, "tone": "warn" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "errStream": 6.7, "ciStream": [ 3.1, 11 ], "finalsPerClip": 0.3, "clipsStream": 14, "tone": "bad" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "err": 11.5, "ci": [ 7.6, 15.9 ], "errStream": 17.6, "ciStream": [ 13.1, 22.5 ], "finalsPerClip": 2.5, "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 20.7, "ci": [ 17.1, 24.5 ], "errStream": 22, "ciStream": [ 18, 26 ], "finalsPerClip": 6.3, "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 30.3, "ci": [ 21.9, 39 ], "errStream": 35, "ciStream": [ 25.1, 46.6 ], "finalsPerClip": 1.5, "clipsStream": 46, "tone": "bad" }, { "provider": "xAI", "model": "Grok STT", "id": "xai:stt", "err": 42.9, "ci": [ 31.5, 54.8 ], "errStream": 34.6, "ciStream": [ 25.3, 44.8 ], "finalsPerClip": 1.5, "clipsStream": 48, "tone": "bad" } ] }, "th": { "label": "Thai", "metric": "CER", "meta": "FLEURS th_th · batch + streaming · n=50 · pooled CER-th · n=50 streaming · Finalize + TTFT from US East (Virginia) and Singapore · batch 2026-09-09 · streaming 2026-09-09", "path": "batch", "rows": [ { "provider": "Alibaba", "model": "Qwen-ASR", "id": "alibaba:qwen-asr", "err": 3.5, "ci": [ 2.3, 4.8 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 3.5, "ci": [ 2.4, 5 ], "errStream": 4.3, "ciStream": [ 2.9, 6 ], "finalsPerClip": 2.4, "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 3.5, "ci": [ 2.4, 4.9 ], "tone": "good" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "errStream": 3.8, "ciStream": [ 2.7, 5.2 ], "clipsStream": 48, "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/meta-direct/2026-09-29-meta-muse-th-fil/ml-stt-meta-th-20260929-retry.json", "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 3.9, "ci": [ 2.5, 5.7 ], "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v1", "id": "elevenlabs:scribe_v1", "err": 4.3, "ci": [ 2.8, 6.1 ], "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "id": "alibaba:qwen3-asr-flash-realtime", "errStream": 4.4, "ciStream": [ 3.1, 6.1 ], "finalsPerClip": 1.2, "tone": "good" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "id": "openai:gpt-live-transcribe", "errStream": 4.5, "ciStream": [ 3.1, 6.2 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 4.7, "ci": [ 3.2, 6.7 ], "errStream": 4.7, "ciStream": [ 3.2, 6.8 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 4.9, "ciStream": [ 3.3, 7 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "err": 5.1, "ci": [ 3.2, 7.3 ], "errStream": 3.9, "ciStream": [ 2.6, 5.2 ], "finalsPerClip": 1, "tone": "warn" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 5.1, "ciStream": [ 3.6, 7.1 ], "runDateStream": "2026-09-29", "provenanceStream": "gs://speko-benchmark-logs/stt/nari-direct/2026-09-29-nari-qwen3-asr-th-fil/ml-stt-nari-th-20260929.json", "tone": "warn" }, { "provider": "Nari", "model": "Qwen3-ASR Fast", "id": "nari:qwen3-asr-fast", "errStream": 5.2, "ciStream": [ 3.6, 7.2 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 5.2, "ci": [ 3.9, 6.8 ], "errStream": 6.9, "ciStream": [ 5.4, 8.6 ], "finalsPerClip": 2.4, "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 6.1, "ci": [ 3.2, 10.2 ], "errStream": 5.7, "ciStream": [ 3.9, 7.8 ], "finalsPerClip": 2.4, "tone": "warn" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 6.4, "ci": [ 4.4, 9 ], "errStream": 8.2, "ciStream": [ 5, 12.8 ], "finalsPerClip": 3.3, "tone": "warn" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "err": 7.9, "ci": [ 5.9, 10.4 ], "errStream": 9.2, "ciStream": [ 6.9, 12.5 ], "finalsPerClip": 2.5, "tone": "warn" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 8.4, "ci": [ 7.1, 10 ], "errStream": 8, "ciStream": [ 6.5, 9.8 ], "finalsPerClip": 3.6, "tone": "warn" }, { "provider": "AssemblyAI", "model": "Universal (batch tier)", "err": 11.2, "ci": [ 8.8, 14.1 ], "tone": "bad" }, { "provider": "xAI", "model": "Grok STT", "id": "xai:stt", "err": 16.6, "ci": [ 10, 23.7 ], "errStream": 9.8, "ciStream": [ 5.6, 15.3 ], "finalsPerClip": 1.2, "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 24.1, "ci": [ 21.7, 26.9 ], "errStream": 25.1, "ciStream": [ 22.9, 27.4 ], "finalsPerClip": 2.6, "tone": "bad" } ] }, "lt": { "label": "Lithuanian", "metric": "WER", "meta": "FLEURS lt_lt · n=50 · corpus WER · batch + streaming (1.0x pacing) · 2026-10-02", "path": "batch", "rows": [ { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 4.9, "ci": [ 3, 7 ], "tone": "good", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct2.json" }, { "provider": "ElevenLabs", "model": "Scribe v1", "id": "elevenlabs:scribe_v1", "err": 5.6, "ci": [ 3.6, 7.9 ], "tone": "good", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct.json" }, { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 5.8, "ci": [ 3.9, 8.2 ], "tone": "good" }, { "provider": "Google", "model": "Gemini 3.5 Transcribe", "id": "gemini:gemini-3.5-transcribe", "err": 6, "ci": [ 3.9, 8.1 ], "tone": "good" }, { "provider": "Azure", "model": "MAI-Transcribe-2", "id": "azure:MAI-Transcribe-2", "err": 6.1, "ci": [ 4.1, 8.3 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 8.1, "ci": [ 5.7, 10.8 ], "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 9, "ci": [ 6.2, 12.2 ], "tone": "warn" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 10.4, "ci": [ 5.9, 16.1 ], "errStream": 11.1, "ciStream": [ 6.3, 17.1 ], "finalsPerClip": 1.1, "tone": "warn" }, { "provider": "Azure", "model": "Fast Transcription", "id": "azure:fast-transcription", "err": 10.6, "ci": [ 7.9, 13.6 ], "tone": "warn", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-azure-fast-ltLT.json" }, { "provider": "OpenAI", "model": "GPT Realtime Whisper", "errStream": 11.3, "ciStream": [ 8.8, 14.4 ], "tone": "warn", "runDateStream": "2026-10-02", "provenanceStream": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct3.json" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "id": "openai:gpt-live-transcribe", "errStream": 11.7, "ciStream": [ 9, 14.6 ], "finalsPerClip": 1, "tone": "warn" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 12, "ciStream": [ 8.9, 15.3 ], "finalsPerClip": 1, "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 13.4, "ci": [ 10.4, 16.7 ], "tone": "warn" }, { "provider": "xAI", "model": "Grok STT", "id": "xai:stt", "err": 20.8, "ci": [ 17.1, 24.5 ], "errStream": 24.6, "ciStream": [ 20.4, 29.3 ], "finalsPerClip": 1, "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 23.7, "ci": [ 20.2, 27.4 ], "errStream": 21.8, "ciStream": [ 17.8, 25.7 ], "finalsPerClip": 3.3, "tone": "bad" }, { "provider": "Groq", "model": "Whisper Large v3", "err": 24.4, "ci": [ 20.9, 27.9 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct.json" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "err": 24.9, "ci": [ 21.4, 28.6 ], "errStream": 8.4, "ciStream": [ 5.1, 12.2 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct3.json", "provenanceStream": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct2.json", "runDateStream": "2026-10-02" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "err": 25.8, "ci": [ 22.1, 29.8 ], "errStream": 35.4, "ciStream": [ 31.1, 39.8 ], "finalsPerClip": 2.5, "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 27.5, "ci": [ 24.2, 30.6 ], "errStream": 36.1, "ciStream": [ 32.5, 39.8 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct2.json", "provenanceStream": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct2.json", "runDateStream": "2026-10-02" }, { "provider": "AssemblyAI", "model": "Universal-2", "err": 28.4, "ci": [ 24.9, 31.8 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct.json" }, { "provider": "OpenAI", "model": "Whisper-1", "id": "openai:whisper-1", "err": 28.7, "ci": [ 24.8, 32.6 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/lt-20261002/lt-batch-direct.json" } ] }, "id": { "label": "Indonesian", "metric": "WER", "meta": "FLEURS id_id · n=50 · corpus WER · batch + streaming (1.0x pacing) · 2026-10-09", "path": "batch", "rows": [ { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 1.7, "ci": [ 0.6, 3.1 ], "tone": "good" }, { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 1.8, "ci": [ 0.8, 2.9 ], "tone": "good", "provenance": "apps/benchmark-runner/results/id-20261009/id-batch-direct.json" }, { "provider": "ElevenLabs", "model": "Scribe v1", "id": "elevenlabs:scribe_v1", "err": 2.1, "ci": [ 0.9, 3.6 ], "tone": "good", "provenance": "apps/benchmark-runner/results/id-20261009/id-batch-direct-scribe-v1.json" }, { "provider": "Azure", "model": "MAI-Transcribe-2", "id": "azure:MAI-Transcribe-2", "err": 2.1, "ci": [ 1.2, 3.4 ], "tone": "good" }, { "provider": "Google", "model": "Gemini 3.5 Transcribe", "id": "gemini:gemini-3.5-transcribe", "err": 2.2, "ci": [ 1.1, 3.5 ], "tone": "good" }, { "provider": "Google", "model": "Gemini 3.5 Transcribe Live", "id": "gemini:gemini-3.5-transcribe-live", "errStream": 2.6, "ciStream": [ 1.3, 4.1 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 3, "ci": [ 1.5, 5 ], "tone": "good" }, { "provider": "xAI", "model": "Grok STT", "id": "xai:stt", "err": 3.1, "ci": [ 1.6, 4.7 ], "errStream": 4.6, "ciStream": [ 2.7, 6.7 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 3.3, "ci": [ 1.8, 5.4 ], "tone": "good" }, { "provider": "Meta", "model": "Muse Voice Transcribe", "id": "meta:muse-voice-transcribe-1.0", "err": 3.6, "ci": [ 2.2, 5.4 ], "errStream": 3.6, "ciStream": [ 2.2, 5.5 ], "tone": "good", "runDateStream": "2026-10-09", "provenanceStream": "apps/benchmark-runner/results/id-20261009/id-stream-direct.json" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 3.7, "ci": [ 2.3, 5.4 ], "errStream": 3.6, "ciStream": [ 2.2, 5.4 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 3.8, "ci": [ 2.1, 5.8 ], "tone": "good" }, { "provider": "Alibaba", "model": "Qwen-ASR", "id": "alibaba:qwen-asr", "err": 3.8, "ci": [ 2.1, 5.8 ], "tone": "good", "provenance": "apps/benchmark-runner/results/id-20261009/id-batch-direct.json" }, { "provider": "OpenAI", "model": "GPT-4o mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 4, "ci": [ 2.3, 6.4 ], "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 4.2, "ciStream": [ 2.7, 6 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Alibaba", "model": "Qwen3-ASR Realtime", "id": "alibaba:qwen3-asr-flash-realtime", "errStream": 4.4, "ciStream": [ 2, 7.1 ], "clipsStream": 36, "finalsPerClip": 1.1, "tone": "good" }, { "provider": "Fish Audio", "model": "Transcribe-1 Pro", "id": "fishaudio:transcribe-1-pro", "err": 4.9, "ci": [ 3, 6.9 ], "tone": "good" }, { "provider": "Nari", "model": "Qwen3-ASR Fast", "id": "nari:qwen3-asr-fast", "errStream": 5.7, "ciStream": [ 3.8, 8.2 ], "tone": "good", "runDateStream": "2026-10-09", "provenanceStream": "apps/benchmark-runner/results/id-20261009/id-batch-direct-nari.json" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "id": "openai:gpt-live-transcribe", "errStream": 5.7, "ciStream": [ 3.8, 8.1 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Azure", "model": "Fast Transcription", "id": "azure:fast-transcription", "err": 5.7, "ci": [ 3.9, 7.7 ], "tone": "good" }, { "provider": "Fish Audio", "model": "Transcribe-1", "id": "fishaudio:transcribe-1", "err": 5.7, "ci": [ 3.8, 8.2 ], "tone": "good" }, { "provider": "Nari", "model": "Qwen3-ASR", "id": "nari:qwen3-asr", "errStream": 5.8, "ciStream": [ 3.9, 8.2 ], "tone": "good", "runDateStream": "2026-10-09", "provenanceStream": "apps/benchmark-runner/results/id-20261009/id-batch-direct-nari.json" }, { "provider": "OpenAI", "model": "GPT Realtime Whisper", "errStream": 5.8, "ciStream": [ 3.8, 8.6 ], "tone": "good", "runDateStream": "2026-10-09", "provenanceStream": "apps/benchmark-runner/results/id-20261009/id-stream-direct.json" }, { "provider": "OpenAI", "model": "Whisper-1", "id": "openai:whisper-1", "err": 6.5, "ci": [ 4.3, 9 ], "tone": "good", "provenance": "apps/benchmark-runner/results/id-20261009/id-batch-direct.json" }, { "provider": "AssemblyAI", "model": "Universal-2", "err": 7.1, "ci": [ 5.1, 9.3 ], "tone": "good", "provenance": "apps/benchmark-runner/results/id-20261009/id-batch-direct-aai.json" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "errStream": 7.1, "ciStream": [ 5.1, 9.7 ], "finalsPerClip": 1, "tone": "good" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 7.5, "ci": [ 5.2, 10.3 ], "errStream": 9.2, "ciStream": [ 6.8, 11.8 ], "tone": "good", "provenance": "apps/benchmark-runner/results/id-20261009/id-batch-direct.json", "provenanceStream": "apps/benchmark-runner/results/id-20261009/id-stream-direct.json", "runDateStream": "2026-10-09" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "err": 8.3, "ci": [ 5.9, 11 ], "errStream": 9.9, "ciStream": [ 7.4, 12.7 ], "clipsStream": 42, "finalsPerClip": 2.5, "tone": "good" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 13.9, "ci": [ 11.6, 16.2 ], "errStream": 14.6, "ciStream": [ 12.4, 17.1 ], "finalsPerClip": 3.2, "tone": "warn" } ] }, "sw": { "label": "Swahili", "metric": "WER", "meta": "FLEURS sw_ke · n=50 · corpus WER · batch + streaming (1.0x pacing) · 2026-10-09", "path": "batch", "rows": [ { "provider": "Google", "model": "Chirp 3", "id": "google:chirp_3", "err": 6.7, "ci": [ 4.8, 8.9 ], "tone": "good", "provenance": "apps/benchmark-runner/results/sw-20261009/sw-batch-direct.json" }, { "provider": "Azure", "model": "Fast Transcription", "id": "azure:fast-transcription", "err": 7.7, "ci": [ 5.7, 10 ], "tone": "good", "provenance": "apps/benchmark-runner/results/sw-20261009/sw-batch-direct.json" }, { "provider": "Google", "model": "Gemini 3.5 Transcribe", "id": "gemini:gemini-3.5-transcribe", "err": 8.2, "ci": [ 6.2, 10 ], "tone": "good" }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "err": 8.8, "ci": [ 6.2, 11.6 ], "errStream": 8.8, "ciStream": [ 6.2, 11.6 ], "finalsPerClip": 1.1, "tone": "good" }, { "provider": "Modulate", "model": "Velma 2 multilingual", "id": "modulate:velma-2-stt-streaming", "errStream": 8.8, "ciStream": [ 6.3, 11.5 ], "finalsPerClip": 1.1, "tone": "good" }, { "provider": "ElevenLabs", "model": "Scribe v1", "id": "elevenlabs:scribe_v1", "err": 8.9, "ci": [ 6.1, 11.8 ], "tone": "good", "provenance": "apps/benchmark-runner/results/sw-20261009/sw-batch-direct.json" }, { "provider": "ElevenLabs", "model": "Scribe v2", "id": "elevenlabs:scribe_v2", "err": 9.3, "ci": [ 6.4, 12.3 ], "tone": "warn" }, { "provider": "Azure", "model": "MAI-Transcribe-2", "id": "azure:MAI-Transcribe-2", "err": 9.6, "ci": [ 6.7, 12.5 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT Transcribe", "id": "openai:gpt-transcribe", "err": 13.5, "ci": [ 10.5, 16.5 ], "tone": "warn" }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "id": "openai:gpt-4o-transcribe", "err": 15, "ci": [ 11.7, 19.1 ], "tone": "bad" }, { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "errStream": 15.9, "ciStream": [ 12.7, 18.8 ], "finalsPerClip": 1.1, "tone": "bad" }, { "provider": "OpenAI", "model": "GPT Live Transcribe", "id": "openai:gpt-live-transcribe", "errStream": 17, "ciStream": [ 13.5, 20.8 ], "finalsPerClip": 1, "tone": "bad" }, { "provider": "Deepgram", "model": "Nova-3", "id": "deepgram:nova-3", "err": 18, "ci": [ 14.3, 22.1 ], "errStream": 18.3, "ciStream": [ 14.3, 22.4 ], "finalsPerClip": 4.3, "tone": "bad" }, { "provider": "Meta", "model": "Muse Voice Transcribe 1.0", "id": "meta:muse-voice-transcribe-1.0", "err": 19.7, "ci": [ 16.5, 23.4 ], "tone": "bad" }, { "provider": "OpenAI", "model": "GPT-4o mini Transcribe", "id": "openai:gpt-4o-mini-transcribe", "err": 21.2, "ci": [ 17.8, 25 ], "tone": "bad" }, { "provider": "Cartesia", "model": "Ink-Whisper", "id": "cartesia:ink-whisper", "err": 31.4, "ci": [ 27.5, 35.2 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/sw-20261009/sw-batch-direct.json" }, { "provider": "xAI", "model": "Grok STT", "id": "xai:stt", "err": 33.9, "ci": [ 27.2, 41.1 ], "errStream": 29, "ciStream": [ 23.9, 35.7 ], "finalsPerClip": 1.2, "tone": "bad" }, { "provider": "Alibaba", "model": "Qwen3-ASR", "id": "alibaba:qwen3-asr-flash", "err": 35.8, "ci": [ 30.8, 41.5 ], "tone": "bad" }, { "provider": "Gladia", "model": "Solaria-1", "id": "gladia:solaria-1", "err": 36, "ci": [ 31.7, 40.5 ], "tone": "bad" }, { "provider": "AssemblyAI", "model": "Universal-2", "err": 37.6, "ci": [ 33.4, 41.9 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/sw-20261009/sw-batch-gw.json" }, { "provider": "OpenAI", "model": "Whisper-1", "id": "openai:whisper-1", "err": 38.2, "ci": [ 34.4, 42.6 ], "tone": "bad", "provenance": "apps/benchmark-runner/results/sw-20261009/sw-batch-direct.json" }, { "provider": "Fish Audio", "model": "transcribe-1", "id": "fishaudio:transcribe-1", "err": 53.9, "ci": [ 49.1, 59.3 ], "tone": "bad" }, { "provider": "Fish Audio", "model": "transcribe-1-pro", "id": "fishaudio:transcribe-1-pro", "err": 55.2, "ci": [ 49.9, 60.3 ], "clips": 49, "tone": "bad" } ] } }, "tts": { "es": { "label": "Spanish", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-07-21", "rows": [ { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1951, "ci": [ 1859, 2142 ], "p": 0.9188, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1751, "ci": [ 1662, 1858 ], "p": 0.7796, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v2", "elo": 1696, "ci": [ 1621, 1785 ], "p": 0.7134, "tone": "good", "estimated": true }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "elo": 1634, "ci": [ 1567, 1718 ], "p": 0.6601, "tone": "good" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1598, "ci": [ 1532, 1671 ], "p": 0.619, "tone": "warn" }, { "provider": "Hume", "model": "octave-2", "id": "hume:octave-2", "elo": 1568, "ci": [ 1485, 1655 ], "p": 0.5826, "tone": "warn" }, { "provider": "Soniox", "model": "tts-rt-v1", "id": "soniox:tts-rt-v1", "elo": 1525, "ci": [ 1451, 1608 ], "p": 0.5302, "tone": "warn" }, { "provider": "Smallest", "model": "lightning_v3.1", "id": "smallest:lightning_v3.1", "elo": 1487, "ci": [ 1410, 1566 ], "p": 0.4833, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "elo": 1460, "ci": [ 1396, 1534 ], "p": 0.4492, "tone": "warn" }, { "provider": "Qwen", "model": "qwen3-tts-flash", "id": "alibaba:qwen3-tts-flash", "elo": 1437, "ci": [ 1384, 1495 ], "p": 0.422, "tone": "warn" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1408, "ci": [ 1355, 1453 ], "p": 0.3867, "tone": "warn" }, { "provider": "Rime", "model": "arcanav3", "id": "rime:arcanav3", "elo": 1172, "ci": [ 1010, 1271 ], "p": 0.1574, "tone": "bad" }, { "provider": "Deepgram", "model": "aura-2", "id": "deepgram:aura-2", "elo": 1055, "ci": [ 835, 1181 ], "p": 0.0852, "tone": "bad" } ] }, "de": { "label": "German", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-07-21", "rows": [ { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1798, "ci": [ 1724, 1879 ], "p": 0.8125, "tone": "good" }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "elo": 1762, "ci": [ 1666, 1881 ], "p": 0.7795, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v1", "id": "soniox:tts-rt-v1", "elo": 1717, "ci": [ 1638, 1804 ], "p": 0.7348, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v2", "elo": 1700, "ci": [ 1612, 1784 ], "p": 0.7053, "tone": "good", "estimated": true }, { "provider": "Hume", "model": "octave-2", "id": "hume:octave-2", "elo": 1687, "ci": [ 1623, 1762 ], "p": 0.7032, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1665, "ci": [ 1592, 1741 ], "p": 0.6795, "tone": "good" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1576, "ci": [ 1480, 1689 ], "p": 0.5772, "tone": "warn" }, { "provider": "Qwen", "model": "qwen3-tts-flash", "id": "alibaba:qwen3-tts-flash", "elo": 1573, "ci": [ 1507, 1654 ], "p": 0.5736, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "elo": 1528, "ci": [ 1470, 1611 ], "p": 0.5208, "tone": "warn" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1497, "ci": [ 1408, 1586 ], "p": 0.4835, "tone": "warn" }, { "provider": "Rime", "model": "arcanav3", "id": "rime:arcanav3", "elo": 1257, "ci": [ 1121, 1379 ], "p": 0.2343, "tone": "bad" }, { "provider": "Smallest", "model": "lightning_v3.1", "id": "smallest:lightning_v3.1", "elo": 1254, "ci": [ 1146, 1333 ], "p": 0.2315, "tone": "bad" }, { "provider": "Gradium", "model": "Gradium TTS", "id": "gradium:default", "elo": 1245, "ci": [ 1114, 1338 ], "p": 0.2244, "tone": "bad" }, { "provider": "Deepgram", "model": "aura-2", "id": "deepgram:aura-2", "elo": 1062, "ci": [ 912, 1153 ], "p": 0.0973, "tone": "bad" } ] }, "fr": { "label": "French", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-07-21", "rows": [ { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1775, "ci": [ 1691, 1923 ], "p": 0.8068, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v2", "elo": 1695, "ci": [ 1596, 1798 ], "p": 0.715, "tone": "good", "estimated": true }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "elo": 1692, "ci": [ 1595, 1799 ], "p": 0.7242, "tone": "good" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1684, "ci": [ 1606, 1770 ], "p": 0.7154, "tone": "good" }, { "provider": "Hume", "model": "octave-2", "id": "hume:octave-2", "elo": 1671, "ci": [ 1612, 1762 ], "p": 0.7004, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v1", "id": "soniox:tts-rt-v1", "elo": 1628, "ci": [ 1535, 1739 ], "p": 0.651, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1622, "ci": [ 1533, 1731 ], "p": 0.6432, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1535, "ci": [ 1460, 1631 ], "p": 0.5365, "tone": "warn" }, { "provider": "Qwen", "model": "qwen3-tts-flash", "id": "alibaba:qwen3-tts-flash", "elo": 1529, "ci": [ 1465, 1615 ], "p": 0.5285, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "elo": 1419, "ci": [ 1328, 1503 ], "p": 0.393, "tone": "warn" }, { "provider": "Smallest", "model": "lightning_v3.1", "id": "smallest:lightning_v3.1", "elo": 1416, "ci": [ 1318, 1491 ], "p": 0.3896, "tone": "warn" }, { "provider": "Rime", "model": "arcanav3", "id": "rime:arcanav3", "elo": 1316, "ci": [ 1215, 1410 ], "p": 0.2786, "tone": "bad" }, { "provider": "Gradium", "model": "Gradium TTS", "id": "gradium:default", "elo": 1233, "ci": [ 1057, 1321 ], "p": 0.1999, "tone": "bad" }, { "provider": "Deepgram", "model": "aura-2", "id": "deepgram:aura-2", "elo": 1114, "ci": [ 959, 1196 ], "p": 0.1123, "tone": "bad" } ] }, "ar": { "label": "Arabic", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-07-21", "rows": [ { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "elo": 1736, "ci": [ 1656, 1824 ], "p": 0.7874, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1644, "ci": [ 1542, 1767 ], "p": 0.6831, "tone": "good" }, { "provider": "Hamsa", "model": "default", "id": "hamsa:default", "measured": "2026-10-08", "per1mCharsUsd": 264, "per1mCharsBasis": "Billed per audio second, not per character: $0.15 per audio minute at 9.9 characters per second, rounded up to a whole second per request.", "elo": 1614, "ci": [ 1528, 1707 ], "p": 0.628, "tone": "good", "estimated": true }, { "provider": "Soniox", "model": "tts-rt-v2", "elo": 1612, "ci": [ 1538, 1699 ], "p": 0.6259, "tone": "good", "estimated": true }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1564, "ci": [ 1511, 1621 ], "p": 0.5831, "tone": "warn" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1454, "ci": [ 1383, 1528 ], "p": 0.4403, "tone": "warn" }, { "provider": "Munsit", "model": "faseeh-v1-preview", "id": "munsit:faseeh-v1-preview", "measured": "2026-10-08", "per1mCharsUsd": 100, "elo": 1401, "ci": [ 1319, 1480 ], "p": 0.3883, "tone": "warn", "estimated": true } ] }, "fil": { "label": "Filipino", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-07-21", "rows": [ { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 2012, "ci": [ 1854, 2289 ], "p": 0.9276, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v2", "elo": 1790, "ci": [ 1693, 1923 ], "p": 0.7659, "tone": "good", "estimated": true }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1693, "ci": [ 1597, 1815 ], "p": 0.6925, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1676, "ci": [ 1552, 1793 ], "p": 0.6756, "tone": "good" }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "elo": 1657, "ci": [ 1534, 1873 ], "p": 0.6562, "tone": "good" }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "elo": 1598, "ci": [ 1497, 1717 ], "p": 0.5943, "tone": "warn" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1597, "ci": [ 1509, 1728 ], "p": 0.5937, "tone": "warn" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1251, "ci": [ 1101, 1346 ], "p": 0.2538, "tone": "bad" } ] }, "nb": { "label": "Norwegian", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-07-21", "rows": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1836, "ci": [ 1756, 1949 ], "p": 0.8724, "tone": "good" }, { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1705, "ci": [ 1638, 1790 ], "p": 0.7519, "tone": "good" }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1628, "ci": [ 1533, 1733 ], "p": 0.6674, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v2", "elo": 1621, "ci": [ 1510, 1743 ], "p": 0.6384, "tone": "good", "estimated": true }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1467, "ci": [ 1391, 1537 ], "p": 0.4665, "tone": "warn" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1420, "ci": [ 1343, 1482 ], "p": 0.4053, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "elo": 1386, "ci": [ 1264, 1487 ], "p": 0.362, "tone": "bad" } ] }, "hi": { "label": "Hindi", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-08-05", "rows": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1736, "ci": [ 1637, 1903 ], "p": 0.8197, "tone": "good" }, { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1563, "ci": [ 1467, 1683 ], "p": 0.5977, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.8-hd", "id": "minimax:speech-2.8-hd", "elo": 1522, "ci": [ 1432, 1636 ], "p": 0.5355, "tone": "warn" }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1490, "ci": [ 1362, 1600 ], "p": 0.4873, "tone": "warn" }, { "provider": "Maya", "model": "Maya 2 Native", "elo": 1478, "ci": [ 1295, 1666 ], "p": 0.47, "tone": "warn" }, { "provider": "Google Chirp", "model": "chirp-3-hd", "elo": 1468, "ci": [ 1378, 1557 ], "p": 0.4547, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1412, "ci": [ 1272, 1515 ], "p": 0.3715, "tone": "bad" }, { "provider": "Speechify", "model": "simba-multilingual", "elo": 1310, "ci": [ 1134, 1394 ], "p": 0.2336, "tone": "bad" } ] }, "ta": { "label": "Tamil", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-08-05", "rows": [ { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1676, "ci": [ 1590, 1772 ], "p": 0.7313, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1620, "ci": [ 1565, 1701 ], "p": 0.6554, "tone": "good" }, { "provider": "Maya", "model": "Maya 2 Native", "elo": 1603, "ci": [ 1541, 1668 ], "p": 0.63, "tone": "good" }, { "provider": "Google Chirp", "model": "chirp-3-hd", "elo": 1555, "ci": [ 1507, 1614 ], "p": 0.56, "tone": "warn" }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1543, "ci": [ 1484, 1616 ], "p": 0.5427, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1462, "ci": [ 1364, 1555 ], "p": 0.4262, "tone": "warn" }, { "provider": "Speechify", "model": "simba-multilingual", "elo": 1144, "ci": [ 933, 1239 ], "p": 0.0843, "tone": "bad" } ] }, "te": { "label": "Telugu", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-08-05", "rows": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1639, "ci": [ 1558, 1743 ], "p": 0.706, "tone": "good" }, { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1625, "ci": [ 1516, 1743 ], "p": 0.6852, "tone": "good" }, { "provider": "Google Chirp", "model": "chirp-3-hd", "elo": 1548, "ci": [ 1472, 1645 ], "p": 0.5678, "tone": "warn" }, { "provider": "Maya", "model": "Maya 2 Native", "elo": 1540, "ci": [ 1453, 1625 ], "p": 0.555, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1419, "ci": [ 1308, 1524 ], "p": 0.3698, "tone": "bad" }, { "provider": "Speechify", "model": "simba-multilingual", "elo": 1269, "ci": [ 1137, 1356 ], "p": 0.1711, "tone": "bad" } ] }, "zh": { "label": "Chinese (Mandarin)", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-08-24", "rows": [ { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1688, "ci": [ 1629, 1754 ], "p": 0.738, "tone": "good" }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1675, "ci": [ 1587, 1767 ], "p": 0.7231, "tone": "good" }, { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1649, "ci": [ 1586, 1724 ], "p": 0.6918, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1564, "ci": [ 1475, 1659 ], "p": 0.5814, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1552, "ci": [ 1470, 1688 ], "p": 0.5654, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.6-hd", "id": "minimax:speech-2.6-hd", "elo": 1467, "ci": [ 1411, 1553 ], "p": 0.4511, "tone": "warn" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1365, "ci": [ 1263, 1454 ], "p": 0.3196, "tone": "bad" } ] }, "ko": { "label": "Korean", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-08-24", "rows": [ { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1917, "ci": [ 1849, 2035 ], "p": 0.8839, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1859, "ci": [ 1767, 1981 ], "p": 0.8438, "tone": "good" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1722, "ci": [ 1646, 1821 ], "p": 0.728, "tone": "good" }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "elo": 1628, "ci": [ 1565, 1706 ], "p": 0.6333, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1498, "ci": [ 1433, 1576 ], "p": 0.4908, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1438, "ci": [ 1362, 1498 ], "p": 0.424, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.6-hd", "id": "minimax:speech-2.6-hd", "elo": 1407, "ci": [ 1339, 1486 ], "p": 0.3911, "tone": "warn" }, { "provider": "Rime", "model": "coda", "id": "rime:coda", "elo": 1187, "ci": [ 1105, 1261 ], "p": 0.1866, "tone": "bad" }, { "provider": "AWS Polly", "model": "generative", "id": "polly:generative", "elo": 953, "ci": [ 696, 1073 ], "p": 0.0527, "tone": "bad" } ] }, "ja": { "label": "Japanese", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-09-01", "rows": [ { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1756, "ci": [ 1684, 1857 ], "p": 0.8134, "tone": "good" }, { "provider": "ElevenLabs", "model": "eleven_v3", "id": "elevenlabs:eleven_v3", "elo": 1646, "ci": [ 1589, 1726 ], "p": 0.6912, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1600, "ci": [ 1534, 1687 ], "p": 0.6339, "tone": "good" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1597, "ci": [ 1542, 1663 ], "p": 0.63, "tone": "good" }, { "provider": "Azure", "model": "DragonHDLatestNeural", "id": "azure:DragonHDLatestNeural", "elo": 1539, "ci": [ 1478, 1613 ], "p": 0.5526, "tone": "warn" }, { "provider": "Soniox", "model": "tts-rt-v2", "id": "soniox:tts-rt-v2", "elo": 1447, "ci": [ 1391, 1496 ], "p": 0.4274, "tone": "warn" }, { "provider": "MiniMax", "model": "speech-2.6-hd", "id": "minimax:speech-2.6-hd", "elo": 1441, "ci": [ 1361, 1521 ], "p": 0.4185, "tone": "warn" }, { "provider": "Google Chirp 3 HD", "model": "chirp-3-hd", "id": "google-chirp:chirp-3-hd", "elo": 1407, "ci": [ 1317, 1477 ], "p": 0.3737, "tone": "bad" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1338, "ci": [ 1247, 1413 ], "p": 0.2883, "tone": "bad" }, { "provider": "Hume", "model": "octave-2", "id": "hume:octave-2", "elo": 1228, "ci": [ 1107, 1304 ], "p": 0.171, "tone": "bad" } ] }, "th": { "label": "Thai", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-09-10", "rows": [ { "provider": "Gemini", "model": "gemini-3.1-flash-tts-preview", "id": "google-tts:gemini-3.1-flash-tts-preview", "elo": 1722, "ci": [ 1628, 1844 ], "p": 0.7431, "tone": "good" }, { "provider": "xAI Grok", "model": "tts", "id": "xai:tts", "elo": 1719, "ci": [ 1648, 1838 ], "p": 0.7401, "tone": "good" }, { "provider": "MiniMax", "model": "speech-2.6-hd", "id": "minimax:speech-2.6-hd", "elo": 1699, "ci": [ 1615, 1796 ], "p": 0.7178, "tone": "good" }, { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "elo": 1688, "ci": [ 1618, 1793 ], "p": 0.7052, "tone": "good" }, { "provider": "ElevenLabs", "model": "eleven_v3", "id": "elevenlabs:eleven_v3", "elo": 1605, "ci": [ 1504, 1692 ], "p": 0.6082, "tone": "warn" }, { "provider": "Soniox", "model": "tts-rt-v2", "id": "soniox:tts-rt-v2", "elo": 1591, "ci": [ 1515, 1674 ], "p": 0.592, "tone": "warn" }, { "provider": "Paxa", "model": "paxa-tts-flash-v1", "id": "paxa:paxa-tts-flash-v1", "elo": 1467, "ci": [ 1382, 1548 ], "p": 0.4452, "tone": "warn" }, { "provider": "Google Chirp 3 HD", "model": "chirp-3-hd", "id": "google-chirp:chirp-3-hd", "elo": 1442, "ci": [ 1369, 1526 ], "p": 0.4164, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1280, "ci": [ 1174, 1363 ], "p": 0.2494, "tone": "bad" }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "elo": 1202, "ci": [ 1104, 1278 ], "p": 0.1824, "tone": "bad" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1084, "ci": [ 931, 1180 ], "p": 0.1002, "tone": "bad" } ] }, "lt": { "label": "Lithuanian", "meta": "Blind A/B listening study · arena Elo · 95% CI · 2026-10-01", "rows": [ { "provider": "Gemini", "model": "gemini-3.8-flash-lite-tts", "id": "google-tts:gemini-3.8-flash-lite-tts", "elo": 1767, "ci": [ 1710, 1858 ], "p": 0.7912, "tone": "good" }, { "provider": "ElevenLabs", "model": "eleven_v4_turbo", "id": "elevenlabs:eleven_v4_turbo", "elo": 1735, "ci": [ 1658, 1843 ], "p": 0.7591, "tone": "good" }, { "provider": "Gemini", "model": "gemini-3.8-flash-tts", "id": "google-tts:gemini-3.8-flash-tts", "elo": 1727, "ci": [ 1669, 1812 ], "p": 0.7505, "tone": "good" }, { "provider": "Soniox", "model": "tts-rt-v2", "id": "soniox:tts-rt-v2", "elo": 1623, "ci": [ 1555, 1707 ], "p": 0.6375, "tone": "good" }, { "provider": "Google Chirp 3 HD", "model": "chirp-3-hd", "id": "google-chirp:chirp-3-hd", "elo": 1418, "ci": [ 1307, 1503 ], "p": 0.4036, "tone": "warn" }, { "provider": "Azure", "model": "Azure AI Speech (native)", "elo": 1288, "ci": [ 1168, 1363 ], "p": 0.2637, "tone": "bad" }, { "provider": "xAI Grok", "model": "tts", "id": "xai:tts", "elo": 1235, "ci": [ 1129, 1309 ], "p": 0.2106, "tone": "bad" }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "elo": 1206, "ci": [ 1082, 1284 ], "p": 0.1838, "tone": "bad" } ] }, "id": { "label": "Indonesian", "meta": "Robustness only · 2026-10-09", "rows": [] }, "sw": { "label": "Swahili", "meta": "No listening study yet", "rows": [] } }, "llmTask": { "es": { "label": "Spanish", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 2, "toolSilent": 84, "deadAir": 59, "fabrication": null, "adherence": 11, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 131, "nCls": 77, "kStall": 1, "kDeadAir": 110, "kToolSilent": 110, "kAdherence": 77, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 90.35, "taskScore": 90, "stalled": 13, "toolSilent": 99, "deadAir": 65, "fabrication": null, "adherence": 72, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 174, "nToolTurns": 114, "nCls": 61, "kStall": 7, "kDeadAir": 113, "kToolSilent": 113, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 89.47, "taskScore": 89, "stalled": 11, "toolSilent": 97, "deadAir": 64, "fabrication": null, "adherence": 46, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 177, "nToolTurns": 117, "nCls": 60, "kStall": 6, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 89.47, "taskScore": 89, "stalled": 6, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 59, "kStall": 3, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 89.18, "taskScore": 89, "stalled": 7, "toolSilent": 70, "deadAir": 49, "fabrication": null, "adherence": 94, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 138, "nCls": 99, "kStall": 4, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 98, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 88.6, "taskScore": 89, "stalled": 11, "toolSilent": 93, "deadAir": 58, "fabrication": null, "adherence": 13, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 159, "nToolTurns": 99, "nCls": 67, "kStall": 6, "kDeadAir": 92, "kToolSilent": 92, "kAdherence": 67, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 87.13, "taskScore": 87, "stalled": 13, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 136, "nCls": 59, "kStall": 7, "kDeadAir": 136, "kToolSilent": 136, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 87.13, "taskScore": 87, "stalled": 7, "toolSilent": 81, "deadAir": 58, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 139, "nCls": 73, "kStall": 4, "kDeadAir": 113, "kToolSilent": 113, "kAdherence": 73, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 6, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 86.26, "taskScore": 86, "stalled": 19, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 60, "kStall": 10, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 85.96, "taskScore": 86, "stalled": 22, "toolSilent": 50, "deadAir": 32, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 104, "kStall": 12, "kDeadAir": 54, "kToolSilent": 54, "kAdherence": 104, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 85.96, "taskScore": 86, "stalled": 17, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 134, "nCls": 59, "kStall": 9, "kDeadAir": 134, "kToolSilent": 134, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 85.67, "taskScore": 86, "stalled": 22, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 159, "nToolTurns": 99, "nCls": 59, "kStall": 12, "kDeadAir": 99, "kToolSilent": 99, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 85.23, "taskScore": 85, "stalled": 17, "toolSilent": 97, "deadAir": 60, "fabrication": null, "adherence": 12, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 157, "nToolTurns": 97, "nCls": 60, "kStall": 9, "kDeadAir": 94, "kToolSilent": 94, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.09, "taskScore": 85, "stalled": 15, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 175, "nToolTurns": 115, "nCls": 60, "kStall": 8, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 85.09, "taskScore": 85, "stalled": 9, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 142, "nCls": 55, "kStall": 5, "kDeadAir": 142, "kToolSilent": 142, "kAdherence": 55, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 84.06, "taskScore": 84, "stalled": 13, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 205, "nToolTurns": 146, "nCls": 59, "kStall": 7, "kDeadAir": 146, "kToolSilent": 146, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 83.63, "taskScore": 84, "stalled": 11, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 16, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 133, "nCls": 57, "kStall": 6, "kDeadAir": 133, "kToolSilent": 133, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 81.73, "taskScore": 82, "stalled": 13, "toolSilent": 100, "deadAir": 73, "fabrication": null, "adherence": 93, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 137, "nCls": 52, "kStall": 7, "kDeadAir": 141, "kToolSilent": 137, "kAdherence": 52, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 81.14, "taskScore": 81, "stalled": 11, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 16, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 141, "nCls": 56, "kStall": 6, "kDeadAir": 141, "kToolSilent": 141, "kAdherence": 56, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 79.53, "taskScore": 80, "stalled": 22, "toolSilent": 88, "deadAir": 61, "fabrication": null, "adherence": 72, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 136, "nCls": 72, "kStall": 12, "kDeadAir": 119, "kToolSilent": 119, "kAdherence": 72, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 77.34, "taskScore": 77, "stalled": 35, "toolSilent": 1, "deadAir": 1, "fabrication": null, "adherence": 91, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 147, "nToolTurns": 87, "nCls": 143, "kStall": 19, "kDeadAir": 1, "kToolSilent": 1, "kAdherence": 143, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 74.56, "taskScore": 75, "stalled": 20, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 132, "nCls": 57, "kStall": 11, "kDeadAir": 132, "kToolSilent": 132, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 58.33, "taskScore": 58, "stalled": 65, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 19, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 35, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 35.23, "taskScore": 35, "stalled": 74, "toolSilent": 94, "deadAir": 35, "fabrication": null, "adherence": 90, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 96, "nToolTurns": 36, "nCls": 62, "kStall": 40, "kDeadAir": 34, "kToolSilent": 34, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 13, "iCls": 19, "iFab": 0 } ] }, "ar": { "label": "Arabic", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 92.98, "taskScore": 93, "stalled": 6, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 191, "nToolTurns": 131, "nCls": 60, "kStall": 3, "kDeadAir": 131, "kToolSilent": 131, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 92.69, "taskScore": 93, "stalled": 6, "toolSilent": 93, "deadAir": 58, "fabrication": null, "adherence": 27, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 158, "nToolTurns": 98, "nCls": 67, "kStall": 3, "kDeadAir": 91, "kToolSilent": 91, "kAdherence": 67, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 4, "toolSilent": 89, "deadAir": 60, "fabrication": null, "adherence": 5, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 181, "nToolTurns": 121, "nCls": 73, "kStall": 2, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 73, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 89.47, "taskScore": 89, "stalled": 11, "toolSilent": 98, "deadAir": 65, "fabrication": null, "adherence": 25, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 176, "nToolTurns": 116, "nCls": 62, "kStall": 6, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 89.47, "taskScore": 89, "stalled": 17, "toolSilent": 98, "deadAir": 64, "fabrication": null, "adherence": 83, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 62, "kStall": 9, "kDeadAir": 110, "kToolSilent": 110, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 89.47, "taskScore": 89, "stalled": 6, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 60, "kStall": 3, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 88.89, "taskScore": 89, "stalled": 11, "toolSilent": 91, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 199, "nToolTurns": 139, "nCls": 72, "kStall": 6, "kDeadAir": 127, "kToolSilent": 127, "kAdherence": 72, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 88.3, "taskScore": 88, "stalled": 19, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 156, "nToolTurns": 96, "nCls": 60, "kStall": 10, "kDeadAir": 96, "kToolSilent": 96, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 87.72, "taskScore": 88, "stalled": 9, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 5, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 87.13, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 137, "nCls": 58, "kStall": 6, "kDeadAir": 137, "kToolSilent": 137, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 86.84, "taskScore": 87, "stalled": 15, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 139, "nCls": 59, "kStall": 8, "kDeadAir": 139, "kToolSilent": 139, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 85.96, "taskScore": 86, "stalled": 19, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 10, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 85.38, "taskScore": 85, "stalled": 9, "toolSilent": 76, "deadAir": 53, "fabrication": null, "adherence": 86, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 136, "nCls": 92, "kStall": 5, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 92, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.09, "taskScore": 85, "stalled": 15, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 174, "nToolTurns": 114, "nCls": 60, "kStall": 8, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 85.09, "taskScore": 85, "stalled": 19, "toolSilent": 94, "deadAir": 58, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 156, "nToolTurns": 96, "nCls": 66, "kStall": 10, "kDeadAir": 90, "kToolSilent": 90, "kAdherence": 66, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 84.36, "taskScore": 84, "stalled": 22, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 32, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 154, "nToolTurns": 94, "nCls": 154, "kStall": 12, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 154, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 84.06, "taskScore": 84, "stalled": 13, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 141, "nCls": 55, "kStall": 7, "kDeadAir": 141, "kToolSilent": 141, "kAdherence": 55, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 79.82, "taskScore": 80, "stalled": 22, "toolSilent": 91, "deadAir": 64, "fabrication": null, "adherence": 68, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 136, "nCls": 71, "kStall": 12, "kDeadAir": 124, "kToolSilent": 124, "kAdherence": 70, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 79.53, "taskScore": 80, "stalled": 26, "toolSilent": 69, "deadAir": 43, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 92, "kStall": 14, "kDeadAir": 70, "kToolSilent": 70, "kAdherence": 92, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 79.39, "taskScore": 79, "stalled": 20, "toolSilent": 100, "deadAir": 67, "fabrication": null, "adherence": 4, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 177, "nToolTurns": 119, "nCls": 58, "kStall": 11, "kDeadAir": 119, "kToolSilent": 119, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 79.39, "taskScore": 79, "stalled": 17, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 132, "nCls": 57, "kStall": 9, "kDeadAir": 132, "kToolSilent": 132, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 78.65, "taskScore": 79, "stalled": 20, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 85, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 135, "nCls": 57, "kStall": 11, "kDeadAir": 136, "kToolSilent": 135, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 77.19, "taskScore": 77, "stalled": 19, "toolSilent": 99, "deadAir": 70, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 137, "nCls": 58, "kStall": 10, "kDeadAir": 136, "kToolSilent": 136, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 40.35, "taskScore": 40, "stalled": 81, "toolSilent": 100, "deadAir": 53, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 128, "nToolTurns": 68, "nCls": 60, "kStall": 44, "kDeadAir": 68, "kToolSilent": 68, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 33.77, "taskScore": 34, "stalled": 76, "toolSilent": 91, "deadAir": 32, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 92, "nToolTurns": 32, "nCls": 63, "kStall": 41, "kDeadAir": 29, "kToolSilent": 29, "kAdherence": 63, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 10, "iCls": 19, "iFab": 0 } ] }, "fil": { "label": "Filipino", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 94.74, "taskScore": 95, "stalled": 6, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 93, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 158, "nToolTurns": 98, "nCls": 60, "kStall": 3, "kDeadAir": 98, "kToolSilent": 98, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 92.69, "taskScore": 93, "stalled": 11, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 158, "nToolTurns": 98, "nCls": 60, "kStall": 6, "kDeadAir": 98, "kToolSilent": 98, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 91.81, "taskScore": 92, "stalled": 6, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 164, "nToolTurns": 104, "nCls": 60, "kStall": 3, "kDeadAir": 104, "kToolSilent": 104, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 90.35, "taskScore": 90, "stalled": 7, "toolSilent": 55, "deadAir": 37, "fabrication": null, "adherence": 10, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 130, "nCls": 119, "kStall": 4, "kDeadAir": 71, "kToolSilent": 71, "kAdherence": 119, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 88.89, "taskScore": 89, "stalled": 7, "toolSilent": 71, "deadAir": 50, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 139, "nCls": 97, "kStall": 4, "kDeadAir": 98, "kToolSilent": 98, "kAdherence": 96, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 88.6, "taskScore": 89, "stalled": 17, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 60, "kStall": 9, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 87.72, "taskScore": 88, "stalled": 13, "toolSilent": 67, "deadAir": 46, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 126, "nCls": 101, "kStall": 7, "kDeadAir": 85, "kToolSilent": 85, "kAdherence": 101, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 86.84, "taskScore": 87, "stalled": 22, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 60, "kStall": 12, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 6, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 175, "nToolTurns": 115, "nCls": 60, "kStall": 6, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 84.21, "taskScore": 84, "stalled": 15, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 124, "nCls": 59, "kStall": 8, "kDeadAir": 124, "kToolSilent": 124, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 83.63, "taskScore": 84, "stalled": 28, "toolSilent": 43, "deadAir": 27, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 157, "nToolTurns": 97, "nCls": 115, "kStall": 15, "kDeadAir": 42, "kToolSilent": 42, "kAdherence": 115, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 82.6, "taskScore": 83, "stalled": 19, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 127, "nCls": 59, "kStall": 10, "kDeadAir": 127, "kToolSilent": 127, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 81.58, "taskScore": 82, "stalled": 19, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 191, "nToolTurns": 134, "nCls": 57, "kStall": 10, "kDeadAir": 134, "kToolSilent": 134, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 81.29, "taskScore": 81, "stalled": 20, "toolSilent": 79, "deadAir": 55, "fabrication": null, "adherence": 50, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 133, "nCls": 86, "kStall": 11, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 86, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 80.7, "taskScore": 81, "stalled": 22, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 140, "nCls": 58, "kStall": 12, "kDeadAir": 140, "kToolSilent": 140, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 79.82, "taskScore": 80, "stalled": 26, "toolSilent": 98, "deadAir": 59, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 152, "nToolTurns": 92, "nCls": 62, "kStall": 14, "kDeadAir": 90, "kToolSilent": 90, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 78.07, "taskScore": 78, "stalled": 24, "toolSilent": 88, "deadAir": 52, "fabrication": null, "adherence": 3, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 145, "nToolTurns": 85, "nCls": 70, "kStall": 13, "kDeadAir": 75, "kToolSilent": 75, "kAdherence": 70, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 76.61, "taskScore": 77, "stalled": 20, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 128, "nCls": 59, "kStall": 11, "kDeadAir": 128, "kToolSilent": 128, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 75.88, "taskScore": 76, "stalled": 24, "toolSilent": 100, "deadAir": 73, "fabrication": null, "adherence": 62, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 180, "nToolTurns": 124, "nCls": 49, "kStall": 13, "kDeadAir": 131, "kToolSilent": 124, "kAdherence": 49, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 75.44, "taskScore": 75, "stalled": 41, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 9, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 140, "nToolTurns": 80, "nCls": 140, "kStall": 22, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 140, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 75.44, "taskScore": 75, "stalled": 26, "toolSilent": 99, "deadAir": 67, "fabrication": null, "adherence": 96, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 123, "nCls": 61, "kStall": 14, "kDeadAir": 122, "kToolSilent": 122, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 75.44, "taskScore": 75, "stalled": 26, "toolSilent": 100, "deadAir": 67, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 180, "nToolTurns": 120, "nCls": 60, "kStall": 14, "kDeadAir": 120, "kToolSilent": 120, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 52.19, "taskScore": 52, "stalled": 76, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 41, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 11.84, "taskScore": 12, "stalled": 94, "toolSilent": 100, "deadAir": 20, "fabrication": null, "adherence": 52, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 75, "nToolTurns": 15, "nCls": 60, "kStall": 51, "kDeadAir": 15, "kToolSilent": 15, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 5, "iCls": 19, "iFab": 0 } ] }, "de": { "label": "German", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 94.74, "taskScore": 95, "stalled": 6, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 72, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 174, "nToolTurns": 114, "nCls": 60, "kStall": 3, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 93.86, "taskScore": 94, "stalled": 2, "toolSilent": 82, "deadAir": 56, "fabrication": null, "adherence": 9, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 128, "nCls": 80, "kStall": 1, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 80, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 92.98, "taskScore": 93, "stalled": 4, "toolSilent": 92, "deadAir": 60, "fabrication": null, "adherence": 6, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 173, "nToolTurns": 113, "nCls": 69, "kStall": 2, "kDeadAir": 104, "kToolSilent": 104, "kAdherence": 69, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 88.3, "taskScore": 88, "stalled": 9, "toolSilent": 98, "deadAir": 62, "fabrication": null, "adherence": 92, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 163, "nToolTurns": 103, "nCls": 62, "kStall": 5, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 6, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 173, "nToolTurns": 113, "nCls": 60, "kStall": 6, "kDeadAir": 113, "kToolSilent": 113, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 86.84, "taskScore": 87, "stalled": 24, "toolSilent": 98, "deadAir": 61, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 62, "kStall": 13, "kDeadAir": 98, "kToolSilent": 98, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 85.09, "taskScore": 85, "stalled": 20, "toolSilent": 98, "deadAir": 65, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 180, "nToolTurns": 120, "nCls": 60, "kStall": 11, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 84.94, "taskScore": 85, "stalled": 13, "toolSilent": 68, "deadAir": 48, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 138, "nCls": 99, "kStall": 7, "kDeadAir": 94, "kToolSilent": 94, "kAdherence": 99, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 84.65, "taskScore": 85, "stalled": 15, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 182, "nToolTurns": 125, "nCls": 57, "kStall": 8, "kDeadAir": 125, "kToolSilent": 125, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 84.65, "taskScore": 85, "stalled": 13, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 134, "nCls": 59, "kStall": 7, "kDeadAir": 134, "kToolSilent": 134, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 84.5, "taskScore": 85, "stalled": 20, "toolSilent": 99, "deadAir": 68, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 130, "nCls": 61, "kStall": 11, "kDeadAir": 129, "kToolSilent": 129, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 83.33, "taskScore": 83, "stalled": 17, "toolSilent": 84, "deadAir": 58, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 133, "nCls": 79, "kStall": 9, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 79, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 82.46, "taskScore": 82, "stalled": 20, "toolSilent": 93, "deadAir": 58, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 55, "kStall": 11, "kDeadAir": 93, "kToolSilent": 93, "kAdherence": 55, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 82.02, "taskScore": 82, "stalled": 24, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 58, "kStall": 13, "kDeadAir": 102, "kToolSilent": 102, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 81.43, "taskScore": 81, "stalled": 30, "toolSilent": 48, "deadAir": 30, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 163, "nToolTurns": 103, "nCls": 114, "kStall": 16, "kDeadAir": 49, "kToolSilent": 49, "kAdherence": 114, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 81.14, "taskScore": 81, "stalled": 15, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 140, "nCls": 55, "kStall": 8, "kDeadAir": 140, "kToolSilent": 140, "kAdherence": 55, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 80.7, "taskScore": 81, "stalled": 17, "toolSilent": 76, "deadAir": 53, "fabrication": null, "adherence": 80, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 139, "nCls": 91, "kStall": 9, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 91, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 80.7, "taskScore": 81, "stalled": 13, "toolSilent": 99, "deadAir": 69, "fabrication": null, "adherence": 94, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 133, "nCls": 58, "kStall": 7, "kDeadAir": 132, "kToolSilent": 132, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 80.12, "taskScore": 80, "stalled": 17, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 202, "nToolTurns": 143, "nCls": 59, "kStall": 9, "kDeadAir": 143, "kToolSilent": 143, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 78.07, "taskScore": 78, "stalled": 26, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 200, "nToolTurns": 140, "nCls": 60, "kStall": 14, "kDeadAir": 140, "kToolSilent": 140, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 77.78, "taskScore": 78, "stalled": 19, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 77, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 139, "nCls": 54, "kStall": 10, "kDeadAir": 142, "kToolSilent": 139, "kAdherence": 54, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 76.75, "taskScore": 77, "stalled": 30, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 72, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 150, "nToolTurns": 90, "nCls": 149, "kStall": 16, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 149, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 62.72, "taskScore": 63, "stalled": 63, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 34, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 39.04, "taskScore": 39, "stalled": 67, "toolSilent": 100, "deadAir": 37, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 95, "nToolTurns": 35, "nCls": 60, "kStall": 36, "kDeadAir": 35, "kToolSilent": 35, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 12, "iCls": 19, "iFab": 0 } ] }, "fr": { "label": "French", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 92.98, "taskScore": 93, "stalled": 2, "toolSilent": 84, "deadAir": 57, "fabrication": null, "adherence": 25, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 129, "nCls": 80, "kStall": 1, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 80, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 92.11, "taskScore": 92, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 60, "kStall": 6, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 92.11, "taskScore": 92, "stalled": 4, "toolSilent": 92, "deadAir": 59, "fabrication": null, "adherence": 5, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 167, "nToolTurns": 107, "nCls": 69, "kStall": 2, "kDeadAir": 98, "kToolSilent": 98, "kAdherence": 69, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 90.35, "taskScore": 90, "stalled": 7, "toolSilent": 85, "deadAir": 59, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 136, "nCls": 81, "kStall": 4, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 81, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 89.47, "taskScore": 89, "stalled": 6, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 3, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 88.6, "taskScore": 89, "stalled": 15, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 170, "nToolTurns": 110, "nCls": 60, "kStall": 8, "kDeadAir": 110, "kToolSilent": 110, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 88.3, "taskScore": 88, "stalled": 9, "toolSilent": 69, "deadAir": 49, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 199, "nToolTurns": 141, "nCls": 100, "kStall": 5, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 99, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 87.43, "taskScore": 87, "stalled": 13, "toolSilent": 81, "deadAir": 56, "fabrication": null, "adherence": 77, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 133, "nCls": 84, "kStall": 7, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 84, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 86.84, "taskScore": 87, "stalled": 17, "toolSilent": 97, "deadAir": 65, "fabrication": null, "adherence": 11, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 181, "nToolTurns": 121, "nCls": 63, "kStall": 9, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 63, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 6, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 86.26, "taskScore": 86, "stalled": 19, "toolSilent": 97, "deadAir": 60, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 159, "nToolTurns": 99, "nCls": 62, "kStall": 10, "kDeadAir": 96, "kToolSilent": 96, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 60, "kStall": 7, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 84.8, "taskScore": 85, "stalled": 17, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 133, "nCls": 60, "kStall": 9, "kDeadAir": 133, "kToolSilent": 133, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 84.06, "taskScore": 84, "stalled": 15, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 205, "nToolTurns": 148, "nCls": 56, "kStall": 8, "kDeadAir": 148, "kToolSilent": 148, "kAdherence": 56, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 82.75, "taskScore": 83, "stalled": 17, "toolSilent": 51, "deadAir": 33, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 114, "kStall": 9, "kDeadAir": 57, "kToolSilent": 57, "kAdherence": 114, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 81.29, "taskScore": 81, "stalled": 28, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 76, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 152, "nToolTurns": 92, "nCls": 152, "kStall": 15, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 152, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 81.14, "taskScore": 81, "stalled": 28, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 161, "nToolTurns": 101, "nCls": 58, "kStall": 15, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 80.7, "taskScore": 81, "stalled": 15, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 201, "nToolTurns": 145, "nCls": 56, "kStall": 8, "kDeadAir": 145, "kToolSilent": 145, "kAdherence": 56, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 79.39, "taskScore": 79, "stalled": 24, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 202, "nToolTurns": 142, "nCls": 60, "kStall": 13, "kDeadAir": 142, "kToolSilent": 142, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 79.24, "taskScore": 79, "stalled": 19, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 205, "nToolTurns": 147, "nCls": 58, "kStall": 10, "kDeadAir": 147, "kToolSilent": 147, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 78.07, "taskScore": 78, "stalled": 15, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 184, "nToolTurns": 127, "nCls": 57, "kStall": 8, "kDeadAir": 127, "kToolSilent": 127, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 73.68, "taskScore": 74, "stalled": 26, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 132, "nCls": 58, "kStall": 14, "kDeadAir": 132, "kToolSilent": 132, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 73.25, "taskScore": 73, "stalled": 22, "toolSilent": 100, "deadAir": 74, "fabrication": null, "adherence": 83, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 191, "nToolTurns": 137, "nCls": 49, "kStall": 12, "kDeadAir": 142, "kToolSilent": 137, "kAdherence": 49, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 55.99, "taskScore": 56, "stalled": 63, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 34, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 48.98, "taskScore": 49, "stalled": 63, "toolSilent": 86, "deadAir": 40, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 111, "nToolTurns": 51, "nCls": 67, "kStall": 34, "kDeadAir": 44, "kToolSilent": 44, "kAdherence": 67, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 } ] }, "nb": { "label": "Norwegian", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 93.86, "taskScore": 94, "stalled": 2, "toolSilent": 98, "deadAir": 62, "fabrication": null, "adherence": 5, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 62, "kStall": 1, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 92.11, "taskScore": 92, "stalled": 4, "toolSilent": 91, "deadAir": 63, "fabrication": null, "adherence": 8, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 128, "nCls": 70, "kStall": 2, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 70, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 92.11, "taskScore": 92, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 73, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 60, "kStall": 6, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 90.64, "taskScore": 91, "stalled": 6, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 3, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 88.01, "taskScore": 88, "stalled": 13, "toolSilent": 95, "deadAir": 59, "fabrication": null, "adherence": 1, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 59, "kStall": 7, "kDeadAir": 95, "kToolSilent": 95, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 87.72, "taskScore": 88, "stalled": 17, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 56, "kStall": 9, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 56, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 86.84, "taskScore": 87, "stalled": 17, "toolSilent": 99, "deadAir": 65, "fabrication": null, "adherence": 22, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 175, "nToolTurns": 115, "nCls": 61, "kStall": 9, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 6, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 86.26, "taskScore": 86, "stalled": 20, "toolSilent": 61, "deadAir": 39, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 101, "kStall": 11, "kDeadAir": 64, "kToolSilent": 64, "kAdherence": 101, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 173, "nToolTurns": 113, "nCls": 60, "kStall": 7, "kDeadAir": 113, "kToolSilent": 113, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 85.67, "taskScore": 86, "stalled": 11, "toolSilent": 91, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 131, "nCls": 71, "kStall": 6, "kDeadAir": 119, "kToolSilent": 119, "kAdherence": 71, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 85.09, "taskScore": 85, "stalled": 15, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 131, "nCls": 57, "kStall": 8, "kDeadAir": 131, "kToolSilent": 131, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 84.5, "taskScore": 85, "stalled": 15, "toolSilent": 91, "deadAir": 64, "fabrication": null, "adherence": 57, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 200, "nToolTurns": 140, "nCls": 72, "kStall": 8, "kDeadAir": 128, "kToolSilent": 128, "kAdherence": 72, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 83.33, "taskScore": 83, "stalled": 26, "toolSilent": 100, "deadAir": 61, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 153, "nToolTurns": 93, "nCls": 60, "kStall": 14, "kDeadAir": 93, "kToolSilent": 93, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 82.46, "taskScore": 82, "stalled": 24, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 67, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 154, "nToolTurns": 94, "nCls": 150, "kStall": 13, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 150, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 81.87, "taskScore": 82, "stalled": 20, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 126, "nCls": 60, "kStall": 11, "kDeadAir": 126, "kToolSilent": 126, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 79.39, "taskScore": 79, "stalled": 13, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 132, "nCls": 58, "kStall": 7, "kDeadAir": 132, "kToolSilent": 132, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 78.36, "taskScore": 78, "stalled": 20, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 135, "nCls": 57, "kStall": 11, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 77.49, "taskScore": 77, "stalled": 19, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 138, "nCls": 59, "kStall": 10, "kDeadAir": 138, "kToolSilent": 138, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 77.19, "taskScore": 77, "stalled": 22, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 131, "nCls": 59, "kStall": 12, "kDeadAir": 131, "kToolSilent": 131, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 76.9, "taskScore": 77, "stalled": 28, "toolSilent": 71, "deadAir": 49, "fabrication": null, "adherence": 89, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 129, "nCls": 93, "kStall": 15, "kDeadAir": 92, "kToolSilent": 92, "kAdherence": 93, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 74.12, "taskScore": 74, "stalled": 17, "toolSilent": 100, "deadAir": 75, "fabrication": null, "adherence": 74, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 144, "nCls": 49, "kStall": 9, "kDeadAir": 149, "kToolSilent": 144, "kAdherence": 49, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 72.81, "taskScore": 73, "stalled": 30, "toolSilent": 98, "deadAir": 68, "fabrication": null, "adherence": 91, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 133, "nCls": 62, "kStall": 16, "kDeadAir": 130, "kToolSilent": 130, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 50.44, "taskScore": 50, "stalled": 72, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 4, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 39, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 39.77, "taskScore": 40, "stalled": 74, "toolSilent": 94, "deadAir": 43, "fabrication": null, "adherence": 85, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 111, "nToolTurns": 51, "nCls": 63, "kStall": 40, "kDeadAir": 48, "kToolSilent": 48, "kAdherence": 63, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 16, "iCls": 19, "iFab": 0 } ] }, "hi": { "label": "Hindi", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 95.91, "taskScore": 96, "stalled": 6, "toolSilent": 99, "deadAir": 61, "fabrication": null, "adherence": 74, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 157, "nToolTurns": 97, "nCls": 61, "kStall": 3, "kDeadAir": 96, "kToolSilent": 96, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 95.32, "taskScore": 95, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 166, "nToolTurns": 106, "nCls": 60, "kStall": 6, "kDeadAir": 106, "kToolSilent": 106, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 92.11, "taskScore": 92, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 93, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 60, "kStall": 6, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 7, "toolSilent": 75, "deadAir": 51, "fabrication": null, "adherence": 11, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 182, "nToolTurns": 122, "nCls": 90, "kStall": 4, "kDeadAir": 92, "kToolSilent": 92, "kAdherence": 90, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 91.23, "taskScore": 91, "stalled": 4, "toolSilent": 99, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 161, "nToolTurns": 101, "nCls": 61, "kStall": 2, "kDeadAir": 100, "kToolSilent": 100, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 89.18, "taskScore": 89, "stalled": 19, "toolSilent": 56, "deadAir": 36, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 107, "kStall": 10, "kDeadAir": 61, "kToolSilent": 61, "kAdherence": 107, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 88.6, "taskScore": 89, "stalled": 13, "toolSilent": 98, "deadAir": 66, "fabrication": null, "adherence": 20, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 123, "nCls": 63, "kStall": 7, "kDeadAir": 120, "kToolSilent": 120, "kAdherence": 63, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 87.72, "taskScore": 88, "stalled": 15, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 60, "kStall": 8, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 6, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 86.84, "taskScore": 87, "stalled": 7, "toolSilent": 85, "deadAir": 60, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 140, "nCls": 79, "kStall": 4, "kDeadAir": 119, "kToolSilent": 119, "kAdherence": 79, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 86.84, "taskScore": 87, "stalled": 19, "toolSilent": 100, "deadAir": 61, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 153, "nToolTurns": 93, "nCls": 60, "kStall": 10, "kDeadAir": 93, "kToolSilent": 93, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 86.55, "taskScore": 87, "stalled": 13, "toolSilent": 81, "deadAir": 56, "fabrication": null, "adherence": 85, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 199, "nToolTurns": 139, "nCls": 87, "kStall": 7, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 86, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 85.96, "taskScore": 86, "stalled": 11, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 200, "nToolTurns": 143, "nCls": 57, "kStall": 6, "kDeadAir": 143, "kToolSilent": 143, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.09, "taskScore": 85, "stalled": 15, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 176, "nToolTurns": 116, "nCls": 60, "kStall": 8, "kDeadAir": 116, "kToolSilent": 116, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 84.65, "taskScore": 85, "stalled": 11, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 203, "nToolTurns": 146, "nCls": 57, "kStall": 6, "kDeadAir": 146, "kToolSilent": 146, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 82.46, "taskScore": 82, "stalled": 20, "toolSilent": 91, "deadAir": 62, "fabrication": null, "adherence": 69, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 128, "nCls": 71, "kStall": 11, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 71, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 82.46, "taskScore": 82, "stalled": 13, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 200, "nToolTurns": 144, "nCls": 56, "kStall": 7, "kDeadAir": 144, "kToolSilent": 144, "kAdherence": 56, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 80.7, "taskScore": 81, "stalled": 20, "toolSilent": 98, "deadAir": 68, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 191, "nToolTurns": 131, "nCls": 62, "kStall": 11, "kDeadAir": 129, "kToolSilent": 129, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 80.41, "taskScore": 80, "stalled": 20, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 137, "nCls": 59, "kStall": 11, "kDeadAir": 137, "kToolSilent": 137, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 79.82, "taskScore": 80, "stalled": 19, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 130, "nCls": 58, "kStall": 10, "kDeadAir": 130, "kToolSilent": 130, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 78.65, "taskScore": 79, "stalled": 33, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 73, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 148, "nToolTurns": 88, "nCls": 148, "kStall": 18, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 148, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 76.75, "taskScore": 77, "stalled": 17, "toolSilent": 100, "deadAir": 76, "fabrication": null, "adherence": 53, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 134, "nCls": 46, "kStall": 9, "kDeadAir": 143, "kToolSilent": 134, "kAdherence": 44, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 72.81, "taskScore": 73, "stalled": 26, "toolSilent": 99, "deadAir": 68, "fabrication": null, "adherence": 96, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 124, "nCls": 59, "kStall": 14, "kDeadAir": 124, "kToolSilent": 123, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 49.56, "taskScore": 50, "stalled": 70, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 38, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 46.64, "taskScore": 47, "stalled": 63, "toolSilent": 83, "deadAir": 38, "fabrication": null, "adherence": 86, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 112, "nToolTurns": 52, "nCls": 69, "kStall": 34, "kDeadAir": 43, "kToolSilent": 43, "kAdherence": 69, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 15, "iCls": 19, "iFab": 0 } ] }, "ta": { "label": "Tamil", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 98.25, "taskScore": 98, "stalled": 6, "toolSilent": 99, "deadAir": 59, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 150, "nToolTurns": 90, "nCls": 61, "kStall": 3, "kDeadAir": 89, "kToolSilent": 89, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 92.11, "taskScore": 92, "stalled": 11, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 83, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 6, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 91.81, "taskScore": 92, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 173, "nToolTurns": 113, "nCls": 60, "kStall": 6, "kDeadAir": 113, "kToolSilent": 113, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 4, "toolSilent": 93, "deadAir": 65, "fabrication": null, "adherence": 4, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 132, "nCls": 66, "kStall": 2, "kDeadAir": 123, "kToolSilent": 123, "kAdherence": 66, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 91.23, "taskScore": 91, "stalled": 2, "toolSilent": 90, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 135, "nCls": 73, "kStall": 1, "kDeadAir": 121, "kToolSilent": 121, "kAdherence": 73, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 89.47, "taskScore": 89, "stalled": 6, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 60, "kStall": 3, "kDeadAir": 100, "kToolSilent": 100, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 88.6, "taskScore": 89, "stalled": 11, "toolSilent": 92, "deadAir": 63, "fabrication": null, "adherence": 25, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 184, "nToolTurns": 125, "nCls": 69, "kStall": 6, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 69, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 88.6, "taskScore": 89, "stalled": 7, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 4, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 87.72, "taskScore": 88, "stalled": 13, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 133, "nCls": 59, "kStall": 7, "kDeadAir": 133, "kToolSilent": 133, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 60, "kStall": 6, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 86.26, "taskScore": 86, "stalled": 20, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 13, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 158, "nToolTurns": 98, "nCls": 158, "kStall": 11, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 158, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 85.96, "taskScore": 86, "stalled": 9, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 139, "nCls": 57, "kStall": 5, "kDeadAir": 139, "kToolSilent": 139, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 85.09, "taskScore": 85, "stalled": 24, "toolSilent": 98, "deadAir": 60, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 154, "nToolTurns": 94, "nCls": 62, "kStall": 13, "kDeadAir": 92, "kToolSilent": 92, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 83.33, "taskScore": 83, "stalled": 15, "toolSilent": 96, "deadAir": 68, "fabrication": null, "adherence": 49, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 139, "nCls": 64, "kStall": 8, "kDeadAir": 133, "kToolSilent": 133, "kAdherence": 64, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 82.75, "taskScore": 83, "stalled": 17, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 132, "nCls": 58, "kStall": 9, "kDeadAir": 132, "kToolSilent": 132, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 82.31, "taskScore": 82, "stalled": 17, "toolSilent": 89, "deadAir": 63, "fabrication": null, "adherence": 66, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 138, "nCls": 74, "kStall": 9, "kDeadAir": 124, "kToolSilent": 123, "kAdherence": 74, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 81.43, "taskScore": 81, "stalled": 19, "toolSilent": 95, "deadAir": 63, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 179, "nToolTurns": 119, "nCls": 66, "kStall": 10, "kDeadAir": 113, "kToolSilent": 113, "kAdherence": 66, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 79.53, "taskScore": 80, "stalled": 31, "toolSilent": 67, "deadAir": 43, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 164, "nToolTurns": 104, "nCls": 94, "kStall": 17, "kDeadAir": 70, "kToolSilent": 70, "kAdherence": 94, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 78.95, "taskScore": 79, "stalled": 20, "toolSilent": 100, "deadAir": 67, "fabrication": null, "adherence": 80, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 184, "nToolTurns": 124, "nCls": 60, "kStall": 11, "kDeadAir": 124, "kToolSilent": 124, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 78.07, "taskScore": 78, "stalled": 24, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 135, "nCls": 60, "kStall": 13, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 75.73, "taskScore": 76, "stalled": 24, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 135, "nCls": 59, "kStall": 13, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 72.51, "taskScore": 73, "stalled": 28, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 157, "nToolTurns": 97, "nCls": 60, "kStall": 15, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 66.08, "taskScore": 66, "stalled": 31, "toolSilent": 100, "deadAir": 74, "fabrication": null, "adherence": 77, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 176, "nToolTurns": 118, "nCls": 45, "kStall": 17, "kDeadAir": 131, "kToolSilent": 118, "kAdherence": 45, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 52.19, "taskScore": 52, "stalled": 61, "toolSilent": 91, "deadAir": 44, "fabrication": null, "adherence": 65, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 116, "nToolTurns": 56, "nCls": 65, "kStall": 33, "kDeadAir": 51, "kToolSilent": 51, "kAdherence": 65, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 16, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 49.56, "taskScore": 50, "stalled": 76, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 41, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 } ] }, "te": { "label": "Telugu", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 94.74, "taskScore": 95, "stalled": 0, "toolSilent": 86, "deadAir": 61, "fabrication": null, "adherence": 8, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 133, "nCls": 75, "kStall": 0, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 75, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 92.69, "taskScore": 93, "stalled": 13, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 156, "nToolTurns": 96, "nCls": 60, "kStall": 7, "kDeadAir": 96, "kToolSilent": 96, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 92.11, "taskScore": 92, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 60, "kStall": 6, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 91.23, "taskScore": 91, "stalled": 13, "toolSilent": 54, "deadAir": 34, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 107, "kStall": 7, "kDeadAir": 55, "kToolSilent": 55, "kAdherence": 107, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 90.64, "taskScore": 91, "stalled": 9, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 137, "nCls": 57, "kStall": 5, "kDeadAir": 137, "kToolSilent": 137, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 89.91, "taskScore": 90, "stalled": 17, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 166, "nToolTurns": 106, "nCls": 60, "kStall": 9, "kDeadAir": 106, "kToolSilent": 106, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 89.47, "taskScore": 89, "stalled": 6, "toolSilent": 99, "deadAir": 62, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 61, "kStall": 3, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 88.3, "taskScore": 88, "stalled": 15, "toolSilent": 95, "deadAir": 58, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 156, "nToolTurns": 96, "nCls": 65, "kStall": 8, "kDeadAir": 91, "kToolSilent": 91, "kAdherence": 65, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 86.84, "taskScore": 87, "stalled": 17, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 9, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 177, "nToolTurns": 117, "nCls": 60, "kStall": 9, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 170, "nToolTurns": 110, "nCls": 60, "kStall": 6, "kDeadAir": 110, "kToolSilent": 110, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 60, "kStall": 6, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 83.77, "taskScore": 84, "stalled": 17, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 124, "nCls": 59, "kStall": 9, "kDeadAir": 124, "kToolSilent": 124, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 83.04, "taskScore": 83, "stalled": 17, "toolSilent": 88, "deadAir": 62, "fabrication": null, "adherence": 56, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 138, "nCls": 75, "kStall": 9, "kDeadAir": 122, "kToolSilent": 122, "kAdherence": 75, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 82.75, "taskScore": 83, "stalled": 17, "toolSilent": 69, "deadAir": 48, "fabrication": null, "adherence": 79, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 140, "nCls": 102, "kStall": 9, "kDeadAir": 96, "kToolSilent": 96, "kAdherence": 100, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 82.75, "taskScore": 83, "stalled": 19, "toolSilent": 99, "deadAir": 68, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 128, "nCls": 61, "kStall": 10, "kDeadAir": 127, "kToolSilent": 127, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 82.31, "taskScore": 82, "stalled": 13, "toolSilent": 88, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 138, "nCls": 74, "kStall": 7, "kDeadAir": 122, "kToolSilent": 122, "kAdherence": 74, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 81.29, "taskScore": 81, "stalled": 17, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 130, "nCls": 60, "kStall": 9, "kDeadAir": 130, "kToolSilent": 130, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 80.7, "taskScore": 81, "stalled": 24, "toolSilent": 100, "deadAir": 60, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 151, "nToolTurns": 91, "nCls": 60, "kStall": 13, "kDeadAir": 91, "kToolSilent": 91, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 80.41, "taskScore": 80, "stalled": 28, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 11, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 148, "nToolTurns": 88, "nCls": 148, "kStall": 15, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 146, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 79.39, "taskScore": 79, "stalled": 17, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 199, "nToolTurns": 144, "nCls": 55, "kStall": 9, "kDeadAir": 144, "kToolSilent": 144, "kAdherence": 55, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 77.63, "taskScore": 78, "stalled": 20, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 200, "nToolTurns": 141, "nCls": 59, "kStall": 11, "kDeadAir": 141, "kToolSilent": 141, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 75.44, "taskScore": 75, "stalled": 26, "toolSilent": 99, "deadAir": 67, "fabrication": null, "adherence": 60, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 126, "nCls": 61, "kStall": 14, "kDeadAir": 125, "kToolSilent": 125, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 74.56, "taskScore": 75, "stalled": 20, "toolSilent": 100, "deadAir": 80, "fabrication": null, "adherence": 80, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 137, "nCls": 38, "kStall": 11, "kDeadAir": 155, "kToolSilent": 137, "kAdherence": 38, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 48.39, "taskScore": 48, "stalled": 74, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 40, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 24.12, "taskScore": 24, "stalled": 83, "toolSilent": 96, "deadAir": 28, "fabrication": null, "adherence": 80, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 85, "nToolTurns": 25, "nCls": 61, "kStall": 45, "kDeadAir": 24, "kToolSilent": 24, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 9, "iCls": 19, "iFab": 0 } ] }, "zh": { "label": "Chinese", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 95.91, "taskScore": 96, "stalled": 2, "toolSilent": 98, "deadAir": 62, "fabrication": null, "adherence": 8, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 62, "kStall": 1, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 92.98, "taskScore": 93, "stalled": 11, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 60, "kStall": 6, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 7, "toolSilent": 84, "deadAir": 57, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 180, "nToolTurns": 123, "nCls": 77, "kStall": 4, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 77, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 91.23, "taskScore": 91, "stalled": 17, "toolSilent": 99, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 61, "kStall": 9, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 89.47, "taskScore": 89, "stalled": 6, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 164, "nToolTurns": 104, "nCls": 60, "kStall": 3, "kDeadAir": 104, "kToolSilent": 104, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 87.72, "taskScore": 88, "stalled": 7, "toolSilent": 64, "deadAir": 45, "fabrication": null, "adherence": 78, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 135, "nCls": 107, "kStall": 4, "kDeadAir": 87, "kToolSilent": 87, "kAdherence": 104, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 87.28, "taskScore": 87, "stalled": 9, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 203, "nToolTurns": 145, "nCls": 58, "kStall": 5, "kDeadAir": 145, "kToolSilent": 145, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 86.84, "taskScore": 87, "stalled": 17, "toolSilent": 100, "deadAir": 67, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 123, "nCls": 60, "kStall": 9, "kDeadAir": 123, "kToolSilent": 123, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 86.84, "taskScore": 87, "stalled": 11, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 2, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 125, "nCls": 58, "kStall": 6, "kDeadAir": 125, "kToolSilent": 125, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 99, "deadAir": 66, "fabrication": null, "adherence": 65, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 182, "nToolTurns": 122, "nCls": 61, "kStall": 7, "kDeadAir": 121, "kToolSilent": 121, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 7, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 175, "nToolTurns": 115, "nCls": 60, "kStall": 7, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 84.8, "taskScore": 85, "stalled": 17, "toolSilent": 81, "deadAir": 56, "fabrication": null, "adherence": 45, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 137, "nCls": 86, "kStall": 9, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 85, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 82.02, "taskScore": 82, "stalled": 24, "toolSilent": 98, "deadAir": 59, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 153, "nToolTurns": 93, "nCls": 62, "kStall": 13, "kDeadAir": 91, "kToolSilent": 91, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 81.58, "taskScore": 82, "stalled": 17, "toolSilent": 60, "deadAir": 39, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 170, "nToolTurns": 110, "nCls": 104, "kStall": 9, "kDeadAir": 66, "kToolSilent": 66, "kAdherence": 104, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 80.85, "taskScore": 81, "stalled": 19, "toolSilent": 100, "deadAir": 73, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 203, "nToolTurns": 149, "nCls": 54, "kStall": 10, "kDeadAir": 149, "kToolSilent": 149, "kAdherence": 54, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 80.7, "taskScore": 81, "stalled": 20, "toolSilent": 82, "deadAir": 57, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 133, "nCls": 83, "kStall": 11, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 83, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 79.82, "taskScore": 80, "stalled": 20, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 86, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 128, "nCls": 60, "kStall": 11, "kDeadAir": 128, "kToolSilent": 128, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 79.39, "taskScore": 79, "stalled": 20, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 205, "nToolTurns": 147, "nCls": 58, "kStall": 11, "kDeadAir": 147, "kToolSilent": 147, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 79.09, "taskScore": 79, "stalled": 24, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 129, "nCls": 60, "kStall": 13, "kDeadAir": 129, "kToolSilent": 129, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 78.51, "taskScore": 79, "stalled": 22, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 126, "nCls": 60, "kStall": 12, "kDeadAir": 126, "kToolSilent": 126, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 78.36, "taskScore": 78, "stalled": 20, "toolSilent": 100, "deadAir": 73, "fabrication": null, "adherence": 88, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 131, "nCls": 50, "kStall": 11, "kDeadAir": 138, "kToolSilent": 131, "kAdherence": 50, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 73.54, "taskScore": 74, "stalled": 31, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 152, "nToolTurns": 92, "nCls": 152, "kStall": 17, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 152, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 56.87, "taskScore": 57, "stalled": 69, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 37, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 42.54, "taskScore": 43, "stalled": 70, "toolSilent": 100, "deadAir": 43, "fabrication": null, "adherence": 91, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 106, "nToolTurns": 46, "nCls": 60, "kStall": 38, "kDeadAir": 46, "kToolSilent": 46, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 14, "iCls": 19, "iFab": 0 } ] }, "ko": { "label": "Korean", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 92.11, "taskScore": 92, "stalled": 13, "toolSilent": 96, "deadAir": 61, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 163, "nToolTurns": 103, "nCls": 64, "kStall": 7, "kDeadAir": 99, "kToolSilent": 99, "kAdherence": 64, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 90.64, "taskScore": 91, "stalled": 9, "toolSilent": 78, "deadAir": 53, "fabrication": null, "adherence": 1, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 182, "nToolTurns": 124, "nCls": 85, "kStall": 5, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 85, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 87.72, "taskScore": 88, "stalled": 20, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 161, "nToolTurns": 101, "nCls": 60, "kStall": 11, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 85.38, "taskScore": 85, "stalled": 17, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 60, "kStall": 9, "kDeadAir": 105, "kToolSilent": 105, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 84.21, "taskScore": 84, "stalled": 22, "toolSilent": 98, "deadAir": 67, "fabrication": null, "adherence": 55, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 180, "nToolTurns": 123, "nCls": 60, "kStall": 12, "kDeadAir": 120, "kToolSilent": 120, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 84.21, "taskScore": 84, "stalled": 17, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 9, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 84.21, "taskScore": 84, "stalled": 19, "toolSilent": 94, "deadAir": 60, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 163, "nToolTurns": 103, "nCls": 66, "kStall": 10, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 66, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 83.92, "taskScore": 84, "stalled": 30, "toolSilent": 100, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 156, "nToolTurns": 96, "nCls": 60, "kStall": 16, "kDeadAir": 96, "kToolSilent": 96, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 83.33, "taskScore": 83, "stalled": 13, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 203, "nToolTurns": 146, "nCls": 57, "kStall": 7, "kDeadAir": 146, "kToolSilent": 146, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 82.46, "taskScore": 82, "stalled": 19, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 174, "nToolTurns": 115, "nCls": 59, "kStall": 10, "kDeadAir": 115, "kToolSilent": 115, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 80.7, "taskScore": 81, "stalled": 19, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 203, "nToolTurns": 146, "nCls": 57, "kStall": 10, "kDeadAir": 146, "kToolSilent": 146, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 80.7, "taskScore": 81, "stalled": 31, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 93, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 170, "nToolTurns": 110, "nCls": 60, "kStall": 17, "kDeadAir": 110, "kToolSilent": 110, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 79.97, "taskScore": 80, "stalled": 17, "toolSilent": 100, "deadAir": 74, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 143, "nCls": 51, "kStall": 9, "kDeadAir": 147, "kToolSilent": 143, "kAdherence": 51, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 78.95, "taskScore": 79, "stalled": 35, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 65, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 154, "nToolTurns": 94, "nCls": 154, "kStall": 19, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 154, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 78.51, "taskScore": 79, "stalled": 20, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 138, "nCls": 57, "kStall": 11, "kDeadAir": 138, "kToolSilent": 138, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 78.36, "taskScore": 78, "stalled": 19, "toolSilent": 83, "deadAir": 60, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 201, "nToolTurns": 146, "nCls": 80, "kStall": 10, "kDeadAir": 121, "kToolSilent": 121, "kAdherence": 80, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 75.44, "taskScore": 75, "stalled": 28, "toolSilent": 58, "deadAir": 41, "fabrication": null, "adherence": 91, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 137, "nCls": 116, "kStall": 15, "kDeadAir": 80, "kToolSilent": 80, "kAdherence": 114, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 75.15, "taskScore": 75, "stalled": 33, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 135, "nCls": 60, "kStall": 18, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 73.68, "taskScore": 74, "stalled": 20, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 93, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 140, "nCls": 57, "kStall": 11, "kDeadAir": 140, "kToolSilent": 140, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 72.81, "taskScore": 73, "stalled": 30, "toolSilent": 99, "deadAir": 68, "fabrication": null, "adherence": 92, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 128, "nCls": 59, "kStall": 16, "kDeadAir": 128, "kToolSilent": 127, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 72.51, "taskScore": 73, "stalled": 41, "toolSilent": 47, "deadAir": 29, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 161, "nToolTurns": 101, "nCls": 114, "kStall": 22, "kDeadAir": 47, "kToolSilent": 47, "kAdherence": 114, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 71.93, "taskScore": 72, "stalled": 35, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 137, "nCls": 58, "kStall": 19, "kDeadAir": 137, "kToolSilent": 137, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 69.3, "taskScore": 69, "stalled": 35, "toolSilent": 75, "deadAir": 53, "fabrication": null, "adherence": 79, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 137, "nCls": 91, "kStall": 19, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 91, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 53.95, "taskScore": 54, "stalled": 74, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 6, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 40, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 43.13, "taskScore": 43, "stalled": 74, "toolSilent": 98, "deadAir": 45, "fabrication": null, "adherence": 70, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 111, "nToolTurns": 51, "nCls": 61, "kStall": 40, "kDeadAir": 50, "kToolSilent": 50, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 } ] }, "th": { "label": "Thai", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 89.47, "taskScore": 89, "stalled": 11, "toolSilent": 80, "deadAir": 55, "fabrication": null, "adherence": 14, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 130, "nCls": 84, "kStall": 6, "kDeadAir": 104, "kToolSilent": 104, "kAdherence": 84, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 87.72, "taskScore": 88, "stalled": 17, "toolSilent": 98, "deadAir": 63, "fabrication": null, "adherence": 79, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 166, "nToolTurns": 106, "nCls": 62, "kStall": 9, "kDeadAir": 104, "kToolSilent": 104, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 87.72, "taskScore": 88, "stalled": 13, "toolSilent": 90, "deadAir": 57, "fabrication": null, "adherence": 12, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 71, "kStall": 7, "kDeadAir": 94, "kToolSilent": 94, "kAdherence": 71, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 85.38, "taskScore": 85, "stalled": 15, "toolSilent": 88, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 137, "nCls": 75, "kStall": 8, "kDeadAir": 120, "kToolSilent": 120, "kAdherence": 75, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 84.21, "taskScore": 84, "stalled": 22, "toolSilent": 99, "deadAir": 66, "fabrication": null, "adherence": 32, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 178, "nToolTurns": 118, "nCls": 61, "kStall": 12, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 84.21, "taskScore": 84, "stalled": 17, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 108, "nCls": 60, "kStall": 9, "kDeadAir": 108, "kToolSilent": 108, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 83.63, "taskScore": 84, "stalled": 19, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 10, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 81.58, "taskScore": 82, "stalled": 20, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 140, "nCls": 57, "kStall": 11, "kDeadAir": 140, "kToolSilent": 140, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 81.58, "taskScore": 82, "stalled": 22, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 174, "nToolTurns": 114, "nCls": 60, "kStall": 12, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 80.7, "taskScore": 81, "stalled": 26, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 130, "nCls": 60, "kStall": 14, "kDeadAir": 130, "kToolSilent": 130, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 80.7, "taskScore": 81, "stalled": 28, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 167, "nToolTurns": 107, "nCls": 60, "kStall": 15, "kDeadAir": 107, "kToolSilent": 107, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 80.7, "taskScore": 81, "stalled": 26, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 60, "kStall": 14, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 80.12, "taskScore": 80, "stalled": 20, "toolSilent": 100, "deadAir": 71, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 141, "nCls": 57, "kStall": 11, "kDeadAir": 141, "kToolSilent": 141, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 78.95, "taskScore": 79, "stalled": 19, "toolSilent": 100, "deadAir": 75, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 210, "nToolTurns": 157, "nCls": 53, "kStall": 10, "kDeadAir": 157, "kToolSilent": 157, "kAdherence": 53, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 78.36, "taskScore": 78, "stalled": 24, "toolSilent": 63, "deadAir": 44, "fabrication": null, "adherence": 83, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 136, "nCls": 108, "kStall": 13, "kDeadAir": 85, "kToolSilent": 85, "kAdherence": 107, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 77.78, "taskScore": 78, "stalled": 24, "toolSilent": 92, "deadAir": 65, "fabrication": null, "adherence": 57, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 139, "nCls": 70, "kStall": 13, "kDeadAir": 128, "kToolSilent": 128, "kAdherence": 70, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 77.34, "taskScore": 77, "stalled": 35, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 21, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 147, "nToolTurns": 87, "nCls": 147, "kStall": 19, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 147, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 77.19, "taskScore": 77, "stalled": 31, "toolSilent": 100, "deadAir": 61, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 152, "nToolTurns": 92, "nCls": 60, "kStall": 17, "kDeadAir": 92, "kToolSilent": 92, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 74.71, "taskScore": 75, "stalled": 24, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 135, "nCls": 58, "kStall": 13, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 73.68, "taskScore": 74, "stalled": 31, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 135, "nCls": 60, "kStall": 17, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 73.25, "taskScore": 73, "stalled": 33, "toolSilent": 57, "deadAir": 36, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 105, "kStall": 18, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 105, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 69.88, "taskScore": 70, "stalled": 22, "toolSilent": 100, "deadAir": 76, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 142, "nCls": 48, "kStall": 12, "kDeadAir": 149, "kToolSilent": 142, "kAdherence": 48, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 69.15, "taskScore": 69, "stalled": 33, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 126, "nCls": 60, "kStall": 18, "kDeadAir": 126, "kToolSilent": 126, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 53.07, "taskScore": 53, "stalled": 67, "toolSilent": 100, "deadAir": 49, "fabrication": null, "adherence": 91, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 117, "nToolTurns": 57, "nCls": 60, "kStall": 36, "kDeadAir": 57, "kToolSilent": 57, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 16, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 43.42, "taskScore": 43, "stalled": 76, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 41, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 } ] }, "ja": { "label": "Japanese", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-05", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 90.35, "taskScore": 90, "stalled": 9, "toolSilent": 99, "deadAir": 65, "fabrication": null, "adherence": 1, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 173, "nToolTurns": 113, "nCls": 61, "kStall": 5, "kDeadAir": 112, "kToolSilent": 112, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 89.47, "taskScore": 89, "stalled": 11, "toolSilent": 96, "deadAir": 64, "fabrication": null, "adherence": 21, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 182, "nToolTurns": 122, "nCls": 65, "kStall": 6, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 65, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 88.6, "taskScore": 89, "stalled": 13, "toolSilent": 61, "deadAir": 41, "fabrication": null, "adherence": 8, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 182, "nToolTurns": 122, "nCls": 108, "kStall": 7, "kDeadAir": 74, "kToolSilent": 74, "kAdherence": 108, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 64, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 60, "kStall": 7, "kDeadAir": 109, "kToolSilent": 109, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 85.09, "taskScore": 85, "stalled": 15, "toolSilent": 99, "deadAir": 64, "fabrication": null, "adherence": 92, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 61, "kStall": 8, "kDeadAir": 110, "kToolSilent": 110, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 84.21, "taskScore": 84, "stalled": 17, "toolSilent": 97, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 169, "nToolTurns": 109, "nCls": 63, "kStall": 9, "kDeadAir": 106, "kToolSilent": 106, "kAdherence": 63, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 82.46, "taskScore": 82, "stalled": 24, "toolSilent": 99, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 164, "nToolTurns": 104, "nCls": 61, "kStall": 13, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 81.58, "taskScore": 82, "stalled": 22, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 174, "nToolTurns": 114, "nCls": 60, "kStall": 12, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 81.58, "taskScore": 82, "stalled": 26, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 60, "kStall": 14, "kDeadAir": 100, "kToolSilent": 100, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 81.29, "taskScore": 81, "stalled": 22, "toolSilent": 55, "deadAir": 38, "fabrication": null, "adherence": 88, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 127, "nCls": 116, "kStall": 12, "kDeadAir": 71, "kToolSilent": 70, "kAdherence": 116, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 79.24, "taskScore": 79, "stalled": 20, "toolSilent": 82, "deadAir": 57, "fabrication": null, "adherence": 43, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 136, "nCls": 83, "kStall": 11, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 83, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 77.19, "taskScore": 77, "stalled": 22, "toolSilent": 99, "deadAir": 70, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 139, "nCls": 58, "kStall": 12, "kDeadAir": 138, "kToolSilent": 138, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 77.05, "taskScore": 77, "stalled": 24, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 130, "nCls": 58, "kStall": 13, "kDeadAir": 130, "kToolSilent": 130, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Inception", "model": "Mercury Voice", "taskDone": 76.75, "taskScore": 77, "stalled": 24, "toolSilent": 100, "deadAir": 73, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 205, "nToolTurns": 149, "nCls": 56, "kStall": 13, "kDeadAir": 149, "kToolSilent": 149, "kAdherence": 56, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 74.27, "taskScore": 74, "stalled": 28, "toolSilent": 86, "deadAir": 61, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 137, "nCls": 77, "kStall": 15, "kDeadAir": 118, "kToolSilent": 118, "kAdherence": 77, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 73.98, "taskScore": 74, "stalled": 28, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 194, "nToolTurns": 135, "nCls": 59, "kStall": 15, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 73.68, "taskScore": 74, "stalled": 31, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 161, "nToolTurns": 101, "nCls": 60, "kStall": 17, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 73.1, "taskScore": 73, "stalled": 26, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 77, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 189, "nToolTurns": 133, "nCls": 53, "kStall": 14, "kDeadAir": 136, "kToolSilent": 133, "kAdherence": 53, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 71.93, "taskScore": 72, "stalled": 43, "toolSilent": 44, "deadAir": 28, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 159, "nToolTurns": 99, "nCls": 115, "kStall": 23, "kDeadAir": 44, "kToolSilent": 44, "kAdherence": 115, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 71.93, "taskScore": 72, "stalled": 33, "toolSilent": 97, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 128, "nCls": 64, "kStall": 18, "kDeadAir": 124, "kToolSilent": 124, "kAdherence": 64, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 71.93, "taskScore": 72, "stalled": 35, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 203, "nToolTurns": 146, "nCls": 57, "kStall": 19, "kDeadAir": 146, "kToolSilent": 146, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 71.35, "taskScore": 71, "stalled": 33, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 195, "nToolTurns": 135, "nCls": 60, "kStall": 18, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 70.61, "taskScore": 71, "stalled": 41, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 25, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 152, "nToolTurns": 92, "nCls": 152, "kStall": 22, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 152, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 52.49, "taskScore": 52, "stalled": 74, "toolSilent": 100, "deadAir": 51, "fabrication": null, "adherence": 6, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 122, "nToolTurns": 62, "nCls": 60, "kStall": 40, "kDeadAir": 62, "kToolSilent": 62, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 36.4, "taskScore": 36, "stalled": 87, "toolSilent": 100, "deadAir": 40, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 100, "nToolTurns": 40, "nCls": 60, "kStall": 47, "kDeadAir": 40, "kToolSilent": 40, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 12, "iCls": 19, "iFab": 0 } ] }, "id": { "label": "Indonesian", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-09", "rows": [ { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 7, "toolSilent": 63, "deadAir": 43, "fabrication": null, "adherence": 31, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 128, "nCls": 106, "kStall": 4, "kDeadAir": 80, "kToolSilent": 80, "kAdherence": 106, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 87.72, "taskScore": 88, "stalled": 11, "toolSilent": 95, "deadAir": 62, "fabrication": null, "adherence": 87, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 65, "kStall": 6, "kDeadAir": 106, "kToolSilent": 106, "kAdherence": 65, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 87.72, "taskScore": 88, "stalled": 22, "toolSilent": 99, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 61, "kStall": 12, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 61, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 86.26, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 161, "nToolTurns": 101, "nCls": 60, "kStall": 7, "kDeadAir": 101, "kToolSilent": 101, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 85.96, "taskScore": 86, "stalled": 17, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 164, "nToolTurns": 104, "nCls": 59, "kStall": 9, "kDeadAir": 104, "kToolSilent": 104, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 176, "nToolTurns": 116, "nCls": 60, "kStall": 7, "kDeadAir": 116, "kToolSilent": 116, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 85.96, "taskScore": 86, "stalled": 13, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 176, "nToolTurns": 116, "nCls": 60, "kStall": 7, "kDeadAir": 116, "kToolSilent": 116, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 85.09, "taskScore": 85, "stalled": 17, "toolSilent": 88, "deadAir": 61, "fabrication": null, "adherence": 68, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 192, "nToolTurns": 133, "nCls": 73, "kStall": 9, "kDeadAir": 117, "kToolSilent": 117, "kAdherence": 73, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 84.5, "taskScore": 85, "stalled": 15, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 8, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 135, "nCls": 58, "kStall": 8, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 83.92, "taskScore": 84, "stalled": 13, "toolSilent": 85, "deadAir": 61, "fabrication": null, "adherence": 99, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 199, "nToolTurns": 142, "nCls": 78, "kStall": 7, "kDeadAir": 121, "kToolSilent": 121, "kAdherence": 78, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 82.02, "taskScore": 82, "stalled": 20, "toolSilent": 100, "deadAir": 67, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 123, "nCls": 51, "kStall": 11, "kDeadAir": 123, "kToolSilent": 123, "kAdherence": 51, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 79.68, "taskScore": 80, "stalled": 20, "toolSilent": 100, "deadAir": 72, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 190, "nToolTurns": 132, "nCls": 53, "kStall": 11, "kDeadAir": 137, "kToolSilent": 132, "kAdherence": 53, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 79.53, "taskScore": 80, "stalled": 24, "toolSilent": 42, "deadAir": 27, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 172, "nToolTurns": 112, "nCls": 125, "kStall": 13, "kDeadAir": 47, "kToolSilent": 47, "kAdherence": 125, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 78.8, "taskScore": 79, "stalled": 30, "toolSilent": 95, "deadAir": 60, "fabrication": null, "adherence": 41, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 162, "nToolTurns": 102, "nCls": 65, "kStall": 16, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 65, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 78.65, "taskScore": 79, "stalled": 24, "toolSilent": 55, "deadAir": 39, "fabrication": null, "adherence": 94, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 205, "nToolTurns": 146, "nCls": 114, "kStall": 13, "kDeadAir": 80, "kToolSilent": 80, "kAdherence": 114, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 78.51, "taskScore": 79, "stalled": 31, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 163, "nToolTurns": 103, "nCls": 57, "kStall": 17, "kDeadAir": 103, "kToolSilent": 103, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 77.19, "taskScore": 77, "stalled": 15, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 12, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 136, "nCls": 57, "kStall": 8, "kDeadAir": 136, "kToolSilent": 136, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 76.9, "taskScore": 77, "stalled": 37, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 56, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 146, "nToolTurns": 86, "nCls": 146, "kStall": 20, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 146, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 74.56, "taskScore": 75, "stalled": 35, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 187, "nToolTurns": 127, "nCls": 60, "kStall": 19, "kDeadAir": 127, "kToolSilent": 127, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 74.12, "taskScore": 74, "stalled": 24, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 197, "nToolTurns": 137, "nCls": 60, "kStall": 13, "kDeadAir": 137, "kToolSilent": 137, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 73.25, "taskScore": 73, "stalled": 28, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 193, "nToolTurns": 135, "nCls": 58, "kStall": 15, "kDeadAir": 135, "kToolSilent": 135, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 48.39, "taskScore": 48, "stalled": 74, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 88, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 60, "kStall": 40, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 21.93, "taskScore": 22, "stalled": 89, "toolSilent": 93, "deadAir": 29, "fabrication": null, "adherence": 88, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 87, "nToolTurns": 27, "nCls": 62, "kStall": 48, "kDeadAir": 25, "kToolSilent": 25, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 10, "iCls": 19, "iFab": 0 } ] }, "sw": { "label": "Swahili", "meta": "same scenarios + v3 grader as the English board · n=57 task / 0 fab · 2026-10-09", "rows": [ { "provider": "Baseten", "model": "DeepSeek-V4-Flash-0731", "taskDone": 91.23, "taskScore": 91, "stalled": 7, "toolSilent": 90, "deadAir": 61, "fabrication": null, "adherence": 33, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 126, "nCls": 71, "kStall": 4, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 71, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.5-flash-lite", "taskDone": 89.47, "taskScore": 89, "stalled": 17, "toolSilent": 100, "deadAir": 65, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 171, "nToolTurns": 111, "nCls": 60, "kStall": 9, "kDeadAir": 111, "kToolSilent": 111, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.4-mini", "taskDone": 88.6, "taskScore": 89, "stalled": 22, "toolSilent": 100, "deadAir": 63, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 58, "kStall": 12, "kDeadAir": 100, "kToolSilent": 100, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1", "taskDone": 83.63, "taskScore": 84, "stalled": 19, "toolSilent": 98, "deadAir": 61, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 157, "nToolTurns": 97, "nCls": 62, "kStall": 10, "kDeadAir": 95, "kToolSilent": 95, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "xAI", "model": "Grok-4.3", "taskDone": 83.63, "taskScore": 84, "stalled": 17, "toolSilent": 97, "deadAir": 62, "fabrication": null, "adherence": 0, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 165, "nToolTurns": 105, "nCls": 55, "kStall": 9, "kDeadAir": 102, "kToolSilent": 102, "kAdherence": 55, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Sonnet 5", "taskDone": 83.33, "taskScore": 83, "stalled": 22, "toolSilent": 58, "deadAir": 37, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 164, "nToolTurns": 104, "nCls": 102, "kStall": 12, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 102, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.7-flash", "taskDone": 83.33, "taskScore": 83, "stalled": 19, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 179, "nToolTurns": 119, "nCls": 60, "kStall": 10, "kDeadAir": 119, "kToolSilent": 119, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Gemini", "model": "gemini-3.8-flash", "taskDone": 82.46, "taskScore": 82, "stalled": 20, "toolSilent": 100, "deadAir": 66, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 178, "nToolTurns": 118, "nCls": 60, "kStall": 11, "kDeadAir": 118, "kToolSilent": 118, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "DeepSeek-V4.1-Flash", "taskDone": 81.14, "taskScore": 81, "stalled": 22, "toolSilent": 93, "deadAir": 62, "fabrication": null, "adherence": 97, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 183, "nToolTurns": 123, "nCls": 68, "kStall": 12, "kDeadAir": 114, "kToolSilent": 114, "kAdherence": 68, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "DeepSeek-V4.1-Flash", "taskDone": 79.82, "taskScore": 80, "stalled": 20, "toolSilent": 91, "deadAir": 62, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 188, "nToolTurns": 128, "nCls": 69, "kStall": 11, "kDeadAir": 116, "kToolSilent": 116, "kAdherence": 69, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5", "taskDone": 78.65, "taskScore": 79, "stalled": 24, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 136, "nCls": 59, "kStall": 13, "kDeadAir": 136, "kToolSilent": 136, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-mini", "taskDone": 78.65, "taskScore": 79, "stalled": 26, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 136, "nCls": 60, "kStall": 14, "kDeadAir": 136, "kToolSilent": 136, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "Qwen3.8-27B", "taskDone": 78.07, "taskScore": 78, "stalled": 22, "toolSilent": 70, "deadAir": 51, "fabrication": null, "adherence": 96, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 198, "nToolTurns": 140, "nCls": 89, "kStall": 12, "kDeadAir": 100, "kToolSilent": 98, "kAdherence": 89, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "taskDone": 78.07, "taskScore": 78, "stalled": 30, "toolSilent": 90, "deadAir": 51, "fabrication": null, "adherence": 59, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 138, "nToolTurns": 78, "nCls": 67, "kStall": 16, "kDeadAir": 70, "kToolSilent": 70, "kAdherence": 67, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.2", "taskDone": 76.75, "taskScore": 77, "stalled": 28, "toolSilent": 100, "deadAir": 69, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 191, "nToolTurns": 131, "nCls": 60, "kStall": 15, "kDeadAir": 131, "kToolSilent": 131, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Cerebras", "model": "gpt-oss-120b", "taskDone": 74.85, "taskScore": 75, "stalled": 30, "toolSilent": 100, "deadAir": 68, "fabrication": null, "adherence": 3, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 186, "nToolTurns": 127, "nCls": 57, "kStall": 16, "kDeadAir": 127, "kToolSilent": 127, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5-nano", "taskDone": 74.42, "taskScore": 74, "stalled": 26, "toolSilent": 100, "deadAir": 73, "fabrication": null, "adherence": 95, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 168, "nToolTurns": 109, "nCls": 46, "kStall": 14, "kDeadAir": 122, "kToolSilent": 109, "kAdherence": 46, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "gpt-oss-120b", "taskDone": 74.12, "taskScore": 74, "stalled": 24, "toolSilent": 100, "deadAir": 70, "fabrication": null, "adherence": 3, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 196, "nToolTurns": 138, "nCls": 58, "kStall": 13, "kDeadAir": 138, "kToolSilent": 138, "kAdherence": 58, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-luna", "taskDone": 71.49, "taskScore": 71, "stalled": 30, "toolSilent": 100, "deadAir": 67, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 181, "nToolTurns": 121, "nCls": 57, "kStall": 16, "kDeadAir": 121, "kToolSilent": 121, "kAdherence": 57, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Anthropic", "model": "Claude Haiku 4.5", "taskDone": 70.61, "taskScore": 71, "stalled": 39, "toolSilent": 0, "deadAir": 0, "fabrication": null, "adherence": 15, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 136, "nToolTurns": 76, "nCls": 131, "kStall": 21, "kDeadAir": 0, "kToolSilent": 0, "kAdherence": 131, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 17, "iCls": 19, "iFab": 0 }, { "provider": "Baseten", "model": "Nemotron-3-Ultra", "taskDone": 68.86, "taskScore": 69, "stalled": 35, "toolSilent": 97, "deadAir": 61, "fabrication": null, "adherence": 22, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 160, "nToolTurns": 100, "nCls": 62, "kStall": 19, "kDeadAir": 97, "kToolSilent": 97, "kAdherence": 62, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "OpenAI", "model": "gpt-5.6-terra", "taskDone": 68.42, "taskScore": 68, "stalled": 43, "toolSilent": 100, "deadAir": 61, "fabrication": null, "adherence": 100, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 153, "nToolTurns": 93, "nCls": 50, "kStall": 23, "kDeadAir": 93, "kToolSilent": 93, "kAdherence": 50, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 18, "iCls": 19, "iFab": 0 }, { "provider": "Together", "model": "Llama-3.3-70B", "taskDone": 46.05, "taskScore": 46, "stalled": 78, "toolSilent": 100, "deadAir": 50, "fabrication": null, "adherence": 98, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 120, "nToolTurns": 60, "nCls": 59, "kStall": 42, "kDeadAir": 60, "kToolSilent": 60, "kAdherence": 59, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 19, "iCls": 19, "iFab": 0 }, { "provider": "Alibaba", "model": "qwen-turbo", "taskDone": 12.28, "taskScore": 12, "stalled": 96, "toolSilent": 100, "deadAir": 13, "fabrication": null, "adherence": 90, "nTask": 57, "nFab": 0, "nStall": 54, "nTurns": 69, "nToolTurns": 9, "nCls": 60, "kStall": 52, "kDeadAir": 9, "kToolSilent": 9, "kAdherence": 60, "kFab": 0, "iStall": 18, "iTurns": 19, "iToolTurns": 3, "iCls": 19, "iFab": 0 } ] } }, "llmFidelity": { "en": { "label": "English", "meta": "control column · n=21 values / model · 2026-07-28", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1-mini", "id": "openai:gpt-4.1-mini", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5", "id": "openai:gpt-5", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5-mini", "id": "openai:gpt-5-mini", "fidelity": 100, "tone": "good" }, { "provider": "Cerebras", "model": "gpt-oss-120b", "id": "cerebras:gpt-oss-120b", "fidelity": 100, "tone": "good" }, { "provider": "Cerebras", "model": "gemma-4-31b", "id": "cerebras:gemma-4-31b", "fidelity": 100, "tone": "good" }, { "provider": "Together", "model": "DeepSeek-V4-Pro", "id": "together:deepseek-ai/DeepSeek-V4-Pro", "fidelity": 100, "tone": "good" }, { "provider": "Together", "model": "Llama-3.3-70B", "id": "together:meta-llama/Llama-3.3-70B-Instruct-Turbo", "fidelity": 100, "tone": "good" }, { "provider": "xAI", "model": "Grok-4.3", "id": "xai:grok-4.3", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4.1", "id": "openai:gpt-4.1", "fidelity": 95, "tone": "good" }, { "provider": "Alibaba", "model": "qwen-turbo", "id": "alibaba:qwen-turbo", "fidelity": 86, "tone": "warn" } ] }, "es": { "label": "Spanish", "meta": "n=21 values / model · 2026-07-28", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1", "id": "openai:gpt-4.1", "fidelity": 100, "band": [ 94, 100 ], "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "id": "openai:gpt-4.1-mini", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5", "id": "openai:gpt-5", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5-mini", "id": "openai:gpt-5-mini", "fidelity": 100, "tone": "good" }, { "provider": "Cerebras", "model": "gpt-oss-120b", "id": "cerebras:gpt-oss-120b", "fidelity": 100, "tone": "good" }, { "provider": "Cerebras", "model": "gemma-4-31b", "id": "cerebras:gemma-4-31b", "fidelity": 100, "tone": "good" }, { "provider": "Together", "model": "Llama-3.3-70B", "id": "together:meta-llama/Llama-3.3-70B-Instruct-Turbo", "fidelity": 100, "tone": "good" }, { "provider": "xAI", "model": "Grok-4.3", "id": "xai:grok-4.3", "fidelity": 100, "tone": "good" }, { "provider": "Together", "model": "DeepSeek-V4-Pro", "id": "together:deepseek-ai/DeepSeek-V4-Pro", "fidelity": 90, "band": [ 90, 100 ], "tone": "warn" }, { "provider": "Alibaba", "model": "qwen-turbo", "id": "alibaba:qwen-turbo", "fidelity": 86, "tone": "warn" } ] }, "ar": { "label": "Arabic", "meta": "n=21 values / model · 2026-07-28", "rows": [ { "provider": "Cerebras", "model": "gemma-4-31b", "id": "cerebras:gemma-4-31b", "fidelity": 100, "tone": "good" }, { "provider": "xAI", "model": "Grok-4.3", "id": "xai:grok-4.3", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4.1", "id": "openai:gpt-4.1", "fidelity": 95, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5", "id": "openai:gpt-5", "fidelity": 90, "band": [ 90, 95 ], "tone": "warn" }, { "provider": "Together", "model": "Llama-3.3-70B", "id": "together:meta-llama/Llama-3.3-70B-Instruct-Turbo", "fidelity": 90, "band": [ 86, 90 ], "tone": "warn" }, { "provider": "Cerebras", "model": "gpt-oss-120b", "id": "cerebras:gpt-oss-120b", "fidelity": 81, "band": [ 81, 86 ], "tone": "warn" }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "id": "openai:gpt-4.1-mini", "fidelity": 81, "band": [ 76, 81 ], "tone": "warn" }, { "provider": "OpenAI", "model": "gpt-5-mini", "id": "openai:gpt-5-mini", "fidelity": 71, "tone": "warn" }, { "provider": "Alibaba", "model": "qwen-turbo", "id": "alibaba:qwen-turbo", "fidelity": 43, "tone": "bad" } ] }, "fil": { "label": "Filipino", "meta": "n=21 values / model · 2026-07-28", "rows": [ { "provider": "OpenAI", "model": "gpt-4.1", "id": "openai:gpt-4.1", "fidelity": 100, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5", "id": "openai:gpt-5", "fidelity": 100, "tone": "good" }, { "provider": "xAI", "model": "Grok-4.3", "id": "xai:grok-4.3", "fidelity": 100, "tone": "good" }, { "provider": "Together", "model": "Llama-3.3-70B", "id": "together:meta-llama/Llama-3.3-70B-Instruct-Turbo", "fidelity": 100, "band": [ 95, 100 ], "tone": "good" }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "id": "openai:gpt-4.1-mini", "fidelity": 95, "tone": "good" }, { "provider": "Cerebras", "model": "gemma-4-31b", "id": "cerebras:gemma-4-31b", "fidelity": 95, "tone": "good" }, { "provider": "OpenAI", "model": "gpt-5-mini", "id": "openai:gpt-5-mini", "fidelity": 90, "tone": "warn" }, { "provider": "Cerebras", "model": "gpt-oss-120b", "id": "cerebras:gpt-oss-120b", "fidelity": 86, "tone": "warn" }, { "provider": "Alibaba", "model": "qwen-turbo", "id": "alibaba:qwen-turbo", "fidelity": 67, "band": [ 52, 67 ], "tone": "bad" } ] } } }, "recommended": [ { "n": "01", "title": "Default", "objective": "balanced", "legs": [ { "mod": "STT", "pick": "Meta Muse Voice Transcribe", "why": "8.5% WER" }, { "mod": "LLM", "pick": "Cerebras Qwen3.8-27B", "why": "86% task success" }, { "mod": "TTS", "pick": "ElevenLabs eleven_v4_turbo", "why": "Elo 1670" } ], "note": "What the Router and new agents use when nobody picks." }, { "n": "02", "title": "Real-time phone agent", "objective": "latency", "legs": [ { "mod": "STT", "pick": "Meta Muse Voice Transcribe", "why": "301ms" }, { "mod": "LLM", "pick": "Cerebras Qwen3.8-27B", "why": "246ms" }, { "mod": "TTS", "pick": "Inworld inworld-tts-2", "why": "84ms" } ], "note": "Fastest legs the router serves, timed direct from US East." }, { "n": "03", "title": "Accuracy-critical", "objective": "quality", "legs": [ { "mod": "STT", "pick": "AssemblyAI Universal-3.5 Pro", "why": "7.2% WER" }, { "mod": "LLM", "pick": "Baseten DeepSeek-V4-Flash-0731", "why": "93% task success" }, { "mod": "TTS", "pick": "ElevenLabs eleven_v4_turbo", "why": "0.90 robustness" } ], "note": "Support & healthcare. A correct answer misread back is still wrong." }, { "n": "04", "title": "Lowest cost", "objective": "cost", "legs": [ { "mod": "STT", "pick": "Modulate Velma 2", "why": "$0.0008/min" }, { "mod": "LLM", "pick": "Baseten DeepSeek-V4-Flash-0731", "why": "$0.0002/min" }, { "mod": "TTS", "pick": "Speechify simba-3.2", "why": "$0.0090/min" } ], "note": "Cheapest legs that still clear every router floor." } ], "recommendedByLanguage": [ { "code": "es", "label": "Spanish", "legs": [ { "mod": "STT", "picks": [ { "provider": "ElevenLabs", "model": "scribe_v2_realtime", "id": "elevenlabs:scribe_v2_realtime", "value": 3.8 }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "value": 4.9 } ], "unit": "% WER", "note": "English-Spanish code-switch. ElevenLabs 3.8% at 100% both-language coverage, Soniox 4.9% at 97%. This board publishes no confidence interval, so neither is shown ahead of the other. Monolingual Spanish is measured only on the batch path, which is not what a live agent runs on.", "caption": "EN-ES code-switch, no CI published" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "value": 1951 } ], "unit": "Elo", "note": "Blind A/B listening study, arena Elo with 95% CI. The only language where one voice separates: ElevenLabs 1951 (95% CI 1859-2142) sits 200 Elo above Cartesia 1751 (1662-1858), but the intervals miss each other by a single point, so a refit could turn this into a tie." } ] }, { "code": "de", "label": "German", "legs": [ { "mod": "STT", "picks": [ { "provider": "ElevenLabs", "model": "scribe_v2_realtime", "id": "elevenlabs:scribe_v2_realtime", "value": 3.1 }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "value": 5.1 } ], "unit": "% WER", "note": "English-German code-switch. ElevenLabs 3.1% at 98% both-language coverage, Soniox 5.1% at 100%. Soniox is last of 11 on the monolingual German batch board (13.4% WER) and the stronger of the two on coverage here: monolingual rank does not predict bilingual behaviour.", "caption": "EN-DE code-switch, no CI published" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "value": 1665 }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "value": 1798 }, { "provider": "Soniox", "model": "tts-rt-v1", "id": "soniox:tts-rt-v1", "value": 1717 }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "value": 1762 } ], "unit": "Elo", "note": "Blind A/B listening study, arena Elo with 95% CI. Every confidence interval shown overlaps the leader, so these four are a statistical tie and none is the German winner. Hume octave-2 also ties on score but is not on the Speko roster.", "caption": "statistical tie, no winner" } ] }, { "code": "fr", "label": "French", "legs": [ { "mod": "STT", "picks": [ { "provider": "ElevenLabs", "model": "scribe_v2_realtime", "id": "elevenlabs:scribe_v2_realtime", "value": 7.8 }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "value": 8.8 } ], "unit": "% WER", "note": "English-French code-switch. ElevenLabs 7.8% and Soniox 8.8%, both at 89% both-language coverage.", "caption": "EN-FR code-switch, no CI published" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "value": 1622 }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "value": 1775 }, { "provider": "Soniox", "model": "tts-rt-v1", "id": "soniox:tts-rt-v1", "value": 1628 }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "value": 1692 } ], "unit": "Elo", "note": "Blind A/B listening study, arena Elo with 95% CI. The widest tie band on the board: the leader interval spans 232 Elo, so every system shown overlaps it and none of them is the French winner. Pick on latency and cost, not on this number. Hume octave-2 and Inworld inworld-tts-2 also tie on score; Hume is not on the Speko roster and Inworld is English-only on Speko.", "caption": "statistical tie, no winner" } ] }, { "code": "ar", "label": "Arabic", "legs": [ { "mod": "STT", "picks": [], "unit": "% WER", "note": "No Arabic measurement exists on the streaming path. The Arabic board at /stt-multilingual is batch, is scored by character error rate rather than word error rate, and its leader has no valid streaming number of ours.", "blank": "no streaming measurement" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "value": 1644 }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "value": 1736 } ], "unit": "Elo", "note": "Blind A/B listening study, arena Elo with 95% CI; the smallest rater panel on the board. Only four systems ran, so the blanks elsewhere are not losses. One FLEURS Arabic corpus, no dialect split. ElevenLabs, which leads Spanish, German and French, falls below the leader here at 1564.", "caption": "statistical tie, no winner" } ] }, { "code": "fil", "label": "Filipino", "legs": [ { "mod": "STT", "picks": [], "unit": "% WER", "note": "Native Filipino streaming eval, n=30: ElevenLabs Scribe v2 Realtime measured 11.7% WER (95% CI 8.9-14.7) on 2026-07-19. It is the only measured Filipino STT row on this site, so there is no comparative ranking.", "blank": "not measured" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "value": 1597 }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "value": 1693 }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "id": "openai:gpt-4o-mini-tts", "value": 1676 }, { "provider": "xAI Grok", "model": "grok-tts", "id": "xai:tts", "value": 1657 } ], "unit": "Elo", "note": "Blind A/B listening study, arena Elo with 95% CI; the thinnest study on the board. Every system shown cleared the measured Filipino language-identification gate and their intervals overlap, so none of them is the Filipino winner. MiniMax speech-2.8-hd, 1598 (1497-1717), also overlaps them and is not shown because the roster maps Alibaba to a different model. Gemini scored 2012 (1854-2289), above every system shown rather than tied with them; it was removed from the Speko roster on 2026-07-17 because its adapter returns one full clip instead of streaming, and it never ran the gate.", "caption": "statistical tie, no winner" } ] }, { "code": "nb", "label": "Norwegian", "legs": [ { "mod": "STT", "picks": [ { "provider": "ElevenLabs", "model": "Scribe v2 Realtime", "id": "elevenlabs:scribe_v2_realtime", "value": 8.3 }, { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "value": 8 } ], "unit": "% WER", "note": "Monolingual FLEURS nb_no, n=50 clips, pooled corpus WER, measured on the live streaming socket. Soniox 8.0% (95% CI 6.2-10.0) and ElevenLabs Scribe v2 Realtime 8.3% (6.3-10.5) overlap, so neither leads. OpenAI is absent because our own streaming adapter commits audio too early, not because the model is weak. Norwegian is the one language whose board also carries a batch arm on the same 50 clips (leader 5.6%, ElevenLabs Scribe v2, 2026-08-21) - it is the lower number but not the one a call runs on, and the streaming socket serves scribe_v2_realtime for any ElevenLabs pin regardless. The pick above stays on the streaming measurement.", "caption": "statistical tie, no winner" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "value": 1836 } ], "unit": "Elo", "note": "Blind A/B listening study over six systems, arena Elo with 95% CI. Cartesia clears ElevenLabs 1628 (95% CI 1533-1733). Gemini scored 1705 within a hair of it but does not stream on Speko. Cartesia sonic-3.5 default voice drifts roughly 5x the rest of the field in every language we have probed for drift; Norwegian was not one of them." } ] }, { "code": "zh", "label": "Mandarin", "legs": [ { "mod": "STT", "picks": [ { "provider": "Soniox", "model": "stt-rt-v5", "id": "soniox:stt-rt-v5", "value": 11.8 } ], "unit": "% MTER", "note": "Mandarin-English code-switch. Soniox is the only provider that survives the pair: 11.8% mixed-token error at 88% both-language coverage. ElevenLabs anglicizes Mandarin, dropping it to English at 34% coverage, so it is not shown.", "caption": "EN-ZH code-switch, mixed-token error" }, { "mod": "LLM", "picks": [ { "provider": "Cerebras", "model": "Qwen3.8-27B", "id": "cerebras:qwen-3.8-27b", "value": 86.14 } ], "unit": "%", "note": "The same recommendation on every language, carried across from our LLM board. Task completion is now measured per language: 21 models across ten languages, 19 scenarios at 3 iterations each, with a confidence interval on every cell — see the Multilingual LLM board. Nothing scores 100%; the English leader is gpt-5.6-luna at 94.21%, and the ten-language means run 68.8% to 74.0%, so language moves this axis far less than model choice does. Claude Haiku 4.5 is picked for behaviour rather than for topping the axis: the board publishes no composite score and no board-wide rank.", "caption": "measured in English" }, { "mod": "TTS", "picks": [ { "provider": "Cartesia", "model": "sonic-3.5", "id": "cartesia:sonic-3.5", "value": 1564 }, { "provider": "ElevenLabs", "model": "eleven_v3_conversational", "id": "elevenlabs:eleven_v3_conversational", "value": 1675 }, { "provider": "Inworld", "model": "inworld-tts-2", "id": "inworld:inworld-tts-2", "value": 1688 } ], "unit": "Elo", "note": "Blind A/B listening study (2026-08-24), arena Elo with 95% CI, raters recruited on first language = Mandarin specifically rather than the generic Chinese pool. The top three intervals overlap — Inworld 1688 (1629–1754), ElevenLabs 1675 (1587–1767) and Gemini 1649 (1586–1724) are a statistical tie and none is the Mandarin winner. Gemini is not shown: it was removed from the Speko roster on 2026-07-17 because its adapter returns one full clip instead of streaming. MiniMax, the home-market vendor, measured 1467 — below the tie band.", "blank": "no listening study", "caption": "statistical tie, no winner" } ] } ], "projector": [ { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · elevenlabs:eleven_flash_v2_5", "c": 0.055, "fab": true }, { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · cartesia:sonic-3.5", "c": 0.058 }, { "name": "deepgram:nova-3 · openai:gpt-4.1-mini · cartesia:sonic-3.5 (never books)", "c": 0.079, "fab": true }, { "name": "cartesia:ink-2 · cerebras:gpt-oss-120b · cartesia:sonic-3.5", "c": 0.087 }, { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · minimax:speech-2.6-hd", "c": 0.092 }, { "name": "deepgram:nova-3 · openai:gpt-4.1 · cartesia:sonic-3.5", "c": 0.111 }, { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · deepgram:aura-2", "c": 0.123 }, { "name": "deepgram:nova-3 · openai:gpt-5-nano · cartesia:sonic-3.5", "c": 0.145 }, { "name": "elevenlabs:scribe_v2_realtime · openai:gpt-4.1 · cartesia:sonic-3.5", "c": 0.147 }, { "name": "elevenlabs:scribe_v2_realtime · openai:gpt-4.1 · elevenlabs:eleven_flash_v2_5", "c": 0.469 } ], "turntakingSynthetic": [ { "label": "Turnsense", "kind": "text", "acc": 98.8, "cut": 0 }, { "label": "LiveKit Intl", "kind": "text", "acc": 90.6, "cut": 6.2 }, { "label": "LiveKit EN", "kind": "text", "acc": 79.2, "cut": 26.2 }, { "label": "Smart Turn", "kind": "audio", "acc": 62.4, "cut": 57.7 } ], "turntakingProsody": [ { "cue": "Final F0 slope", "unit": "Hz/s", "end": -119.7, "wait": 42.9, "want": "complete falls, incomplete rises/holds" }, { "cue": "Final energy slope", "unit": "dB/s", "end": -14.9, "wait": -2.5, "want": "complete tapers off" } ] }