{ "site": { "measured": "2026-07-03", "run": "run 07-03", "scope": "110 models · 42 integrations" }, "boards": [ { "id": "stt", "navLabel": "Speech-to-Text", "identityLabel": "Provider / Model", "title": "Speech-to-Text", "subtitle": "How each system turns speech to text for a live agent — how fast it ends the turn in a live call, accuracy on clean English, and price. Streaming models a caller actually hears rank first; batch transcription APIs sit at the end. Measured through one gateway, no human scoring.", "live": true, "meta": "English · clean read (FLEURS, n=50) · measured 2026-07-03 · run 07-03", "columns": [ { "id": "wer", "label": "WER", "dir": "lower", "info": "Word Error Rate on FLEURS clean read-English, n=50. Lower is better. Clean read-speech is near-solved — not the whole story for live audio." }, { "id": "endpoint", "label": "End-of-turn", "dir": "lower", "info": "Streaming turn-finalization p50 — ms from the caller stopping to the FINAL transcript (the wait before a live agent can reply), not time-to-first-word. Measured from us-east4 through the gateway, real-time-paced, English (FLEURS). \"—\" = not measured as a live stream. † Smallest is flushed at end-of-speech, so its final lands on the turn signal instead of the vendor silence timer (~1–3s)." }, { "id": "cost", "label": "Cost / min", "dir": "lower", "info": "$ per minute of audio (vendor pricing). Lower = cheaper. \"—\" where a vendor does not publish a clean per-minute rate." } ], "rows": [ { "provider": "Smallest AI", "model": "Pulse", "status": { "label": "FAIR", "tone": "warn" }, "flag": "Streaming-first model (~64 ms TTFT) scored on a whole-clip batch harness. Some of its WER is inverse-text-normalization — it spells out numbers and dates (\"800\" → \"eight hundred\") that the digit-style FLEURS references count as errors, not mishearing — likely a touch better than 5.1% in real use. Measured 2026-07-06, same 50 FLEURS clips, same gateway.", "rec": [ { "label": "Streaming" }, { "label": "Real-time phone" } ], "cells": { "wer": { "v": "5.1%", "s": 5.1, "tone": "warn" }, "endpoint": { "v": "183ms †", "s": 183, "tone": "good", "ci": "n=22", "note": "Flushed at end-of-speech: Speko forces the final on the turn signal for flush-capable providers, instead of waiting the vendor silence timer (~1–3s)." }, "cost": { "v": "—", "s": null, "dim": true } } }, { "provider": "Smallest AI", "model": "Pulse Pro", "status": { "label": "BATCH-ONLY", "tone": "warn" }, "flag": "Smallest's highest-English-accuracy model — but pre-recorded/HTTP ONLY (cannot drive a live socket), so end-of-turn is \"—\". On this whole-clip FLEURS harness Pulse Pro (6.4%) is statistically tied with the streaming Pulse (5.1%) — overlapping CIs — so \"Pro\" buys no clean-read accuracy here; its edge would show on harder audio, not clean read-speech. Batch latency ~2.4s. Measured 2026-07-08 via the gateway, same 50 FLEURS clips.", "rec": [ { "label": "Batch" } ], "cells": { "wer": { "v": "6.4%", "s": 6.4, "tone": "warn", "ci": "95% CI 4.4–8.8 · n=50" }, "endpoint": { "v": "—", "s": null, "dim": true, "note": "Batch / pre-recorded HTTP model — no streaming socket, so it cannot carry a live turn (like the other batch APIs)." }, "cost": { "v": "—", "s": null, "dim": true } } }, { "provider": "Deepgram", "model": "Nova-3", "status": { "label": "FAIR", "tone": "warn" }, "flag": "Streaming-first model scored on a whole-clip batch harness; truncation can inflate WER (Cartesia often returns only the tail). Treat as a lower bound on field accuracy.", "rec": [ { "label": "Real-time phone" }, { "label": "Streaming" } ], "cells": { "wer": { "v": "9.8%", "s": 9.8, "tone": "warn" }, "endpoint": { "v": "368ms", "s": 368, "tone": "good", "ci": "n=19" }, "cost": { "v": "$0.0048", "s": 0.0048 } } }, { "provider": "Deepgram", "model": "Nova-2", "status": { "label": "WEAK", "tone": "bad" }, "flag": "Streaming-first model scored on a whole-clip batch harness; truncation can inflate WER (Cartesia often returns only the tail). Treat as a lower bound on field accuracy.", "rec": [], "cells": { "wer": { "v": "13.9%", "s": 13.9, "tone": "bad" }, "endpoint": { "v": "387ms", "s": 387, "tone": "good", "ci": "n=18" }, "cost": { "v": "$0.0043", "s": 0.0043 } } }, { "provider": "Cartesia", "model": "Ink-2", "status": { "label": "WEAK", "tone": "bad" }, "flag": "Streaming-first model scored on a whole-clip batch harness; truncation can inflate WER (Cartesia often returns only the tail). Treat as a lower bound on field accuracy.", "rec": [], "cells": { "wer": { "v": "11.0%", "s": 11, "tone": "bad" }, "endpoint": { "v": "575ms", "s": 575, "tone": "good", "ci": "n=22" }, "cost": { "v": "$0.0022", "s": 0.0022 } } }, { "provider": "Alibaba", "model": "Qwen3-ASR", "status": { "label": "EXCELLENT", "tone": "good" }, "flag": "End-of-turn measured through the gateway (qwen3-asr-flash-realtime, server_vad). DashScope is Asia-hosted, so the p50 carries the gateway→Singapore round-trip it cannot shed — a US-region live agent pays that network cost.", "rec": [ { "label": "Multilingual" }, { "label": "Streaming" } ], "cells": { "wer": { "v": "2.8%", "s": 2.8, "tone": "good" }, "endpoint": { "v": "710ms", "s": 710, "tone": "good", "ci": "n=12" }, "cost": { "v": "—", "s": null, "dim": true } } }, { "provider": "Google", "model": "Chirp 3", "status": { "label": "GOOD", "tone": "good" }, "rec": [ { "label": "Multilingual" }, { "label": "Global" } ], "cells": { "wer": { "v": "3.9%", "s": 3.9, "tone": "good" }, "endpoint": { "v": "995ms", "s": 995, "tone": "warn", "ci": "n=14" }, "cost": { "v": "—", "s": null, "dim": true } } }, { "provider": "ElevenLabs", "model": "Scribe v2", "status": { "label": "EXCELLENT", "tone": "good" }, "flag": "End-of-turn reflects ElevenLabs realtime VAD at its vendor-default silence window (~1.5s) — patient by default and tunable, not a hard floor. Measured through the gateway (scribe_v2_realtime).", "rec": [ { "label": "Balanced" }, { "label": "Streaming" } ], "cells": { "wer": { "v": "3.1%", "s": 3.1, "tone": "good" }, "endpoint": { "v": "1750ms", "s": 1750, "tone": "warn", "ci": "n=14" }, "cost": { "v": "$0.0067", "s": 0.0067 } } }, { "provider": "Soniox", "model": "stt-rt-v5", "status": { "label": "FAIR", "tone": "warn" }, "flag": "Streaming-first real-time model (60+ languages) scored on the whole-clip batch harness — same 50 FLEURS clips + gateway as the rest, measured 2026-07-08. End-of-turn 269ms (us-east4) is fast, BUT on native endpointing only 11/20 clips finalized: Soniox does not reliably finalize from silence alone, so it needs the flush path (on in prod) for dependable turn-taking (expect ~180ms flush-forced there, like Smallest). Like other streaming-first ASRs, whole-clip WER is a slight lower bound.", "rec": [ { "label": "Streaming" }, { "label": "Multilingual" } ], "cells": { "wer": { "v": "7.5%", "s": 7.5, "tone": "warn", "ci": "95% CI 4.8–10.7 · n=50" }, "endpoint": { "v": "269ms", "s": 269, "tone": "warn", "ci": "n=11/20 · native", "note": "us-east4 p50 over the 11/20 clips that finalized on Soniox NATIVE endpointing (flush off on staging). 9/20 did not finalize from silence alone — native turn-detection is unreliable; the flush path (on in prod) is needed for a dependable final, where Soniox should reach the ~180ms flush-forced class like Smallest. 5/11 were early finalizes." }, "cost": { "v": "$0.002", "s": 0.002 } } }, { "provider": "OpenAI", "model": "GPT-4o Transcribe", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Accuracy" }, { "label": "Healthcare", "caution": true } ], "cells": { "wer": { "v": "2.3%", "s": 2.3, "tone": "good" }, "cost": { "v": "$0.006", "s": 0.006 } } }, { "provider": "OpenAI", "model": "GPT-4o-mini Transcribe", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [ { "label": "Value" } ], "cells": { "wer": { "v": "2.7%", "s": 2.7, "tone": "good" }, "cost": { "v": "$0.003", "s": 0.003 } } }, { "provider": "ElevenLabs", "model": "Scribe v1", "status": { "label": "EXCELLENT", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "3.1%", "s": 3.1, "tone": "good" }, "cost": { "v": "$0.0067", "s": 0.0067 } } }, { "provider": "OpenAI", "model": "Whisper-1", "status": { "label": "GOOD", "tone": "good" }, "rec": [ { "label": "Media" } ], "cells": { "wer": { "v": "3.9%", "s": 3.9, "tone": "good" }, "cost": { "v": "$0.006", "s": 0.006 } } }, { "provider": "xAI", "model": "Grok STT", "status": { "label": "FAIR", "tone": "warn" }, "flag": "Conditional — measured only on loudness-normalized input. On raw-level audio it degrades; a best-case reading, not field performance.", "rec": [], "cells": { "wer": { "v": "4.8%", "s": 4.8, "tone": "warn" }, "cost": { "v": "—", "s": null, "dim": true } } }, { "provider": "Gradium", "model": "Gradium ASR", "status": { "label": "WEAK", "tone": "bad" }, "flag": "FLEURS includes many very-low-volume clips that Gradium does not yet auto-gain — its streaming loudness-normalization has not shipped. Low-level audio depresses this score; treat it as a worst-case on quiet input, not field performance.", "rec": [], "cells": { "wer": { "v": "13.7%", "s": 13.7, "tone": "bad" }, "cost": { "v": "—", "s": null, "dim": true } } }, { "provider": "Cartesia", "model": "Ink-Whisper", "status": { "label": "WEAK", "tone": "bad" }, "flag": "Streaming-first model scored on a whole-clip batch harness; truncation can inflate WER (Cartesia often returns only the tail). Treat as a lower bound on field accuracy.", "rec": [], "cells": { "wer": { "v": "25.2%", "s": 25.2, "tone": "bad" }, "cost": { "v": "$0.0022", "s": 0.0022 } } } ], "reading": "For a live agent the number a caller actually feels is end-of-turn latency, not clean-read WER — so this board leads with the models that stream a live turn through the gateway, ordered by how fast they finalize it. Smallest Pulse ends the turn in ~183 ms (flushed at end-of-speech); Deepgram nova-3/nova-2 and Cartesia ink-2 sit in a 368–575 ms band; Alibaba Qwen3-ASR lands at ~710 ms — fast, and at 2.8% WER, though its DashScope endpoint is Asia-hosted; Google Chirp 3 trails at ~995 ms; and ElevenLabs Scribe v2 sits near 1.75 s on its patient default VAD (tunable, not a floor). Soniox streams too, but its live end-of-turn is not yet measured. The batch transcription APIs below (OpenAI, ElevenLabs Scribe v1, xAI Grok, Gradium) win clean-read WER — a near-solved 2.3–3.9% cluster — but do not stream a live turn here, so end-of-turn reads as a dash; click WER to rank the whole board by accuracy instead. Caveats: streaming-first ASRs like nova-3 and ink-2 score worse on whole-clip WER because batch scoring truncates them (treat as a lower bound), and Grok and Gradium were measured on loudness-normalized input (see flags)." }, { "id": "tts", "navLabel": "Text-to-Speech", "identityLabel": "System / Model", "title": "Text-to-Speech", "subtitle": "Ranked by how human it sounds — third-party blind-listening preference (Artificial Analysis Elo). Intelligibility, voice drift, synth latency and cost sit alongside as the practical trade-offs a voice build actually makes.", "live": true, "meta": "English · naturalness via Artificial Analysis · measured 2026-07-03 · run 07-03", "columns": [ { "id": "natural", "label": "Naturalness", "dir": "higher", "info": "How human it sounds — third-party blind-listening preference Elo (Artificial Analysis). Higher = preferred by listeners. This is the ranking axis." }, { "id": "wer", "label": "Intelligibility", "dir": "lower", "info": "Round-trip WER through 2 reference STTs at native sample rate. A near-solved gate, not the ranking. Lower = clearer." }, { "id": "drift", "label": "Drift", "dir": "lower", "info": "Voice steadiness: mean pairwise embedding distance over 20 repeats. Lower = steadier. \"det\" = deterministic output." }, { "id": "synth", "label": "Synth p50", "dir": "lower", "info": "Generation latency to first audio, p50 (us-east4)." }, { "id": "cost", "label": "Cost / 1M chars", "dir": "lower", "info": "$ per 1M characters (vendor / Artificial Analysis). Lower = cheaper." } ], "tabs": [ { "label": "Naturalness", "sortCol": "natural", "dir": "higher" }, { "label": "Latency", "sortCol": "synth", "dir": "lower" }, { "label": "Cost", "sortCol": "cost", "dir": "lower" } ], "rows": [ { "provider": "Hume", "model": "octave-2", "status": { "label": "CLEAREST", "tone": "good" }, "rec": [ { "label": "Accuracy" } ], "cells": { "wer": { "v": "0.71%", "s": 0.71, "tone": "good" }, "drift": { "v": "25", "s": 25, "tone": "warn" }, "synth": { "v": "833ms", "s": 833 }, "natural": { "v": "1060", "s": 1060 }, "cost": { "v": "$87.5", "s": 87.5, "tone": "warn" } } }, { "provider": "Rime", "model": "coda", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "0.79%", "s": 0.79, "tone": "good" }, "drift": { "v": "21", "s": 21 }, "synth": { "v": "357ms", "s": 357, "tone": "good" }, "natural": { "v": "1031", "s": 1031 }, "cost": { "v": "$50.0", "s": 50, "tone": "warn" } } }, { "provider": "Alibaba", "model": "cosyvoice-v3-flash", "status": { "label": "CLEAR", "tone": "good" }, "rec": [ { "label": "Value" } ], "cells": { "wer": { "v": "0.79%", "s": 0.79, "tone": "good" }, "drift": { "v": "det", "s": 0, "dim": true, "note": "Deterministic: identical audio every run — the steadiest possible result." }, "synth": { "v": "1.67s", "s": 1670, "tone": "warn" }, "natural": { "v": "1206", "s": 1206, "tone": "good" }, "cost": { "v": "$27.6", "s": 27.6, "tone": "good" } } }, { "provider": "xAI", "model": "tts", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "0.79%", "s": 0.79, "tone": "good" }, "drift": { "v": "15", "s": 15 }, "synth": { "v": "618ms", "s": 618 }, "natural": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$15.0", "s": 15, "tone": "good" } } }, { "provider": "Soniox", "model": "tts-rt-v1", "status": { "label": "DRIFTS", "tone": "warn" }, "flag": "Real-time streaming TTS (60+ languages). Intelligibility 0.79% via open Whisper-large-v3 (50 prompts, 48/50 flawless) — at the near-solved floor with ElevenLabs & Cartesia (both 0.5% on the same open-ASR run); an earlier single-Deepgram read of 2.08% was STT-method noise, not the voice. Drift 24 — stochastic (20/20 distinct renders, ECAPA over 20 repeats on Modal), so the voice wanders more than the steady pack. Naturalness \"—\": not on the Artificial Analysis leaderboard. Synth p50 472ms (Modal us-east-1, n=25) — fast, 3rd behind Cartesia/Rime.", "rec": [ { "label": "Streaming" }, { "label": "Multilingual" } ], "cells": { "natural": { "v": "—", "s": null, "dim": true, "note": "Not rated on the Artificial Analysis Speech Arena." }, "wer": { "v": "0.79%", "s": 0.79, "tone": "good", "ci": "Whisper-lg-v3 · n=50", "note": "Round-trip via open Whisper-large-v3 (50 prompts, 48/50 flawless). On the same open-ASR run ElevenLabs & Cartesia both scored 0.5% — Soniox is at the near-solved floor with the pack. An earlier single-Deepgram read showed 2.08%; that was STT-method noise, not the voice." }, "drift": { "v": "24", "s": 24, "tone": "warn", "ci": "mpd 0.238 · n=20", "note": "Mean pairwise ECAPA embedding distance ×100 over 20 repeats (Modal, speechbrain/spkrec-ecapa-voxceleb, 2026-07-08). Stochastic — 20/20 distinct renders; wanders more than the steady pack (Cartesia 13, Rime 21), near Hume 25." }, "synth": { "v": "472ms", "s": 472, "tone": "good", "ci": "n=25 · us-east-1", "note": "Time-to-first-audio p50 from Modal us-east-1 (N. Virginia) through the gateway — same vantage as the rest of the TTS column. n=25, paced, 0 errors; tight 428–522ms. 3rd-fastest here, behind only Cartesia/Rime (357ms)." }, "cost": { "v": "—", "s": null, "dim": true, "note": "Token-based pricing (~$0.70/hr of audio); no clean per-1M-char rate published." } } }, { "provider": "Cartesia", "model": "sonic-3.5", "lead": true, "status": { "label": "CLEAR", "tone": "good" }, "rec": [ { "label": "Balanced" }, { "label": "Real-time" } ], "cells": { "wer": { "v": "0.79%", "s": 0.79, "tone": "good" }, "drift": { "v": "13", "s": 13, "tone": "good" }, "synth": { "v": "357ms", "s": 357, "tone": "good" }, "natural": { "v": "1209", "s": 1209, "tone": "good" }, "cost": { "v": "$39.0", "s": 39 } } }, { "provider": "Google", "model": "gemini-3.1-flash-tts", "batchOnly": true, "status": { "label": "BATCH-ONLY", "tone": "warn" }, "flag": "Batch-only API — no streaming, so it renders the whole clip before the first byte (4.04s p50). Set aside below the real-time models despite topping raw naturalness Elo; it cannot carry a live turn.", "rec": [], "cells": { "wer": { "v": "—", "s": null, "dim": true, "note": "Not yet run through our English round-trip intelligibility harness." }, "drift": { "v": "—", "s": null, "dim": true, "note": "Not yet run through our 20-repeat voice-drift harness." }, "synth": { "v": "4.04s", "s": 4042, "tone": "bad", "note": "Batch-only API: full render before first byte (p50, us-east4, n=30). ~10× the real-time tier — no partial-audio streaming today." }, "natural": { "v": "1214", "s": 1214, "tone": "good", "note": "Artificial Analysis Speech Arena Elo — blind 30-second clips, #2 behind Speechify Simba 3.2. Our 10-minute take shows the quality holds short but degrades badly at length." }, "cost": { "v": "$75.0", "s": 75, "tone": "warn" } } }, { "provider": "OpenAI", "model": "gpt-4o-mini-tts", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "0.79%", "s": 0.79, "tone": "good" }, "drift": { "v": "22", "s": 22, "tone": "warn" }, "synth": { "v": "1.02s", "s": 1020, "tone": "warn" }, "natural": { "v": "—", "s": null, "dim": true }, "cost": { "v": "—", "s": null, "dim": true, "note": "Billed per audio token, not per character." } } }, { "provider": "ElevenLabs", "model": "eleven_v3", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "0.93%", "s": 0.93, "tone": "good" }, "drift": { "v": "20", "s": 20 }, "synth": { "v": "750ms", "s": 750 }, "natural": { "v": "1172", "s": 1172 }, "cost": { "v": "$100", "s": 100, "tone": "warn" } } }, { "provider": "Amazon", "model": "polly-generative", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "0.93%", "s": 0.93, "tone": "good" }, "drift": { "v": "det", "s": 0, "dim": true, "note": "Deterministic: identical audio every run — the steadiest possible result." }, "synth": { "v": "488ms", "s": 488, "tone": "good" }, "natural": { "v": "1069", "s": 1069 }, "cost": { "v": "$30.0", "s": 30, "tone": "good" } } }, { "provider": "Gradium", "model": "default", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "0.96%", "s": 0.96, "tone": "good" }, "drift": { "v": "12", "s": 12, "tone": "good" }, "synth": { "v": "568ms", "s": 568 }, "natural": { "v": "1087", "s": 1087 }, "cost": { "v": "$47.2", "s": 47.2 } } }, { "provider": "Inworld", "model": "inworld-tts-2", "status": { "label": "CLEAR", "tone": "good" }, "rec": [ { "label": "Value" } ], "cells": { "wer": { "v": "0.96%", "s": 0.96, "tone": "good" }, "drift": { "v": "18", "s": 18 }, "synth": { "v": "369ms", "s": 369, "tone": "good" }, "natural": { "v": "1201", "s": 1201, "tone": "good" }, "cost": { "v": "$20.8", "s": 20.8, "tone": "good" } } }, { "provider": "MiniMax", "model": "speech-2.6-hd", "status": { "label": "CLEAR", "tone": "good" }, "rec": [ { "label": "Expressive" } ], "cells": { "wer": { "v": "1.04%", "s": 1.04, "tone": "good" }, "drift": { "v": "15", "s": 15 }, "synth": { "v": "592ms", "s": 592 }, "natural": { "v": "1141", "s": 1141 }, "cost": { "v": "$100", "s": 100, "tone": "warn" } } }, { "provider": "Alibaba", "model": "qwen3-tts-flash", "status": { "label": "CLEAR", "tone": "good" }, "rec": [ { "label": "Value" } ], "cells": { "wer": { "v": "1.51%", "s": 1.51, "tone": "good" }, "drift": { "v": "21", "s": 21 }, "synth": { "v": "670ms", "s": 670 }, "natural": { "v": "929", "s": 929, "dim": true }, "cost": { "v": "$10.0", "s": 10, "tone": "good" } } }, { "provider": "Deepgram", "model": "aura-2", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "wer": { "v": "4.61%", "s": 4.61, "tone": "warn" }, "drift": { "v": "9", "s": 9, "tone": "good" }, "synth": { "v": "362ms", "s": 362, "tone": "good" }, "natural": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$30.0", "s": 30, "tone": "good" } } }, { "provider": "Speechify", "model": "simba-3.2", "status": { "label": "CLEAR", "tone": "good" }, "rec": [], "cells": { "wer": { "v": "1.89%", "s": 1.89, "tone": "good" }, "drift": { "v": "7", "s": 7, "tone": "good" }, "synth": { "v": "730ms", "s": 730 }, "natural": { "v": "—", "s": null, "dim": true, "note": "Not rated by Artificial Analysis." }, "cost": { "v": "$10.0", "s": 10, "tone": "good", "note": "Vendor list (Starter tier); $6–8 per 1M at Pro/Scale volume." } } } ], "reading": "We rank on naturalness because intelligibility is a near-solved gate — twelve of the fourteen models we scored sit in a 0.7–1.6% WER band you cannot separate, so clarity no longer picks a winner. What a listener actually notices is how human it sounds. Cartesia sonic-3.5 leads the real-time field, with Alibaba cosyvoice and Inworld close behind (and Inworld at $20.8 is the value pick of that top cluster). Hume tops clarity but lands further down on naturalness. Google gemini-3.1-flash-tts actually posts the highest raw blind-listening Elo (1214), but it is a batch-only API — 4.04s to first audio, no streaming — so we set it aside at the bottom rather than rank it: it cannot carry a live turn, and our 10-minute take shows it degrades badly at length. Three models (xAI, OpenAI gpt-4o-mini-tts, Deepgram aura-2) are not rated by Artificial Analysis and sit at the bottom, unranked rather than judged." }, { "id": "llm", "navLabel": "LLM", "identityLabel": "Provider / Model", "title": "LLM · voice-agent reliability", "subtitle": "For a live agent the question is not just \"is it reliable\" — it is \"reliable AND fast enough for a turn, at a price you can run.\" Grouped by reliability; within a tier the fastest, cheapest model leads. TTFT is time-to-first-token from us-east4 (p50, whisker = p50→p90).", "live": true, "meta": "English · TTFT from us-east4 · measured 2026-07-03 · run 07-03", "columns": [ { "id": "overall", "label": "Overall", "dir": "higher", "info": "Mean of Tool and Grounded on the same scripted voice-agent runs. Higher is better." }, { "id": "tool", "label": "Tool", "dir": "higher", "info": "Tool-calling correctness + restraint (right tool, right args, no spurious calls), % of runs." }, { "id": "ground", "label": "Grounded", "dir": "higher", "info": "% of runs with no fabricated facts." }, { "id": "ttft", "label": "TTFT p50", "dir": "lower", "info": "Time-to-first-token, p50 (us-east4, n=12). Whisker spans p50→p90." }, { "id": "cost", "label": "Cost / 1M tok", "dir": "lower", "info": "$ per 1M output tokens (Artificial Analysis / host pricing; flat per-token where the host prices that way)." } ], "rows": [ { "provider": "Cerebras", "model": "gpt-oss-120b", "lead": true, "status": { "label": "PERFECT", "tone": "good" }, "flag": "TTFT here is from the 2026-07-03 run; a same-run re-probe on 2026-07-08 alongside gemma-4-31b read 225ms p50 — so the 3ms gap to gemma is run-to-run noise, not a real lead. gpt-oss holds the top slot on its clean $0.75/1M output price, not on speed.", "rec": [ { "label": "Real-time" }, { "label": "Value" } ], "cells": { "overall": { "v": "100%", "s": 100, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "195ms", "s": 195, "lo": 195, "hi": 260, "ci": "195–260 p50–p90", "tone": "good" }, "cost": { "v": "$0.75", "s": 0.75, "tone": "good" } } }, { "provider": "Cerebras", "model": "gemma-4-31b", "status": { "label": "PERFECT", "tone": "good" }, "flag": "Non-reasoning Cerebras model and the fastest reliable pick measured: TTFT p50 192ms from us-east4 (n=15, measured 2026-07-08), where the gpt-oss-120b anchor in the SAME run read 225ms — so the two Cerebras models are effectively tied on first-token and gemma is at least as fast as the leader. Ranked #2 within the perfect tier because gpt-oss-120b has a published $0.75/1M output price while Gemma has none, keeping the value edge with gpt-oss. Reliability is 9/9 tool-calling + 9/9 grounded on the same 6-scenario voice-agent harness (judge gpt-4.1). p90 is from n=15 — a thin tail, read the whisker as indicative.", "rec": [ { "label": "Real-time" } ], "cells": { "overall": { "v": "100%", "s": 100, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "192ms", "s": 192, "lo": 192, "hi": 232, "ci": "192–232 p50–p90 · n=15", "tone": "good" }, "cost": { "v": "—", "s": null, "dim": true, "note": "Cerebras publishes no output-only rate for Gemma 4 31B. Artificial Analysis lists a blended ~$1.04/1M (input+output mix) — a different basis than the output-only prices in this column, so shown as — rather than mislabeled as an output rate." } } }, { "provider": "Together", "model": "Llama-3.3-70B", "status": { "label": "PERFECT", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "100%", "s": 100, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "698ms", "s": 698, "lo": 698, "hi": 860, "ci": "698–860 p50–p90", "tone": "good" }, "cost": { "v": "$1.04", "s": 1.04, "tone": "good" } } }, { "provider": "xAI", "model": "Grok-4.3", "status": { "label": "PERFECT", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "100%", "s": 100, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "2.15s", "s": 2150, "lo": 2150, "hi": 2640, "ci": "2.15–2.64s p50–p90", "tone": "warn" }, "cost": { "v": "$2.50", "s": 2.5 } } }, { "provider": "Together", "model": "Kimi-K2.6", "status": { "label": "PERFECT", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "100%", "s": 100, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$4.50", "s": 4.5, "tone": "warn" } } }, { "provider": "OpenAI", "model": "gpt-4.1-mini", "status": { "label": "STRONG", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "94%", "s": 94, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "89%", "s": 89, "tone": "warn" }, "ttft": { "v": "592ms", "s": 592, "lo": 592, "hi": 990, "ci": "592–990 p50–p90", "tone": "good" }, "cost": { "v": "$1.60", "s": 1.6, "tone": "good" } } }, { "provider": "Together", "model": "Qwen3.7-Plus", "status": { "label": "STRONG", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "94%", "s": 94, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "89%", "s": 89, "tone": "warn" }, "ttft": { "v": "1.53s", "s": 1530, "lo": 1530, "hi": 2100, "ci": "1.53–2.10s p50–p90", "tone": "warn" }, "cost": { "v": "$1.28", "s": 1.28, "tone": "good" } } }, { "provider": "Fireworks", "model": "DeepSeek-V4-Pro", "status": { "label": "STRONG", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "94%", "s": 94, "tone": "good" }, "tool": { "v": "89%", "s": 89, "tone": "warn" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$3.48", "s": 3.48 } } }, { "provider": "Fireworks", "model": "Kimi-K2.6", "status": { "label": "STRONG", "tone": "good" }, "rec": [], "cells": { "overall": { "v": "94%", "s": 94, "tone": "good" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "86%", "s": 86, "tone": "warn" }, "ttft": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$4.00", "s": 4, "tone": "warn" } } }, { "provider": "OpenAI", "model": "gpt-4.1", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "overall": { "v": "83%", "s": 83, "tone": "warn" }, "tool": { "v": "67%", "s": 67, "tone": "bad" }, "ground": { "v": "100%", "s": 100, "tone": "good" }, "ttft": { "v": "484ms", "s": 484, "lo": 484, "hi": 756, "ci": "484–756 p50–p90", "tone": "good" }, "cost": { "v": "$8.00", "s": 8, "tone": "warn" } } }, { "provider": "Together", "model": "DeepSeek-V4-Pro", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "overall": { "v": "83%", "s": 83, "tone": "warn" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "67%", "s": 67, "tone": "bad" }, "ttft": { "v": "—", "s": null, "dim": true }, "cost": { "v": "$3.48", "s": 3.48 } } }, { "provider": "Together", "model": "gpt-oss-120b", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "overall": { "v": "78%", "s": 78, "tone": "warn" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "56%", "s": 56, "tone": "bad" }, "ttft": { "v": "431ms", "s": 431, "lo": 431, "hi": 580, "ci": "431–580 p50–p90", "tone": "good" }, "cost": { "v": "$0.60", "s": 0.6, "tone": "good" } } }, { "provider": "Fireworks", "model": "gpt-oss-120b", "status": { "label": "FAIR", "tone": "warn" }, "rec": [], "cells": { "overall": { "v": "78%", "s": 78, "tone": "warn" }, "tool": { "v": "100%", "s": 100, "tone": "good" }, "ground": { "v": "56%", "s": 56, "tone": "bad" }, "ttft": { "v": "484ms", "s": 484, "lo": 484, "hi": 594, "ci": "484–594 p50–p90", "tone": "good" }, "cost": { "v": "$0.60", "s": 0.6, "tone": "good" } } }, { "provider": "OpenAI", "model": "gpt-5-nano", "meta": "reasoning", "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "overall": { "v": "56%", "s": 56, "tone": "bad" }, "tool": { "v": "56%", "s": 56, "tone": "bad" }, "ground": { "v": "56%", "s": 56, "tone": "bad" }, "ttft": { "v": "525ms", "s": 525, "lo": 525, "hi": 1020, "ci": "525–1020 p50–p90", "tone": "good" }, "cost": { "v": "$0.40", "s": 0.4, "tone": "good" } } }, { "provider": "OpenAI", "model": "gpt-5", "meta": "reasoning", "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "overall": { "v": "50%", "s": 50, "tone": "bad" }, "tool": { "v": "67%", "s": 67, "tone": "bad" }, "ground": { "v": "33%", "s": 33, "tone": "bad" }, "ttft": { "v": "635ms", "s": 635, "lo": 635, "hi": 679, "ci": "635–679 p50–p90", "tone": "good" }, "cost": { "v": "$10.00", "s": 10, "tone": "warn" } } }, { "provider": "OpenAI", "model": "gpt-5-mini", "meta": "reasoning", "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "overall": { "v": "50%", "s": 50, "tone": "bad" }, "tool": { "v": "67%", "s": 67, "tone": "bad" }, "ground": { "v": "33%", "s": 33, "tone": "bad" }, "ttft": { "v": "746ms", "s": 746, "lo": 746, "hi": 794, "ci": "746–794 p50–p90", "tone": "good" }, "cost": { "v": "$2.00", "s": 2 } } }, { "provider": "Alibaba", "model": "qwen-turbo", "status": { "label": "WEAK", "tone": "bad" }, "rec": [], "cells": { "overall": { "v": "33%", "s": 33, "tone": "bad" }, "tool": { "v": "0%", "s": 0, "tone": "bad" }, "ground": { "v": "67%", "s": 67, "tone": "bad" }, "ttft": { "v": "483ms", "s": 483, "lo": 483, "hi": 497, "ci": "483–497 p50–p90", "tone": "good" }, "cost": { "v": "$0.20", "s": 0.2, "tone": "good" } } } ], "reading": "When reliability ties, speed and cost decide. Five models are 100% reliable — two of them Cerebras: gpt-oss-120b at 195ms/$0.75 and the non-reasoning gemma-4-31b, which measured an even faster 192ms p50 on a same-run basis (Gemma has no published output-only price, so gpt-oss keeps the value edge and the lead). Grok-4.3 matches their reliability at 2.15s, too slow to lead a live call. That is why the board ranks the reliable cluster by latency, not alphabetically. Reasoning models (gpt-5, gpt-5-mini) finish near the bottom: under a voice latency cap they burn the turn thinking before they fire the tool." }, { "id": "s2s", "navLabel": "Speech-to-Speech", "identityLabel": "Provider / Model", "title": "Speech-to-Speech · realtime", "subtitle": "Native voice-to-voice models, each driven through one live 10-turn conversation — normal turns, a tool call with the result fed back, a mid-utterance barge-in, and a late callback that probes memory. Conversational latency is time to first audio; the judge grades score what actually happened in the call. Ranked by latency among models that complete the job.", "live": true, "meta": "English · us-east4 · multi-turn · n=15 · measured 2026-07-08", "columns": [ { "id": "lat", "label": "Conversational latency", "dir": "lower", "info": "Time from end-of-speech to first audio back — p50 over the clean (non-interrupted) turns, pooled across 15 sessions. Caption shows p90 · p95. Lower is better." }, { "id": "task", "label": "Task success", "dir": "higher", "info": "Whether the agent accomplished the scenario goal across the whole 10-turn call. LLM-judged (gpt-4.1)." }, { "id": "instr", "label": "Instruction-following", "dir": "higher", "info": "Adherence to the system prompt: persona, brevity, and using the tool instead of answering from memory. 0-1, LLM-judged — the axis that actually separates these models." }, { "id": "tool", "label": "Tool use", "dir": "higher", "info": "Right tool, right arguments, called when needed and not when not. 0-1, LLM-judged." }, { "id": "ctx", "label": "Context retention", "dir": "higher", "info": "Correctly recalled an earlier-stated detail (a name) at a late callback turn. 0-1, LLM-judged." }, { "id": "price", "label": "Audio price", "info": "Vendor-published audio pricing in the vendor’s own denomination (per minute, or per 1M audio tokens in/out), read from official pricing pages 2026-07-05. Not directly comparable across rows — hover a value for its basis." } ], "rows": [ { "provider": "OpenAI", "model": "gpt-realtime", "lead": true, "status": { "label": "PASS", "tone": "good" }, "rec": [ { "label": "Fastest" } ], "cells": { "lat": { "v": "494ms", "s": 494, "tone": "good", "ci": "p90 768 · p95 870" }, "task": { "v": "100%", "s": 100, "tone": "good" }, "instr": { "v": "0.81", "s": 81, "tone": "good" }, "tool": { "v": "1.00", "s": 100, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$32 / $64", "s": null, "ci": "per 1M audio tok, in/out", "note": "OpenAI audio-token pricing (input/output per 1M) from the official model page, 2026-07-05. No official per-minute rate is published." } } }, { "provider": "OpenAI", "model": "gpt-realtime-mini", "status": { "label": "PASS", "tone": "good" }, "rec": [ { "label": "Best value" } ], "cells": { "lat": { "v": "614ms", "s": 614, "tone": "good", "ci": "p90 812 · p95 847" }, "task": { "v": "99%", "s": 99, "tone": "good" }, "instr": { "v": "0.82", "s": 82, "tone": "good" }, "tool": { "v": "1.00", "s": 100, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$10 / $20", "s": null, "ci": "per 1M audio tok, in/out", "note": "OpenAI mini realtime audio-token pricing (input/output per 1M) — developers.openai.com pricing, 2026-07-08." } } }, { "provider": "Inworld", "model": "Realtime", "status": { "label": "PASS", "tone": "good" }, "rec": [], "cells": { "lat": { "v": "973ms", "s": 973, "tone": "warn", "ci": "p90 1214 · p95 1303" }, "task": { "v": "90%", "s": 90, "tone": "good" }, "instr": { "v": "0.45", "s": 45, "tone": "bad" }, "tool": { "v": "0.91", "s": 91, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "~$0.01 / min", "s": null, "ci": "TTS only · est.", "dim": true, "note": "Inworld realtime TTS ≈ $0.01/min at scale (vendor-stated; Realtime TTS-2 down to $10/1M chars). STT ($0.15/hr) + LLM billed separately — no single all-in vendor rate, so this is the TTS component only." } } }, { "provider": "xAI", "model": "grok-voice-think-fast", "status": { "label": "PASS", "tone": "good" }, "rec": [], "cells": { "lat": { "v": "1007ms", "s": 1007, "tone": "warn", "ci": "p90 1360 · p95 1426" }, "task": { "v": "100%", "s": 100, "tone": "good" }, "instr": { "v": "0.50", "s": 50, "tone": "warn" }, "tool": { "v": "0.98", "s": 98, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$0.05", "s": null, "ci": "per min, flat", "note": "xAI Voice API flat rate: $0.05 per minute of audio sent or received — billed API-wide, not per model (docs.x.ai, 2026-07-05)." } } }, { "provider": "OpenAI", "model": "gpt-realtime-2.1-mini", "status": { "label": "PASS", "tone": "good" }, "rec": [], "cells": { "lat": { "v": "1011ms", "s": 1011, "tone": "warn", "ci": "p90 1360 · p95 1542" }, "task": { "v": "99%", "s": 99, "tone": "good" }, "instr": { "v": "0.75", "s": 75, "tone": "warn" }, "tool": { "v": "0.97", "s": 97, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$10 / $20", "s": null, "ci": "per 1M audio tok, in/out", "note": "OpenAI audio-token pricing (input/output per 1M) for the mini realtime tier, official pricing page (2026-07-06 release)." } } }, { "provider": "OpenAI", "model": "gpt-realtime-2", "status": { "label": "PASS", "tone": "good" }, "rec": [], "cells": { "lat": { "v": "1103ms", "s": 1103, "tone": "warn", "ci": "p90 1468 · p95 1626" }, "task": { "v": "100%", "s": 100, "tone": "good" }, "instr": { "v": "0.85", "s": 85, "tone": "good" }, "tool": { "v": "0.98", "s": 98, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$32 / $64", "s": null, "ci": "per 1M audio tok, in/out", "note": "OpenAI audio-token pricing (input/output per 1M), 2026-07-05. Reasoning effort can increase output-token spend." } } }, { "provider": "OpenAI", "model": "gpt-realtime-2.1", "status": { "label": "PASS", "tone": "good" }, "rec": [], "cells": { "lat": { "v": "1104ms", "s": 1104, "tone": "warn", "ci": "p90 1514 · p95 2146" }, "task": { "v": "100%", "s": 100, "tone": "good" }, "instr": { "v": "0.81", "s": 81, "tone": "good" }, "tool": { "v": "0.98", "s": 98, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$32 / $64", "s": null, "ci": "per 1M audio tok, in/out", "note": "OpenAI audio-token pricing (input/output per 1M), 2026-07-06 release. Reasoning effort can increase output-token spend." } } }, { "provider": "Google", "model": "Gemini 3.1 Live", "status": { "label": "PASS", "tone": "good" }, "rec": [], "flag": "Barge-in not scored — Gemini emits no interruption signal over the gateway (a gateway normalization gap, not a model failure); it was judged via an STT-fallback transcript because it sends no assistant transcript frames.", "cells": { "lat": { "v": "1108ms", "s": 1108, "tone": "warn", "ci": "p90 1209 · p95 1285" }, "task": { "v": "100%", "s": 100, "tone": "good" }, "instr": { "v": "0.80", "s": 80, "tone": "good" }, "tool": { "v": "0.99", "s": 99, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$0.005 / $0.018", "s": null, "ci": "per min, in/out", "note": "Google-published audio per-minute prices (paid tier), input/output — ai.google.dev pricing, 2026-07-05." } } }, { "provider": "xAI", "model": "grok-voice-fast", "status": { "label": "PASS", "tone": "good" }, "rec": [], "cells": { "lat": { "v": "1111ms", "s": 1111, "tone": "warn", "ci": "p90 1576 · p95 2752" }, "task": { "v": "100%", "s": 100, "tone": "good" }, "instr": { "v": "0.81", "s": 81, "tone": "good" }, "tool": { "v": "1.00", "s": 100, "tone": "good" }, "ctx": { "v": "1.00", "s": 100, "tone": "good" }, "price": { "v": "$0.05", "s": null, "ci": "per min, flat", "note": "xAI Voice API flat rate: $0.05 per minute of audio sent or received — billed API-wide, not per model (docs.x.ai, 2026-07-05)." } } } ], "reading": "This board no longer stops at \"did it answer.\" Each model runs a full 10-turn call and is graded on what happened. Task success is near-universal — the easy part — so the real separation is instruction-following, which ranges 0.45 (Inworld) to 0.85 (gpt-realtime-2): that is where these models actually differ. gpt-realtime leads on speed (494ms) with clean grades across the board, and gpt-realtime-mini matches its quality at a fraction of the price. The newer gpt-realtime-2 / 2.1 are ~2x slower than the original for no quality gain. Gemini scores well once judged via an STT fallback (it emits no transcript over the gateway), though its barge-in is left unscored — a gateway gap, not a model failure." }, { "id": "stacks", "navLabel": "Cost / solve", "identityLabel": "Stack · STT / LLM / TTS", "title": "True cost-per-solve · whole stack", "subtitle": "The number no component leaderboard can show: what one grounded, resolved outcome costs across a full STT + LLM + TTS stack — not per minute. A solve counts only when the agent actually fires the tool.", "live": true, "meta": "dental-booking task · n=3/stack · grounded (tool fired) · measured 2026-07-03", "columns": [ { "id": "grounded", "label": "Grounded", "dir": "higher", "info": "% of runs the required tool actually fired. Higher is better." }, { "id": "turns", "label": "Turns", "info": "Mean turns to resolution." }, { "id": "call", "label": "$/call", "dir": "lower", "info": "Cost per call: real consumption × billed rates." }, { "id": "solve", "label": "$/solve", "dir": "lower", "info": "Cost per grounded solve. Lower = cheaper." } ], "rows": [ { "provider": "", "model": "CHEAPEST", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "cartesia:sonic-3.5" ], "lead": true, "status": { "label": "GROUNDED", "tone": "good" }, "rec": [], "cells": { "grounded": { "v": "100%", "s": 100, "tone": "good" }, "turns": { "v": "5.3", "s": 5.3, "dim": true }, "call": { "v": "$0.058", "s": 0.058 }, "solve": { "v": "$0.058", "s": 0.058, "tone": "good" } } }, { "provider": "", "model": "DEEPGRAM-NATIVE", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "deepgram:aura-2" ], "status": { "label": "GROUNDED", "tone": "good" }, "rec": [], "cells": { "grounded": { "v": "100%", "s": 100, "tone": "good" }, "turns": { "v": "5.0", "s": 5, "dim": true }, "call": { "v": "$0.123", "s": 0.123 }, "solve": { "v": "$0.123", "s": 0.123, "tone": "good" } } }, { "provider": "", "model": "SINGLE-VENDOR", "stack": [ "cartesia:ink-2", "cerebras:gpt-oss-120b", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "flag": "Weak grounding (67%) — a low bill here is only meaningful when the tool actually fires.", "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "6.0", "s": 6, "dim": true }, "call": { "v": "$0.087", "s": 0.087 }, "solve": { "v": "$0.131", "s": 0.131, "tone": "good" } } }, { "provider": "", "model": "EXPRESSIVE VOICE", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "minimax:speech-2.6-hd" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "4.3", "s": 4.3, "dim": true }, "call": { "v": "$0.092", "s": 0.092 }, "solve": { "v": "$0.137", "s": 0.137, "tone": "good" } } }, { "provider": "", "model": "PREMIUM VOICE / CHEAP BRAIN", "stack": [ "deepgram:nova-3", "cerebras:gpt-oss-120b", "elevenlabs:eleven_flash_v2_5" ], "status": { "label": "WEAK", "tone": "bad" }, "flag": "Only 33% grounded — the cheap-looking $/call is misleading; it rarely closes the booking.", "rec": [], "cells": { "grounded": { "v": "33%", "s": 33, "tone": "bad" }, "turns": { "v": "1.7", "s": 1.7, "dim": true }, "call": { "v": "$0.055", "s": 0.055 }, "solve": { "v": "$0.166", "s": 0.166, "tone": "warn" } } }, { "provider": "", "model": "PREMIUM BRAIN / CHEAP VOICE", "stack": [ "deepgram:nova-3", "openai:gpt-4.1", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "4.3", "s": 4.3, "dim": true }, "call": { "v": "$0.111", "s": 0.111 }, "solve": { "v": "$0.166", "s": 0.166, "tone": "warn" } } }, { "provider": "", "model": "CHEAP-OPENAI", "stack": [ "deepgram:nova-3", "openai:gpt-5-nano", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "10.0", "s": 10, "dim": true }, "call": { "v": "$0.145", "s": 0.145 }, "solve": { "v": "$0.218", "s": 0.218, "tone": "warn" } } }, { "provider": "", "model": "ACCURACY", "stack": [ "elevenlabs:scribe_v2_realtime", "openai:gpt-4.1", "cartesia:sonic-3.5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "5.7", "s": 5.7, "dim": true }, "call": { "v": "$0.147", "s": 0.147 }, "solve": { "v": "$0.221", "s": 0.221, "tone": "warn" } } }, { "provider": "", "model": "PREMIUM (ALL-BRAND)", "stack": [ "elevenlabs:scribe_v2_realtime", "openai:gpt-4.1", "elevenlabs:eleven_flash_v2_5" ], "status": { "label": "PARTIAL", "tone": "warn" }, "rec": [], "cells": { "grounded": { "v": "67%", "s": 67, "tone": "warn" }, "turns": { "v": "7.3", "s": 7.3, "dim": true }, "call": { "v": "$0.469", "s": 0.469 }, "solve": { "v": "$0.703", "s": 0.703, "tone": "bad" } } }, { "provider": "", "model": "BALANCED", "stack": [ "deepgram:nova-3", "openai:gpt-4.1-mini", "cartesia:sonic-3.5" ], "status": { "label": "NO SOLVE", "tone": "bad" }, "flag": "Never booked (0% grounded) — gpt-4.1-mini kept clarifying and never fired the tool.", "rec": [], "cells": { "grounded": { "v": "0%", "s": 0, "tone": "bad" }, "turns": { "v": "3.3", "s": 3.3, "dim": true }, "call": { "v": "$0.079", "s": 0.079 }, "solve": { "v": "never booked", "s": null, "tone": "bad" } } } ], "reading": "Wiring a real tool kills the fabrication trap — a naive cost ÷ resolution crowns whichever model most confidently lies \"you're booked.\" The cheapest honest stack (Deepgram · Cerebras · Cartesia) solves for $0.058, while all-brand ElevenLabs + GPT-4.1 costs ~12× more. Three stacks (⚠) look cheap only because they rarely close the booking." }, { "id": "turntaking", "navLabel": "Turn-taking", "identityLabel": "Model", "title": "Turn-taking · end-of-turn detection", "subtitle": "Has the caller actually finished — or just paused mid-thought? Ranked on 200 REAL human clips (Pipecat’s Smart Turn eval set), where an incomplete turn carries genuine continuation prosody — the pitch stays up and the energy doesn’t taper. The audio model (Smart Turn) hears that; the text models read the STT transcript. The costly error is a FALSE CUTOFF — ending the turn on someone who wasn’t done. The headline finding: on synthetic TTS clips this ranking flips completely (Smart Turn goes from first to last) — which is the whole point. See “why it flips” and the prosody proof below.", "live": true, "meta": "English · 200 real human clips · pipecat smart-turn-v3.1-test · transcripts via Deepgram nova-3", "columns": [ { "id": "acc", "label": "EOT accuracy", "dir": "higher", "info": "Correct END-vs-WAIT decisions across 200 real clips, at each model’s best operating threshold (min false-cutoff s.t. END-recall ≥ 85%)." }, { "id": "recall", "label": "END-recall", "dir": "higher", "info": "Completed turns correctly ended — the agent didn’t leave the caller hanging." }, { "id": "cut", "label": "False-cutoff", "dir": "lower", "info": "WAIT clips wrongly ended = talked over the caller mid-thought. The costly live-call error." }, { "id": "lat", "label": "Inference", "dir": "lower", "info": "Model forward pass, warm CPU. Text models additionally wait on the STT transcript; audio runs in parallel with STT." } ], "rows": [ { "provider": "Smart Turn v3.2", "model": "Pipecat · Whisper-tiny", "meta": "audio · prosody", "rec": [], "lead": true, "flag": "Wins on REAL audio — an incomplete turn actually sounds incomplete (pitch stays up, energy holds), and an audio model hears it. Caveat: this eval set is Smart Turn’s home distribution (held-out split of the data it trained on), so treat 94% as a ceiling, not the exact prod gap; the FLIP vs the synthetic board is the durable result. This is what Speko runs on live calls.", "cells": { "acc": { "v": "94.0%", "s": 94, "tone": "good" }, "recall": { "v": "89.0%", "s": 89, "tone": "good" }, "cut": { "v": "1.0%", "s": 1, "tone": "good", "note": "One false cutoff in 100 incomplete clips. The SAME model scored 57.7% on synthetic TTS — the audio, not the model, was the difference." }, "lat": { "v": "32ms", "s": 32, "tone": "good", "ci": "49 p90", "note": "Runs in-process on the audio buffer, in parallel with STT — no transcript wait, so it decides first on a live call." } } }, { "provider": "LiveKit Intl", "model": "turn-detector v0.4.1", "meta": "text · transcript", "rec": [], "cells": { "acc": { "v": "87.0%", "s": 87, "tone": "good" }, "recall": { "v": "86.0%", "s": 86, "tone": "good" }, "cut": { "v": "12.0%", "s": 12, "tone": "warn" }, "lat": { "v": "29ms", "s": 29, "tone": "good", "ci": "57 p90", "note": "Text model: can’t run until the STT transcript lands (~0–125ms on Nova-3), then adds this forward pass." } } }, { "provider": "Turnsense", "model": "SmolLM2-135M", "meta": "text · transcript", "rec": [], "cells": { "acc": { "v": "86.0%", "s": 86, "tone": "good" }, "recall": { "v": "88.0%", "s": 88, "tone": "good" }, "cut": { "v": "16.0%", "s": 16, "tone": "warn" }, "lat": { "v": "127ms", "s": 127, "tone": "warn", "ci": "193 p90", "note": "Pads every input to a fixed 256 tokens → full pass regardless of length. Fixable (dynamic length), but as-shipped it’s the slowest." } } }, { "provider": "LiveKit EN", "model": "turn-detector v1.2.2", "meta": "text · transcript", "rec": [], "cells": { "acc": { "v": "82.0%", "s": 82, "tone": "warn" }, "recall": { "v": "87.0%", "s": 87, "tone": "good" }, "cut": { "v": "23.0%", "s": 23, "tone": "warn" }, "lat": { "v": "5ms", "s": 5, "tone": "good", "ci": "24 p90" } } }, { "provider": "VAD + silence timer", "model": "baseline", "meta": "audio · energy", "rec": [], "cells": { "acc": { "v": "46.9%", "s": 46.9, "tone": "bad", "dim": true }, "recall": { "v": "100%", "s": 100, "tone": "neutral", "dim": true }, "cut": { "v": "100%", "s": 100, "tone": "bad", "dim": true, "note": "A silence timer ends every turn — the floor semantic detection has to beat." }, "lat": { "v": "~1ms", "s": 1, "tone": "neutral", "dim": true } } } ], "reading": "This is the board that flips. On REAL human speech the audio model wins outright — Smart Turn hits 94% with a 1% false-cutoff, because an incomplete turn actually SOUNDS incomplete: the pitch stays up and the energy doesn’t taper (see the prosody proof below). On synthetic TTS clips the exact same models rank the opposite way — Smart Turn drops to LAST (62% / 57.7% false-cutoff) — because a TTS voice gives every clip a finished-sounding contour no matter what the words say, so an audio model can’t hear the difference and the text models (reading the words) win. Two honest caveats: (1) this real set is Smart Turn’s home distribution — held out, but the same collection it trained on — so treat 94% as a ceiling; a truly neutral ranking needs a third-party corpus, which is the next step. (2) The text models here run on real STT transcripts (Deepgram nova-3), messier than clean text, which is why they land below their synthetic scores. The durable lesson stands on its own: you cannot benchmark audio turn-taking on synthetic speech — prosody is the signal that decides it, and it is measurable (below)." } ], "recommended": [ { "n": "01", "title": "Real-time phone agent", "recipe": "weights · TTFT ×0.5 · synth-latency ×0.3 · reliability ×0.2", "legs": [ { "mod": "STT", "pick": "Deepgram nova-3 †", "why": "streaming leader" }, { "mod": "LLM", "pick": "Cerebras gpt-oss-120b", "why": "195ms · 100%" }, { "mod": "TTS", "pick": "Cartesia sonic-3.5", "why": "357ms · 0.79%" } ], "note": "Lowest end-to-end turn latency — every leg picked for speed without giving up reliability." }, { "n": "02", "title": "Accuracy-critical", "recipe": "weights · STT-WER ×0.35 · grounding ×0.35 · TTS-intel ×0.2 · TTFT ×0.1", "legs": [ { "mod": "STT", "pick": "OpenAI gpt-4o-transcribe", "why": "2.3% · floor" }, { "mod": "LLM", "pick": "Cerebras gpt-oss-120b", "why": "100% ground" }, { "mod": "TTS", "pick": "Hume octave-2", "why": "0.71% · clearest" } ], "note": "Support & healthcare — grounding + the lowest error at every leg. Cerebras matches Grok-4.3’s grounding at 195ms, not 2.15s." }, { "n": "03", "title": "Natural conversation", "recipe": "weights · naturalness ×0.5 · TTS-drift ×0.25 · latency ×0.25", "legs": [ { "mod": "STT", "pick": "OpenAI gpt-4o-transcribe", "why": "2.3% WER" }, { "mod": "LLM", "pick": "Cerebras gpt-oss-120b †", "why": "100% reliable" }, { "mod": "TTS", "pick": "Cartesia sonic-3.5", "why": "Elo 1209" } ], "note": "Companion, sales, brand voice — weighted on the Artificial-Analysis naturalness Elo plus voice steadiness." }, { "n": "04", "title": "Tool-heavy agent", "recipe": "weights · tool-calling ×0.5 · reliability ×0.3 · TTFT ×0.2", "legs": [ { "mod": "STT", "pick": "OpenAI gpt-4o-transcribe", "why": "2.3% WER" }, { "mod": "LLM", "pick": "Cerebras gpt-oss-120b", "why": "100% tool" }, { "mod": "TTS", "pick": "Cartesia sonic-3.5", "why": "357ms" } ], "note": "Scheduling & transactions — the model must fire the right tool under a latency cap. Cerebras does both." } ], "projector": [ { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · elevenlabs:eleven_flash_v2_5", "c": 0.055, "fab": true }, { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · cartesia:sonic-3.5", "c": 0.058 }, { "name": "deepgram:nova-3 · openai:gpt-4.1-mini · cartesia:sonic-3.5 (never books)", "c": 0.079, "fab": true }, { "name": "cartesia:ink-2 · cerebras:gpt-oss-120b · cartesia:sonic-3.5", "c": 0.087 }, { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · minimax:speech-2.6-hd", "c": 0.092 }, { "name": "deepgram:nova-3 · openai:gpt-4.1 · cartesia:sonic-3.5", "c": 0.111 }, { "name": "deepgram:nova-3 · cerebras:gpt-oss-120b · deepgram:aura-2", "c": 0.123 }, { "name": "deepgram:nova-3 · openai:gpt-5-nano · cartesia:sonic-3.5", "c": 0.145 }, { "name": "elevenlabs:scribe_v2_realtime · openai:gpt-4.1 · cartesia:sonic-3.5", "c": 0.147 }, { "name": "elevenlabs:scribe_v2_realtime · openai:gpt-4.1 · elevenlabs:eleven_flash_v2_5", "c": 0.469 } ], "turntakingSynthetic": [ { "label": "Turnsense", "kind": "text", "acc": 98.8, "cut": 0 }, { "label": "LiveKit Intl", "kind": "text", "acc": 90.6, "cut": 6.2 }, { "label": "LiveKit EN", "kind": "text", "acc": 79.2, "cut": 26.2 }, { "label": "Smart Turn", "kind": "audio", "acc": 62.4, "cut": 57.7 } ], "turntakingProsody": [ { "cue": "Final F0 slope", "unit": "Hz/s", "end": -119.7, "wait": 42.9, "want": "complete falls, incomplete rises/holds" }, { "cue": "Final energy slope", "unit": "dB/s", "end": -14.9, "wait": -2.5, "want": "complete tapers off" } ] }