{"name":"GPU Economy - published inference throughput per card","description":"The inputs of the one modelled figure on this site: cost per million output tokens = a listed rate per GPU-hour / (output tokens per second per GPU x 3,600) x 1,000,000. Throughput as NVIDIA published it, per card and model. Cards are not comparable on it: NVIDIA runs each card in its own configuration.","source":"NVIDIA published inference benchmarks","source_page":"https://developer.nvidia.com/deep-learning-performance-training-inference/ai-inference","source_data":"https://developer.nvidia.com/search-data/nv_inference_benchmark.json","reference_concurrency":64,"formula":"usd_per_million_output_tokens = usd_per_gpu_hour / (output_tokens_per_second_per_gpu * 3600) * 1e6","license":"CC BY 4.0 for the selection and formula; the throughput figures are NVIDIA's, linked","data_updated_at":"2026-10-23T22:40:01.505772+00:00","cards":[{"gpu_id":"b200","gpu":"NVIDIA B200","runs":[{"model":"Qwen 3.5","output_tokens_per_second_per_gpu":2041.6,"framework":"TRT-LLM","framework_version":null,"precision":"FP4","sequence":"1K/1K","concurrency":64,"gpus":8,"speculative_decoding":"mtp","date":"2026-06-24","config":"Qwen 3.5, 1K/1K in/out, FP4, TRT-LLM, concurrency 64, 8 GPUs, speculative decoding mtp"},{"model":"Nemotron 3 Super 120B","output_tokens_per_second_per_gpu":672.9,"framework":"vLLM","framework_version":null,"precision":"NVFP4","sequence":"1K/1K","concurrency":64,"gpus":8,"speculative_decoding":"off","date":"2026-05-06","config":"Nemotron 3 Super 120B, 1K/1K in/out, NVFP4, vLLM, concurrency 64, 8 GPUs"},{"model":"gpt-oss-120b","output_tokens_per_second_per_gpu":14200.4,"framework":"TRT-LLM","framework_version":null,"precision":"FP4","sequence":"1K/1K","concurrency":64,"gpus":2,"speculative_decoding":"eagle","date":"2026-02-05","config":"gpt-oss-120b, 1K/1K in/out, FP4, TRT-LLM, concurrency 64, 2 GPUs, speculative decoding eagle"}]},{"gpu_id":"b300","gpu":"NVIDIA B300","runs":[{"model":"Qwen 3.5","output_tokens_per_second_per_gpu":14782.3,"framework":"SGLang","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":4,"speculative_decoding":"mtp","date":"2026-08-10","config":"Qwen 3.5, 8K/1K in/out, FP4, SGLang, concurrency 64, 4 GPUs, speculative decoding mtp"},{"model":"DeepSeek-V4-Pro","output_tokens_per_second_per_gpu":1598.9,"framework":"Dynamo SGLang","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":28,"speculative_decoding":"mtp","date":"2026-07-30","config":"DeepSeek-V4-Pro, 8K/1K in/out, FP4, Dynamo SGLang, concurrency 64, 28 GPUs, speculative decoding mtp"},{"model":"MiniMax-M3","output_tokens_per_second_per_gpu":11672,"framework":"vLLM","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":4,"speculative_decoding":"mtp","date":"2026-07-29","config":"MiniMax-M3, 8K/1K in/out, FP4, vLLM, concurrency 64, 4 GPUs, speculative decoding mtp"},{"model":"MiniMax-M2.5","output_tokens_per_second_per_gpu":3005.4,"framework":"vLLM","framework_version":null,"precision":"FP4","sequence":"1K/1K","concurrency":64,"gpus":4,"speculative_decoding":"off","date":"2026-06-04","config":"MiniMax-M2.5, 1K/1K in/out, FP4, vLLM, concurrency 64, 4 GPUs"}]},{"gpu_id":"gb300","gpu":"NVIDIA GB300 NVL72","runs":[{"model":"Kimi-K3","output_tokens_per_second_per_gpu":8291.4,"framework":"Dynamo vLLM","framework_version":null,"precision":"FP4","sequence":null,"concurrency":64,"gpus":16,"speculative_decoding":"mtp","date":"2026-08-18","config":"Kimi-K3, FP4, Dynamo vLLM, concurrency 64, 16 GPUs, speculative decoding mtp"},{"model":"DeepSeek-V4-Pro","output_tokens_per_second_per_gpu":1599.4,"framework":"Dynamo SGLang","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":28,"speculative_decoding":"mtp","date":"2026-06-03","config":"DeepSeek-V4-Pro, 8K/1K in/out, FP4, Dynamo SGLang, concurrency 64, 28 GPUs, speculative decoding mtp"}]},{"gpu_id":"h100-sxm","gpu":"NVIDIA H100 SXM","runs":[{"model":"Nemotron 3 Super 120B","output_tokens_per_second_per_gpu":1196.1,"framework":"vLLM","framework_version":null,"precision":"FP8","sequence":"1K/1K","concurrency":64,"gpus":4,"speculative_decoding":"off","date":"2026-05-06","config":"Nemotron 3 Super 120B, 1K/1K in/out, FP8, vLLM, concurrency 64, 4 GPUs"},{"model":"Qwen 3.5","output_tokens_per_second_per_gpu":1409.9,"framework":"vLLM","framework_version":null,"precision":"NVFP4","sequence":"1K/1K","concurrency":64,"gpus":4,"speculative_decoding":"on","date":"2026-04-09","config":"Qwen 3.5, 1K/1K in/out, NVFP4, vLLM, concurrency 64, 4 GPUs, speculative decoding on"}]},{"gpu_id":"h200","gpu":"NVIDIA H200","runs":[{"model":"DeepSeek-V4-Pro","output_tokens_per_second_per_gpu":329,"framework":"SGLang","framework_version":null,"precision":"FP4","sequence":"1K/1K","concurrency":64,"gpus":8,"speculative_decoding":"off","date":"2026-07-14","config":"DeepSeek-V4-Pro, 1K/1K in/out, FP4, SGLang, concurrency 64, 8 GPUs"},{"model":"Qwen 3.5","output_tokens_per_second_per_gpu":3973.7,"framework":"SGLang","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":4,"speculative_decoding":"on","date":"2026-07-04","config":"Qwen 3.5, 8K/1K in/out, FP4, SGLang, concurrency 64, 4 GPUs, speculative decoding on"},{"model":"MiniMax-M3","output_tokens_per_second_per_gpu":3209.1,"framework":"Dynamo vLLM","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":4,"speculative_decoding":"off","date":"2026-06-25","config":"MiniMax-M3, 8K/1K in/out, FP4, Dynamo vLLM, concurrency 64, 4 GPUs"},{"model":"MiniMax-M2.5","output_tokens_per_second_per_gpu":1430.1,"framework":"vLLM","framework_version":null,"precision":"FP8","sequence":"1K/1K","concurrency":64,"gpus":4,"speculative_decoding":"off","date":"2026-06-05","config":"MiniMax-M2.5, 1K/1K in/out, FP8, vLLM, concurrency 64, 4 GPUs"},{"model":"Kimi K2.5/2.6","output_tokens_per_second_per_gpu":521.3,"framework":"vLLM","framework_version":null,"precision":"INT4","sequence":"1K/1K","concurrency":64,"gpus":8,"speculative_decoding":"off","date":"2026-05-30","config":"Kimi K2.5/2.6, 1K/1K in/out, INT4, vLLM, concurrency 64, 8 GPUs"},{"model":"Nemotron 3 Super 120B","output_tokens_per_second_per_gpu":8692.8,"framework":"Dynamo vLLM","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":1,"speculative_decoding":"off","date":"2026-05-06","config":"Nemotron 3 Super 120B, 8K/1K in/out, FP4, Dynamo vLLM, concurrency 64, 1 GPU"},{"model":"gpt-oss-120b","output_tokens_per_second_per_gpu":2343.3,"framework":"vLLM","framework_version":null,"precision":"FP4","sequence":"1K/1K","concurrency":64,"gpus":8,"speculative_decoding":"off","date":"2026-03-27","config":"gpt-oss-120b, 1K/1K in/out, FP4, vLLM, concurrency 64, 8 GPUs"},{"model":"DeepSeek-R1","output_tokens_per_second_per_gpu":1490.8,"framework":"Dynamo TRT-LLM","framework_version":null,"precision":"FP8","sequence":"8K/1K","concurrency":64,"gpus":16,"speculative_decoding":"mtp","date":"2026-03-23","config":"DeepSeek-R1, 8K/1K in/out, FP8, Dynamo TRT-LLM, concurrency 64, 16 GPUs, speculative decoding mtp"}]},{"gpu_id":"rtx-pro-6000","gpu":"NVIDIA RTX PRO 6000 Blackwell","runs":[{"model":"Qwen 3.5","output_tokens_per_second_per_gpu":1729.3,"framework":"SGLang","framework_version":null,"precision":"FP4","sequence":"8K/1K","concurrency":64,"gpus":16,"speculative_decoding":"mtp","date":"2026-07-30","config":"Qwen 3.5, 8K/1K in/out, FP4, SGLang, concurrency 64, 16 GPUs, speculative decoding mtp"}]}]}