{ "metadata": { "version": "0.4.29", "engine": "vllm", "model": "DeepSeek-V4-Flash", "server": "http://192.168.0.213", "timestamp": "2026-07-10T19:43:02.952838", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 20.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "standalone_cold", "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 5.0, "decode_warmup_context": 262144, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 0.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 8192, "temperature": null, "ignore_eos": true, "max_total_tokens": 0, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1, 2, 8, 16 ], "context_lengths": [ 8192, 16384, 65536, 131072, 262144 ], "startup_diagnostics_available": true, "nvidia_p2p_override_effective": false, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.29", "server_url": "http://192.168.0.213:8000", "hostname": "DESKTOP-RMAJ6JK", "uname": "", "env": {}, "args": { "concurrency": "1,2,8,16", "contexts": "8k,16k,64k,128k,256k", "max_tokens": 8192, "duration": 20.0, "request_count": 0, "run_burst": false, "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "prefill_contexts": "8k,16k,32k,64k,128k,256k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 0 }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 595.79, 00000000:01:00.0, 5, 16, 450.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 255, "stdout": "ERROR: Option -m is missing its value. Please run 'nvidia-smi -h' for help.", "stderr": "" } }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": { "samples": 352, "duration_seconds": 770.953, "gpu_count": 1, "cpu_util_avg_pct": 7.58, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.76, "gpu_util_max_pct": 90.0, "mem_util_avg_pct": 6.68, "mem_util_max_pct": 58.0, "temp_avg_c": 34.05, "temp_max_c": 43.0, "power_total_avg_w": 47.91, "power_total_max_w": 339.22, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95469.19, "vram_used_max_mb": 95690.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.53, "vram_used_max_pct": 97.76, "pcie_rx_avg_mb_s": 20.13, "pcie_rx_max_mb_s": 381.0, "pcie_tx_avg_mb_s": 12.02, "pcie_tx_max_mb_s": 2765.0 }, "event_log": [ "19:30:11 startup startup preparation done", "19:30:11 hardware monitor interval=2s", "19:30:11 prefill warmup start", "19:30:12 prefill start ctx=8k", "19:30:22 prefill done ctx=8k 11,910 tok/s", "19:30:23 prefill start ctx=16k", "19:30:34 prefill done ctx=16k 11,654 tok/s", "19:30:35 prefill start ctx=32k", "19:30:47 prefill done ctx=32k 11,275 tok/s", "19:30:48 prefill start ctx=64k", "19:31:01 prefill done ctx=64k 10,672 tok/s", "19:31:02 prefill start ctx=128k", "19:31:17 prefill done ctx=128k 9,178 tok/s", "19:31:18 prefill start ctx=256k", "19:31:54 prefill done ctx=256k 7,346 tok/s", "19:31:55 decode warmup start C=1 ctx=256k 5s", "19:31:55 cell start C=1 ctx=256k", "19:32:35 ready C=1 ctx=256k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:32:40 cell done C=1 ctx=256k 202.6 tok/s", "19:32:40 decode warmup done C=1 ctx=256k", "19:32:43 cell start C=1 ctx=8k", "19:32:49 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:33:09 cell done C=1 ctx=8k 235.4 tok/s", "19:33:11 cell start C=1 ctx=16k", "19:33:16 ready C=1 ctx=16k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:33:36 cell done C=1 ctx=16k 227.4 tok/s", "19:33:38 cell start C=1 ctx=64k", "19:33:48 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:34:08 cell done C=1 ctx=64k 242.4 tok/s", "19:34:10 cell start C=1 ctx=128k", "19:34:28 ready C=1 ctx=128k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:34:48 cell done C=1 ctx=128k 212.3 tok/s", "19:34:50 cell start C=1 ctx=256k", "19:35:31 ready C=1 ctx=256k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:35:51 cell done C=1 ctx=256k 206.1 tok/s", "19:35:53 cell start C=2 ctx=8k", "19:35:59 ready C=2 ctx=8k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:36:19 cell done C=2 ctx=8k 346.9 tok/s", "19:36:21 cell start C=8 ctx=8k", "19:36:27 ready C=8 ctx=8k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:36:47 cell done C=8 ctx=8k 778.2 tok/s", "19:36:49 cell start C=16 ctx=8k", "19:36:55 ready C=16 ctx=8k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "19:37:15 cell done C=16 ctx=8k 1110.9 tok/s", "19:37:18 cell start C=2 ctx=16k", "19:37:23 ready C=2 ctx=16k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:37:43 cell done C=2 ctx=16k 353.6 tok/s", "19:37:45 cell start C=8 ctx=16k", "19:37:51 ready C=8 ctx=16k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:38:11 cell done C=8 ctx=16k 770.2 tok/s", "19:38:13 cell start C=16 ctx=16k", "19:38:19 ready C=16 ctx=16k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "19:38:39 cell done C=16 ctx=16k 1023.7 tok/s", "19:38:41 cell start C=2 ctx=64k", "19:38:47 ready C=2 ctx=64k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:39:07 cell done C=2 ctx=64k 339.7 tok/s", "19:39:09 cell start C=8 ctx=64k", "19:39:15 ready C=8 ctx=64k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:39:35 cell done C=8 ctx=64k 724.5 tok/s", "19:39:38 cell start C=16 ctx=64k", "19:39:43 ready C=16 ctx=64k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "19:40:03 cell done C=16 ctx=64k 1046.5 tok/s", "19:40:05 cell start C=2 ctx=128k", "19:40:10 ready C=2 ctx=128k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:40:30 cell done C=2 ctx=128k 335.7 tok/s", "19:40:33 cell start C=8 ctx=128k", "19:40:40 ready C=8 ctx=128k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:41:00 cell done C=8 ctx=128k 649.2 tok/s", "19:41:02 cell start C=16 ctx=128k", "19:41:08 ready C=16 ctx=128k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "19:41:28 cell done C=16 ctx=128k 939.9 tok/s", "19:41:31 cell start C=2 ctx=256k", "19:41:36 ready C=2 ctx=256k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:41:56 cell done C=2 ctx=256k 326.7 tok/s", "19:41:58 cell start C=8 ctx=256k", "19:42:07 ready C=8 ctx=256k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:42:27 cell done C=8 ctx=256k 620.4 tok/s", "19:42:30 cell start C=16 ctx=256k", "19:42:40 ready C=16 ctx=256k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "19:43:00 cell done C=16 ctx=256k 873.0 tok/s" ], "prefill": { "8192": { "ttft_seconds": 0.688, "prefill_seconds": 0.688, "tok_per_sec": 11910.0, "client_ttft_seconds": 0.688, "client_tok_per_sec": 11910.0, "prompt_tokens": 8194, "samples": 10, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 4, "duration_seconds": 6.968, "gpu_count": 1, "cpu_util_avg_pct": 7.47, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.0, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 36.75, "temp_max_c": 37.0, "power_total_avg_w": 62.37, "power_total_max_w": 62.81, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95081.5, "vram_used_max_mb": 95096.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.13, "vram_used_max_pct": 97.15, "pcie_rx_avg_mb_s": 9.75, "pcie_rx_max_mb_s": 16.0, "pcie_tx_avg_mb_s": 6.75, "pcie_tx_max_mb_s": 18.0 } }, "16384": { "ttft_seconds": 1.406, "prefill_seconds": 1.406, "tok_per_sec": 11654.0, "client_ttft_seconds": 1.406, "client_tok_per_sec": 11654.0, "prompt_tokens": 16386, "samples": 7, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 5, "duration_seconds": 8.656, "gpu_count": 1, "cpu_util_avg_pct": 6.34, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.0, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 36.0, "temp_max_c": 36.0, "power_total_avg_w": 62.14, "power_total_max_w": 62.81, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95180.2, "vram_used_max_mb": 95188.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.23, "vram_used_max_pct": 97.24, "pcie_rx_avg_mb_s": 6.6, "pcie_rx_max_mb_s": 8.0, "pcie_tx_avg_mb_s": 3.0, "pcie_tx_max_mb_s": 3.0 } }, "32768": { "ttft_seconds": 2.906, "prefill_seconds": 2.906, "tok_per_sec": 11275.0, "client_ttft_seconds": 2.906, "client_tok_per_sec": 11275.0, "prompt_tokens": 32770, "samples": 4, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 6, "duration_seconds": 10.843, "gpu_count": 1, "cpu_util_avg_pct": 6.1, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.83, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 36.0, "temp_max_c": 36.0, "power_total_avg_w": 61.77, "power_total_max_w": 61.98, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95155.17, "vram_used_max_mb": 95183.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.21, "vram_used_max_pct": 97.24, "pcie_rx_avg_mb_s": 7.17, "pcie_rx_max_mb_s": 9.0, "pcie_tx_avg_mb_s": 2.5, "pcie_tx_max_mb_s": 3.0 } }, "65536": { "ttft_seconds": 6.141, "prefill_seconds": 6.141, "tok_per_sec": 10672.0, "client_ttft_seconds": 6.141, "client_tok_per_sec": 10672.0, "prompt_tokens": 65538, "samples": 2, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 5, "duration_seconds": 8.657, "gpu_count": 1, "cpu_util_avg_pct": 5.62, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.0, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 36.0, "temp_max_c": 36.0, "power_total_avg_w": 61.98, "power_total_max_w": 62.19, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95132.0, "vram_used_max_mb": 95132.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.19, "vram_used_max_pct": 97.19, "pcie_rx_avg_mb_s": 6.4, "pcie_rx_max_mb_s": 8.0, "pcie_tx_avg_mb_s": 3.8, "pcie_tx_max_mb_s": 6.0 } }, "131072": { "ttft_seconds": 14.282, "prefill_seconds": 14.282, "tok_per_sec": 9178.0, "client_ttft_seconds": 14.282, "client_tok_per_sec": 9178.0, "prompt_tokens": 131074, "samples": 1, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 7, "duration_seconds": 12.984, "gpu_count": 1, "cpu_util_avg_pct": 7.59, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.0, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 36.0, "temp_max_c": 36.0, "power_total_avg_w": 62.08, "power_total_max_w": 62.63, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95134.57, "vram_used_max_mb": 95140.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.19, "vram_used_max_pct": 97.19, "pcie_rx_avg_mb_s": 7.43, "pcie_rx_max_mb_s": 9.0, "pcie_tx_avg_mb_s": 5.57, "pcie_tx_max_mb_s": 6.0 } }, "262144": { "ttft_seconds": 35.687, "prefill_seconds": 35.687, "tok_per_sec": 7346.0, "client_ttft_seconds": 35.687, "client_tok_per_sec": 7346.0, "prompt_tokens": 262146, "samples": 1, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 16, "duration_seconds": 32.953, "gpu_count": 1, "cpu_util_avg_pct": 7.46, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.94, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 36.0, "temp_max_c": 36.0, "power_total_avg_w": 62.52, "power_total_max_w": 64.67, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95147.31, "vram_used_max_mb": 95169.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.2, "vram_used_max_pct": 97.22, "pcie_rx_avg_mb_s": 43.19, "pcie_rx_max_mb_s": 357.0, "pcie_tx_avg_mb_s": 3.62, "pcie_tx_max_mb_s": 6.0 } } }, "results": [ { "concurrency": 1, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4709, "server_output_tokens": 4709, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 235.45, "per_request_avg_tps": 235.45, "ttft_avg": 0.07800000021234155, "ttft_p50": 0.07800000021234155, "ttft_p90": 0.07800000021234155, "ttft_p99": 0.07800000021234155, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004312900432881403, "inter_token_latency_p50": 0.004312900432881403, "inter_token_latency_p90": 0.004312900432881403, "inter_token_latency_p99": 0.004312900432881403, "output_tps_per_user_avg": 231.86252860743892, "output_tps_per_user_p50": 231.86252860743892, "output_tps_per_user_p90": 231.86252860743892, "output_tps_per_user_p99": 231.86252860743892, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.011440973817129125, "chunk_inter_token_latency_p50": 0.011440973817129125, "chunk_inter_token_latency_p90": 0.011440973817129125, "chunk_inter_token_latency_p99": 0.011440973817129125, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 5776.0, "output_seq_len_p50": 5776.0, "output_seq_len_p90": 5776.0, "output_seq_len_p99": 5776.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.07800000021234155, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004312900432881403, "chunk_inter_token_latency_avg": 0.011440973817129125, "input_tokens": 8192, "output_tokens": 5776, "output_tps_per_user": 231.86252860743892, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4709, "wall_time": 26.016000000061467, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 235.45, "server_utilization": 0.0076314564913912974, "server_spec_accept_rate": 0.851145038167939, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.016, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.39, "gpu_count": 1, "cpu_util_avg_pct": 12.62, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.56, "gpu_util_max_pct": 9.0, "mem_util_avg_pct": 0.67, "mem_util_max_pct": 1.0, "temp_avg_c": 36.78, "temp_max_c": 37.0, "power_total_avg_w": 68.69, "power_total_max_w": 93.99, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95294.56, "vram_used_max_mb": 95346.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.35, "vram_used_max_pct": 97.4, "pcie_rx_avg_mb_s": 33.22, "pcie_rx_max_mb_s": 210.0, "pcie_tx_avg_mb_s": 313.44, "pcie_tx_max_mb_s": 2765.0 } }, { "concurrency": 1, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4548, "server_output_tokens": 4548, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 227.4, "per_request_avg_tps": 227.4, "ttft_avg": 0.09399999980814755, "ttft_p50": 0.09399999980814755, "ttft_p90": 0.09399999980814755, "ttft_p99": 0.09399999980814755, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004459675881803902, "inter_token_latency_p50": 0.004459675881803902, "inter_token_latency_p90": 0.004459675881803902, "inter_token_latency_p99": 0.004459675881803902, "output_tps_per_user_avg": 224.23154204549687, "output_tps_per_user_p50": 224.23154204549687, "output_tps_per_user_p90": 224.23154204549687, "output_tps_per_user_p99": 224.23154204549687, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.011410243902469008, "chunk_inter_token_latency_p50": 0.011410243902469008, "chunk_inter_token_latency_p90": 0.011410243902469008, "chunk_inter_token_latency_p99": 0.011410243902469008, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 5246.0, "output_seq_len_p50": 5246.0, "output_seq_len_p90": 5246.0, "output_seq_len_p99": 5246.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.09399999980814755, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004459675881803902, "chunk_inter_token_latency_avg": 0.011410243902469008, "input_tokens": 16384, "output_tokens": 5246, "output_tps_per_user": 224.23154204549687, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4548, "wall_time": 25.031000000191852, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 227.22957781515726, "server_utilization": 0.01042345276872969, "server_spec_accept_rate": 0.8115384615384615, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.016, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.375, "gpu_count": 1, "cpu_util_avg_pct": 6.92, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.56, "gpu_util_max_pct": 12.0, "mem_util_avg_pct": 0.89, "mem_util_max_pct": 2.0, "temp_avg_c": 36.11, "temp_max_c": 37.0, "power_total_avg_w": 63.04, "power_total_max_w": 69.44, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95302.78, "vram_used_max_mb": 95415.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.36, "vram_used_max_pct": 97.47, "pcie_rx_avg_mb_s": 33.22, "pcie_rx_max_mb_s": 106.0, "pcie_tx_avg_mb_s": 6.89, "pcie_tx_max_mb_s": 19.0 } }, { "concurrency": 1, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4849, "server_output_tokens": 4849, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 242.45, "per_request_avg_tps": 242.45, "ttft_avg": 0.18700000015087426, "ttft_p50": 0.18700000015087426, "ttft_p90": 0.18700000015087426, "ttft_p99": 0.18700000015087426, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004218783930483012, "inter_token_latency_p50": 0.004218783930483012, "inter_token_latency_p90": 0.004218783930483012, "inter_token_latency_p99": 0.004218783930483012, "output_tps_per_user_avg": 237.03513061535463, "output_tps_per_user_p50": 237.03513061535463, "output_tps_per_user_p90": 237.03513061535463, "output_tps_per_user_p99": 237.03513061535463, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01125144787637506, "chunk_inter_token_latency_p50": 0.01125144787637506, "chunk_inter_token_latency_p90": 0.01125144787637506, "chunk_inter_token_latency_p99": 0.01125144787637506, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 5527.0, "output_seq_len_p50": 5527.0, "output_seq_len_p90": 5527.0, "output_seq_len_p99": 5527.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.18700000015087426, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004218783930483012, "chunk_inter_token_latency_avg": 0.01125144787637506, "input_tokens": 65536, "output_tokens": 5527, "output_tps_per_user": 237.03513061535463, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4849, "wall_time": 29.718000000109896, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 242.45, "server_utilization": 0.028385295486272666, "server_spec_accept_rate": 1.0, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 9.703, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.843, "gpu_count": 1, "cpu_util_avg_pct": 7.51, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.0, "gpu_util_max_pct": 12.0, "mem_util_avg_pct": 2.89, "mem_util_max_pct": 12.0, "temp_avg_c": 36.33, "temp_max_c": 37.0, "power_total_avg_w": 60.81, "power_total_max_w": 90.85, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95432.0, "vram_used_max_mb": 95509.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.49, "vram_used_max_pct": 97.57, "pcie_rx_avg_mb_s": 23.33, "pcie_rx_max_mb_s": 101.0, "pcie_tx_avg_mb_s": 7.67, "pcie_tx_max_mb_s": 33.0 } }, { "concurrency": 1, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4247, "server_output_tokens": 4247, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 212.35, "per_request_avg_tps": 212.35, "ttft_avg": 0.3279999999795109, "ttft_p50": 0.3279999999795109, "ttft_p90": 0.3279999999795109, "ttft_p99": 0.3279999999795109, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004733646770214657, "inter_token_latency_p50": 0.004733646770214657, "inter_token_latency_p90": 0.004733646770214657, "inter_token_latency_p99": 0.004733646770214657, "output_tps_per_user_avg": 211.2536166180082, "output_tps_per_user_p50": 211.2536166180082, "output_tps_per_user_p90": 211.2536166180082, "output_tps_per_user_p99": 211.2536166180082, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.011296097560922001, "chunk_inter_token_latency_p50": 0.011296097560922001, "chunk_inter_token_latency_p90": 0.011296097560922001, "chunk_inter_token_latency_p99": 0.011296097560922001, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 4893.0, "output_seq_len_p50": 4893.0, "output_seq_len_p90": 4893.0, "output_seq_len_p99": 4893.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.3279999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004733646770214657, "chunk_inter_token_latency_avg": 0.011296097560922001, "input_tokens": 131072, "output_tokens": 4893, "output_tps_per_user": 211.2536166180082, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4247, "wall_time": 37.90600000019185, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 212.35, "server_utilization": 0.051931130758492317, "server_spec_accept_rate": 0.7105263157894737, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 17.891, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.437, "gpu_count": 1, "cpu_util_avg_pct": 6.29, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.56, "gpu_util_max_pct": 3.0, "mem_util_avg_pct": 11.56, "mem_util_max_pct": 15.0, "temp_avg_c": 34.89, "temp_max_c": 35.0, "power_total_avg_w": 31.45, "power_total_max_w": 50.53, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95584.0, "vram_used_max_mb": 95586.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.65, "vram_used_max_pct": 97.65, "pcie_rx_avg_mb_s": 2.33, "pcie_rx_max_mb_s": 7.0, "pcie_tx_avg_mb_s": 0.56, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 1, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4122, "server_output_tokens": 4122, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 206.1, "per_request_avg_tps": 206.1, "ttft_avg": 0.6090000001713634, "ttft_p50": 0.6090000001713634, "ttft_p90": 0.6090000001713634, "ttft_p99": 0.6090000001713634, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.00487749800157676, "inter_token_latency_p50": 0.00487749800157676, "inter_token_latency_p90": 0.00487749800157676, "inter_token_latency_p99": 0.00487749800157676, "output_tps_per_user_avg": 205.02314909749379, "output_tps_per_user_p50": 205.02314909749379, "output_tps_per_user_p90": 205.02314909749379, "output_tps_per_user_p99": 205.02314909749379, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.011700383509055659, "chunk_inter_token_latency_p50": 0.011700383509055659, "chunk_inter_token_latency_p90": 0.011700383509055659, "chunk_inter_token_latency_p99": 0.011700383509055659, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 5005.0, "output_seq_len_p50": 5005.0, "output_seq_len_p90": 5005.0, "output_seq_len_p99": 5005.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.6090000001713634, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.00487749800157676, "chunk_inter_token_latency_avg": 0.011700383509055659, "input_tokens": 262144, "output_tokens": 5005, "output_tps_per_user": 205.02314909749379, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4122, "wall_time": 61.10900000017136, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 206.1, "server_utilization": 0.09958120055839925, "server_spec_accept_rate": 0.9140625, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 41.094, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.359, "gpu_count": 1, "cpu_util_avg_pct": 5.61, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.67, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 8.0, "mem_util_max_pct": 12.0, "temp_avg_c": 33.78, "temp_max_c": 34.0, "power_total_avg_w": 32.97, "power_total_max_w": 50.83, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95563.0, "vram_used_max_mb": 95563.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.63, "vram_used_max_pct": 97.63, "pcie_rx_avg_mb_s": 7.56, "pcie_rx_max_mb_s": 49.0, "pcie_tx_avg_mb_s": 1.22, "pcie_tx_max_mb_s": 4.0 } }, { "concurrency": 2, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 6939, "server_output_tokens": 6939, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 346.95, "per_request_avg_tps": 173.475, "ttft_avg": 0.11699999996926636, "ttft_p50": 0.11699999996926636, "ttft_p90": 0.14819999996107072, "ttft_p99": 0.1552199999592267, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005828866098053786, "inter_token_latency_p50": 0.005828866098053786, "inter_token_latency_p90": 0.005985110903927023, "inter_token_latency_p99": 0.006020265985248501, "output_tps_per_user_avg": 171.75278060572188, "output_tps_per_user_p50": 171.75278060572188, "output_tps_per_user_p90": 176.35667429833728, "output_tps_per_user_p99": 177.39255037917576, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013944019879217097, "chunk_inter_token_latency_p50": 0.013944019879217097, "chunk_inter_token_latency_p90": 0.013977131697513672, "chunk_inter_token_latency_p99": 0.013984581856630403, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 4274.5, "output_seq_len_p50": 4274.5, "output_seq_len_p90": 4383.7, "output_seq_len_p99": 4408.27, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.006024172105395332, "chunk_inter_token_latency_avg": 0.013985409652087817, "input_tokens": 8192, "output_tokens": 4138, "output_tps_per_user": 165.99791348995262, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1559999999590218, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.00563356009071224, "chunk_inter_token_latency_avg": 0.013902630106346379, "input_tokens": 8192, "output_tokens": 4411, "output_tps_per_user": 177.50764772149114, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 6939, "wall_time": 25.531999999890104, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 346.95, "server_utilization": 0.011726384364820874, "server_spec_accept_rate": 0.7384259259259259, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.516, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.546, "gpu_count": 1, "cpu_util_avg_pct": 7.67, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.78, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 2.44, "mem_util_max_pct": 9.0, "temp_avg_c": 33.67, "temp_max_c": 34.0, "power_total_avg_w": 52.0, "power_total_max_w": 80.85, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95606.89, "vram_used_max_mb": 95656.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.67, "vram_used_max_pct": 97.72, "pcie_rx_avg_mb_s": 5.0, "pcie_rx_max_mb_s": 27.0, "pcie_tx_avg_mb_s": 1.11, "pcie_tx_max_mb_s": 5.0 } }, { "concurrency": 8, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.016, "measurement_wall_seconds": 20.016, "client_output_tokens": 15576, "server_output_tokens": 15579, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 778.1774580402354, "per_request_avg_tps": 97.27218225502942, "ttft_avg": 0.2436250001192093, "ttft_p50": 0.26500000013038516, "ttft_p90": 0.26500000013038516, "ttft_p99": 0.26500000013038516, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01022154550881964, "inter_token_latency_p50": 0.010067584558997995, "inter_token_latency_p90": 0.010999251639785526, "inter_token_latency_p99": 0.01126960316033176, "output_tps_per_user_avg": 98.23233962622275, "output_tps_per_user_p50": 99.36278002125493, "output_tps_per_user_p90": 104.80091823074326, "output_tps_per_user_p99": 105.37441621222423, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.024830530081252716, "chunk_inter_token_latency_p50": 0.02483145562744313, "chunk_inter_token_latency_p90": 0.02492836740907339, "chunk_inter_token_latency_p99": 0.02500345620986136, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 2484.5, "output_seq_len_p50": 2511.5, "output_seq_len_p90": 2648.9, "output_seq_len_p99": 2663.39, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01129964221817023, "chunk_inter_token_latency_avg": 0.02486811023605181, "input_tokens": 8192, "output_tokens": 2237, "output_tps_per_user": 88.49837726649115, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009576850094827179, "chunk_inter_token_latency_avg": 0.024715964740322834, "input_tokens": 8192, "output_tokens": 2636, "output_tps_per_user": 104.418466416232, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009484234234169909, "chunk_inter_token_latency_avg": 0.024697947213908736, "input_tokens": 8192, "output_tokens": 2665, "output_tps_per_user": 105.43813821016656, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010870512820477796, "chunk_inter_token_latency_avg": 0.025011799409948913, "input_tokens": 8192, "output_tokens": 2341, "output_tps_per_user": 91.99198018663913, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010839124839051324, "chunk_inter_token_latency_avg": 0.024892610837269593, "input_tokens": 8192, "output_tokens": 2332, "output_tps_per_user": 92.25837093389575, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010254058441488894, "chunk_inter_token_latency_avg": 0.024819253437945617, "input_tokens": 8192, "output_tokens": 2465, "output_tps_per_user": 97.52236206826217, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009881110676507094, "chunk_inter_token_latency_avg": 0.024843657816940645, "input_tokens": 8192, "output_tokens": 2558, "output_tps_per_user": 101.2031979742477, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.26500000013038516, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009566830745864686, "chunk_inter_token_latency_avg": 0.024794896957633598, "input_tokens": 8192, "output_tokens": 2642, "output_tps_per_user": 104.52782395384756, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 15576, "wall_time": 26.07800000021234, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 778.3273381361599, "server_utilization": 0.027640763145649117, "server_spec_accept_rate": 0.7779017857142857, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.031, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.328, "gpu_count": 1, "cpu_util_avg_pct": 6.31, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.89, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 11.56, "mem_util_max_pct": 15.0, "temp_avg_c": 33.11, "temp_max_c": 34.0, "power_total_avg_w": 35.58, "power_total_max_w": 67.33, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95604.0, "vram_used_max_mb": 95604.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.67, "vram_used_max_pct": 97.67, "pcie_rx_avg_mb_s": 0.44, "pcie_rx_max_mb_s": 2.0, "pcie_tx_avg_mb_s": 0.67, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 16, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.984, "measurement_wall_seconds": 20.0, "client_output_tokens": 22200, "server_output_tokens": 22200, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1110.888710972192, "per_request_avg_tps": 69.430544435762, "ttft_avg": 0.35968750006577466, "ttft_p50": 0.3040000001201406, "ttft_p90": 0.546000000089407, "ttft_p99": 0.546000000089407, "time_to_second_token_avg": 0.06813333326329787, "time_to_second_token_p50": 0.07899999991059303, "time_to_second_token_p90": 0.09700000006705521, "time_to_second_token_p99": 0.10900000017136335, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.014212213727561079, "inter_token_latency_p50": 0.013967254777582724, "inter_token_latency_p90": 0.015221839551522908, "inter_token_latency_p99": 0.016670762822967897, "output_tps_per_user_avg": 70.67604757573697, "output_tps_per_user_p50": 71.60823042794637, "output_tps_per_user_p90": 75.48167033431736, "output_tps_per_user_p99": 76.8707482993813, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03396922814587391, "chunk_inter_token_latency_p50": 0.03393007915541972, "chunk_inter_token_latency_p90": 0.034200797872085305, "chunk_inter_token_latency_p99": 0.03423672470038942, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 1814.375, "output_seq_len_p50": 1847.0, "output_seq_len_p90": 1934.0, "output_seq_len_p99": 1967.2, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.10900000017136335, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014149563318661386, "chunk_inter_token_latency_avg": 0.03424306472891368, "input_tokens": 8192, "output_tokens": 1833, "output_tps_per_user": 70.67355913953425, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.31200000015087426, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.014663055872182524, "chunk_inter_token_latency_avg": 0.03397490092444934, "input_tokens": 8192, "output_tokens": 1755, "output_tps_per_user": 68.19860803348047, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.546000000089407, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014612162937354073, "chunk_inter_token_latency_avg": 0.033868351063574664, "input_tokens": 8192, "output_tokens": 1744, "output_tps_per_user": 68.43613805069415, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.013784946236504062, "chunk_inter_token_latency_avg": 0.0338705416114895, "input_tokens": 8192, "output_tokens": 1861, "output_tps_per_user": 72.54290171635849, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.013687599787018705, "chunk_inter_token_latency_avg": 0.03393007915541972, "input_tokens": 8192, "output_tokens": 1880, "output_tps_per_user": 73.05882810428153, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.016898160315248455, "chunk_inter_token_latency_avg": 0.0340198412695875, "input_tokens": 8192, "output_tokens": 1523, "output_tps_per_user": 59.178039582073694, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.013493704092239323, "chunk_inter_token_latency_avg": 0.03393007915541972, "input_tokens": 8192, "output_tokens": 1907, "output_tps_per_user": 74.10863563957454, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.546000000089407, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014798954096344071, "chunk_inter_token_latency_avg": 0.033868351063574664, "input_tokens": 8192, "output_tokens": 1722, "output_tps_per_user": 67.57234284867737, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.015382177033378078, "chunk_inter_token_latency_avg": 0.034200797872085305, "input_tokens": 8192, "output_tokens": 1673, "output_tps_per_user": 65.01030366703498, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.01340928050042135, "chunk_inter_token_latency_avg": 0.03393007915541972, "input_tokens": 8192, "output_tokens": 1919, "output_tps_per_user": 74.57521676637145, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.012977168949761295, "chunk_inter_token_latency_avg": 0.0338781456953371, "input_tokens": 8192, "output_tokens": 1972, "output_tps_per_user": 77.05840957078657, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.013644031830136948, "chunk_inter_token_latency_avg": 0.03393007915541972, "input_tokens": 8192, "output_tokens": 1886, "output_tps_per_user": 73.29211866767997, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.014336120401230851, "chunk_inter_token_latency_avg": 0.034200797872085305, "input_tokens": 8192, "output_tokens": 1795, "output_tps_per_user": 69.7538784561368, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.013305656460766764, "chunk_inter_token_latency_avg": 0.0338705416114895, "input_tokens": 8192, "output_tokens": 1928, "output_tps_per_user": 75.1560062405499, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.013191335740061635, "chunk_inter_token_latency_avg": 0.03383333333330623, "input_tokens": 8192, "output_tokens": 1940, "output_tps_per_user": 75.8073344280848, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.546000000089407, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.01506150206966774, "chunk_inter_token_latency_avg": 0.03395866666641086, "input_tokens": 8192, "output_tokens": 1692, "output_tps_per_user": 66.39444030047265, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 22200, "wall_time": 26.593999999808148, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1109.1681238998442, "server_utilization": 0.056491391344811515, "server_spec_accept_rate": 0.81328125, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.563, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.328, "gpu_count": 1, "cpu_util_avg_pct": 5.99, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.0, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 9.78, "mem_util_max_pct": 16.0, "temp_avg_c": 33.11, "temp_max_c": 34.0, "power_total_avg_w": 36.63, "power_total_max_w": 61.53, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95604.0, "vram_used_max_mb": 95604.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.67, "vram_used_max_pct": 97.67, "pcie_rx_avg_mb_s": 48.56, "pcie_rx_max_mb_s": 344.0, "pcie_tx_avg_mb_s": 0.67, "pcie_tx_max_mb_s": 1.0 } }, { "concurrency": 2, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 7071, "server_output_tokens": 7071, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 353.55, "per_request_avg_tps": 176.775, "ttft_avg": 0.13300000003073364, "ttft_p50": 0.13300000003073364, "ttft_p90": 0.164200000022538, "ttft_p99": 0.17122000002069399, "time_to_second_token_avg": 0.01500000013038516, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.01500000013038516, "time_to_second_token_p99": 0.01500000013038516, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005708226577586073, "inter_token_latency_p50": 0.005708226577586073, "inter_token_latency_p90": 0.005841573148092723, "inter_token_latency_p99": 0.00587157612645672, "output_tps_per_user_avg": 175.3352612631905, "output_tps_per_user_p50": 175.3352612631905, "output_tps_per_user_p90": 179.43116661322324, "output_tps_per_user_p99": 180.3527453169806, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013965065850284645, "chunk_inter_token_latency_p50": 0.013965065850284645, "chunk_inter_token_latency_p90": 0.013966392401286293, "chunk_inter_token_latency_p99": 0.013966690875261666, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 4276.0, "output_seq_len_p50": 4276.0, "output_seq_len_p90": 4370.4, "output_seq_len_p99": 4391.64, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0940000000409782, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.005874909790719386, "chunk_inter_token_latency_avg": 0.013963407661532585, "input_tokens": 16384, "output_tokens": 4158, "output_tps_per_user": 170.21537957564954, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005541543364452761, "chunk_inter_token_latency_avg": 0.013966724039036706, "input_tokens": 16384, "output_tokens": 4394, "output_tps_per_user": 180.4551429507314, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7071, "wall_time": 25.061999999918044, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 353.55, "server_utilization": 0.014425314099581255, "server_spec_accept_rate": 0.7971698113207547, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.047, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.328, "gpu_count": 1, "cpu_util_avg_pct": 7.71, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.11, "gpu_util_max_pct": 1.0, "mem_util_avg_pct": 6.89, "mem_util_max_pct": 16.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 40.84, "power_total_max_w": 54.42, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95606.67, "vram_used_max_mb": 95615.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.67, "vram_used_max_pct": 97.68, "pcie_rx_avg_mb_s": 5.11, "pcie_rx_max_mb_s": 37.0, "pcie_tx_avg_mb_s": 0.67, "pcie_tx_max_mb_s": 1.0 } }, { "concurrency": 8, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.985, "measurement_wall_seconds": 20.0, "client_output_tokens": 15392, "server_output_tokens": 15412, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 770.1776332299435, "per_request_avg_tps": 96.27220415374293, "ttft_avg": 0.2653750001336448, "ttft_p50": 0.23400000017136335, "ttft_p90": 0.376700000022538, "ttft_p99": 0.4453699999838136, "time_to_second_token_avg": 0.04028571430327637, "time_to_second_token_p50": 0.0470000000204891, "time_to_second_token_p90": 0.053399999951943763, "time_to_second_token_p99": 0.06203999985940754, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.010404014273249664, "inter_token_latency_p50": 0.010556058429773797, "inter_token_latency_p90": 0.01124234021194426, "inter_token_latency_p99": 0.011908025490341864, "output_tps_per_user_avg": 96.77717966878404, "output_tps_per_user_p50": 94.7907602250778, "output_tps_per_user_p90": 106.81858637205332, "output_tps_per_user_p99": 107.78116837634438, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.025714041948229557, "chunk_inter_token_latency_p50": 0.025693500541473856, "chunk_inter_token_latency_p90": 0.025823031930041967, "chunk_inter_token_latency_p99": 0.02598602220128771, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 2444.5, "output_seq_len_p50": 2397.5, "output_seq_len_p90": 2691.0, "output_seq_len_p99": 2703.6, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.10900000017136335, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009667808219136265, "chunk_inter_token_latency_avg": 0.025715587044423182, "input_tokens": 16384, "output_tokens": 2629, "output_tps_per_user": 103.4360609285381, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4529999999795109, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.009268860946775872, "chunk_inter_token_latency_avg": 0.025548419979696185, "input_tokens": 16384, "output_tokens": 2705, "output_tps_per_user": 107.88812193237672, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23400000017136335, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.010293973941323331, "chunk_inter_token_latency_avg": 0.025640973630720185, "input_tokens": 16384, "output_tokens": 2457, "output_tps_per_user": 97.14421327468855, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.28100000019185245, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.009402011922455147, "chunk_inter_token_latency_avg": 0.02567141403852453, "input_tokens": 16384, "output_tokens": 2685, "output_tps_per_user": 106.36021398905757, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.3440000000409782, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.010925347222231116, "chunk_inter_token_latency_avg": 0.026004132231426124, "input_tokens": 16384, "output_tokens": 2305, "output_tps_per_user": 91.5302717304197, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23400000017136335, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.010873978494576388, "chunk_inter_token_latency_avg": 0.02574541751516304, "input_tokens": 16384, "output_tokens": 2326, "output_tps_per_user": 91.96266118226828, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23400000017136335, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.010818142918224263, "chunk_inter_token_latency_avg": 0.025640973630720185, "input_tokens": 16384, "output_tokens": 2338, "output_tps_per_user": 92.43730717546707, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23400000017136335, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.01198199052127493, "chunk_inter_token_latency_avg": 0.02574541751516304, "input_tokens": 16384, "output_tokens": 2111, "output_tps_per_user": 83.45858713745636, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 15392, "wall_time": 26.07799999997951, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 769.9840127963603, "server_utilization": 0.03396928804094923, "server_spec_accept_rate": 0.7866379310344828, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.062, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.75, "gpu_count": 1, "cpu_util_avg_pct": 5.86, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.56, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 8.0, "mem_util_max_pct": 16.0, "temp_avg_c": 32.56, "temp_max_c": 33.0, "power_total_avg_w": 36.48, "power_total_max_w": 59.08, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95589.56, "vram_used_max_mb": 95591.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.65, "vram_used_max_pct": 97.65, "pcie_rx_avg_mb_s": 46.78, "pcie_rx_max_mb_s": 344.0, "pcie_tx_avg_mb_s": 4.89, "pcie_tx_max_mb_s": 26.0 } }, { "concurrency": 16, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.984, "measurement_wall_seconds": 20.015, "client_output_tokens": 20457, "server_output_tokens": 20456, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1023.6689351512671, "per_request_avg_tps": 63.97930844695419, "ttft_avg": 0.39931250002700835, "ttft_p50": 0.4690000000409782, "ttft_p90": 0.5385000001406297, "ttft_p99": 0.5995499999611639, "time_to_second_token_avg": 0.06025000006775372, "time_to_second_token_p50": 0.06200000015087426, "time_to_second_token_p90": 0.06299999984912574, "time_to_second_token_p99": 0.1692499998491257, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.015119325023466013, "inter_token_latency_p50": 0.015018686619780291, "inter_token_latency_p90": 0.016337055056262775, "inter_token_latency_p99": 0.017165588986595755, "output_tps_per_user_avg": 66.39069496410784, "output_tps_per_user_p50": 66.58376151926726, "output_tps_per_user_p90": 71.04292675914606, "output_tps_per_user_p99": 73.07618695334456, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.035204719510364364, "chunk_inter_token_latency_p50": 0.035205344585033785, "chunk_inter_token_latency_p90": 0.03554573888259438, "chunk_inter_token_latency_p99": 0.03571132297014839, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 1668.125, "output_seq_len_p50": 1667.5, "output_seq_len_p90": 1787.5, "output_seq_len_p99": 1844.75, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.6089999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.015030778515364527, "chunk_inter_token_latency_avg": 0.03488235294111908, "input_tokens": 16384, "output_tokens": 1658, "output_tps_per_user": 66.53015337680584, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.014831290134243445, "chunk_inter_token_latency_avg": 0.035286111111054196, "input_tokens": 16384, "output_tokens": 1714, "output_tps_per_user": 67.42501771246016, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.04600000008940697, "latency": 0.0, "inter_token_latency_avg": 0.015275090689176273, "chunk_inter_token_latency_avg": 0.03528631284901893, "input_tokens": 16384, "output_tokens": 1655, "output_tps_per_user": 65.46605976674081, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4839999999385327, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.015033633633609022, "chunk_inter_token_latency_avg": 0.035205344585033785, "input_tokens": 16384, "output_tokens": 1666, "output_tps_per_user": 66.51751827744499, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4839999999385327, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.01456984866121014, "chunk_inter_token_latency_avg": 0.03515589887634694, "input_tokens": 16384, "output_tokens": 1719, "output_tps_per_user": 68.63489273312344, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.04600000008940697, "latency": 0.0, "inter_token_latency_avg": 0.01364200863925354, "chunk_inter_token_latency_avg": 0.03533566433552106, "input_tokens": 16384, "output_tokens": 1853, "output_tps_per_user": 73.30298832406528, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4839999999385327, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.013929326655514203, "chunk_inter_token_latency_avg": 0.03495949720664668, "input_tokens": 16384, "output_tokens": 1798, "output_tps_per_user": 71.79097918592713, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.546000000089407, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.014366513233491454, "chunk_inter_token_latency_avg": 0.034921678321409994, "input_tokens": 16384, "output_tokens": 1739, "output_tps_per_user": 69.60631182719989, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29600000008940697, "time_to_second_token": 0.18799999984912574, "latency": 0.0, "inter_token_latency_avg": 0.015712772585550247, "chunk_inter_token_latency_avg": 0.03572096317253279, "input_tokens": 16384, "output_tokens": 1606, "output_tps_per_user": 63.64249177256077, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4690000000409782, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.017203436426088674, "chunk_inter_token_latency_avg": 0.03510659186530017, "input_tokens": 16384, "output_tokens": 1456, "output_tps_per_user": 58.12792137758707, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23500000010244548, "time_to_second_token": 0.04600000008940697, "latency": 0.0, "inter_token_latency_avg": 0.014225788288230606, "chunk_inter_token_latency_avg": 0.035237099023567024, "input_tokens": 16384, "output_tokens": 1777, "output_tps_per_user": 70.29487433236498, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5310000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.016951120162802546, "chunk_inter_token_latency_avg": 0.034921678321409994, "input_tokens": 16384, "output_tokens": 1474, "output_tps_per_user": 58.99315150832303, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4690000000409782, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.015006594724196055, "chunk_inter_token_latency_avg": 0.03495949720664668, "input_tokens": 16384, "output_tokens": 1669, "output_tps_per_user": 66.63736966172868, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23500000010244548, "time_to_second_token": 0.04600000008940697, "latency": 0.0, "inter_token_latency_avg": 0.0156925465837873, "chunk_inter_token_latency_avg": 0.03543478260855197, "input_tokens": 16384, "output_tokens": 1611, "output_tps_per_user": 63.72452008733538, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4690000000409782, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.01471546149321518, "chunk_inter_token_latency_avg": 0.035205344585033785, "input_tokens": 16384, "output_tokens": 1702, "output_tps_per_user": 67.95573488884922, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4690000000409782, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.015722989949723005, "chunk_inter_token_latency_avg": 0.03565669515663678, "input_tokens": 16384, "output_tokens": 1593, "output_tps_per_user": 63.60113459320867, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 20457, "wall_time": 26.07799999997951, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1021.2171134762043, "server_utilization": 0.05835272219637044, "server_spec_accept_rate": 0.6875, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.031, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.797, "gpu_count": 1, "cpu_util_avg_pct": 6.08, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.56, "gpu_util_max_pct": 2.0, "mem_util_avg_pct": 5.67, "mem_util_max_pct": 12.0, "temp_avg_c": 32.78, "temp_max_c": 33.0, "power_total_avg_w": 36.94, "power_total_max_w": 55.61, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95594.56, "vram_used_max_mb": 95600.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.66, "vram_used_max_pct": 97.66, "pcie_rx_avg_mb_s": 10.78, "pcie_rx_max_mb_s": 78.0, "pcie_tx_avg_mb_s": 6.11, "pcie_tx_max_mb_s": 37.0 } }, { "concurrency": 2, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 6794, "server_output_tokens": 6794, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 339.7, "per_request_avg_tps": 169.85, "ttft_avg": 0.2184999999590218, "ttft_p50": 0.2184999999590218, "ttft_p90": 0.24369999999180436, "ttft_p99": 0.24936999999918044, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005915543913334653, "inter_token_latency_p50": 0.005915543913334653, "inter_token_latency_p90": 0.0061097074390491155, "inter_token_latency_p99": 0.00615339423233487, "output_tps_per_user_avg": 169.33120101970206, "output_tps_per_user_p50": 169.33120101970206, "output_tps_per_user_p90": 174.88909112839303, "output_tps_per_user_p99": 176.13961640284847, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.014475342003063866, "chunk_inter_token_latency_p50": 0.014475342003063866, "chunk_inter_token_latency_p90": 0.014507971626419225, "chunk_inter_token_latency_p99": 0.014515313291674182, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 4197.5, "output_seq_len_p50": 4197.5, "output_seq_len_p90": 4339.5, "output_seq_len_p99": 4371.45, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.25, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.006158248320477731, "chunk_inter_token_latency_avg": 0.014516129032258065, "input_tokens": 65536, "output_tokens": 4020, "output_tps_per_user": 162.38383838383837, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1869999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005672839506191576, "chunk_inter_token_latency_avg": 0.014434554973869666, "input_tokens": 65536, "output_tokens": 4375, "output_tps_per_user": 176.27856365556576, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 6794, "wall_time": 25.530999999959022, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 339.7, "server_utilization": 0.032201023731968315, "server_spec_accept_rate": 0.7818627450980392, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.516, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.359, "gpu_count": 1, "cpu_util_avg_pct": 7.78, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.78, "gpu_util_max_pct": 3.0, "mem_util_avg_pct": 3.11, "mem_util_max_pct": 13.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 45.46, "power_total_max_w": 57.19, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95594.11, "vram_used_max_mb": 95598.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.66, "vram_used_max_pct": 97.66, "pcie_rx_avg_mb_s": 43.56, "pcie_rx_max_mb_s": 342.0, "pcie_tx_avg_mb_s": 0.89, "pcie_tx_max_mb_s": 3.0 } }, { "concurrency": 8, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 14491, "server_output_tokens": 14494, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 724.55, "per_request_avg_tps": 90.56875, "ttft_avg": 0.5022499999613501, "ttft_p50": 0.5080000000307336, "ttft_p90": 0.74239999987185, "ttft_p99": 0.7915400000056252, "time_to_second_token_avg": 0.015999999828636646, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999828636646, "time_to_second_token_p99": 0.015999999828636646, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.010938256498684358, "inter_token_latency_p50": 0.01079653707486964, "inter_token_latency_p90": 0.011651876794131152, "inter_token_latency_p99": 0.0124810032470297, "output_tps_per_user_avg": 91.80517142671297, "output_tps_per_user_p50": 92.67625668591714, "output_tps_per_user_p90": 97.12067027394664, "output_tps_per_user_p99": 99.1920005346574, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0268994228205205, "chunk_inter_token_latency_p50": 0.026912394988443188, "chunk_inter_token_latency_p90": 0.026996392071084955, "chunk_inter_token_latency_p99": 0.027017582543939555, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 2340.375, "output_seq_len_p50": 2368.0, "output_seq_len_p90": 2478.0, "output_seq_len_p99": 2509.5, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5620000001508743, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01048125257513355, "chunk_inter_token_latency_avg": 0.02686166842645103, "input_tokens": 65536, "output_tokens": 2428, "output_tps_per_user": 95.40844406063349, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.20399999991059303, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010536006546644845, "chunk_inter_token_latency_avg": 0.0270199370409234, "input_tokens": 65536, "output_tokens": 2445, "output_tps_per_user": 94.9126213592233, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.28199999989010394, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0111413090593756, "chunk_inter_token_latency_avg": 0.02697061909756507, "input_tokens": 65536, "output_tokens": 2308, "output_tps_per_user": 89.75605960400884, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.45399999991059303, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011257054673703273, "chunk_inter_token_latency_avg": 0.026874736842062128, "input_tokens": 65536, "output_tokens": 2269, "output_tps_per_user": 88.83318318920685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010402112103927545, "chunk_inter_token_latency_avg": 0.026986301369725622, "input_tokens": 65536, "output_tokens": 2463, "output_tps_per_user": 96.13432253075105, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012573128408462872, "chunk_inter_token_latency_avg": 0.026950053134824244, "input_tokens": 65536, "output_tokens": 2018, "output_tps_per_user": 79.53470031586633, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7189999998081475, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010058121019132749, "chunk_inter_token_latency_avg": 0.02679321314958798, "input_tokens": 65536, "output_tokens": 2513, "output_tps_per_user": 99.42214834140303, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7970000000204891, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.011057067603094436, "chunk_inter_token_latency_avg": 0.02673885350302455, "input_tokens": 65536, "output_tokens": 2279, "output_tps_per_user": 90.43989201261097, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 14491, "wall_time": 26.530999999959022, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 724.7, "server_utilization": 0.048394602140530485, "server_spec_accept_rate": 0.7974537037037037, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.5, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.828, "gpu_count": 1, "cpu_util_avg_pct": 5.72, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.67, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 7.89, "mem_util_max_pct": 13.0, "temp_avg_c": 32.33, "temp_max_c": 33.0, "power_total_avg_w": 26.16, "power_total_max_w": 28.91, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95600.0, "vram_used_max_mb": 95600.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.66, "vram_used_max_pct": 97.66, "pcie_rx_avg_mb_s": 24.11, "pcie_rx_max_mb_s": 212.0, "pcie_tx_avg_mb_s": 1.89, "pcie_tx_max_mb_s": 14.0 } }, { "concurrency": 16, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 20929, "server_output_tokens": 20954, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1046.45, "per_request_avg_tps": 65.403125, "ttft_avg": 0.8313124998530839, "ttft_p50": 0.8519999998388812, "ttft_p90": 1.3049999999348074, "ttft_p99": 1.395149999903515, "time_to_second_token_avg": 0.0493333333482345, "time_to_second_token_p50": 0.05400000012014061, "time_to_second_token_p90": 0.07849999994505197, "time_to_second_token_p99": 0.07894999991403892, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.014864409420472874, "inter_token_latency_p50": 0.014836615872091943, "inter_token_latency_p90": 0.015467143671921674, "inter_token_latency_p99": 0.01595969247782403, "output_tps_per_user_avg": 67.38122089752005, "output_tps_per_user_p50": 67.40517332063439, "output_tps_per_user_p90": 70.47326003666174, "output_tps_per_user_p99": 72.39155872842038, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03614120259352501, "chunk_inter_token_latency_p50": 0.036202166310988694, "chunk_inter_token_latency_p90": 0.036335111773563765, "chunk_inter_token_latency_p99": 0.03655423471561624, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 1632.25, "output_seq_len_p50": 1636.5, "output_seq_len_p90": 1707.5, "output_seq_len_p99": 1739.5, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.21899999980814755, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015479101684392985, "chunk_inter_token_latency_avg": 0.036276315789593505, "input_tokens": 65536, "output_tokens": 1604, "output_tps_per_user": 64.6032321764682, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.39099999982863665, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015239036442260956, "chunk_inter_token_latency_avg": 0.036229074889897926, "input_tokens": 65536, "output_tokens": 1620, "output_tps_per_user": 65.6209468222542, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5469999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016044502617841273, "chunk_inter_token_latency_avg": 0.036320000000091064, "input_tokens": 65536, "output_tokens": 1529, "output_tps_per_user": 62.326643824284915, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29699999978765845, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014550000000060261, "chunk_inter_token_latency_avg": 0.03659023668654208, "input_tokens": 65536, "output_tokens": 1701, "output_tps_per_user": 68.72852233648511, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1719999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01417706476536093, "chunk_inter_token_latency_avg": 0.035879699248274353, "input_tokens": 65536, "output_tokens": 1684, "output_tps_per_user": 70.5364626987751, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9529999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014594430992657151, "chunk_inter_token_latency_avg": 0.03598507462667107, "input_tokens": 65536, "output_tokens": 1653, "output_tps_per_user": 68.51928660343981, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7189999998081475, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.014717312348717891, "chunk_inter_token_latency_avg": 0.03623397913574062, "input_tokens": 65536, "output_tokens": 1653, "output_tps_per_user": 67.94718874653195, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8909999998286366, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.015455185659450364, "chunk_inter_token_latency_avg": 0.03630225563919018, "input_tokens": 65536, "output_tokens": 1563, "output_tps_per_user": 64.70320202129253, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46899999980814755, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014339171044998223, "chunk_inter_token_latency_avg": 0.03617525773207946, "input_tokens": 65536, "output_tokens": 1714, "output_tps_per_user": 69.73903839084332, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8129999998491257, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.015280403276622558, "chunk_inter_token_latency_avg": 0.03608630952380952, "input_tokens": 65536, "output_tokens": 1588, "output_tps_per_user": 65.44329896907216, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0939999998081475, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015325447570358681, "chunk_inter_token_latency_avg": 0.03604360902261801, "input_tokens": 65536, "output_tokens": 1565, "output_tps_per_user": 65.25094914253103, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.406999999890104, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.015252095422281767, "chunk_inter_token_latency_avg": 0.03589681335350383, "input_tokens": 65536, "output_tokens": 1552, "output_tps_per_user": 65.56476158279874, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.281999999890104, "time_to_second_token": 0.04600000008940697, "latency": 0.0, "inter_token_latency_avg": 0.014955919395465995, "chunk_inter_token_latency_avg": 0.03582202111613876, "input_tokens": 65536, "output_tokens": 1589, "output_tps_per_user": 66.86315789473684, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.327999999979511, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.014202516476878726, "chunk_inter_token_latency_avg": 0.0358066465255447, "input_tokens": 65536, "output_tokens": 1670, "output_tps_per_user": 70.41005737454839, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0939999998081475, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.013751577739553057, "chunk_inter_token_latency_avg": 0.03626172465966865, "input_tokens": 65536, "output_tokens": 1744, "output_tps_per_user": 72.71892861600485, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014466785290665165, "chunk_inter_token_latency_avg": 0.036350223547036466, "input_tokens": 65536, "output_tokens": 1687, "output_tps_per_user": 69.12385716025382, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 20929, "wall_time": 25.641000000061467, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1046.914813882763, "server_utilization": 0.04634713820381575, "server_spec_accept_rate": 0.7916666666666666, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.594, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.765, "gpu_count": 1, "cpu_util_avg_pct": 6.52, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.11, "gpu_util_max_pct": 10.0, "mem_util_avg_pct": 7.89, "mem_util_max_pct": 19.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 31.87, "power_total_max_w": 47.55, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95603.33, "vram_used_max_mb": 95609.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.66, "vram_used_max_pct": 97.67, "pcie_rx_avg_mb_s": 24.33, "pcie_rx_max_mb_s": 211.0, "pcie_tx_avg_mb_s": 2.0, "pcie_tx_max_mb_s": 15.0 } }, { "concurrency": 2, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 6714, "server_output_tokens": 6714, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 335.7, "per_request_avg_tps": 167.85, "ttft_avg": 0.41399999998975545, "ttft_p50": 0.41399999998975545, "ttft_p90": 0.4699999999487773, "ttft_p99": 0.48259999993955716, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006001478488070193, "inter_token_latency_p50": 0.006001478488070193, "inter_token_latency_p90": 0.006381860542205783, "inter_token_latency_p99": 0.006467446504386291, "output_tps_per_user_avg": 167.67810234704865, "output_tps_per_user_p50": 167.67810234704865, "output_tps_per_user_p90": 178.30577370021842, "output_tps_per_user_p99": 180.6969997546816, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.014615673110986015, "chunk_inter_token_latency_p50": 0.014615673110986015, "chunk_inter_token_latency_p90": 0.014635793616755889, "chunk_inter_token_latency_p99": 0.01464032073055411, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 4040.0, "output_seq_len_p50": 4040.0, "output_seq_len_p90": 4285.6, "output_seq_len_p99": 4340.86, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.3440000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.006476956055739681, "chunk_inter_token_latency_avg": 0.014640823743198358, "input_tokens": 131072, "output_tokens": 3733, "output_tps_per_user": 154.39351315558648, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4839999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005526000920400706, "chunk_inter_token_latency_avg": 0.014590522478773673, "input_tokens": 131072, "output_tokens": 4347, "output_tps_per_user": 180.96269153851085, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 6714, "wall_time": 25.046000000089407, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 335.7, "server_utilization": 0.05602605863192178, "server_spec_accept_rate": 0.8316831683168316, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.031, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.797, "gpu_count": 1, "cpu_util_avg_pct": 9.0, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.33, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 4.33, "mem_util_max_pct": 19.0, "temp_avg_c": 32.56, "temp_max_c": 33.0, "power_total_avg_w": 47.84, "power_total_max_w": 63.1, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95613.11, "vram_used_max_mb": 95690.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.68, "vram_used_max_pct": 97.76, "pcie_rx_avg_mb_s": 3.44, "pcie_rx_max_mb_s": 13.0, "pcie_tx_avg_mb_s": 1.67, "pcie_tx_max_mb_s": 4.0 } }, { "concurrency": 8, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.016, "measurement_wall_seconds": 20.016, "client_output_tokens": 12995, "server_output_tokens": 12997, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 649.2306155131522, "per_request_avg_tps": 81.15382693914403, "ttft_avg": 1.0041249999194406, "ttft_p50": 0.9924999999348074, "ttft_p90": 1.5045999999856576, "ttft_p99": 1.6129599999985658, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.012042741192547488, "inter_token_latency_p50": 0.012109782504217558, "inter_token_latency_p90": 0.012652823107608098, "inter_token_latency_p99": 0.012887902576960516, "output_tps_per_user_avg": 83.19462349472948, "output_tps_per_user_p50": 82.58704723824926, "output_tps_per_user_p90": 87.7228308310621, "output_tps_per_user_p99": 88.63396401114214, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.02821425445114633, "chunk_inter_token_latency_p50": 0.02818260820453366, "chunk_inter_token_latency_p90": 0.02838303977902114, "chunk_inter_token_latency_p99": 0.028471035355051355, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 2122.5, "output_seq_len_p50": 2110.0, "output_seq_len_p90": 2256.6, "output_seq_len_p99": 2270.46, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012540880503154567, "chunk_inter_token_latency_avg": 0.02820674646356963, "input_tokens": 131072, "output_tokens": 2068, "output_tps_per_user": 79.73921765289585, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.39099999982863665, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011982101881675707, "chunk_inter_token_latency_avg": 0.02807419354857136, "input_tokens": 131072, "output_tokens": 2180, "output_tps_per_user": 83.45781148208275, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9069999998901039, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011269484808502817, "chunk_inter_token_latency_avg": 0.028279558011171158, "input_tokens": 131072, "output_tokens": 2272, "output_tps_per_user": 88.73520103115104, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012237463126759409, "chunk_inter_token_latency_avg": 0.0280620067641811, "input_tokens": 131072, "output_tokens": 2035, "output_tps_per_user": 81.71628299441578, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.077999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012281159420299753, "chunk_inter_token_latency_avg": 0.02834113712376866, "input_tokens": 131072, "output_tokens": 2071, "output_tps_per_user": 81.42553693644605, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.452999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011660614525149203, "chunk_inter_token_latency_avg": 0.028111111111134107, "input_tokens": 131072, "output_tokens": 2149, "output_tps_per_user": 85.75877350573893, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7349999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011456202756838766, "chunk_inter_token_latency_avg": 0.02815846994549769, "input_tokens": 131072, "output_tokens": 2250, "output_tps_per_user": 87.28895788816685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2659999998286366, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012914022517999673, "chunk_inter_token_latency_avg": 0.028480812641276936, "input_tokens": 131072, "output_tokens": 1955, "output_tps_per_user": 77.43520646693868, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 12995, "wall_time": 27.07799999997951, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 649.3305355771018, "server_utilization": 0.07175430432759422, "server_spec_accept_rate": 0.6936274509803921, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 7.031, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.766, "gpu_count": 1, "cpu_util_avg_pct": 6.0, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.44, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 20.22, "mem_util_max_pct": 21.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 27.6, "power_total_max_w": 31.05, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95497.89, "vram_used_max_mb": 95553.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.56, "vram_used_max_pct": 97.62, "pcie_rx_avg_mb_s": 39.22, "pcie_rx_max_mb_s": 344.0, "pcie_tx_avg_mb_s": 0.33, "pcie_tx_max_mb_s": 1.0 } }, { "concurrency": 16, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.985, "measurement_wall_seconds": 20.0, "client_output_tokens": 18783, "server_output_tokens": 18821, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 939.8548911635585, "per_request_avg_tps": 58.740930697722405, "ttft_avg": 1.762812499931897, "ttft_p50": 1.7734999998938292, "ttft_p90": 2.827999999979511, "ttft_p99": 3.0084499999065883, "time_to_second_token_avg": 0.014999999897554517, "time_to_second_token_p50": 0.014999999897554517, "time_to_second_token_p90": 0.014999999897554517, "time_to_second_token_p99": 0.014999999897554517, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.016451548538805906, "inter_token_latency_p50": 0.0161772066816792, "inter_token_latency_p90": 0.017849567099597632, "inter_token_latency_p99": 0.018077234218916393, "output_tps_per_user_avg": 60.936302454179604, "output_tps_per_user_p50": 61.81560551723361, "output_tps_per_user_p90": 64.22283767949646, "output_tps_per_user_p99": 65.12872535166974, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.038643904873378786, "chunk_inter_token_latency_p50": 0.03869411708852359, "chunk_inter_token_latency_p90": 0.03915997051093866, "chunk_inter_token_latency_p99": 0.03927841502645575, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 1480.875, "output_seq_len_p50": 1496.0, "output_seq_len_p90": 1616.0, "output_seq_len_p99": 1631.65, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01581953951463627, "chunk_inter_token_latency_avg": 0.03817117117120194, "input_tokens": 131072, "output_tokens": 1608, "output_tps_per_user": 63.21296514824581, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01569178921564861, "chunk_inter_token_latency_avg": 0.037827178729599016, "input_tokens": 131072, "output_tokens": 1633, "output_tps_per_user": 63.72759576726608, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01614556135776922, "chunk_inter_token_latency_avg": 0.0391376582280102, "input_tokens": 131072, "output_tokens": 1533, "output_tps_per_user": 61.936527187938346, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.2339999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01774441132643961, "chunk_inter_token_latency_avg": 0.03929537953808904, "input_tokens": 131072, "output_tokens": 1343, "output_tps_per_user": 56.35577205708568, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9529999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015451632778792058, "chunk_inter_token_latency_avg": 0.03822865853655413, "input_tokens": 131072, "output_tokens": 1624, "output_tps_per_user": 64.71807959172685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018098853868238875, "chunk_inter_token_latency_avg": 0.038281818181911315, "input_tokens": 131072, "output_tokens": 1397, "output_tps_per_user": 55.252117469983524, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.780999999959022, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016847212165142265, "chunk_inter_token_latency_avg": 0.03877666666676911, "input_tokens": 131072, "output_tokens": 1382, "output_tps_per_user": 59.3570016331278, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.875, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015337142857116839, "chunk_inter_token_latency_avg": 0.03896129032251455, "input_tokens": 131072, "output_tokens": 1576, "output_tps_per_user": 65.20119225048319, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.031999999890104, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01795472287275566, "chunk_inter_token_latency_avg": 0.039182282793867124, "input_tokens": 131072, "output_tokens": 1282, "output_tps_per_user": 55.69565217391305, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.125, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01599678869614008, "chunk_inter_token_latency_avg": 0.038318461538292464, "input_tokens": 131072, "output_tokens": 1558, "output_tps_per_user": 62.5125466739017, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4849999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016375583722495323, "chunk_inter_token_latency_avg": 0.03871766561517427, "input_tokens": 131072, "output_tokens": 1500, "output_tps_per_user": 61.06652544093978, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6719999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0161218543046629, "chunk_inter_token_latency_avg": 0.03858003169578602, "input_tokens": 131072, "output_tokens": 1511, "output_tps_per_user": 62.0276043377201, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.406999999890104, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.015834339369509412, "chunk_inter_token_latency_avg": 0.03857679738552048, "input_tokens": 131072, "output_tokens": 1492, "output_tps_per_user": 63.15388199431919, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.031999999890104, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016655100624566273, "chunk_inter_token_latency_avg": 0.03877221324717286, "input_tokens": 131072, "output_tokens": 1442, "output_tps_per_user": 60.04166666666667, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.5939999998081475, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01620885200558918, "chunk_inter_token_latency_avg": 0.0388046357617251, "input_tokens": 131072, "output_tokens": 1447, "output_tps_per_user": 61.694683846528875, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.875, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01694139194139194, "chunk_inter_token_latency_avg": 0.03867056856187291, "input_tokens": 131072, "output_tokens": 1366, "output_tps_per_user": 59.02702702702703, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 18783, "wall_time": 26.625, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 941.05, "server_utilization": 0.0988366682177757, "server_spec_accept_rate": 0.7013888888888888, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.594, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.812, "gpu_count": 1, "cpu_util_avg_pct": 8.56, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.0, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 6.67, "mem_util_max_pct": 21.0, "temp_avg_c": 32.11, "temp_max_c": 33.0, "power_total_avg_w": 44.63, "power_total_max_w": 57.84, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95602.78, "vram_used_max_mb": 95676.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.67, "vram_used_max_pct": 97.74, "pcie_rx_avg_mb_s": 4.11, "pcie_rx_max_mb_s": 22.0, "pcie_tx_avg_mb_s": 2.0, "pcie_tx_max_mb_s": 8.0 } }, { "concurrency": 2, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 6534, "server_output_tokens": 6534, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 326.7, "per_request_avg_tps": 163.35, "ttft_avg": 0.8595000000204891, "ttft_p50": 0.8595000000204891, "ttft_p90": 1.009500000020489, "ttft_p99": 1.043250000020489, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006114527358410098, "inter_token_latency_p50": 0.006114527358410098, "inter_token_latency_p90": 0.006118738805015353, "inter_token_latency_p99": 0.006119686380501535, "output_tps_per_user_avg": 163.54505959674725, "output_tps_per_user_p50": 163.54505959674725, "output_tps_per_user_p90": 163.65770301882588, "output_tps_per_user_p99": 163.6830477887936, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.014635282856084869, "chunk_inter_token_latency_p50": 0.014635282856084869, "chunk_inter_token_latency_p90": 0.01470264226788775, "chunk_inter_token_latency_p99": 0.014717798135543398, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 3875.0, "output_seq_len_p50": 3875.0, "output_seq_len_p90": 3902.2, "output_seq_len_p99": 3908.32, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 1.047000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.006119791666666667, "chunk_inter_token_latency_avg": 0.01455108359133127, "input_tokens": 262144, "output_tokens": 3841, "output_tps_per_user": 163.40425531914894, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6720000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.006109263050153531, "chunk_inter_token_latency_avg": 0.01471948212083847, "input_tokens": 262144, "output_tokens": 3909, "output_tps_per_user": 163.68586387434556, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 6534, "wall_time": 25.609000000171363, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 326.7, "server_utilization": 0.1032107957189391, "server_spec_accept_rate": 0.5931372549019608, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.594, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.843, "gpu_count": 1, "cpu_util_avg_pct": 7.5, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.0, "gpu_util_max_pct": 0.0, "mem_util_avg_pct": 8.56, "mem_util_max_pct": 11.0, "temp_avg_c": 32.11, "temp_max_c": 33.0, "power_total_avg_w": 29.57, "power_total_max_w": 58.51, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95512.78, "vram_used_max_mb": 95615.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.57, "vram_used_max_pct": 97.68, "pcie_rx_avg_mb_s": 0.78, "pcie_rx_max_mb_s": 2.0, "pcie_tx_avg_mb_s": 0.78, "pcie_tx_max_mb_s": 4.0 } }, { "concurrency": 8, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 12408, "server_output_tokens": 12408, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 620.4, "per_request_avg_tps": 77.55, "ttft_avg": 2.0703750000393484, "ttft_p50": 2.093999999924563, "ttft_p90": 3.1333000000100584, "ttft_p99": 3.3796299999020993, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.0125826065051163, "inter_token_latency_p50": 0.012555396611037734, "inter_token_latency_p90": 0.013920094310045115, "inter_token_latency_p99": 0.014000083309531631, "output_tps_per_user_avg": 80.00900693649774, "output_tps_per_user_p50": 79.64705338757587, "output_tps_per_user_p90": 86.610630691806, "output_tps_per_user_p99": 93.62758532237888, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.029795425273043064, "chunk_inter_token_latency_p50": 0.02977779738409294, "chunk_inter_token_latency_p90": 0.030072316858443488, "chunk_inter_token_latency_p99": 0.030160041990186036, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 2077.25, "output_seq_len_p50": 2043.0, "output_seq_len_p90": 2311.2, "output_seq_len_p99": 2389.32, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 1.9219999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01254803073967339, "chunk_inter_token_latency_avg": 0.029755125284738042, "input_tokens": 262144, "output_tokens": 2083, "output_tps_per_user": 79.69377990430623, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.406999999890104, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012307692307641137, "chunk_inter_token_latency_avg": 0.02972255729782576, "input_tokens": 262144, "output_tokens": 2003, "output_tps_per_user": 81.2500000003378, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.6410000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010592407175593474, "chunk_inter_token_latency_avg": 0.029800469483447833, "input_tokens": 262144, "output_tokens": 2398, "output_tps_per_user": 94.40724694799809, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4840000001713634, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014008970976141245, "chunk_inter_token_latency_avg": 0.030030542986185134, "input_tokens": 262144, "output_tokens": 1896, "output_tps_per_user": 71.38283045222275, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.2660000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01388200431028963, "chunk_inter_token_latency_avg": 0.030169789227046318, "input_tokens": 262144, "output_tokens": 1857, "output_tps_per_user": 72.0357073552253, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.0160000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012749745157949823, "chunk_inter_token_latency_avg": 0.029850835322073454, "input_tokens": 262144, "output_tokens": 1963, "output_tps_per_user": 78.4329402361797, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1090000001713634, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01256276248240208, "chunk_inter_token_latency_avg": 0.029682469680030493, "input_tokens": 262144, "output_tokens": 2144, "output_tps_per_user": 79.6003268708455, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7180000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01200923889123962, "chunk_inter_token_latency_avg": 0.02935161290299748, "input_tokens": 262144, "output_tokens": 2274, "output_tps_per_user": 83.26922372486652, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 12408, "wall_time": 29.125, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 619.9040767367054, "server_utilization": 0.12275476966030707, "server_spec_accept_rate": 0.678030303030303, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 9.078, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.344, "gpu_count": 1, "cpu_util_avg_pct": 6.83, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.22, "gpu_util_max_pct": 14.0, "mem_util_avg_pct": 6.11, "mem_util_max_pct": 25.0, "temp_avg_c": 32.22, "temp_max_c": 33.0, "power_total_avg_w": 43.92, "power_total_max_w": 57.88, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95494.11, "vram_used_max_mb": 95572.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.56, "vram_used_max_pct": 97.64, "pcie_rx_avg_mb_s": 8.56, "pcie_rx_max_mb_s": 45.0, "pcie_tx_avg_mb_s": 4.11, "pcie_tx_max_mb_s": 24.0 } }, { "concurrency": 16, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.968, "measurement_wall_seconds": 20.0, "client_output_tokens": 17433, "server_output_tokens": 17467, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 873.0468749951951, "per_request_avg_tps": 54.565429687199696, "ttft_avg": 3.432500000024447, "ttft_p50": 3.4300000000512227, "ttft_p90": 5.5774999998975545, "ttft_p99": 6.0310500001651235, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.017548040562367414, "inter_token_latency_p50": 0.017962054791369166, "inter_token_latency_p90": 0.018425126433494587, "inter_token_latency_p99": 0.01844518106856948, "output_tps_per_user_avg": 57.11220406057569, "output_tps_per_user_p50": 55.67299114948371, "output_tps_per_user_p90": 60.76292030120385, "output_tps_per_user_p99": 62.93850323594217, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.04189232421682446, "chunk_inter_token_latency_p50": 0.04214864256736728, "chunk_inter_token_latency_p90": 0.04245392409687572, "chunk_inter_token_latency_p99": 0.04262353257774213, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 1497.5625, "output_seq_len_p50": 1439.5, "output_seq_len_p90": 1688.5, "output_seq_len_p99": 1790.7, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 3.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018441447835465835, "chunk_inter_token_latency_avg": 0.04231921824132144, "input_tokens": 262144, "output_tokens": 1410, "output_tps_per_user": 54.225677339543864, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016605066206165742, "chunk_inter_token_latency_avg": 0.040227336122886884, "input_tokens": 262144, "output_tokens": 1738, "output_tps_per_user": 60.222584335657935, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8440000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0180526657998247, "chunk_inter_token_latency_avg": 0.04162668665686715, "input_tokens": 262144, "output_tokens": 1539, "output_tps_per_user": 55.393481001000445, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.219000000040978, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017313527180866237, "chunk_inter_token_latency_avg": 0.04181679389332883, "input_tokens": 262144, "output_tokens": 1583, "output_tps_per_user": 57.75830595080208, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 5.7649999998975545, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.018161585366015504, "chunk_inter_token_latency_avg": 0.042323268206416235, "input_tokens": 262144, "output_tokens": 1313, "output_tps_per_user": 55.06127245208613, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.968000000109896, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018212508884150675, "chunk_inter_token_latency_avg": 0.042215815485996705, "input_tokens": 262144, "output_tokens": 1408, "output_tps_per_user": 54.90731707317073, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 5.3899999998975545, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017941438102455406, "chunk_inter_token_latency_avg": 0.04231293706330829, "input_tokens": 262144, "output_tokens": 1350, "output_tps_per_user": 55.736892120322466, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.311999999918044, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018096635647954083, "chunk_inter_token_latency_avg": 0.042489075630574545, "input_tokens": 262144, "output_tokens": 1398, "output_tps_per_user": 55.25889007513146, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4839999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017160561660666276, "chunk_inter_token_latency_avg": 0.04115519765764475, "input_tokens": 262144, "output_tokens": 1639, "output_tps_per_user": 58.273150947739666, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.547000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01684174454840614, "chunk_inter_token_latency_avg": 0.041778979907560826, "input_tokens": 262144, "output_tokens": 1606, "output_tps_per_user": 59.37627168764043, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.2350000001024455, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018408805031523336, "chunk_inter_token_latency_avg": 0.04208146964873785, "input_tokens": 262144, "output_tokens": 1432, "output_tps_per_user": 54.32183122628518, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 6.0780000002123415, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018445839874411302, "chunk_inter_token_latency_avg": 0.042418772563176894, "input_tokens": 262144, "output_tokens": 1275, "output_tps_per_user": 54.21276595744681, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.672000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017982671480282927, "chunk_inter_token_latency_avg": 0.042647260274301116, "input_tokens": 262144, "output_tokens": 1386, "output_tps_per_user": 55.609090178644955, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 5.0310000001918525, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016975795297386232, "chunk_inter_token_latency_avg": 0.04232241379313877, "input_tokens": 262144, "output_tokens": 1447, "output_tps_per_user": 58.907402126483596, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1089999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01581600889394794, "chunk_inter_token_latency_avg": 0.0406471428574462, "input_tokens": 262144, "output_tokens": 1800, "output_tps_per_user": 63.22707623050554, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.875, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01631234718835628, "chunk_inter_token_latency_avg": 0.04189481946648489, "input_tokens": 262144, "output_tokens": 1637, "output_tps_per_user": 61.30325626674976, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 17433, "wall_time": 30.79700000002049, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 872.6954783956734, "server_utilization": 0.09846440204746398, "server_spec_accept_rate": 0.6831896551724138, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 10.688, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.75, "gpu_count": 1, "cpu_util_avg_pct": 6.44, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.67, "gpu_util_max_pct": 9.0, "mem_util_avg_pct": 7.11, "mem_util_max_pct": 15.0, "temp_avg_c": 32.11, "temp_max_c": 33.0, "power_total_avg_w": 34.15, "power_total_max_w": 57.47, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95437.22, "vram_used_max_mb": 95438.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.5, "vram_used_max_pct": 97.5, "pcie_rx_avg_mb_s": 14.44, "pcie_rx_max_mb_s": 78.0, "pcie_tx_avg_mb_s": 0.44, "pcie_tx_max_mb_s": 2.0 } } ], "summary_table": { "8192": { "1": 235.45, "2": 346.95, "8": 778.1774580402354, "16": 1110.888710972192 }, "16384": { "1": 227.4, "2": 353.55, "8": 770.1776332299435, "16": 1023.6689351512671 }, "65536": { "1": 242.45, "2": 339.7, "8": 724.55, "16": 1046.45 }, "131072": { "1": 212.35, "2": 335.7, "8": 649.2306155131522, "16": 939.8548911635585 }, "262144": { "1": 206.1, "2": 326.7, "8": 620.4, "16": 873.0468749951951 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": true, "mode": "standalone_cold", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." } } }