{ "metadata": { "version": "0.4.29", "engine": "vllm", "model": "DeepSeek-V4-Flash", "server": "http://192.168.0.213", "timestamp": "2026-07-10T18:48:54.637673", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 20.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "standalone_cold", "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 5.0, "decode_warmup_context": 262144, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 0.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 8192, "temperature": null, "ignore_eos": true, "max_total_tokens": -1, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1, 2, 8, 16 ], "context_lengths": [ 8192, 16384, 65536, 131072, 262144 ], "startup_diagnostics_available": true, "nvidia_p2p_override_effective": false, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.29", "server_url": "http://192.168.0.213:8000", "hostname": "DESKTOP-RMAJ6JK", "uname": "", "env": {}, "args": { "concurrency": "1,2,8,16", "contexts": "8k,16k,64k,128k,256k", "max_tokens": 8192, "duration": 20.0, "request_count": 0, "run_burst": false, "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "prefill_contexts": "8k,16k,32k,64k,128k,256k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 0 }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 595.79, 00000000:01:00.0, 1, 16, 450.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 255, "stdout": "ERROR: Option -m is missing its value. Please run 'nvidia-smi -h' for help.", "stderr": "" } }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": { "samples": 356, "duration_seconds": 804.969, "gpu_count": 1, "cpu_util_avg_pct": 9.65, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.31, "gpu_util_max_pct": 99.0, "mem_util_avg_pct": 12.09, "mem_util_max_pct": 59.0, "temp_avg_c": 33.08, "temp_max_c": 49.0, "power_total_avg_w": 43.79, "power_total_max_w": 450.02, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95069.57, "vram_used_max_mb": 95316.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.12, "vram_used_max_pct": 97.37, "pcie_rx_avg_mb_s": 9.1, "pcie_rx_max_mb_s": 1039.0, "pcie_tx_avg_mb_s": 13.67, "pcie_tx_max_mb_s": 3573.0 }, "event_log": [ "18:35:28 startup startup preparation done", "18:35:28 hardware monitor interval=2s", "18:35:28 prefill warmup start", "18:35:29 prefill start ctx=8k", "18:35:45 prefill done ctx=8k 10,925 tok/s", "18:35:46 prefill start ctx=16k", "18:36:02 prefill done ctx=16k 10,703 tok/s", "18:36:03 prefill start ctx=32k", "18:36:21 prefill done ctx=32k 10,436 tok/s", "18:36:22 prefill start ctx=64k", "18:36:42 prefill done ctx=64k 9,963 tok/s", "18:36:43 prefill start ctx=128k", "18:37:12 prefill done ctx=128k 9,123 tok/s", "18:37:13 prefill start ctx=256k", "18:37:46 prefill done ctx=256k 7,851 tok/s", "18:37:47 decode warmup start C=1 ctx=256k 5s", "18:37:48 cell start C=1 ctx=256k", "18:38:27 ready C=1 ctx=256k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "18:38:32 cell done C=1 ctx=256k 230.2 tok/s", "18:38:32 decode warmup done C=1 ctx=256k", "18:38:34 cell start C=1 ctx=8k", "18:38:40 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "18:39:00 cell done C=1 ctx=8k 238.4 tok/s", "18:39:02 cell start C=1 ctx=16k", "18:39:08 ready C=1 ctx=16k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "18:39:28 cell done C=1 ctx=16k 224.0 tok/s", "18:39:30 cell start C=1 ctx=64k", "18:39:40 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "18:40:00 cell done C=1 ctx=64k 212.8 tok/s", "18:40:03 cell start C=1 ctx=128k", "18:40:21 ready C=1 ctx=128k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "18:40:41 cell done C=1 ctx=128k 215.6 tok/s", "18:40:43 cell start C=1 ctx=256k", "18:41:22 ready C=1 ctx=256k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "18:41:42 cell done C=1 ctx=256k 215.3 tok/s", "18:41:44 cell start C=2 ctx=8k", "18:41:50 ready C=2 ctx=8k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "18:42:10 cell done C=2 ctx=8k 399.0 tok/s", "18:42:12 cell start C=8 ctx=8k", "18:42:19 ready C=8 ctx=8k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "18:42:39 cell done C=8 ctx=8k 758.7 tok/s", "18:42:41 cell start C=16 ctx=8k", "18:42:46 ready C=16 ctx=8k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "18:43:06 cell done C=16 ctx=8k 999.8 tok/s", "18:43:09 cell start C=2 ctx=16k", "18:43:14 ready C=2 ctx=16k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "18:43:34 cell done C=2 ctx=16k 372.6 tok/s", "18:43:36 cell start C=8 ctx=16k", "18:43:42 ready C=8 ctx=16k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "18:44:02 cell done C=8 ctx=16k 719.8 tok/s", "18:44:04 cell start C=16 ctx=16k", "18:44:10 ready C=16 ctx=16k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "18:44:30 cell done C=16 ctx=16k 987.1 tok/s", "18:44:32 cell start C=2 ctx=64k", "18:44:37 ready C=2 ctx=64k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "18:44:57 cell done C=2 ctx=64k 391.6 tok/s", "18:44:59 cell start C=8 ctx=64k", "18:45:05 ready C=8 ctx=64k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "18:45:25 cell done C=8 ctx=64k 730.1 tok/s", "18:45:27 cell start C=16 ctx=64k", "18:45:34 ready C=16 ctx=64k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "18:45:54 cell done C=16 ctx=64k 976.4 tok/s", "18:45:56 cell start C=2 ctx=128k", "18:46:01 ready C=2 ctx=128k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "18:46:21 cell done C=2 ctx=128k 332.4 tok/s", "18:46:23 cell start C=8 ctx=128k", "18:46:29 ready C=8 ctx=128k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "18:46:49 cell done C=8 ctx=128k 668.7 tok/s", "18:46:51 cell start C=16 ctx=128k", "18:47:00 ready C=16 ctx=128k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "18:47:20 cell done C=16 ctx=128k 980.6 tok/s", "18:47:22 cell start C=2 ctx=256k", "18:47:28 ready C=2 ctx=256k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "18:47:48 cell done C=2 ctx=256k 356.4 tok/s", "18:47:50 cell start C=8 ctx=256k", "18:47:58 ready C=8 ctx=256k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "18:48:18 cell done C=8 ctx=256k 651.2 tok/s", "18:48:20 cell start C=16 ctx=256k", "18:48:32 ready C=16 ctx=256k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "18:48:52 cell done C=16 ctx=256k 845.8 tok/s" ], "prefill": { "8192": { "ttft_seconds": 0.75, "prefill_seconds": 0.75, "tok_per_sec": 10925.0, "client_ttft_seconds": 0.75, "client_tok_per_sec": 10925.0, "prompt_tokens": 8194, "samples": 15, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 6, "duration_seconds": 12.109, "gpu_count": 1, "cpu_util_avg_pct": 11.1, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.0, "gpu_util_max_pct": 0.0, "mem_util_avg_pct": 15.17, "mem_util_max_pct": 16.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 26.18, "power_total_max_w": 26.35, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94732.0, "vram_used_max_mb": 94732.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.78, "vram_used_max_pct": 96.78, "pcie_rx_avg_mb_s": 0.5, "pcie_rx_max_mb_s": 1.0, "pcie_tx_avg_mb_s": 0.83, "pcie_tx_max_mb_s": 4.0 } }, "16384": { "ttft_seconds": 1.531, "prefill_seconds": 1.531, "tok_per_sec": 10703.0, "client_ttft_seconds": 1.531, "client_tok_per_sec": 10703.0, "prompt_tokens": 16386, "samples": 8, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 6, "duration_seconds": 11.625, "gpu_count": 1, "cpu_util_avg_pct": 11.25, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.67, "gpu_util_max_pct": 2.0, "mem_util_avg_pct": 15.17, "mem_util_max_pct": 17.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 27.2, "power_total_max_w": 31.78, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94729.67, "vram_used_max_mb": 94732.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.78, "vram_used_max_pct": 96.78, "pcie_rx_avg_mb_s": 3.33, "pcie_rx_max_mb_s": 16.0, "pcie_tx_avg_mb_s": 1.17, "pcie_tx_max_mb_s": 4.0 } }, "32768": { "ttft_seconds": 3.14, "prefill_seconds": 3.14, "tok_per_sec": 10436.0, "client_ttft_seconds": 3.14, "client_tok_per_sec": 10436.0, "prompt_tokens": 32770, "samples": 5, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 7, "duration_seconds": 15.25, "gpu_count": 1, "cpu_util_avg_pct": 18.36, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.71, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 32.71, "temp_max_c": 33.0, "power_total_avg_w": 63.61, "power_total_max_w": 85.84, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94678.0, "vram_used_max_mb": 94729.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.72, "vram_used_max_pct": 96.77, "pcie_rx_avg_mb_s": 14.71, "pcie_rx_max_mb_s": 43.0, "pcie_tx_avg_mb_s": 6.14, "pcie_tx_max_mb_s": 25.0 } }, "65536": { "ttft_seconds": 6.578, "prefill_seconds": 6.578, "tok_per_sec": 9963.0, "client_ttft_seconds": 6.578, "client_tok_per_sec": 9963.0, "prompt_tokens": 65538, "samples": 3, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 8, "duration_seconds": 17.266, "gpu_count": 1, "cpu_util_avg_pct": 21.02, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.88, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 2.5, "mem_util_max_pct": 10.0, "temp_avg_c": 33.38, "temp_max_c": 34.0, "power_total_avg_w": 59.21, "power_total_max_w": 88.02, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94759.38, "vram_used_max_mb": 94796.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.81, "vram_used_max_pct": 96.84, "pcie_rx_avg_mb_s": 20.88, "pcie_rx_max_mb_s": 101.0, "pcie_tx_avg_mb_s": 4.75, "pcie_tx_max_mb_s": 12.0 } }, "131072": { "ttft_seconds": 14.367, "prefill_seconds": 14.367, "tok_per_sec": 9123.0, "client_ttft_seconds": 14.367, "client_tok_per_sec": 9123.0, "prompt_tokens": 131074, "samples": 2, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 12, "duration_seconds": 26.203, "gpu_count": 1, "cpu_util_avg_pct": 17.84, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.67, "gpu_util_max_pct": 13.0, "mem_util_avg_pct": 5.0, "mem_util_max_pct": 25.0, "temp_avg_c": 33.67, "temp_max_c": 34.0, "power_total_avg_w": 58.52, "power_total_max_w": 90.01, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95064.67, "vram_used_max_mb": 95162.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.12, "vram_used_max_pct": 97.22, "pcie_rx_avg_mb_s": 94.0, "pcie_rx_max_mb_s": 1039.0, "pcie_tx_avg_mb_s": 310.33, "pcie_tx_max_mb_s": 3573.0 } }, "262144": { "ttft_seconds": 33.39, "prefill_seconds": 33.39, "tok_per_sec": 7851.0, "client_ttft_seconds": 33.39, "client_tok_per_sec": 7851.0, "prompt_tokens": 262146, "samples": 1, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 14, "duration_seconds": 30.875, "gpu_count": 1, "cpu_util_avg_pct": 14.39, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.36, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 5.71, "mem_util_max_pct": 17.0, "temp_avg_c": 33.64, "temp_max_c": 34.0, "power_total_avg_w": 49.5, "power_total_max_w": 63.02, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95166.79, "vram_used_max_mb": 95189.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.22, "vram_used_max_pct": 97.24, "pcie_rx_avg_mb_s": 8.0, "pcie_rx_max_mb_s": 55.0, "pcie_tx_avg_mb_s": 2.36, "pcie_tx_max_mb_s": 8.0 } } }, "results": [ { "concurrency": 1, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4769, "server_output_tokens": 4769, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 238.45, "per_request_avg_tps": 238.45, "ttft_avg": 0.10899999993853271, "ttft_p50": 0.10899999993853271, "ttft_p90": 0.10899999993853271, "ttft_p99": 0.10899999993853271, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004252690927739871, "inter_token_latency_p50": 0.004252690927739871, "inter_token_latency_p90": 0.004252690927739871, "inter_token_latency_p99": 0.004252690927739871, "output_tps_per_user_avg": 235.1452332162447, "output_tps_per_user_p50": 235.1452332162447, "output_tps_per_user_p90": 235.1452332162447, "output_tps_per_user_p99": 235.1452332162447, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.010779991338268284, "chunk_inter_token_latency_p50": 0.010779991338268284, "chunk_inter_token_latency_p90": 0.010779991338268284, "chunk_inter_token_latency_p99": 0.010779991338268284, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 5854.0, "output_seq_len_p50": 5854.0, "output_seq_len_p90": 5854.0, "output_seq_len_p99": 5854.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.10899999993853271, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004252690927739871, "chunk_inter_token_latency_avg": 0.010779991338268284, "input_tokens": 8192, "output_tokens": 5854, "output_tps_per_user": 235.1452332162447, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4769, "wall_time": 26.016000000061467, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 238.45, "server_utilization": 0.007617985878855427, "server_spec_accept_rate": 0.8115942028985508, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.016, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 8, "duration_seconds": 16.875, "gpu_count": 1, "cpu_util_avg_pct": 11.47, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.5, "gpu_util_max_pct": 2.0, "mem_util_avg_pct": 9.62, "mem_util_max_pct": 15.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 37.9, "power_total_max_w": 60.66, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95161.62, "vram_used_max_mb": 95216.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.22, "vram_used_max_pct": 97.27, "pcie_rx_avg_mb_s": 2.88, "pcie_rx_max_mb_s": 10.0, "pcie_tx_avg_mb_s": 4.0, "pcie_tx_max_mb_s": 13.0 } }, { "concurrency": 1, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4480, "server_output_tokens": 4480, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 224.0, "per_request_avg_tps": 224.0, "ttft_avg": 0.125, "ttft_p50": 0.125, "ttft_p90": 0.125, "ttft_p99": 0.125, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004466845021975922, "inter_token_latency_p50": 0.004466845021975922, "inter_token_latency_p90": 0.004466845021975922, "inter_token_latency_p99": 0.004466845021975922, "output_tps_per_user_avg": 223.87165775401073, "output_tps_per_user_p50": 223.87165775401073, "output_tps_per_user_p90": 223.87165775401073, "output_tps_per_user_p99": 223.87165775401073, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.010610531093962778, "chunk_inter_token_latency_p50": 0.010610531093962778, "chunk_inter_token_latency_p90": 0.010610531093962778, "chunk_inter_token_latency_p99": 0.010610531093962778, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 5234.0, "output_seq_len_p50": 5234.0, "output_seq_len_p90": 5234.0, "output_seq_len_p99": 5234.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.125, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004466845021975922, "chunk_inter_token_latency_avg": 0.010610531093962778, "input_tokens": 16384, "output_tokens": 5234, "output_tps_per_user": 223.87165775401073, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4480, "wall_time": 25.562999999849126, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 224.0, "server_utilization": 0.01049795615013005, "server_spec_accept_rate": 0.6546762589928058, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.563, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 8, "duration_seconds": 17.688, "gpu_count": 1, "cpu_util_avg_pct": 12.21, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.0, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 6.88, "mem_util_max_pct": 19.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 43.74, "power_total_max_w": 55.01, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95005.25, "vram_used_max_mb": 95038.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.06, "vram_used_max_pct": 97.09, "pcie_rx_avg_mb_s": 1.88, "pcie_rx_max_mb_s": 6.0, "pcie_tx_avg_mb_s": 1.0, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 1, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4255, "server_output_tokens": 4255, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 212.75, "per_request_avg_tps": 212.75, "ttft_avg": 0.2339999999385327, "ttft_p50": 0.2339999999385327, "ttft_p90": 0.2339999999385327, "ttft_p99": 0.2339999999385327, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004721185064947538, "inter_token_latency_p50": 0.004721185064947538, "inter_token_latency_p90": 0.004721185064947538, "inter_token_latency_p99": 0.004721185064947538, "output_tps_per_user_avg": 211.81122668215338, "output_tps_per_user_p50": 211.81122668215338, "output_tps_per_user_p90": 211.81122668215338, "output_tps_per_user_p99": 211.81122668215338, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.010892322097407054, "chunk_inter_token_latency_p50": 0.010892322097407054, "chunk_inter_token_latency_p90": 0.010892322097407054, "chunk_inter_token_latency_p99": 0.010892322097407054, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 4929.0, "output_seq_len_p50": 4929.0, "output_seq_len_p90": 4929.0, "output_seq_len_p99": 4929.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.2339999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004721185064947538, "chunk_inter_token_latency_avg": 0.010892322097407054, "input_tokens": 65536, "output_tokens": 4929, "output_tps_per_user": 211.81122668215338, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4255, "wall_time": 30.75, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 212.75, "server_utilization": 0.02805648457822374, "server_spec_accept_rate": 0.6510791366906474, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 10.735, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.328, "gpu_count": 1, "cpu_util_avg_pct": 8.86, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.44, "gpu_util_max_pct": 9.0, "mem_util_avg_pct": 16.56, "mem_util_max_pct": 24.0, "temp_avg_c": 32.56, "temp_max_c": 33.0, "power_total_avg_w": 28.15, "power_total_max_w": 34.28, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94941.89, "vram_used_max_mb": 94967.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.99, "vram_used_max_pct": 97.02, "pcie_rx_avg_mb_s": 1.56, "pcie_rx_max_mb_s": 9.0, "pcie_tx_avg_mb_s": 1.22, "pcie_tx_max_mb_s": 4.0 } }, { "concurrency": 1, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4312, "server_output_tokens": 4312, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 215.6, "per_request_avg_tps": 215.6, "ttft_avg": 0.3440000000409782, "ttft_p50": 0.3440000000409782, "ttft_p90": 0.3440000000409782, "ttft_p99": 0.3440000000409782, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004678730287113115, "inter_token_latency_p50": 0.004678730287113115, "inter_token_latency_p90": 0.004678730287113115, "inter_token_latency_p99": 0.004678730287113115, "output_tps_per_user_avg": 213.73320081184315, "output_tps_per_user_p50": 213.73320081184315, "output_tps_per_user_p90": 213.73320081184315, "output_tps_per_user_p99": 213.73320081184315, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01095172740182748, "chunk_inter_token_latency_p50": 0.01095172740182748, "chunk_inter_token_latency_p90": 0.01095172740182748, "chunk_inter_token_latency_p99": 0.01095172740182748, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 4947.0, "output_seq_len_p50": 4947.0, "output_seq_len_p90": 4947.0, "output_seq_len_p99": 4947.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.3440000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004678730287113115, "chunk_inter_token_latency_avg": 0.01095172740182748, "input_tokens": 131072, "output_tokens": 4947, "output_tps_per_user": 213.73320081184315, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4312, "wall_time": 38.39100000006147, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 215.6, "server_utilization": 0.052025269416573816, "server_spec_accept_rate": 0.6485507246376812, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 18.391, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 10, "duration_seconds": 19.594, "gpu_count": 1, "cpu_util_avg_pct": 9.33, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.6, "gpu_util_max_pct": 3.0, "mem_util_avg_pct": 16.5, "mem_util_max_pct": 20.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 26.94, "power_total_max_w": 30.64, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94976.5, "vram_used_max_mb": 94986.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.03, "vram_used_max_pct": 97.04, "pcie_rx_avg_mb_s": 0.6, "pcie_rx_max_mb_s": 1.0, "pcie_tx_avg_mb_s": 1.2, "pcie_tx_max_mb_s": 5.0 } }, { "concurrency": 1, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 4307, "server_output_tokens": 4307, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 215.35, "per_request_avg_tps": 215.35, "ttft_avg": 0.625, "ttft_p50": 0.625, "ttft_p90": 0.625, "ttft_p99": 0.625, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004671140939572314, "inter_token_latency_p50": 0.004671140939572314, "inter_token_latency_p90": 0.004671140939572314, "inter_token_latency_p99": 0.004671140939572314, "output_tps_per_user_avg": 214.08045977126076, "output_tps_per_user_p50": 214.08045977126076, "output_tps_per_user_p90": 214.08045977126076, "output_tps_per_user_p99": 214.08045977126076, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01123098201930365, "chunk_inter_token_latency_p50": 0.01123098201930365, "chunk_inter_token_latency_p90": 0.01123098201930365, "chunk_inter_token_latency_p99": 0.01123098201930365, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 5216.0, "output_seq_len_p50": 5216.0, "output_seq_len_p90": 5216.0, "output_seq_len_p99": 5216.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004671140939572314, "chunk_inter_token_latency_avg": 0.01123098201930365, "input_tokens": 262144, "output_tokens": 5216, "output_tps_per_user": 214.08045977126076, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4307, "wall_time": 58.98400000017136, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 215.18860854219048, "server_utilization": 0.09959123002601267, "server_spec_accept_rate": 0.6578947368421053, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 38.969, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.765, "gpu_count": 1, "cpu_util_avg_pct": 11.81, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.22, "gpu_util_max_pct": 9.0, "mem_util_avg_pct": 5.56, "mem_util_max_pct": 18.0, "temp_avg_c": 32.67, "temp_max_c": 34.0, "power_total_avg_w": 51.83, "power_total_max_w": 92.11, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95175.56, "vram_used_max_mb": 95284.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.23, "vram_used_max_pct": 97.34, "pcie_rx_avg_mb_s": 17.67, "pcie_rx_max_mb_s": 81.0, "pcie_tx_avg_mb_s": 8.0, "pcie_tx_max_mb_s": 29.0 } }, { "concurrency": 2, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 7980, "server_output_tokens": 7980, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 399.0, "per_request_avg_tps": 199.5, "ttft_avg": 0.17949999996926636, "ttft_p50": 0.17949999996926636, "ttft_p90": 0.23589999999385328, "ttft_p99": 0.24858999999938533, "time_to_second_token_avg": 0.01500000013038516, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.01500000013038516, "time_to_second_token_p99": 0.01500000013038516, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005138645940595918, "inter_token_latency_p50": 0.005138645940595918, "inter_token_latency_p90": 0.005382248651501634, "inter_token_latency_p99": 0.00543705926145542, "output_tps_per_user_avg": 195.2895448394883, "output_tps_per_user_p50": 195.2895448394883, "output_tps_per_user_p90": 204.54744333734155, "output_tps_per_user_p99": 206.63047049935852, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0133050231459696, "chunk_inter_token_latency_p50": 0.0133050231459696, "chunk_inter_token_latency_p90": 0.013338109186861038, "chunk_inter_token_latency_p99": 0.013345553546061612, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 4849.0, "output_seq_len_p50": 4849.0, "output_seq_len_p90": 5089.8, "output_seq_len_p99": 5143.98, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.25, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.005443149329228063, "chunk_inter_token_latency_avg": 0.013263665594855305, "input_tokens": 8192, "output_tokens": 4548, "output_tps_per_user": 183.7171717171717, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004834142551963773, "chunk_inter_token_latency_avg": 0.013346380697083897, "input_tokens": 8192, "output_tokens": 5150, "output_tps_per_user": 206.86191796180486, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7980, "wall_time": 25.546000000089407, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 399.0, "server_utilization": 0.011798587885544443, "server_spec_accept_rate": 0.7175925925925926, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.531, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.828, "gpu_count": 1, "cpu_util_avg_pct": 9.87, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.22, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 15.33, "mem_util_max_pct": 24.0, "temp_avg_c": 32.56, "temp_max_c": 33.0, "power_total_avg_w": 36.75, "power_total_max_w": 61.8, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95163.11, "vram_used_max_mb": 95233.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.22, "vram_used_max_pct": 97.29, "pcie_rx_avg_mb_s": 13.33, "pcie_rx_max_mb_s": 98.0, "pcie_tx_avg_mb_s": 2.89, "pcie_tx_max_mb_s": 17.0 } }, { "concurrency": 8, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.984, "measurement_wall_seconds": 20.0, "client_output_tokens": 15161, "server_output_tokens": 15184, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 758.6569255427659, "per_request_avg_tps": 94.83211569284573, "ttft_avg": 0.5363750000833534, "ttft_p50": 0.5930000001098961, "ttft_p90": 0.5930000001098961, "ttft_p99": 0.5930000001098961, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.010413095803733648, "inter_token_latency_p50": 0.010321969417293056, "inter_token_latency_p90": 0.011190740886372093, "inter_token_latency_p99": 0.01155849848326753, "output_tps_per_user_avg": 96.40212051074055, "output_tps_per_user_p50": 96.94320694353848, "output_tps_per_user_p90": 103.2531211853686, "output_tps_per_user_p99": 103.47642865652128, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0254004248387435, "chunk_inter_token_latency_p50": 0.02542920110313831, "chunk_inter_token_latency_p90": 0.02556502952291667, "chunk_inter_token_latency_p99": 0.02560863377492485, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 2454.375, "output_seq_len_p50": 2482.5, "output_seq_len_p90": 2622.7, "output_seq_len_p99": 2628.37, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.00969492172578413, "chunk_inter_token_latency_avg": 0.02546740220644798, "input_tokens": 8192, "output_tokens": 2620, "output_tps_per_user": 103.1467842943435, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011599360438478135, "chunk_inter_token_latency_avg": 0.02554426559338897, "input_tokens": 8192, "output_tokens": 2190, "output_tps_per_user": 86.21164979775406, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.13999999989755452, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010059945504103145, "chunk_inter_token_latency_avg": 0.025613478691814646, "input_tokens": 8192, "output_tokens": 2570, "output_tps_per_user": 99.4041170095932, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01101561822118379, "chunk_inter_token_latency_avg": 0.025518592964651896, "input_tokens": 8192, "output_tokens": 2306, "output_tps_per_user": 90.78019770846191, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010786321155407237, "chunk_inter_token_latency_avg": 0.025390999999828637, "input_tokens": 8192, "output_tokens": 2355, "output_tps_per_user": 92.71001536039884, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010583993330482967, "chunk_inter_token_latency_avg": 0.025189484126814123, "input_tokens": 8192, "output_tokens": 2400, "output_tps_per_user": 94.48229687748378, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009902886115377783, "chunk_inter_token_latency_avg": 0.025264676616744912, "input_tokens": 8192, "output_tokens": 2565, "output_tps_per_user": 100.98066244012857, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009661719939051993, "chunk_inter_token_latency_avg": 0.02521449851025684, "input_tokens": 8192, "output_tokens": 2629, "output_tps_per_user": 103.50124059776047, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 15161, "wall_time": 26.530999999959022, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 758.6310267338355, "server_utilization": 0.0317725752508361, "server_spec_accept_rate": 0.7855603448275862, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.516, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.891, "gpu_count": 1, "cpu_util_avg_pct": 8.92, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.56, "gpu_util_max_pct": 10.0, "mem_util_avg_pct": 13.78, "mem_util_max_pct": 27.0, "temp_avg_c": 32.78, "temp_max_c": 33.0, "power_total_avg_w": 34.49, "power_total_max_w": 50.14, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95021.78, "vram_used_max_mb": 95043.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.07, "vram_used_max_pct": 97.09, "pcie_rx_avg_mb_s": 3.33, "pcie_rx_max_mb_s": 17.0, "pcie_tx_avg_mb_s": 1.11, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 16, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.985, "measurement_wall_seconds": 20.016, "client_output_tokens": 19981, "server_output_tokens": 19981, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 999.7998498822905, "per_request_avg_tps": 62.487490617643154, "ttft_avg": 0.7724374999961583, "ttft_p50": 0.75, "ttft_p90": 1.0309999999590218, "ttft_p99": 1.1372499999590218, "time_to_second_token_avg": 0.22831249998125713, "time_to_second_token_p50": 0.2809999999590218, "time_to_second_token_p90": 0.2809999999590218, "time_to_second_token_p99": 0.2809999999590218, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.015515900101110319, "inter_token_latency_p50": 0.015535572715333259, "inter_token_latency_p90": 0.016511816145582666, "inter_token_latency_p99": 0.01699204704939628, "output_tps_per_user_avg": 64.6872147474115, "output_tps_per_user_p50": 64.37008684194268, "output_tps_per_user_p90": 69.55591225248281, "output_tps_per_user_p99": 73.92350782691653, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.036652919374740654, "chunk_inter_token_latency_p50": 0.03664705882356031, "chunk_inter_token_latency_p90": 0.03686983883561299, "chunk_inter_token_latency_p99": 0.0374985423252856, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 1570.875, "output_seq_len_p50": 1565.0, "output_seq_len_p90": 1691.0, "output_seq_len_p99": 1771.35, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.7660000000614673, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.016472542372895246, "chunk_inter_token_latency_avg": 0.036702416918459956, "input_tokens": 8192, "output_tokens": 1476, "output_tps_per_user": 60.7070831789421, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.125, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.017069863013712664, "chunk_inter_token_latency_avg": 0.037589743589774495, "input_tokens": 8192, "output_tokens": 1461, "output_tps_per_user": 58.58277826814861, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.015963863337726997, "chunk_inter_token_latency_avg": 0.03698173515984854, "input_tokens": 8192, "output_tokens": 1523, "output_tps_per_user": 62.641478371762624, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.014402489626568161, "chunk_inter_token_latency_avg": 0.03664705882356031, "input_tokens": 8192, "output_tokens": 1688, "output_tps_per_user": 69.43244021889852, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.016551089918270086, "chunk_inter_token_latency_avg": 0.036702416918459956, "input_tokens": 8192, "output_tokens": 1469, "output_tps_per_user": 60.41898176724543, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.014351447135274949, "chunk_inter_token_latency_avg": 0.036591867469910375, "input_tokens": 8192, "output_tokens": 1694, "output_tps_per_user": 69.6793842860671, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.015138317757022112, "chunk_inter_token_latency_avg": 0.036591867469910375, "input_tokens": 8192, "output_tokens": 1606, "output_tps_per_user": 66.05753796759463, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.015984868421066112, "chunk_inter_token_latency_avg": 0.036702416918459956, "input_tokens": 8192, "output_tokens": 1521, "output_tps_per_user": 62.55916368270643, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.015615038560424479, "chunk_inter_token_latency_avg": 0.03653684210529397, "input_tokens": 8192, "output_tokens": 1557, "output_tps_per_user": 64.0408280857179, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.016048216644663466, "chunk_inter_token_latency_avg": 0.03664705882356031, "input_tokens": 8192, "output_tokens": 1515, "output_tps_per_user": 62.31221961553786, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.015016687268245048, "chunk_inter_token_latency_avg": 0.03675794251137744, "input_tokens": 8192, "output_tokens": 1619, "output_tps_per_user": 66.59258344645988, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1559999999590218, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.013391816143532213, "chunk_inter_token_latency_avg": 0.03614372163398104, "input_tokens": 8192, "output_tokens": 1785, "output_tps_per_user": 74.6724708047135, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.01545610687024204, "chunk_inter_token_latency_avg": 0.03675794251137744, "input_tokens": 8192, "output_tokens": 1573, "output_tps_per_user": 64.69934559816744, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.015157205240187455, "chunk_inter_token_latency_avg": 0.03664705882356031, "input_tokens": 8192, "output_tokens": 1604, "output_tps_per_user": 65.97522327853842, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0309999999590218, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.01641558441562643, "chunk_inter_token_latency_avg": 0.036277945619428195, "input_tokens": 8192, "output_tokens": 1464, "output_tps_per_user": 60.91772151883143, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0309999999590218, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.015219264892307648, "chunk_inter_token_latency_avg": 0.03616867469888775, "input_tokens": 8192, "output_tokens": 1579, "output_tps_per_user": 65.70619586925221, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 19981, "wall_time": 25.625, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 998.2513988778297, "server_utilization": 0.05425492382014119, "server_spec_accept_rate": 0.6899038461538461, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.594, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.938, "gpu_count": 1, "cpu_util_avg_pct": 8.69, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.44, "gpu_util_max_pct": 1.0, "mem_util_avg_pct": 24.0, "mem_util_max_pct": 25.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 27.27, "power_total_max_w": 27.6, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94982.0, "vram_used_max_mb": 94982.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.03, "vram_used_max_pct": 97.03, "pcie_rx_avg_mb_s": 2.0, "pcie_rx_max_mb_s": 8.0, "pcie_tx_avg_mb_s": 1.78, "pcie_tx_max_mb_s": 5.0 } }, { "concurrency": 2, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 7451, "server_output_tokens": 7451, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 372.55, "per_request_avg_tps": 186.275, "ttft_avg": 0.2029999999795109, "ttft_p50": 0.2029999999795109, "ttft_p90": 0.26539999996311964, "ttft_p99": 0.2794399999594316, "time_to_second_token_avg": 0.014999999897554517, "time_to_second_token_p50": 0.014999999897554517, "time_to_second_token_p90": 0.014999999897554517, "time_to_second_token_p99": 0.014999999897554517, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005497419603512079, "inter_token_latency_p50": 0.005497419603512079, "inter_token_latency_p90": 0.005833070980645327, "inter_token_latency_p99": 0.0059085925405003075, "output_tps_per_user_avg": 182.96927901760452, "output_tps_per_user_p50": 182.96927901760452, "output_tps_per_user_p90": 194.14068213118563, "output_tps_per_user_p99": 196.65424783174137, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013931701721987315, "chunk_inter_token_latency_p50": 0.013931701721987315, "chunk_inter_token_latency_p90": 0.014035633730491302, "chunk_inter_token_latency_p99": 0.014059018432404698, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 4537.0, "output_seq_len_p50": 4537.0, "output_seq_len_p90": 4802.6, "output_seq_len_p99": 4862.36, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005077855382095518, "chunk_inter_token_latency_avg": 0.01380178671135733, "input_tokens": 16384, "output_tokens": 4869, "output_tps_per_user": 196.9335329095809, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.005916983824928639, "chunk_inter_token_latency_avg": 0.014061616732617297, "input_tokens": 16384, "output_tokens": 4205, "output_tps_per_user": 169.00502512562815, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7451, "wall_time": 25.530999999959022, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 372.2707969042287, "server_utilization": 0.014306949089557763, "server_spec_accept_rate": 0.8632075471698113, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.516, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 8, "duration_seconds": 16.141, "gpu_count": 1, "cpu_util_avg_pct": 9.05, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.5, "gpu_util_max_pct": 1.0, "mem_util_avg_pct": 17.38, "mem_util_max_pct": 23.0, "temp_avg_c": 32.25, "temp_max_c": 33.0, "power_total_avg_w": 36.83, "power_total_max_w": 80.5, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94994.25, "vram_used_max_mb": 95046.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.04, "vram_used_max_pct": 97.1, "pcie_rx_avg_mb_s": 6.5, "pcie_rx_max_mb_s": 36.0, "pcie_tx_avg_mb_s": 1.38, "pcie_tx_max_mb_s": 4.0 } }, { "concurrency": 8, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 14395, "server_output_tokens": 14395, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 719.75, "per_request_avg_tps": 89.96875, "ttft_avg": 0.5669999998935964, "ttft_p50": 0.6099999998696148, "ttft_p90": 0.6474999998696148, "ttft_p99": 0.7262499998696148, "time_to_second_token_avg": 0.01500000013038516, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.01500000013038516, "time_to_second_token_p99": 0.01500000013038516, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.010933528324756378, "inter_token_latency_p50": 0.010931997931320738, "inter_token_latency_p90": 0.011565749063734474, "inter_token_latency_p99": 0.011776917111687876, "output_tps_per_user_avg": 91.6518004984611, "output_tps_per_user_p50": 91.47671160454667, "output_tps_per_user_p90": 95.70051282051283, "output_tps_per_user_p99": 99.03466666666665, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.025864813977676704, "chunk_inter_token_latency_p50": 0.025821235357025428, "chunk_inter_token_latency_p90": 0.02609161660821629, "chunk_inter_token_latency_p99": 0.0262548759465164, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 2238.375, "output_seq_len_p50": 2225.0, "output_seq_len_p90": 2333.7, "output_seq_len_p99": 2414.97, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.6099999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01146519285042333, "chunk_inter_token_latency_avg": 0.02590329436769394, "input_tokens": 16384, "output_tokens": 2127, "output_tps_per_user": 87.22051282051282, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.15699999989010394, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011800380228127143, "chunk_inter_token_latency_avg": 0.02627301587299419, "input_tokens": 16384, "output_tokens": 2105, "output_tps_per_user": 84.74303206064671, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6099999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010984677782785038, "chunk_inter_token_latency_avg": 0.025712025316455698, "input_tokens": 16384, "output_tokens": 2220, "output_tps_per_user": 91.03589743589744, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6099999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01100451467268623, "chunk_inter_token_latency_avg": 0.025712025316455698, "input_tokens": 16384, "output_tokens": 2216, "output_tps_per_user": 90.87179487179488, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7349999998696148, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.010879318079856438, "chunk_inter_token_latency_avg": 0.025634249471458773, "input_tokens": 16384, "output_tokens": 2230, "output_tps_per_user": 91.91752577319588, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6099999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01064875491480996, "chunk_inter_token_latency_avg": 0.025930851063829786, "input_tokens": 16384, "output_tokens": 2290, "output_tps_per_user": 93.90769230769232, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6099999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010625544899738448, "chunk_inter_token_latency_avg": 0.026013874066168622, "input_tokens": 16384, "output_tokens": 2295, "output_tps_per_user": 94.11282051282052, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5940000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010059843169624433, "chunk_inter_token_latency_avg": 0.025739176346356918, "input_tokens": 16384, "output_tokens": 2424, "output_tps_per_user": 99.4051282051282, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 14395, "wall_time": 25.54700000002049, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 719.75, "server_utilization": 0.013006317354143482, "server_spec_accept_rate": 0.6578947368421053, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.5, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 8, "duration_seconds": 16.516, "gpu_count": 1, "cpu_util_avg_pct": 12.64, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 46.0, "gpu_util_max_pct": 99.0, "mem_util_avg_pct": 18.62, "mem_util_max_pct": 59.0, "temp_avg_c": 40.75, "temp_max_c": 49.0, "power_total_avg_w": 204.19, "power_total_max_w": 450.02, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95132.62, "vram_used_max_mb": 95204.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.19, "vram_used_max_pct": 97.26, "pcie_rx_avg_mb_s": 60.62, "pcie_rx_max_mb_s": 188.0, "pcie_tx_avg_mb_s": 50.75, "pcie_tx_max_mb_s": 156.0 } }, { "concurrency": 16, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.984, "measurement_wall_seconds": 20.015, "client_output_tokens": 19727, "server_output_tokens": 19727, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 987.1397117724518, "per_request_avg_tps": 61.69623198577824, "ttft_avg": 0.903375000008964, "ttft_p50": 1.172000000020489, "ttft_p90": 1.172000000020489, "ttft_p99": 1.172000000020489, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.015699428746189258, "inter_token_latency_p50": 0.01583055373369361, "inter_token_latency_p90": 0.0164963511145738, "inter_token_latency_p99": 0.017274987846658568, "output_tps_per_user_avg": 63.97479167254328, "output_tps_per_user_p50": 63.169267787036006, "output_tps_per_user_p90": 66.33619603149964, "output_tps_per_user_p99": 77.09471064905544, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03682767926745188, "chunk_inter_token_latency_p50": 0.036577918201277135, "chunk_inter_token_latency_p90": 0.037274696648167786, "chunk_inter_token_latency_p99": 0.03791370597239413, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 1541.25, "output_seq_len_p50": 1538.0, "output_seq_len_p90": 1612.5, "output_seq_len_p99": 1842.3, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016121868652618852, "chunk_inter_token_latency_avg": 0.03646554364459118, "input_tokens": 16384, "output_tokens": 1478, "output_tps_per_user": 62.027549135103456, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1559999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016388118811867664, "chunk_inter_token_latency_avg": 0.038021439509922685, "input_tokens": 16384, "output_tokens": 1516, "output_tps_per_user": 61.01981633644475, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014862111043281594, "chunk_inter_token_latency_avg": 0.037246177369936594, "input_tokens": 16384, "output_tokens": 1640, "output_tps_per_user": 67.28519233154628, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01657063326368688, "chunk_inter_token_latency_avg": 0.03669029275796309, "input_tokens": 16384, "output_tokens": 1438, "output_tps_per_user": 60.34772383692868, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015412297734574785, "chunk_inter_token_latency_avg": 0.03646554364459118, "input_tokens": 16384, "output_tokens": 1546, "output_tps_per_user": 64.88325214200057, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016187627464254277, "chunk_inter_token_latency_avg": 0.036409785932596396, "input_tokens": 16384, "output_tokens": 1472, "output_tps_per_user": 61.77557534037725, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01739928571424181, "chunk_inter_token_latency_avg": 0.037246177369936594, "input_tokens": 16384, "output_tokens": 1401, "output_tps_per_user": 57.47362371212007, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01586409060620789, "chunk_inter_token_latency_avg": 0.036354198473157316, "input_tokens": 16384, "output_tokens": 1502, "output_tps_per_user": 63.03544431400832, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015505410566479015, "chunk_inter_token_latency_avg": 0.037246177369936594, "input_tokens": 16384, "output_tokens": 1572, "output_tps_per_user": 64.49361632267187, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01642206896546072, "chunk_inter_token_latency_avg": 0.036354198473157316, "input_tokens": 16384, "output_tokens": 1451, "output_tps_per_user": 60.89366705883549, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01590013054826275, "chunk_inter_token_latency_avg": 0.037189312977005395, "input_tokens": 16384, "output_tokens": 1533, "output_tps_per_user": 62.89256537640568, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015797016861179333, "chunk_inter_token_latency_avg": 0.03730321592639898, "input_tokens": 16384, "output_tokens": 1543, "output_tps_per_user": 63.30309126006368, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01537815656561776, "chunk_inter_token_latency_avg": 0.037132621951125815, "input_tokens": 16384, "output_tokens": 1585, "output_tps_per_user": 65.02729997142728, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012686201385145469, "chunk_inter_token_latency_avg": 0.036409785932596396, "input_tokens": 16384, "output_tokens": 1878, "output_tps_per_user": 78.82580211685118, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015402328589856433, "chunk_inter_token_latency_avg": 0.036354198473157316, "input_tokens": 16384, "output_tokens": 1547, "output_tps_per_user": 64.92524777445493, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.172000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0152935131662929, "chunk_inter_token_latency_avg": 0.036354198473157316, "input_tokens": 16384, "output_tokens": 1558, "output_tps_per_user": 65.387199731453, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 19727, "wall_time": 25.562999999849126, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 985.6107918996498, "server_utilization": 0.057506503158677114, "server_spec_accept_rate": 0.6298076923076923, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.532, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.36, "gpu_count": 1, "cpu_util_avg_pct": 8.92, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.67, "gpu_util_max_pct": 1.0, "mem_util_avg_pct": 14.78, "mem_util_max_pct": 22.0, "temp_avg_c": 35.0, "temp_max_c": 36.0, "power_total_avg_w": 38.01, "power_total_max_w": 86.52, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95239.67, "vram_used_max_mb": 95316.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.3, "vram_used_max_pct": 97.37, "pcie_rx_avg_mb_s": 1.56, "pcie_rx_max_mb_s": 7.0, "pcie_tx_avg_mb_s": 0.89, "pcie_tx_max_mb_s": 3.0 } }, { "concurrency": 2, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 7831, "server_output_tokens": 7831, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 391.55, "per_request_avg_tps": 195.775, "ttft_avg": 0.3119999999180436, "ttft_p50": 0.3119999999180436, "ttft_p90": 0.37439999990165235, "ttft_p99": 0.3884399998979643, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005272081553888448, "inter_token_latency_p50": 0.005272081553888448, "inter_token_latency_p90": 0.005406317283386433, "inter_token_latency_p99": 0.00543652032252348, "output_tps_per_user_avg": 189.87073251816378, "output_tps_per_user_p50": 189.87073251816378, "output_tps_per_user_p90": 194.7051486836524, "output_tps_per_user_p99": 195.79289232088735, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.014027252074960079, "chunk_inter_token_latency_p50": 0.014027252074960079, "chunk_inter_token_latency_p90": 0.014056330596734874, "chunk_inter_token_latency_p99": 0.014062873264134204, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 4689.0, "output_seq_len_p50": 4689.0, "output_seq_len_p90": 4820.2, "output_seq_len_p99": 4849.72, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.3899999998975545, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.00543987621576093, "chunk_inter_token_latency_avg": 0.013990903922741584, "input_tokens": 65536, "output_tokens": 4525, "output_tps_per_user": 183.82771231130303, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2339999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005104286892015966, "chunk_inter_token_latency_avg": 0.014063600227178574, "input_tokens": 65536, "output_tokens": 4853, "output_tps_per_user": 195.91375272502455, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7831, "wall_time": 25.514999999897555, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 391.2565575838163, "server_utilization": 0.0321441843180974, "server_spec_accept_rate": 0.8891509433962265, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.5, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.859, "gpu_count": 1, "cpu_util_avg_pct": 5.3, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.0, "gpu_util_max_pct": 0.0, "mem_util_avg_pct": 13.44, "mem_util_max_pct": 22.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 35.28, "power_total_max_w": 54.34, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95131.11, "vram_used_max_mb": 95150.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.19, "vram_used_max_pct": 97.2, "pcie_rx_avg_mb_s": 1.11, "pcie_rx_max_mb_s": 4.0, "pcie_tx_avg_mb_s": 0.44, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 8, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.016, "measurement_wall_seconds": 20.016, "client_output_tokens": 14613, "server_output_tokens": 14613, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 730.0659472484566, "per_request_avg_tps": 91.25824340605708, "ttft_avg": 0.8887500000128057, "ttft_p50": 1.0, "ttft_p90": 1.202999999979511, "ttft_p99": 1.202999999979511, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.010911965501128015, "inter_token_latency_p50": 0.010840688698084378, "inter_token_latency_p90": 0.0116941482171727, "inter_token_latency_p99": 0.011968098390681034, "output_tps_per_user_avg": 91.94076481710997, "output_tps_per_user_p50": 92.32930716306898, "output_tps_per_user_p90": 97.47402618728363, "output_tps_per_user_p99": 98.63400055687268, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.026565083934174692, "chunk_inter_token_latency_p50": 0.02645398213918415, "chunk_inter_token_latency_p90": 0.027075743115234367, "chunk_inter_token_latency_p99": 0.027082328409843783, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 2220.875, "output_seq_len_p50": 2217.0, "output_seq_len_p90": 2337.2, "output_seq_len_p99": 2390.12, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.7970000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011206099815157116, "chunk_inter_token_latency_avg": 0.026619099890230515, "input_tokens": 65536, "output_tokens": 2165, "output_tps_per_user": 89.23711340206187, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011563695753597303, "chunk_inter_token_latency_avg": 0.02708306010924483, "input_tokens": 65536, "output_tokens": 2144, "output_tps_per_user": 86.47754327926815, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010317611423643867, "chunk_inter_token_latency_avg": 0.02625991189431826, "input_tokens": 65536, "output_tokens": 2312, "output_tps_per_user": 96.92165743986027, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7970000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01012526096033403, "chunk_inter_token_latency_avg": 0.026648351648351647, "input_tokens": 65536, "output_tokens": 2396, "output_tps_per_user": 98.76288659793813, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010403141361274423, "chunk_inter_token_latency_avg": 0.02628886438813779, "input_tokens": 65536, "output_tokens": 2293, "output_tps_per_user": 96.12481127311109, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010513227513245581, "chunk_inter_token_latency_avg": 0.02625991189431826, "input_tokens": 65536, "output_tokens": 2269, "output_tps_per_user": 95.11826874669109, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011998537298848626, "chunk_inter_token_latency_avg": 0.02707260726065845, "input_tokens": 65536, "output_tokens": 2052, "output_tps_per_user": 83.34349221850229, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011168149882923174, "chunk_inter_token_latency_avg": 0.02628886438813779, "input_tokens": 65536, "output_tokens": 2136, "output_tps_per_user": 89.54034557944686, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 14613, "wall_time": 25.608999999938533, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 730.0659472484566, "server_utilization": 0.05128205128205132, "server_spec_accept_rate": 0.8003472222222222, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.562, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.781, "gpu_count": 1, "cpu_util_avg_pct": 6.6, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.0, "gpu_util_max_pct": 0.0, "mem_util_avg_pct": 18.44, "mem_util_max_pct": 23.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 30.73, "power_total_max_w": 46.76, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95125.67, "vram_used_max_mb": 95129.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.18, "vram_used_max_pct": 97.18, "pcie_rx_avg_mb_s": 0.78, "pcie_rx_max_mb_s": 3.0, "pcie_tx_avg_mb_s": 0.78, "pcie_tx_max_mb_s": 3.0 } }, { "concurrency": 16, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.985, "measurement_wall_seconds": 20.0, "client_output_tokens": 19513, "server_output_tokens": 19535, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 976.3822867214063, "per_request_avg_tps": 61.023892920087896, "ttft_avg": 1.4701874999591382, "ttft_p50": 1.4144999999552965, "ttft_p90": 2.078499999945052, "ttft_p99": 2.216599999833852, "time_to_second_token_avg": 0.22939999995287508, "time_to_second_token_p50": 0.2649999998975545, "time_to_second_token_p90": 0.2853999999118968, "time_to_second_token_p99": 0.4506400000280701, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.016278825963594167, "inter_token_latency_p50": 0.01635960192040587, "inter_token_latency_p90": 0.01727726729288527, "inter_token_latency_p99": 0.017718248920302736, "output_tps_per_user_avg": 61.66045471255664, "output_tps_per_user_p50": 61.13188009601484, "output_tps_per_user_p90": 67.84642745928184, "output_tps_per_user_p99": 69.11706498753882, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.038610168206803214, "chunk_inter_token_latency_p50": 0.038601834794300106, "chunk_inter_token_latency_p90": 0.03977483034404136, "chunk_inter_token_latency_p99": 0.04026604180868752, "input_seq_len_avg": 65536.0, "output_seq_len_avg": 1510.625, "output_seq_len_p50": 1478.5, "output_seq_len_p90": 1641.5, "output_seq_len_p99": 1687.5, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 2.2189999998081475, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014456204379599432, "chunk_inter_token_latency_avg": 0.03754502369677957, "input_tokens": 65536, "output_tokens": 1645, "output_tps_per_user": 69.17445089605941, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3599999998696148, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.01619000657462196, "chunk_inter_token_latency_avg": 0.038718553459119495, "input_tokens": 65536, "output_tokens": 1522, "output_tps_per_user": 61.76649746192893, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.953999999910593, "time_to_second_token": 0.2649999998975545, "latency": 0.0, "inter_token_latency_avg": 0.017069701280227598, "chunk_inter_token_latency_avg": 0.037914691943127965, "input_tokens": 65536, "output_tokens": 1407, "output_tps_per_user": 58.58333333333333, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.31299999984912574, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01591620111735659, "chunk_inter_token_latency_avg": 0.040316037735945706, "input_tokens": 65536, "output_tokens": 1612, "output_tps_per_user": 62.829062828912214, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.953999999910593, "time_to_second_token": 0.2649999998975545, "latency": 0.0, "inter_token_latency_avg": 0.016517549896765314, "chunk_inter_token_latency_avg": 0.0379746835443038, "input_tokens": 65536, "output_tokens": 1454, "output_tps_per_user": 60.541666666666664, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.953999999910593, "time_to_second_token": 0.2649999998975545, "latency": 0.0, "inter_token_latency_avg": 0.015463917525773196, "chunk_inter_token_latency_avg": 0.03785488958990536, "input_tokens": 65536, "output_tokens": 1553, "output_tps_per_user": 64.66666666666666, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3599999998696148, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.016973084886156644, "chunk_inter_token_latency_avg": 0.03866981132081915, "input_tokens": 65536, "output_tokens": 1450, "output_tps_per_user": 58.9168089777013, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3599999998696148, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.017380918727944155, "chunk_inter_token_latency_avg": 0.038791798107320155, "input_tokens": 65536, "output_tokens": 1416, "output_tps_per_user": 57.534357973393604, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.9380000000819564, "time_to_second_token": 0.2649999998975545, "latency": 0.0, "inter_token_latency_avg": 0.01675977653631285, "chunk_inter_token_latency_avg": 0.0380952380952381, "input_tokens": 65536, "output_tokens": 1433, "output_tps_per_user": 59.666666666666664, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4690000000409782, "time_to_second_token": 0.4690000000409782, "latency": 0.0, "inter_token_latency_avg": 0.014947464874795955, "chunk_inter_token_latency_avg": 0.038533858267781065, "input_tokens": 65536, "output_tokens": 1638, "output_tps_per_user": 66.90097674597484, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4690000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016201653944046424, "chunk_inter_token_latency_avg": 0.03998273155422446, "input_tokens": 65536, "output_tokens": 1573, "output_tps_per_user": 61.722093525363015, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01599327052494042, "chunk_inter_token_latency_avg": 0.037604430379844095, "input_tokens": 65536, "output_tokens": 1487, "output_tps_per_user": 62.526298072715505, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.9380000000819564, "time_to_second_token": 0.2649999998975545, "latency": 0.0, "inter_token_latency_avg": 0.017777777777777778, "chunk_inter_token_latency_avg": 0.0380952380952381, "input_tokens": 65536, "output_tokens": 1351, "output_tps_per_user": 56.25, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8440000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017103471749489448, "chunk_inter_token_latency_avg": 0.03931924882629108, "input_tokens": 65536, "output_tokens": 1470, "output_tps_per_user": 58.46766169154229, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8440000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017173615857826383, "chunk_inter_token_latency_avg": 0.039566929133858265, "input_tokens": 65536, "output_tokens": 1464, "output_tps_per_user": 58.22885572139304, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.01453659976387249, "chunk_inter_token_latency_avg": 0.038779527559055116, "input_tokens": 65536, "output_tokens": 1695, "output_tps_per_user": 68.79187817258884, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 19513, "wall_time": 26.57800000021234, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 976.75, "server_utilization": 0.04282794500185805, "server_spec_accept_rate": 0.7294407894736842, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.531, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.343, "gpu_count": 1, "cpu_util_avg_pct": 6.14, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.0, "gpu_util_max_pct": 0.0, "mem_util_avg_pct": 21.78, "mem_util_max_pct": 23.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 26.85, "power_total_max_w": 30.6, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95131.67, "vram_used_max_mb": 95132.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.19, "vram_used_max_pct": 97.19, "pcie_rx_avg_mb_s": 0.56, "pcie_rx_max_mb_s": 1.0, "pcie_tx_avg_mb_s": 0.33, "pcie_tx_max_mb_s": 1.0 } }, { "concurrency": 2, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 6648, "server_output_tokens": 6648, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 332.4, "per_request_avg_tps": 166.2, "ttft_avg": 0.5, "ttft_p50": 0.5, "ttft_p90": 0.6, "ttft_p99": 0.6225, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006010207654167592, "inter_token_latency_p50": 0.006010207654167592, "inter_token_latency_p90": 0.006121681170485815, "inter_token_latency_p99": 0.006146762711657415, "output_tps_per_user_avg": 166.4730828072361, "output_tps_per_user_p50": 166.4730828072361, "output_tps_per_user_p90": 169.5607198708223, "output_tps_per_user_p99": 170.25543821012923, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013974925336276643, "chunk_inter_token_latency_p50": 0.013974925336276643, "chunk_inter_token_latency_p90": 0.014023357160307174, "chunk_inter_token_latency_p99": 0.014034254320714043, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 3999.5, "output_seq_len_p50": 3999.5, "output_seq_len_p90": 4090.3, "output_seq_len_p99": 4110.73, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.006149549549565371, "chunk_inter_token_latency_avg": 0.013914385556238478, "input_tokens": 131072, "output_tokens": 3886, "output_tps_per_user": 162.6135364777533, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005870865758769812, "chunk_inter_token_latency_avg": 0.014035465116314807, "input_tokens": 131072, "output_tokens": 4113, "output_tps_per_user": 170.33262913671888, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 6648, "wall_time": 25.062000000150874, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 332.1508868327101, "server_utilization": 0.05564845782237082, "server_spec_accept_rate": 0.6851851851851852, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.047, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.313, "gpu_count": 1, "cpu_util_avg_pct": 5.53, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.11, "gpu_util_max_pct": 1.0, "mem_util_avg_pct": 9.89, "mem_util_max_pct": 21.0, "temp_avg_c": 32.89, "temp_max_c": 34.0, "power_total_avg_w": 42.21, "power_total_max_w": 85.58, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95185.0, "vram_used_max_mb": 95193.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.24, "vram_used_max_pct": 97.25, "pcie_rx_avg_mb_s": 0.89, "pcie_rx_max_mb_s": 4.0, "pcie_tx_avg_mb_s": 0.44, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 8, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 13374, "server_output_tokens": 13378, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 668.7, "per_request_avg_tps": 83.5875, "ttft_avg": 1.132875000097556, "ttft_p50": 1.1485000001266599, "ttft_p90": 1.6449000000953673, "ttft_p99": 1.7538900000648572, "time_to_second_token_avg": 0.015999999828636646, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999828636646, "time_to_second_token_p99": 0.015999999828636646, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.011855889569122487, "inter_token_latency_p50": 0.011756612081947716, "inter_token_latency_p90": 0.012775652867096417, "inter_token_latency_p99": 0.01323526593746886, "output_tps_per_user_avg": 84.65442104266188, "output_tps_per_user_p50": 85.0799183287213, "output_tps_per_user_p90": 89.7120228742611, "output_tps_per_user_p99": 91.11136208426304, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.02761095269596955, "chunk_inter_token_latency_p50": 0.027649416176676416, "chunk_inter_token_latency_p90": 0.028023324160079854, "chunk_inter_token_latency_p99": 0.028041904005635946, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 2068.0, "output_seq_len_p50": 2068.0, "output_seq_len_p90": 2182.9, "output_seq_len_p99": 2285.59, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 1.0630000000819564, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.013286334056399132, "chunk_inter_token_latency_avg": 0.02771493212669683, "input_tokens": 131072, "output_tokens": 1845, "output_tps_per_user": 75.26530612244898, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012556789500252397, "chunk_inter_token_latency_avg": 0.028043968432919956, "input_tokens": 131072, "output_tokens": 1982, "output_tps_per_user": 79.63819095477388, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011570154856964736, "chunk_inter_token_latency_avg": 0.027765765765981817, "input_tokens": 131072, "output_tokens": 2132, "output_tps_per_user": 86.4292667092561, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4380000000819564, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011943069306930693, "chunk_inter_token_latency_avg": 0.02741477272727273, "input_tokens": 131072, "output_tokens": 2021, "output_tps_per_user": 83.73056994818653, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.40600000019185245, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010956881533053181, "chunk_inter_token_latency_avg": 0.028014476614576952, "input_tokens": 131072, "output_tokens": 2297, "output_tps_per_user": 91.2668442187077, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2340000001713634, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.012061973227521365, "chunk_inter_token_latency_avg": 0.027583900226656, "input_tokens": 131072, "output_tokens": 2018, "output_tps_per_user": 82.9051748944639, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.7660000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011241721854366313, "chunk_inter_token_latency_avg": 0.027191075515023325, "input_tokens": 131072, "output_tokens": 2115, "output_tps_per_user": 88.95434462395967, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.593000000109896, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011230192217492073, "chunk_inter_token_latency_avg": 0.02715873015862879, "input_tokens": 131072, "output_tokens": 2134, "output_tps_per_user": 89.04567086949827, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 13374, "wall_time": 26.109000000171363, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 668.3987009776905, "server_utilization": 0.06753994797473062, "server_spec_accept_rate": 0.7106481481481481, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.063, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.328, "gpu_count": 1, "cpu_util_avg_pct": 6.68, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 0.22, "gpu_util_max_pct": 2.0, "mem_util_avg_pct": 20.11, "mem_util_max_pct": 22.0, "temp_avg_c": 32.11, "temp_max_c": 33.0, "power_total_avg_w": 27.44, "power_total_max_w": 31.24, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95180.22, "vram_used_max_mb": 95193.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.24, "vram_used_max_pct": 97.25, "pcie_rx_avg_mb_s": 1.56, "pcie_rx_max_mb_s": 7.0, "pcie_tx_avg_mb_s": 0.67, "pcie_tx_max_mb_s": 2.0 } }, { "concurrency": 16, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 19613, "server_output_tokens": 19670, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 980.65, "per_request_avg_tps": 61.290625, "ttft_avg": 1.912999999942258, "ttft_p50": 1.9064999999245629, "ttft_p90": 3.007999999797903, "ttft_p99": 3.2529499999363907, "time_to_second_token_avg": 0.015500000095926225, "time_to_second_token_p50": 0.015500000095926225, "time_to_second_token_p90": 0.015900000068359076, "time_to_second_token_p99": 0.015990000062156468, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01658627103325232, "inter_token_latency_p50": 0.016498321634927395, "inter_token_latency_p90": 0.017775558889996726, "inter_token_latency_p99": 0.018265856917298207, "output_tps_per_user_avg": 60.49746270507396, "output_tps_per_user_p50": 60.618064106462725, "output_tps_per_user_p90": 64.6836571900854, "output_tps_per_user_p99": 67.47729014507712, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.040173623921168605, "chunk_inter_token_latency_p50": 0.04016648655027501, "chunk_inter_token_latency_p90": 0.041034104988292755, "chunk_inter_token_latency_p99": 0.04134471647420044, "input_seq_len_avg": 131072.0, "output_seq_len_avg": 1583.0625, "output_seq_len_p50": 1581.5, "output_seq_len_p90": 1717.5, "output_seq_len_p99": 1789.95, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 1.468000000109896, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018342995169020423, "chunk_inter_token_latency_avg": 0.04070290964764256, "input_tokens": 131072, "output_tokens": 1450, "output_tps_per_user": 54.51672372944333, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015634201585561974, "chunk_inter_token_latency_avg": 0.04108630952396197, "input_tokens": 131072, "output_tokens": 1767, "output_tps_per_user": 63.962332487991574, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017326789107138293, "chunk_inter_token_latency_avg": 0.04139031770071311, "input_tokens": 131072, "output_tokens": 1580, "output_tps_per_user": 57.71409773712891, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.75, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.017722377622454474, "chunk_inter_token_latency_avg": 0.03959843750017171, "input_tokens": 131072, "output_tokens": 1431, "output_tps_per_user": 56.42583750912674, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1089999999385327, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017003150598721716, "chunk_inter_token_latency_avg": 0.04082299546168134, "input_tokens": 131072, "output_tokens": 1588, "output_tps_per_user": 58.812629706119246, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016660231660315564, "chunk_inter_token_latency_avg": 0.03989214175674944, "input_tokens": 131072, "output_tokens": 1555, "output_tps_per_user": 60.023174970729, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6559999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014718906860045436, "chunk_inter_token_latency_avg": 0.04047699386512495, "input_tokens": 131072, "output_tokens": 1794, "output_tps_per_user": 67.93982797149877, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.577999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01550121506690789, "chunk_inter_token_latency_avg": 0.03955813953508587, "input_tokens": 131072, "output_tokens": 1647, "output_tps_per_user": 64.51107191814967, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.280999999959022, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016336411609539227, "chunk_inter_token_latency_avg": 0.038940251572423694, "input_tokens": 131072, "output_tokens": 1517, "output_tps_per_user": 61.212953242196456, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016919089759836577, "chunk_inter_token_latency_avg": 0.04080182926838638, "input_tokens": 131072, "output_tokens": 1583, "output_tps_per_user": 59.104834491383365, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.0, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016168944099310623, "chunk_inter_token_latency_avg": 0.0400492307690617, "input_tokens": 131072, "output_tokens": 1611, "output_tps_per_user": 61.84695759091874, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.0939999998081475, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017410452961791893, "chunk_inter_token_latency_avg": 0.03909859154956395, "input_tokens": 131072, "output_tokens": 1436, "output_tps_per_user": 57.43675952570274, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.9219999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016094689699418973, "chunk_inter_token_latency_avg": 0.03936776212862575, "input_tokens": 131072, "output_tokens": 1564, "output_tps_per_user": 62.13229448195578, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8129999998491257, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01629342431769875, "chunk_inter_token_latency_avg": 0.04028374233148832, "input_tokens": 131072, "output_tokens": 1613, "output_tps_per_user": 61.37445269339416, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.375, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015418716256736359, "chunk_inter_token_latency_avg": 0.039726429675393374, "input_tokens": 131072, "output_tokens": 1668, "output_tps_per_user": 64.85624246202113, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9069999998901039, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01782874015753898, "chunk_inter_token_latency_avg": 0.04098190045262354, "input_tokens": 131072, "output_tokens": 1525, "output_tps_per_user": 56.089212763423696, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 19613, "wall_time": 28.687999999849126, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 983.5, "server_utilization": 0.08008175399479744, "server_spec_accept_rate": 0.81875, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 8.625, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.344, "gpu_count": 1, "cpu_util_avg_pct": 5.58, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.67, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 22.0, "mem_util_max_pct": 25.0, "temp_avg_c": 32.0, "temp_max_c": 32.0, "power_total_avg_w": 29.55, "power_total_max_w": 35.13, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95160.89, "vram_used_max_mb": 95288.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.21, "vram_used_max_pct": 97.34, "pcie_rx_avg_mb_s": 2.22, "pcie_rx_max_mb_s": 8.0, "pcie_tx_avg_mb_s": 1.78, "pcie_tx_max_mb_s": 5.0 } }, { "concurrency": 2, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.0, "measurement_wall_seconds": 20.0, "client_output_tokens": 7128, "server_output_tokens": 7128, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 356.4, "per_request_avg_tps": 178.2, "ttft_avg": 0.8674999999348074, "ttft_p50": 0.8674999999348074, "ttft_p90": 1.0358999999705703, "ttft_p99": 1.073789999978617, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.0057004440156647514, "inter_token_latency_p50": 0.0057004440156647514, "inter_token_latency_p90": 0.005706674287805518, "inter_token_latency_p99": 0.005708076099037191, "output_tps_per_user_avg": 175.4252587529869, "output_tps_per_user_p50": 175.4252587529869, "output_tps_per_user_p90": 175.61698892337225, "output_tps_per_user_p99": 175.66012821170895, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.014831360674631882, "chunk_inter_token_latency_p50": 0.014831360674631882, "chunk_inter_token_latency_p90": 0.014874002808243084, "chunk_inter_token_latency_p99": 0.014883597288305605, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 4151.0, "output_seq_len_p50": 4151.0, "output_seq_len_p90": 4186.2, "output_seq_len_p99": 4194.12, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.6569999998901039, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005692656175488794, "chunk_inter_token_latency_avg": 0.014884663341645885, "input_tokens": 262144, "output_tokens": 4195, "output_tps_per_user": 175.66492146596858, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.077999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.00570823185584071, "chunk_inter_token_latency_avg": 0.014778058007617879, "input_tokens": 262144, "output_tokens": 4107, "output_tps_per_user": 175.18559604000524, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7128, "wall_time": 25.609000000171363, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 356.4, "server_utilization": 0.10358602749907098, "server_spec_accept_rate": 0.795, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.594, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.813, "gpu_count": 1, "cpu_util_avg_pct": 5.91, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.89, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 12.56, "mem_util_max_pct": 24.0, "temp_avg_c": 32.44, "temp_max_c": 33.0, "power_total_avg_w": 43.13, "power_total_max_w": 65.47, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95156.0, "vram_used_max_mb": 95275.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.21, "vram_used_max_pct": 97.33, "pcie_rx_avg_mb_s": 2.78, "pcie_rx_max_mb_s": 8.0, "pcie_tx_avg_mb_s": 1.44, "pcie_tx_max_mb_s": 6.0 } }, { "concurrency": 8, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 20.016, "measurement_wall_seconds": 20.016, "client_output_tokens": 13035, "server_output_tokens": 13043, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 651.2290167921461, "per_request_avg_tps": 81.40362709901827, "ttft_avg": 2.12112500006333, "ttft_p50": 2.133000000147149, "ttft_p90": 3.1811999999918044, "ttft_p99": 3.3979200000176206, "time_to_second_token_avg": 0.01500000013038516, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.01500000013038516, "time_to_second_token_p99": 0.01500000013038516, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01225666511375698, "inter_token_latency_p50": 0.012299848310827186, "inter_token_latency_p90": 0.01269531116638152, "inter_token_latency_p99": 0.012705032339104987, "output_tps_per_user_avg": 81.66476756807066, "output_tps_per_user_p50": 81.30456063526242, "output_tps_per_user_p90": 85.55849670279042, "output_tps_per_user_p99": 85.62206523341659, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.029601427450954367, "chunk_inter_token_latency_p50": 0.02961249368995448, "chunk_inter_token_latency_p90": 0.02991031805830845, "chunk_inter_token_latency_p99": 0.02993296728963808, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 2044.625, "output_seq_len_p50": 2054.5, "output_seq_len_p90": 2103.9, "output_seq_len_p99": 2124.69, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 3.077999999979511, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.011692046556711218, "chunk_inter_token_latency_avg": 0.02932968369822206, "input_tokens": 262144, "output_tokens": 2063, "output_tps_per_user": 85.52822597392083, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012706112469407595, "chunk_inter_token_latency_avg": 0.029935483870896926, "input_tokens": 262144, "output_tokens": 2046, "output_tps_per_user": 78.70227832530934, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.9530000002123415, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012371316306409197, "chunk_inter_token_latency_avg": 0.02966784452279049, "input_tokens": 262144, "output_tokens": 2037, "output_tps_per_user": 80.83214229046354, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.422000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012207411219784343, "chunk_inter_token_latency_avg": 0.029103067484712856, "input_tokens": 262144, "output_tokens": 1944, "output_tps_per_user": 81.91745014531149, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.5780000002123415, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012228380315245172, "chunk_inter_token_latency_avg": 0.029899532710056246, "input_tokens": 262144, "output_tokens": 2094, "output_tps_per_user": 81.77697898006132, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3130000000819564, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011678269049849253, "chunk_inter_token_latency_avg": 0.029557142857118465, "input_tokens": 262144, "output_tokens": 2127, "output_tps_per_user": 85.62912840348616, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012690682036513204, "chunk_inter_token_latency_avg": 0.02985536723166157, "input_tokens": 262144, "output_tokens": 2083, "output_tps_per_user": 78.79797138741903, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.672000000020489, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012479102956135848, "chunk_inter_token_latency_avg": 0.029463297232176333, "input_tokens": 262144, "output_tokens": 1963, "output_tps_per_user": 80.13396503859359, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 13035, "wall_time": 28.25, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 651.1082268406327, "server_utilization": 0.1123188405797102, "server_spec_accept_rate": 0.72, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 8.187, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 10, "duration_seconds": 19.516, "gpu_count": 1, "cpu_util_avg_pct": 6.94, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.4, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 33.6, "temp_max_c": 34.0, "power_total_avg_w": 63.19, "power_total_max_w": 65.18, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95078.6, "vram_used_max_mb": 95093.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.13, "vram_used_max_pct": 97.15, "pcie_rx_avg_mb_s": 6.9, "pcie_rx_max_mb_s": 14.0, "pcie_tx_avg_mb_s": 3.2, "pcie_tx_max_mb_s": 9.0 } }, { "concurrency": 16, "context_tokens": 262144, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 19.969, "measurement_wall_seconds": 20.015, "client_output_tokens": 16890, "server_output_tokens": 16890, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 845.8110070592088, "per_request_avg_tps": 52.86318794120055, "ttft_avg": 3.691312500028289, "ttft_p50": 3.7030000000959262, "ttft_p90": 5.968999999924563, "ttft_p99": 6.461999999848194, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.018636410245054823, "inter_token_latency_p50": 0.018559600893043066, "inter_token_latency_p90": 0.01995148870211999, "inter_token_latency_p99": 0.0206232953772821, "output_tps_per_user_avg": 53.80273203530534, "output_tps_per_user_p50": 53.88051127723506, "output_tps_per_user_p90": 56.21319413004376, "output_tps_per_user_p99": 59.16580870387749, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.04480509755656021, "chunk_inter_token_latency_p50": 0.04493175058687651, "chunk_inter_token_latency_p90": 0.0453701626015704, "chunk_inter_token_latency_p99": 0.045800356472643786, "input_seq_len_avg": 262144.0, "output_seq_len_avg": 1455.25, "output_seq_len_p50": 1401.0, "output_seq_len_p90": 1619.5, "output_seq_len_p99": 1761.0, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 5.437000000150874, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01857604702416613, "chunk_inter_token_latency_avg": 0.04435438596471948, "input_tokens": 262144, "output_tokens": 1362, "output_tps_per_user": 53.83276639529769, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.280999999959022, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0184881118881692, "chunk_inter_token_latency_avg": 0.045039182282933486, "input_tokens": 262144, "output_tokens": 1431, "output_tps_per_user": 54.08881155895178, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.765000000130385, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.020271211022415223, "chunk_inter_token_latency_avg": 0.04545365853643999, "input_tokens": 262144, "output_tokens": 1380, "output_tps_per_user": 49.33104385792411, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.890000000130385, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.020685427910493905, "chunk_inter_token_latency_avg": 0.04586153846138563, "input_tokens": 262144, "output_tokens": 1298, "output_tps_per_user": 48.34321070499542, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 6.155999999959022, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018622441243428323, "chunk_inter_token_latency_avg": 0.04401971326179562, "input_tokens": 262144, "output_tokens": 1320, "output_tps_per_user": 53.698652444554774, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.0, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017795412275208054, "chunk_inter_token_latency_avg": 0.04506122448965556, "input_tokens": 262144, "output_tokens": 1614, "output_tps_per_user": 56.19425864008585, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 6.515999999828637, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017930318754693815, "chunk_inter_token_latency_avg": 0.044058287796141994, "input_tokens": 262144, "output_tokens": 1350, "output_tps_per_user": 55.7714569205982, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6099999998696148, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017915024630567104, "chunk_inter_token_latency_avg": 0.04496754250392732, "input_tokens": 262144, "output_tokens": 1625, "output_tps_per_user": 55.81906922381634, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 5.781999999890104, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018543154761920005, "chunk_inter_token_latency_avg": 0.044187943262447675, "input_tokens": 262144, "output_tokens": 1345, "output_tps_per_user": 53.92825615917243, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8129999998491257, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016755044843083784, "chunk_inter_token_latency_avg": 0.044414561664281524, "input_tokens": 262144, "output_tokens": 1785, "output_tps_per_user": 59.68351677750263, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.984000000171363, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018025245441732535, "chunk_inter_token_latency_avg": 0.044547660311803454, "input_tokens": 262144, "output_tokens": 1427, "output_tps_per_user": 55.47774665441021, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.019631766381824754, "chunk_inter_token_latency_avg": 0.04496411092998688, "input_tokens": 262144, "output_tokens": 1405, "output_tps_per_user": 50.93785146739562, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2340000001713634, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018618204803988996, "chunk_inter_token_latency_avg": 0.04489939024376615, "input_tokens": 262144, "output_tokens": 1583, "output_tps_per_user": 53.71087118913567, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.017783427495235777, "chunk_inter_token_latency_avg": 0.0451098726113226, "input_tokens": 262144, "output_tokens": 1594, "output_tps_per_user": 56.23212962000168, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.5160000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.019464183381103502, "chunk_inter_token_latency_avg": 0.045286666666700816, "input_tokens": 262144, "output_tokens": 1397, "output_tps_per_user": 51.376416899711, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.593000000109896, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.019077542062846082, "chunk_inter_token_latency_avg": 0.044655821917655124, "input_tokens": 262144, "output_tokens": 1368, "output_tps_per_user": 52.41765405133197, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 16890, "wall_time": 31.811999999918044, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 843.1930507730294, "server_utilization": 0.14622816796729843, "server_spec_accept_rate": 0.7713068181818182, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 11.781, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 9, "duration_seconds": 17.344, "gpu_count": 1, "cpu_util_avg_pct": 6.92, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.33, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 63.79, "power_total_max_w": 65.52, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95087.67, "vram_used_max_mb": 95125.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.14, "vram_used_max_pct": 97.18, "pcie_rx_avg_mb_s": 6.0, "pcie_rx_max_mb_s": 7.0, "pcie_tx_avg_mb_s": 3.11, "pcie_tx_max_mb_s": 9.0 } } ], "summary_table": { "8192": { "1": 238.45, "2": 399.0, "8": 758.6569255427659, "16": 999.7998498822905 }, "16384": { "1": 224.0, "2": 372.55, "8": 719.75, "16": 987.1397117724518 }, "65536": { "1": 212.75, "2": 391.55, "8": 730.0659472484566, "16": 976.3822867214063 }, "131072": { "1": 215.6, "2": 332.4, "8": 668.7, "16": 980.65 }, "262144": { "1": 215.35, "2": 356.4, "8": 651.2290167921461, "16": 845.8110070592088 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": true, "mode": "standalone_cold", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." } } }