{ "metadata": { "version": "0.4.29", "engine": "vllm", "model": "DeepSeek-V4-Flash-DSpark", "server": "192.168.0.213:8000", "timestamp": "2026-07-11T00:34:20.342536", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 10.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "standalone_cold", "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 3.0, "decode_warmup_context": 8192, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 0.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 8192, "temperature": null, "ignore_eos": true, "max_total_tokens": 2995972, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1, 2, 4, 8, 16, 32, 64 ], "context_lengths": [ 0, 8192 ], "startup_diagnostics_available": true, "nvidia_p2p_override_effective": false, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.29", "server_url": "http://192.168.0.213:8000", "hostname": "DESKTOP-RMAJ6JK", "uname": "", "env": {}, "args": { "concurrency": "1,2,4,8,16,32,64", "contexts": "0,8k", "max_tokens": 8192, "duration": 10.0, "request_count": 0, "run_burst": false, "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "prefill_contexts": "8k,64k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 2995972 }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 595.79, 00000000:01:00.0, 5, 16, 450.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 255, "stdout": "ERROR: Option -m is missing its value. Please run 'nvidia-smi -h' for help.", "stderr": "" } }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": { "samples": 120, "duration_seconds": 290.25, "gpu_count": 1, "cpu_util_avg_pct": 16.02, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.1, "gpu_util_max_pct": 21.0, "mem_util_avg_pct": 3.22, "mem_util_max_pct": 25.0, "temp_avg_c": 33.79, "temp_max_c": 34.0, "power_total_avg_w": 60.1, "power_total_max_w": 84.26, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94987.18, "vram_used_max_mb": 95217.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.04, "vram_used_max_pct": 97.27, "pcie_rx_avg_mb_s": 41.98, "pcie_rx_max_mb_s": 570.0, "pcie_tx_avg_mb_s": 174.23, "pcie_tx_max_mb_s": 6889.0 }, "event_log": [ "00:29:27 benchmark start engine=vllm", "00:29:27 startup server=http://192.168.0.213:8000 model=DeepSeek-V4-Flash-DSpark", "00:29:27 startup decode concurrency=1,2,4,8,16,32,64 contexts=0,8k", "00:29:27 startup NVIDIA P2P override: unknown: /proc/driver/nvidia/params is not readable", "00:29:27 startup KV cache budget manual: 2,995,972 tokens", "00:29:27 startup engine vLLM 0.11.2.dev279+eldritch.enlightenment.v45c1582.b12xf3686b5.patchc1441b5.fi25dd814.cu132.20260704 models=['DeepSeek-V4-Flash-DSpark']", "00:29:27 startup model context length: 524,288 tokens", "00:29:27 startup prefill tests: standalone cold profile ['8k', '64k']", "00:29:27 startup calibrating padding text run=xpycoigjwxuo up_to=64k", "00:29:27 startup token targeting: estimate from 8k", "00:29:27 startup calibrated: 6.25 chars/token (cached, source=8k)", "00:29:27 startup context 8k: 51,188 chars (~8,191 tokens)", "00:29:27 startup context 64k: 409,507 chars (~65,535 tokens)", "00:29:27 startup startup preparation done", "00:29:27 hardware monitor interval=2s", "00:29:27 prefill warmup start", "00:29:28 prefill start ctx=8k", "00:29:38 prefill done ctx=8k 10,923 tok/s", "00:29:39 prefill start ctx=64k", "00:29:51 prefill done ctx=64k 10,595 tok/s", "00:29:52 decode warmup start C=1 ctx=8k 3s", "00:29:52 cell start C=1 ctx=8k", "00:29:58 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "00:30:01 cell done C=1 ctx=8k 257.0 tok/s", "00:30:01 decode warmup done C=1 ctx=8k", "00:30:04 cell start C=1 ctx=0", "00:30:09 ready C=1 ctx=0 running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "00:30:19 cell done C=1 ctx=0 219.7 tok/s", "00:30:21 cell start C=1 ctx=8k", "00:30:27 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "00:30:37 cell done C=1 ctx=8k 396.3 tok/s", "00:30:39 cell start C=2 ctx=0", "00:30:44 ready C=2 ctx=0 running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "00:30:54 cell done C=2 ctx=0 388.5 tok/s", "00:30:56 cell start C=4 ctx=0", "00:31:02 ready C=4 ctx=0 running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "00:31:12 cell done C=4 ctx=0 545.6 tok/s", "00:31:14 cell start C=8 ctx=0", "00:31:20 ready C=8 ctx=0 running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "00:31:30 cell done C=8 ctx=0 813.4 tok/s", "00:31:32 cell start C=16 ctx=0", "00:31:38 ready C=16 ctx=0 running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "00:31:48 cell done C=16 ctx=0 1130.0 tok/s", "00:31:51 cell start C=32 ctx=0", "00:31:56 ready C=32 ctx=0 running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "00:32:06 cell done C=32 ctx=0 1460.9 tok/s", "00:32:08 cell start C=64 ctx=0", "00:32:14 ready C=64 ctx=0 running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "00:32:24 cell done C=64 ctx=0 2282.2 tok/s", "00:32:26 cell start C=2 ctx=8k", "00:32:31 ready C=2 ctx=8k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "00:32:41 cell done C=2 ctx=8k 382.5 tok/s", "00:32:43 cell start C=4 ctx=8k", "00:32:49 ready C=4 ctx=8k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "00:32:59 cell done C=4 ctx=8k 710.9 tok/s", "00:33:01 cell start C=8 ctx=8k", "00:33:07 ready C=8 ctx=8k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "00:33:17 cell done C=8 ctx=8k 947.1 tok/s", "00:33:19 cell start C=16 ctx=8k", "00:33:24 ready C=16 ctx=8k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "00:33:35 cell done C=16 ctx=8k 1156.2 tok/s", "00:33:37 cell start C=32 ctx=8k", "00:33:43 ready C=32 ctx=8k running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "00:33:53 cell done C=32 ctx=8k 1508.7 tok/s", "00:33:55 cell start C=64 ctx=8k", "00:34:08 ready C=64 ctx=8k running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "00:34:18 cell done C=64 ctx=8k 2425.9 tok/s" ], "prefill": { "8192": { "ttft_seconds": 0.75, "prefill_seconds": 0.75, "tok_per_sec": 10923.0, "client_ttft_seconds": 0.75, "client_tok_per_sec": 10923.0, "prompt_tokens": 8192, "samples": 10, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 4, "duration_seconds": 7.484, "gpu_count": 1, "cpu_util_avg_pct": 12.32, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.75, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 65.78, "power_total_max_w": 66.08, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95070.5, "vram_used_max_mb": 95073.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.12, "vram_used_max_pct": 97.13, "pcie_rx_avg_mb_s": 7.25, "pcie_rx_max_mb_s": 8.0, "pcie_tx_avg_mb_s": 98.75, "pcie_tx_max_mb_s": 386.0 } }, "65536": { "ttft_seconds": 6.093, "prefill_seconds": 6.093, "tok_per_sec": 10595.0, "client_ttft_seconds": 6.093, "client_tok_per_sec": 10595.0, "prompt_tokens": 64556, "samples": 2, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 5, "duration_seconds": 9.375, "gpu_count": 1, "cpu_util_avg_pct": 11.64, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.8, "gpu_util_max_pct": 21.0, "mem_util_avg_pct": 6.4, "mem_util_max_pct": 18.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 53.83, "power_total_max_w": 66.43, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95108.6, "vram_used_max_mb": 95180.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.16, "vram_used_max_pct": 97.23, "pcie_rx_avg_mb_s": 9.0, "pcie_rx_max_mb_s": 10.0, "pcie_tx_avg_mb_s": 3.6, "pcie_tx_max_mb_s": 6.0 } } }, "results": [ { "concurrency": 1, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 2197, "server_output_tokens": 2197, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 219.7, "per_request_avg_tps": 219.7, "ttft_avg": 0.0779999999795109, "ttft_p50": 0.0779999999795109, "ttft_p90": 0.0779999999795109, "ttft_p99": 0.0779999999795109, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004682146219021176, "inter_token_latency_p50": 0.004682146219021176, "inter_token_latency_p90": 0.004682146219021176, "inter_token_latency_p99": 0.004682146219021176, "output_tps_per_user_avg": 213.57726846237932, "output_tps_per_user_p50": 213.57726846237932, "output_tps_per_user_p90": 213.57726846237932, "output_tps_per_user_p99": 213.57726846237932, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01332321428573258, "chunk_inter_token_latency_p50": 0.01332321428573258, "chunk_inter_token_latency_p90": 0.01332321428573258, "chunk_inter_token_latency_p99": 0.01332321428573258, "input_seq_len_avg": 65.0, "output_seq_len_avg": 3188.0, "output_seq_len_p50": 3188.0, "output_seq_len_p90": 3188.0, "output_seq_len_p99": 3188.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.004682146219021176, "chunk_inter_token_latency_avg": 0.01332321428573258, "input_tokens": 65, "output_tokens": 3188, "output_tps_per_user": 213.57726846237932, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2197, "wall_time": 15.015000000130385, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 219.7, "server_utilization": 0.0014410144741898723, "server_spec_accept_rate": 0.3912280701754386, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.0, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 8.359, "gpu_count": 1, "cpu_util_avg_pct": 15.3, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.75, "gpu_util_max_pct": 12.0, "mem_util_avg_pct": 5.5, "mem_util_max_pct": 19.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 63.06, "power_total_max_w": 84.26, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95033.25, "vram_used_max_mb": 95071.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.08, "vram_used_max_pct": 97.12, "pcie_rx_avg_mb_s": 3.0, "pcie_rx_max_mb_s": 6.0, "pcie_tx_avg_mb_s": 8.5, "pcie_tx_max_mb_s": 27.0 } }, { "concurrency": 1, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 3963, "server_output_tokens": 3963, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 396.3, "per_request_avg_tps": 396.3, "ttft_avg": 0.09399999980814755, "ttft_p50": 0.09399999980814755, "ttft_p90": 0.09399999980814755, "ttft_p99": 0.09399999980814755, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.002781285020556867, "inter_token_latency_p50": 0.002781285020556867, "inter_token_latency_p90": 0.002781285020556867, "inter_token_latency_p99": 0.002781285020556867, "output_tps_per_user_avg": 359.5460345160097, "output_tps_per_user_p50": 359.5460345160097, "output_tps_per_user_p90": 359.5460345160097, "output_tps_per_user_p99": 359.5460345160097, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013549590536907614, "chunk_inter_token_latency_p50": 0.013549590536907614, "chunk_inter_token_latency_p90": 0.013549590536907614, "chunk_inter_token_latency_p99": 0.013549590536907614, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 5355.0, "output_seq_len_p50": 5355.0, "output_seq_len_p90": 5355.0, "output_seq_len_p99": 5355.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.09399999980814755, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.002781285020556867, "chunk_inter_token_latency_avg": 0.013549590536907614, "input_tokens": 8190, "output_tokens": 5355, "output_tps_per_user": 359.5460345160097, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3963, "wall_time": 16.031000000191852, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 395.70644033433905, "server_utilization": 0.0026898936851543764, "server_spec_accept_rate": 0.9872727272727273, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.016, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 3, "duration_seconds": 4.734, "gpu_count": 1, "cpu_util_avg_pct": 15.17, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.67, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 8.0, "mem_util_max_pct": 22.0, "temp_avg_c": 33.67, "temp_max_c": 34.0, "power_total_avg_w": 49.48, "power_total_max_w": 58.68, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95048.33, "vram_used_max_mb": 95061.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.1, "vram_used_max_pct": 97.11, "pcie_rx_avg_mb_s": 22.67, "pcie_rx_max_mb_s": 35.0, "pcie_tx_avg_mb_s": 38.67, "pcie_tx_max_mb_s": 51.0 } }, { "concurrency": 2, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 3885, "server_output_tokens": 3885, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 388.5, "per_request_avg_tps": 194.25, "ttft_avg": 0.1404999999795109, "ttft_p50": 0.1404999999795109, "ttft_p90": 0.1904999999795109, "ttft_p99": 0.2017499999795109, "time_to_second_token_avg": 0.07000000006519258, "time_to_second_token_p50": 0.07000000006519258, "time_to_second_token_p90": 0.11400000001303852, "time_to_second_token_p99": 0.12390000000130386, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005276101892351294, "inter_token_latency_p50": 0.005276101892351294, "inter_token_latency_p90": 0.005314085879724768, "inter_token_latency_p99": 0.0053226322768838005, "output_tps_per_user_avg": 189.5492183690859, "output_tps_per_user_p50": 189.5492183690859, "output_tps_per_user_p90": 190.9138309683272, "output_tps_per_user_p99": 191.22086880315652, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01547086241024339, "chunk_inter_token_latency_p50": 0.01547086241024339, "chunk_inter_token_latency_p90": 0.015555132816095016, "chunk_inter_token_latency_p99": 0.015574093657411631, "input_seq_len_avg": 65.0, "output_seq_len_avg": 2817.5, "output_seq_len_p50": 2817.5, "output_seq_len_p90": 2828.3, "output_seq_len_p99": 2830.73, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.2029999999795109, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.005228621908134449, "chunk_inter_token_latency_avg": 0.015365524402928857, "input_tokens": 65, "output_tokens": 2831, "output_tps_per_user": 191.25498411813754, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0779999999795109, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.0053235818765681375, "chunk_inter_token_latency_avg": 0.015576200417557922, "input_tokens": 65, "output_tokens": 2804, "output_tps_per_user": 187.84345262003427, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3885, "wall_time": 15.014999999897555, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 388.5, "server_utilization": 0.0027219162290251786, "server_spec_accept_rate": 0.4326530612244898, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.0, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 3, "duration_seconds": 5.328, "gpu_count": 1, "cpu_util_avg_pct": 15.57, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.0, "gpu_util_max_pct": 3.0, "mem_util_avg_pct": 1.33, "mem_util_max_pct": 2.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 58.67, "power_total_max_w": 60.76, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94964.67, "vram_used_max_mb": 94966.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.02, "vram_used_max_pct": 97.02, "pcie_rx_avg_mb_s": 201.33, "pcie_rx_max_mb_s": 204.0, "pcie_tx_avg_mb_s": 161.0, "pcie_tx_max_mb_s": 177.0 } }, { "concurrency": 4, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 5456, "server_output_tokens": 5456, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 545.6, "per_request_avg_tps": 136.4, "ttft_avg": 0.20724999997764826, "ttft_p50": 0.25, "ttft_p90": 0.25, "ttft_p99": 0.25, "time_to_second_token_avg": 0.015749999845866114, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999828636646, "time_to_second_token_p99": 0.015999999828636646, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.0074571152510381905, "inter_token_latency_p50": 0.007530842479085137, "inter_token_latency_p90": 0.007658828787369834, "inter_token_latency_p99": 0.0076880567918193345, "output_tps_per_user_avg": 134.23439841341317, "output_tps_per_user_p50": 132.79365079365078, "output_tps_per_user_p90": 139.04761904761907, "output_tps_per_user_p99": 141.10476190476192, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.020260022205850783, "chunk_inter_token_latency_p50": 0.020218228498074454, "chunk_inter_token_latency_p90": 0.02033525087984818, "chunk_inter_token_latency_p99": 0.020380388084246616, "input_seq_len_avg": 65.0, "output_seq_len_avg": 2120.75, "output_seq_len_p50": 2092.5, "output_seq_len_p90": 2191.0, "output_seq_len_p99": 2223.4, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 0.25, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.007075471698113208, "chunk_inter_token_latency_avg": 0.020218228498074454, "input_tokens": 65, "output_tokens": 2227, "output_tps_per_user": 141.33333333333334, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.07899999991059303, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.007691304347869279, "chunk_inter_token_latency_avg": 0.020385403329179777, "input_tokens": 65, "output_tokens": 2071, "output_tps_per_user": 130.01695873301776, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.007583052479537795, "chunk_inter_token_latency_avg": 0.020218228498074454, "input_tokens": 65, "output_tokens": 2078, "output_tps_per_user": 131.87301587301587, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.007478632478632479, "chunk_inter_token_latency_avg": 0.020218228498074454, "input_tokens": 65, "output_tokens": 2107, "output_tps_per_user": 133.71428571428572, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 5456, "wall_time": 16.015999999828637, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 545.6, "server_utilization": 0.0049634942999872145, "server_spec_accept_rate": 0.37635135135135134, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.0, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.656, "gpu_count": 1, "cpu_util_avg_pct": 15.2, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.75, "gpu_util_max_pct": 3.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 57.66, "power_total_max_w": 60.31, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94959.75, "vram_used_max_mb": 94960.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.01, "vram_used_max_pct": 97.01, "pcie_rx_avg_mb_s": 293.0, "pcie_rx_max_mb_s": 570.0, "pcie_tx_avg_mb_s": 133.5, "pcie_tx_max_mb_s": 177.0 } }, { "concurrency": 8, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.984, "measurement_wall_seconds": 10.0, "client_output_tokens": 8121, "server_output_tokens": 8121, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 813.4014423127, "per_request_avg_tps": 101.6751802890875, "ttft_avg": 0.33787499999743886, "ttft_p50": 0.375, "ttft_p90": 0.375, "ttft_p99": 0.375, "time_to_second_token_avg": 0.028999999980442226, "time_to_second_token_p50": 0.030999999959021807, "time_to_second_token_p90": 0.030999999959021807, "time_to_second_token_p99": 0.030999999959021807, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.009782579350868847, "inter_token_latency_p50": 0.00991678856507894, "inter_token_latency_p90": 0.010016755177714703, "inter_token_latency_p99": 0.010053286342478093, "output_tps_per_user_avg": 102.29364495838199, "output_tps_per_user_p50": 100.83925940202437, "output_tps_per_user_p90": 105.97914172858329, "output_tps_per_user_p99": 106.60022053495076, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.027527795831541666, "chunk_inter_token_latency_p50": 0.027432337433986875, "chunk_inter_token_latency_p90": 0.027695501758964975, "chunk_inter_token_latency_p99": 0.02784030324063051, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1601.625, "output_seq_len_p50": 1575.0, "output_seq_len_p90": 1670.8, "output_seq_len_p99": 1680.88, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.009462224866126722, "chunk_inter_token_latency_avg": 0.0278563922941489, "input_tokens": 65, "output_tokens": 1682, "output_tps_per_user": 105.68338991602734, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.009546788990788093, "chunk_inter_token_latency_avg": 0.027432337433986875, "input_tokens": 65, "output_tokens": 1636, "output_tps_per_user": 104.74726119587665, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.010057345360785137, "chunk_inter_token_latency_avg": 0.02762654867245758, "input_tokens": 65, "output_tokens": 1553, "output_tps_per_user": 99.42981613211043, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.009999359384970232, "chunk_inter_token_latency_avg": 0.027432337433986875, "input_tokens": 65, "output_tokens": 1562, "output_tps_per_user": 100.00640656071158, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.009374774774737858, "chunk_inter_token_latency_avg": 0.027432337433986875, "input_tokens": 65, "output_tokens": 1666, "output_tps_per_user": 106.66922929121382, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.009986564299384857, "chunk_inter_token_latency_avg": 0.027432337433986875, "input_tokens": 65, "output_tokens": 1564, "output_tps_per_user": 100.1345377670674, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.00990418781721988, "chunk_inter_token_latency_avg": 0.027577738515792462, "input_tokens": 65, "output_tokens": 1577, "output_tps_per_user": 100.96739060838019, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.009929389312937999, "chunk_inter_token_latency_avg": 0.027432337433986875, "input_tokens": 65, "output_tokens": 1573, "output_tps_per_user": 100.71112819566855, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 8121, "wall_time": 16.015000000130385, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 810.8836744777108, "server_utilization": 0.00928653772255672, "server_spec_accept_rate": 0.3886792452830189, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.0, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.344, "gpu_count": 1, "cpu_util_avg_pct": 12.4, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 1.75, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 14.25, "mem_util_max_pct": 22.0, "temp_avg_c": 33.0, "temp_max_c": 33.0, "power_total_avg_w": 40.99, "power_total_max_w": 51.78, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94964.0, "vram_used_max_mb": 94967.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.02, "vram_used_max_pct": 97.02, "pcie_rx_avg_mb_s": 12.0, "pcie_rx_max_mb_s": 42.0, "pcie_tx_avg_mb_s": 14.25, "pcie_tx_max_mb_s": 26.0 } }, { "concurrency": 16, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.984, "measurement_wall_seconds": 10.0, "client_output_tokens": 11282, "server_output_tokens": 11282, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1130.0080128274699, "per_request_avg_tps": 70.62550080171687, "ttft_avg": 0.7286250000324799, "ttft_p50": 0.7660000000614673, "ttft_p90": 0.7809999999590218, "ttft_p99": 0.7809999999590218, "time_to_second_token_avg": 0.030062499950872734, "time_to_second_token_p50": 0.030999999959021807, "time_to_second_token_p90": 0.030999999959021807, "time_to_second_token_p99": 0.030999999959021807, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.013913314976285335, "inter_token_latency_p50": 0.013821320303725949, "inter_token_latency_p90": 0.0146552945625173, "inter_token_latency_p99": 0.014977476682595574, "output_tps_per_user_avg": 71.97654513642546, "output_tps_per_user_p50": 72.3541406302363, "output_tps_per_user_p90": 74.98520028951762, "output_tps_per_user_p99": 76.82365322644044, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03796639862286192, "chunk_inter_token_latency_p50": 0.037818407960148036, "chunk_inter_token_latency_p90": 0.03810275689217923, "chunk_inter_token_latency_p99": 0.03914944818693169, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1098.125, "output_seq_len_p50": 1101.0, "output_seq_len_p90": 1141.0, "output_seq_len_p99": 1168.95, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.7809999999590218, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013708746618556818, "chunk_inter_token_latency_avg": 0.03791271820443768, "input_tokens": 65, "output_tokens": 1110, "output_tps_per_user": 72.9461290535746, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7809999999590218, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.014050831792957035, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1083, "output_tps_per_user": 71.1701637835597, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7809999999590218, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013394713656369614, "chunk_inter_token_latency_avg": 0.03810275689217923, "input_tokens": 65, "output_tokens": 1136, "output_tps_per_user": 74.65631783210745, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7809999999590218, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.014492850333631565, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1050, "output_tps_per_user": 68.99953956465261, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7809999999590218, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013598389982092585, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1119, "output_tps_per_user": 73.5381174769129, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.09300000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.015005665722217787, "chunk_inter_token_latency_avg": 0.03933415841541742, "input_tokens": 65, "output_tokens": 1060, "output_tps_per_user": 66.64149518667296, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.012971843003395487, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1173, "output_tps_per_user": 77.09004801694267, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013745931283887442, "chunk_inter_token_latency_avg": 0.03810275689217923, "input_tokens": 65, "output_tokens": 1107, "output_tps_per_user": 72.7487995791285, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.014195144724537359, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1072, "output_tps_per_user": 70.44662237725734, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013277729257624027, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1146, "output_tps_per_user": 75.31408274692778, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013671762589909632, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1113, "output_tps_per_user": 73.1434585280207, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013973345588216462, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1089, "output_tps_per_user": 71.5648227324519, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.014261726078780028, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1067, "output_tps_per_user": 70.11773991984718, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.014817738791403032, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1027, "output_tps_per_user": 67.48668026056586, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013896709323564453, "chunk_inter_token_latency_avg": 0.037818407960148036, "input_tokens": 65, "output_tokens": 1095, "output_tps_per_user": 71.9594816813441, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013549910873422024, "chunk_inter_token_latency_avg": 0.03800749999994878, "input_tokens": 65, "output_tokens": 1123, "output_tps_per_user": 73.80122344284102, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 11282, "wall_time": 16.014999999897555, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1128.2, "server_utilization": 0.01770846676059945, "server_spec_accept_rate": 0.3639423076923077, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.0, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 3, "duration_seconds": 5.234, "gpu_count": 1, "cpu_util_avg_pct": 17.77, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 4.0, "gpu_util_max_pct": 8.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 33.33, "temp_max_c": 34.0, "power_total_avg_w": 68.64, "power_total_max_w": 70.92, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95022.67, "vram_used_max_mb": 95088.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.07, "vram_used_max_pct": 97.14, "pcie_rx_avg_mb_s": 137.67, "pcie_rx_max_mb_s": 212.0, "pcie_tx_avg_mb_s": 161.0, "pcie_tx_max_mb_s": 352.0 } }, { "concurrency": 32, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.016, "measurement_wall_seconds": 10.016, "client_output_tokens": 14632, "server_output_tokens": 14679, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1460.8626197993415, "per_request_avg_tps": 45.65195686872942, "ttft_avg": 1.2730624998439453, "ttft_p50": 1.3439999998081475, "ttft_p90": 1.3589999999385327, "ttft_p99": 1.3589999999385327, "time_to_second_token_avg": 0.08312500001920853, "time_to_second_token_p50": 0.0470000000204891, "time_to_second_token_p90": 0.0470000000204891, "time_to_second_token_p99": 0.820699999961072, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.02163524595420951, "inter_token_latency_p50": 0.021736089030239252, "inter_token_latency_p90": 0.02258380327120332, "inter_token_latency_p99": 0.024065841217233135, "output_tps_per_user_avg": 46.36212982538495, "output_tps_per_user_p50": 46.006436512511506, "output_tps_per_user_p90": 48.65698321216232, "output_tps_per_user_p99": 54.876389701497644, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.052661083403042985, "chunk_inter_token_latency_p50": 0.05238314176253061, "chunk_inter_token_latency_p90": 0.05242366412252842, "chunk_inter_token_latency_p99": 0.05669098976422353, "input_seq_len_avg": 65.0, "output_seq_len_avg": 638.875, "output_seq_len_p50": 630.0, "output_seq_len_p90": 669.0, "output_seq_len_p99": 751.2700000000001, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 1.3589999999385327, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.019728715728745294, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 694, "output_tps_per_user": 50.68753657101825, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02122981366462809, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 645, "output_tps_per_user": 47.10356933872403, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022590460526484286, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 609, "output_tps_per_user": 44.26647251514125, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.11000000010244548, "latency": 0.0, "inter_token_latency_avg": 0.02424757281569975, "chunk_inter_token_latency_avg": 0.05676136363675169, "input_tokens": 65, "output_tokens": 619, "output_tps_per_user": 41.2412412409593, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021905901116590822, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 628, "output_tps_per_user": 45.649799781239416, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022016103059614316, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 622, "output_tps_per_user": 45.421299005198165, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022376432078593272, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 612, "output_tps_per_user": 44.68987712105649, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2190000000409782, "time_to_second_token": 1.1399999998975545, "latency": 0.0, "inter_token_latency_avg": 0.023661341852904222, "chunk_inter_token_latency_avg": 0.05653435114472536, "input_tokens": 65, "output_tokens": 627, "output_tps_per_user": 42.26302997592923, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02153070866144959, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 636, "output_tps_per_user": 46.44528964299652, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022303425774911075, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 614, "output_tps_per_user": 44.83616149788483, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021394080997044308, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 643, "output_tps_per_user": 46.74190025447481, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021528213166304773, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 639, "output_tps_per_user": 46.45067346161204, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021875200000032784, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 626, "output_tps_per_user": 45.71386775885484, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.01761855670105733, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 777, "output_tps_per_user": 56.75833820939417, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021840255591087043, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 627, "output_tps_per_user": 45.787009947269006, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02084146341466538, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 657, "output_tps_per_user": 47.98127559969404, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022523887973674613, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 608, "output_tps_per_user": 44.397308367399816, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02229707792224423, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 617, "output_tps_per_user": 44.84892610086679, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021001536098341766, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 652, "output_tps_per_user": 47.615564657623196, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02208723747983924, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 620, "output_tps_per_user": 45.27501462836983, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021736089030239252, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 630, "output_tps_per_user": 46.006436512511506, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02233986928107923, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 613, "output_tps_per_user": 44.76301930947066, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.020194977843457147, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 678, "output_tps_per_user": 49.51726155639157, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021598736176967597, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 634, "output_tps_per_user": 46.29900526616818, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02122981366462809, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 645, "output_tps_per_user": 47.10356933872403, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.020746585735994673, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 660, "output_tps_per_user": 48.200702164936544, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021770700636975303, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 629, "output_tps_per_user": 45.93329432409734, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021664037855051176, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 635, "output_tps_per_user": 46.15944666874927, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021098310292016045, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 652, "output_tps_per_user": 47.397160538416045, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021736089030239252, "chunk_inter_token_latency_avg": 0.05238314176253061, "input_tokens": 65, "output_tokens": 630, "output_tps_per_user": 46.006436512511506, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023084033613617556, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 596, "output_tps_per_user": 43.31998543833724, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3439999998081475, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.020530642750526823, "chunk_inter_token_latency_avg": 0.05242366412252842, "input_tokens": 65, "output_tokens": 670, "output_tps_per_user": 48.707681106298516, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 14632, "wall_time": 15.140000000130385, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1465.5551118120923, "server_utilization": 0.010407326758037683, "server_spec_accept_rate": 0.2911184210526316, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.078, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.516, "gpu_count": 1, "cpu_util_avg_pct": 15.0, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.75, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 65.72, "power_total_max_w": 66.03, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94935.25, "vram_used_max_mb": 94967.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.98, "vram_used_max_pct": 97.02, "pcie_rx_avg_mb_s": 8.25, "pcie_rx_max_mb_s": 9.0, "pcie_tx_avg_mb_s": 128.25, "pcie_tx_max_mb_s": 500.0 } }, { "concurrency": 64, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 22822, "server_output_tokens": 22822, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 2282.2, "per_request_avg_tps": 35.659375, "ttft_avg": 2.3146718751268054, "ttft_p50": 2.343000000109896, "ttft_p90": 2.3590000001713634, "ttft_p99": 2.3590000001713634, "time_to_second_token_avg": 0.07918749992313678, "time_to_second_token_p50": 0.07899999991059303, "time_to_second_token_p90": 0.07899999991059303, "time_to_second_token_p99": 0.0940000000409782, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.02673708368744956, "inter_token_latency_p50": 0.026410358565566373, "inter_token_latency_p90": 0.028018634088648367, "inter_token_latency_p99": 0.03045070606847271, "output_tps_per_user_avg": 37.47969962744192, "output_tps_per_user_p50": 37.86394499795454, "output_tps_per_user_p90": 39.15563861667449, "output_tps_per_user_p99": 40.479137872867, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.073028562469753, "chunk_inter_token_latency_p50": 0.07288736263725006, "chunk_inter_token_latency_p90": 0.07289010988916833, "chunk_inter_token_latency_p99": 0.0771089966557211, "input_seq_len_avg": 65.0, "output_seq_len_avg": 499.03125, "output_seq_len_p50": 503.0, "output_seq_len_p90": 520.0, "output_seq_len_p99": 537.59, "request_count": 64, "completed_request_count": 0, "request_samples": [ { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026713709677419355, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 497, "output_tps_per_user": 37.43396226415094, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026479041915825622, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 502, "output_tps_per_user": 37.76571687068232, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.125, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.03199173553754414, "chunk_inter_token_latency_avg": 0.0841521739139748, "input_tokens": 65, "output_tokens": 485, "output_tps_per_user": 31.258072849046982, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026114173228009126, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 509, "output_tps_per_user": 38.29338157745832, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026217391304009165, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 507, "output_tps_per_user": 38.14262023266517, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.029545657015208546, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 450, "output_tps_per_user": 33.8459219060606, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025980392156862746, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 511, "output_tps_per_user": 38.490566037735846, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026394422310756973, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 503, "output_tps_per_user": 37.886792452830186, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02587890625, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 513, "output_tps_per_user": 38.64150943396226, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026479041915825622, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 502, "output_tps_per_user": 37.76571687068232, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02562862669245648, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 518, "output_tps_per_user": 39.0188679245283, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026062868369015002, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 510, "output_tps_per_user": 38.368762249854896, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02754677754677755, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 482, "output_tps_per_user": 36.30188679245283, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025809338521067385, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 515, "output_tps_per_user": 38.74566561183775, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02498305084713491, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 532, "output_tps_per_user": 40.02713704257946, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.0273525773192343, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 486, "output_tps_per_user": 36.55962611233718, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025560693641288317, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 520, "output_tps_per_user": 39.122568973820606, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027128834355477784, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 490, "output_tps_per_user": 36.86114880192346, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02543186180422265, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 522, "output_tps_per_user": 39.32075471698113, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.026247035573041547, "chunk_inter_token_latency_avg": 0.07297252747230232, "input_tokens": 65, "output_tokens": 507, "output_tps_per_user": 38.09954069735421, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.025218631178955106, "chunk_inter_token_latency_avg": 0.07288461538533178, "input_tokens": 65, "output_tokens": 527, "output_tps_per_user": 39.65322276628947, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.029026258205974584, "chunk_inter_token_latency_avg": 0.07288461538533178, "input_tokens": 65, "output_tokens": 458, "output_tps_per_user": 34.45156426652906, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025778210116731516, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 515, "output_tps_per_user": 38.79245283018868, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027953586497890294, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 475, "output_tps_per_user": 35.77358490566038, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026447105788423155, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 502, "output_tps_per_user": 37.81132075471698, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026185770750988144, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 507, "output_tps_per_user": 38.18867924528302, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027637499999642993, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 481, "output_tps_per_user": 36.182722750354316, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025334608030592735, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 524, "output_tps_per_user": 39.471698113207545, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02642629482037577, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 503, "output_tps_per_user": 37.841097543078895, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026237623762376237, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 506, "output_tps_per_user": 38.113207547169814, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3590000001713634, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.029220264316803163, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 455, "output_tps_per_user": 34.222825268043465, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02781132075435773, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 478, "output_tps_per_user": 35.9565807331646, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02726337448559671, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 487, "output_tps_per_user": 36.679245283018865, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027128834355477784, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 490, "output_tps_per_user": 36.86114880192346, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025529865125240848, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 520, "output_tps_per_user": 39.16981132075472, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025809338521067385, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 515, "output_tps_per_user": 38.74566561183775, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.028046511627544686, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 474, "output_tps_per_user": 35.65505804357832, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02567829457364341, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 517, "output_tps_per_user": 38.943396226415096, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025778210116731516, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 515, "output_tps_per_user": 38.79245283018868, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026321428571088566, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 505, "output_tps_per_user": 37.991858887872034, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027489626556016597, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 483, "output_tps_per_user": 36.37735849056604, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026321428571088566, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 505, "output_tps_per_user": 37.991858887872034, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.025958904109844198, "chunk_inter_token_latency_avg": 0.07288461538533178, "input_tokens": 65, "output_tokens": 512, "output_tps_per_user": 38.522427440254596, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027869747898799656, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 477, "output_tps_per_user": 35.88120006076803, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02867965367965368, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 463, "output_tps_per_user": 34.867924528301884, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02846781115843055, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 467, "output_tps_per_user": 35.12739333680232, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02658517034033795, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 500, "output_tps_per_user": 37.61495552588917, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.025910156249665306, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 513, "output_tps_per_user": 38.594904267044605, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026321428571088566, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 505, "output_tps_per_user": 37.991858887872034, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027040816326530614, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 491, "output_tps_per_user": 36.9811320754717, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027604166666666666, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 481, "output_tps_per_user": 36.22641509433962, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02452125693129138, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 542, "output_tps_per_user": 40.780943766545185, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027753138074955306, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 479, "output_tps_per_user": 36.03196140556118, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02628968253968254, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 505, "output_tps_per_user": 38.0377358490566, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027661795407098122, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 480, "output_tps_per_user": 36.15094339622642, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026692152917160233, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 498, "output_tps_per_user": 37.46419418109603, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.026009803921824285, "chunk_inter_token_latency_avg": 0.07288461538533178, "input_tokens": 65, "output_tokens": 511, "output_tps_per_user": 38.4470410851856, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027489626556016597, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 483, "output_tps_per_user": 36.37735849056604, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02754677754677755, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 482, "output_tps_per_user": 36.30188679245283, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.024812734082397005, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 535, "output_tps_per_user": 40.30188679245283, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.02567829457364341, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 517, "output_tps_per_user": 38.943396226415096, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.027580041579685317, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 482, "output_tps_per_user": 36.258103422750885, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026447105788423155, "chunk_inter_token_latency_avg": 0.07280219780219781, "input_tokens": 65, "output_tokens": 502, "output_tps_per_user": 37.81132075471698, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.343000000109896, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.026062868369015002, "chunk_inter_token_latency_avg": 0.07289010988916833, "input_tokens": 65, "output_tokens": 510, "output_tps_per_user": 38.368762249854896, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 22822, "wall_time": 15.719000000040978, "num_completed": 64, "num_errors": 0, "server_gen_throughput": 2282.2, "server_utilization": 0.06651082361982841, "server_spec_accept_rate": 0.3299342105263158, "server_spec_accept_length": 0.0, "avg_running_reqs": 64, "max_running_reqs": 64, "effective_concurrency": 64, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.625, "ready_reason": "running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 5, "duration_seconds": 8.875, "gpu_count": 1, "cpu_util_avg_pct": 21.1, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.6, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 68.7, "power_total_max_w": 73.56, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94998.4, "vram_used_max_mb": 95088.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.05, "vram_used_max_pct": 97.14, "pcie_rx_avg_mb_s": 33.0, "pcie_rx_max_mb_s": 86.0, "pcie_tx_avg_mb_s": 1693.6, "pcie_tx_max_mb_s": 6889.0 } }, { "concurrency": 2, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 3825, "server_output_tokens": 3825, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 382.5, "per_request_avg_tps": 191.25, "ttft_avg": 0.1565000000409782, "ttft_p50": 0.1565000000409782, "ttft_p90": 0.20650000004097818, "ttft_p99": 0.2177500000409782, "time_to_second_token_avg": 0.01500000013038516, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.01500000013038516, "time_to_second_token_p99": 0.01500000013038516, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005144647535764763, "inter_token_latency_p50": 0.005144647535764763, "inter_token_latency_p90": 0.005259805928134394, "inter_token_latency_p99": 0.0052857165664175615, "output_tps_per_user_avg": 194.52907026607994, "output_tps_per_user_p50": 194.52907026607994, "output_tps_per_user_p90": 198.8834317350179, "output_tps_per_user_p99": 199.86316306552894, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.015540529413509335, "chunk_inter_token_latency_p50": 0.015540529413509335, "chunk_inter_token_latency_p90": 0.015581186963800668, "chunk_inter_token_latency_p99": 0.015590334912616218, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 2794.0, "output_seq_len_p50": 2794.0, "output_seq_len_p90": 2846.8, "output_seq_len_p99": 2858.68, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.2190000000409782, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.005000699545302725, "chunk_inter_token_latency_avg": 0.015489707475645167, "input_tokens": 8190, "output_tokens": 2860, "output_tps_per_user": 199.9720221022524, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.005288595526226802, "chunk_inter_token_latency_avg": 0.015591351351373501, "input_tokens": 8190, "output_tokens": 2728, "output_tps_per_user": 189.0861184299075, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3825, "wall_time": 15.030999999959022, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 382.5, "server_utilization": 0.003650570001280884, "server_spec_accept_rate": 0.3917525773195876, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.016, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 8.906, "gpu_count": 1, "cpu_util_avg_pct": 18.3, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.5, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 62.31, "power_total_max_w": 63.4, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94897.5, "vram_used_max_mb": 94913.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.94, "vram_used_max_pct": 96.96, "pcie_rx_avg_mb_s": 29.75, "pcie_rx_max_mb_s": 57.0, "pcie_tx_avg_mb_s": 127.75, "pcie_tx_max_mb_s": 135.0 } }, { "concurrency": 4, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.015, "measurement_wall_seconds": 10.015, "client_output_tokens": 7120, "server_output_tokens": 7120, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 710.9335995913434, "per_request_avg_tps": 177.73339989783585, "ttft_avg": 0.3399999999674037, "ttft_p50": 0.4220000000204891, "ttft_p90": 0.4220000000204891, "ttft_p99": 0.4220000000204891, "time_to_second_token_avg": 0.015999999886844307, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999991618098, "time_to_second_token_p99": 0.01600000005448237, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005981629139798319, "inter_token_latency_p50": 0.006334192684043264, "inter_token_latency_p90": 0.0068897907255626905, "inter_token_latency_p99": 0.007031797004931688, "output_tps_per_user_avg": 173.8536140691067, "output_tps_per_user_p50": 158.01167263584279, "output_tps_per_user_p90": 215.05504708876552, "output_tps_per_user_p99": 235.25401246881682, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0221490173539389, "chunk_inter_token_latency_p50": 0.02218567950634584, "chunk_inter_token_latency_p90": 0.022371479626677158, "chunk_inter_token_latency_p99": 0.022400892390574936, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 2634.0, "output_seq_len_p50": 2383.5, "output_seq_len_p90": 3243.6000000000004, "output_seq_len_p99": 3548.16, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 0.4220000000204891, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.006521626297568993, "chunk_inter_token_latency_avg": 0.02240416047545247, "input_tokens": 8190, "output_tokens": 2313, "output_tps_per_user": 153.33598620527533, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.09399999980814755, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.007047575480417133, "chunk_inter_token_latency_avg": 0.022295224312868095, "input_tokens": 8190, "output_tokens": 2187, "output_tps_per_user": 141.89276904925208, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.004210555710689615, "chunk_inter_token_latency_avg": 0.021820549927611448, "input_tokens": 8190, "output_tokens": 3582, "output_tps_per_user": 237.4983419554892, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.006146759070517534, "chunk_inter_token_latency_avg": 0.02207613469982359, "input_tokens": 8190, "output_tokens": 2454, "output_tps_per_user": 162.6873590664102, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7120, "wall_time": 16.04700000002049, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 709.7996211772592, "server_utilization": 0.006308441142564347, "server_spec_accept_rate": 0.5916666666666667, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.0, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 8.219, "gpu_count": 1, "cpu_util_avg_pct": 18.65, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.25, "gpu_util_max_pct": 4.0, "mem_util_avg_pct": 1.25, "mem_util_max_pct": 2.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 61.34, "power_total_max_w": 62.09, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94887.5, "vram_used_max_mb": 94893.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.94, "vram_used_max_pct": 96.94, "pcie_rx_avg_mb_s": 17.25, "pcie_rx_max_mb_s": 20.0, "pcie_tx_avg_mb_s": 98.75, "pcie_tx_max_mb_s": 131.0 } }, { "concurrency": 8, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.969, "measurement_wall_seconds": 10.0, "client_output_tokens": 9442, "server_output_tokens": 9480, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 947.1361219742389, "per_request_avg_tps": 118.39201524677986, "ttft_avg": 0.49600000015925616, "ttft_p50": 0.5310000001918525, "ttft_p90": 0.5733000001404435, "ttft_p99": 0.6621300000324845, "time_to_second_token_avg": 0.01599999986189817, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999921768905, "time_to_second_token_p99": 0.01600000004749745, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.008853886410269999, "inter_token_latency_p50": 0.009282140529278489, "inter_token_latency_p90": 0.010209885887309821, "inter_token_latency_p99": 0.010358651868860475, "output_tps_per_user_avg": 115.92701519139852, "output_tps_per_user_p50": 108.20209997140745, "output_tps_per_user_p90": 142.45047790701568, "output_tps_per_user_p99": 146.77032830185235, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03091748428431753, "chunk_inter_token_latency_p50": 0.030623108411735774, "chunk_inter_token_latency_p90": 0.03190520819773487, "chunk_inter_token_latency_p99": 0.03265582694191456, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 1678.375, "output_seq_len_p50": 1584.5, "output_seq_len_p90": 2057.7, "output_seq_len_p99": 2120.07, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5310000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.006791157102469015, "chunk_inter_token_latency_avg": 0.03039578947336658, "input_tokens": 8190, "output_tokens": 2127, "output_tps_per_user": 147.2503116790564, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.11000000010244548, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009892809587176121, "chunk_inter_token_latency_avg": 0.03154777070050644, "input_tokens": 8190, "output_tokens": 1503, "output_tps_per_user": 101.0835184067712, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5310000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008671471471380856, "chunk_inter_token_latency_avg": 0.030916488222375003, "input_tokens": 8190, "output_tokens": 1666, "output_tps_per_user": 115.3206815360437, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5310000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.010139044943714273, "chunk_inter_token_latency_avg": 0.03033193277279228, "input_tokens": 8190, "output_tokens": 1425, "output_tps_per_user": 98.62861892331907, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5310000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.007720855614892581, "chunk_inter_token_latency_avg": 0.03085042735010497, "input_tokens": 8190, "output_tokens": 1871, "output_tps_per_user": 129.51932400744846, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5310000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.007122841637814073, "chunk_inter_token_latency_avg": 0.03039578947336658, "input_tokens": 8190, "output_tokens": 2028, "output_tps_per_user": 140.39340629042678, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.010375181422366103, "chunk_inter_token_latency_avg": 0.030162447257427193, "input_tokens": 8190, "output_tokens": 1379, "output_tps_per_user": 96.38385675302686, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5310000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.01011772950234697, "chunk_inter_token_latency_avg": 0.03273922902460119, "input_tokens": 8190, "output_tokens": 1428, "output_tps_per_user": 98.8364039350957, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 9442, "wall_time": 15.516000000061467, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 946.4856229973864, "server_utilization": 0.010631484565133853, "server_spec_accept_rate": 0.603125, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.5, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.828, "gpu_count": 1, "cpu_util_avg_pct": 18.32, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.25, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 33.75, "temp_max_c": 34.0, "power_total_avg_w": 62.76, "power_total_max_w": 78.14, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94925.25, "vram_used_max_mb": 94950.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 96.97, "vram_used_max_pct": 97.0, "pcie_rx_avg_mb_s": 9.5, "pcie_rx_max_mb_s": 23.0, "pcie_tx_avg_mb_s": 63.75, "pcie_tx_max_mb_s": 129.0 } }, { "concurrency": 16, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.969, "measurement_wall_seconds": 10.0, "client_output_tokens": 11526, "server_output_tokens": 11526, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1156.18417092513, "per_request_avg_tps": 72.26151068282063, "ttft_avg": 0.7833749998681014, "ttft_p50": 0.6719999997876585, "ttft_p90": 1.2660000000614673, "ttft_p99": 1.2795999999158085, "time_to_second_token_avg": 0.22350000010919757, "time_to_second_token_p50": 0.28100000019185245, "time_to_second_token_p90": 0.30500000005122274, "time_to_second_token_p99": 0.32899999991059303, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.013385339407624704, "inter_token_latency_p50": 0.013592081473412388, "inter_token_latency_p90": 0.015469736929375703, "inter_token_latency_p99": 0.01572315652413574, "output_tps_per_user_avg": 75.80230869355238, "output_tps_per_user_p50": 73.57368277900933, "output_tps_per_user_p90": 88.5736196316488, "output_tps_per_user_p99": 94.47155605104075, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.04389962858445029, "chunk_inter_token_latency_p50": 0.04379857537119994, "chunk_inter_token_latency_p90": 0.045383315469637045, "chunk_inter_token_latency_p99": 0.04733554651763099, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 1080.1875, "output_seq_len_p50": 1026.0, "output_seq_len_p90": 1280.0, "output_seq_len_p99": 1356.1, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.10999999986961484, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.011839555202509152, "chunk_inter_token_latency_avg": 0.04449552238793738, "input_tokens": 8190, "output_tokens": 1260, "output_tps_per_user": 84.46263249721328, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.01050842490845493, "chunk_inter_token_latency_avg": 0.0440000000001257, "input_tokens": 8190, "output_tokens": 1366, "output_tps_per_user": 95.16174010011854, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.015244161358919794, "chunk_inter_token_latency_avg": 0.04754966887451141, "input_tokens": 8190, "output_tokens": 943, "output_tps_per_user": 65.59888579340388, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.015728070175483527, "chunk_inter_token_latency_avg": 0.04612218649530861, "input_tokens": 8190, "output_tokens": 913, "output_tps_per_user": 63.58059118777152, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.011042340261771345, "chunk_inter_token_latency_avg": 0.04307507507519813, "input_tokens": 8190, "output_tokens": 1300, "output_tps_per_user": 90.56051310626667, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.014272636815961172, "chunk_inter_token_latency_avg": 0.043466666666790846, "input_tokens": 8190, "output_tokens": 1006, "output_tps_per_user": 70.0641383154719, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.012830053667299623, "chunk_inter_token_latency_avg": 0.04386544342520177, "input_tokens": 8190, "output_tokens": 1119, "output_tps_per_user": 77.94199665343042, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9529999999795109, "time_to_second_token": 0.32899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.015695312499831613, "chunk_inter_token_latency_avg": 0.04464444444396548, "input_tokens": 8190, "output_tokens": 897, "output_tps_per_user": 63.71329019480998, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.014636734693919365, "chunk_inter_token_latency_avg": 0.0437317073171981, "input_tokens": 8190, "output_tokens": 981, "output_tps_per_user": 68.32124930264922, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9529999999795109, "time_to_second_token": 0.32899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.014291666666513339, "chunk_inter_token_latency_avg": 0.04394687499952852, "input_tokens": 8190, "output_tokens": 985, "output_tps_per_user": 69.97084548180024, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.013532075471736772, "chunk_inter_token_latency_avg": 0.04333534743214797, "input_tokens": 8190, "output_tokens": 1061, "output_tps_per_user": 73.8984941436818, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.281999999890104, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013504424778700622, "chunk_inter_token_latency_avg": 0.04136746987933293, "input_tokens": 8190, "output_tokens": 1018, "output_tps_per_user": 74.04980340793298, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6719999997876585, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.011549114331756021, "chunk_inter_token_latency_avg": 0.04294610778455383, "input_tokens": 8190, "output_tokens": 1243, "output_tps_per_user": 86.58672615703094, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.011953002610912561, "chunk_inter_token_latency_avg": 0.042520123838818986, "input_tokens": 8190, "output_tokens": 1150, "output_tps_per_user": 83.66098733108653, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6559999999590218, "time_to_second_token": 0.28100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.013885769603137442, "chunk_inter_token_latency_avg": 0.04440866873077702, "input_tokens": 8190, "output_tokens": 1034, "output_tps_per_user": 72.0161740098333, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013652087475088004, "chunk_inter_token_latency_avg": 0.042918749999807916, "input_tokens": 8190, "output_tokens": 1007, "output_tps_per_user": 73.24887141433686, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 11526, "wall_time": 15.609000000171363, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1152.6, "server_utilization": 0.018989368515434868, "server_spec_accept_rate": 0.4984375, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.594, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 3, "duration_seconds": 5.172, "gpu_count": 1, "cpu_util_avg_pct": 18.2, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 2.67, "gpu_util_max_pct": 5.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 65.03, "power_total_max_w": 68.6, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95090.67, "vram_used_max_mb": 95126.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.14, "vram_used_max_pct": 97.18, "pcie_rx_avg_mb_s": 12.0, "pcie_rx_max_mb_s": 25.0, "pcie_tx_avg_mb_s": 168.0, "pcie_tx_max_mb_s": 497.0 } }, { "concurrency": 32, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.968, "measurement_wall_seconds": 10.015, "client_output_tokens": 15039, "server_output_tokens": 15039, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1508.7279293573633, "per_request_avg_tps": 47.147747792417604, "ttft_avg": 1.3543437499392894, "ttft_p50": 1.7969999997876585, "ttft_p90": 1.8119999999180436, "ttft_p99": 1.8119999999180436, "time_to_second_token_avg": 0.23849999997764826, "time_to_second_token_p50": 0.25, "time_to_second_token_p90": 0.42820000001229386, "time_to_second_token_p99": 0.8564499999559484, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.021823996697065102, "inter_token_latency_p50": 0.021768742839598617, "inter_token_latency_p90": 0.02428331729301932, "inter_token_latency_p99": 0.028389840692976388, "output_tps_per_user_avg": 46.67443419396653, "output_tps_per_user_p50": 45.93884550757099, "output_tps_per_user_p90": 51.3026344588124, "output_tps_per_user_p99": 71.53406767230199, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.05490922549024426, "chunk_inter_token_latency_p50": 0.053377431907042396, "chunk_inter_token_latency_p90": 0.057189189189771714, "chunk_inter_token_latency_p99": 0.058414855545725654, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 661.03125, "output_seq_len_p50": 648.5, "output_seq_len_p90": 741.0000000000001, "output_seq_len_p99": 999.0800000000004, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.02242716857645228, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 612, "output_tps_per_user": 44.5887761797075, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.020033625731304593, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 685, "output_tps_per_user": 49.91607677073639, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.022752688172274767, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 652, "output_tps_per_user": 43.95085066117803, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.019935397039234016, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 744, "output_tps_per_user": 50.16203078533837, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02159183673491381, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 687, "output_tps_per_user": 46.31379962145642, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.797000000020489, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.023055462185058648, "chunk_inter_token_latency_avg": 0.053377431907042396, "input_tokens": 8190, "output_tokens": 596, "output_tps_per_user": 43.3736696307941, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.019744956772640262, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 695, "output_tps_per_user": 50.645843974986924, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.020920610687347085, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 656, "output_tps_per_user": 47.79975187840985, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.0207741935485987, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 714, "output_tps_per_user": 48.1366459622426, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.797000000020489, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02377469670729618, "chunk_inter_token_latency_avg": 0.053377431907042396, "input_tokens": 8190, "output_tokens": 578, "output_tps_per_user": 42.061525003307885, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.02287646076830107, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 600, "output_tps_per_user": 43.713055534606866, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.021613564669104638, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 635, "output_tps_per_user": 46.26724074948373, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.01926137841372025, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 770, "output_tps_per_user": 51.91736429868803, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.020300740741055322, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 676, "output_tps_per_user": 49.25928628691091, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.019464488636665257, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 705, "output_tps_per_user": 51.37561117923745, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.022041666666891183, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 673, "output_tps_per_user": 45.36862003734506, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.797000000020489, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.024322695035655845, "chunk_inter_token_latency_avg": 0.053377431907042396, "input_tokens": 8190, "output_tokens": 565, "output_tps_per_user": 41.11386499456785, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.125, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.024983766233977897, "chunk_inter_token_latency_avg": 0.05896551724187887, "input_tokens": 8190, "output_tokens": 617, "output_tps_per_user": 40.02599090284478, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.02012187958915175, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 682, "output_tps_per_user": 49.69714660946123, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.022928792569892994, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 647, "output_tps_per_user": 43.6132865239954, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02317996870133157, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 640, "output_tps_per_user": 43.14069673193973, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.02188977635816668, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 627, "output_tps_per_user": 45.683426986083305, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.0193874345551713, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 765, "output_tps_per_user": 51.5798001615054, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9529999999795109, "time_to_second_token": 0.8440000000409782, "latency": 0.0, "inter_token_latency_avg": 0.022824451410894786, "chunk_inter_token_latency_avg": 0.056441860465701064, "input_tokens": 8190, "output_tokens": 639, "output_tps_per_user": 43.81266309527468, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.797000000020489, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02094351145054946, "chunk_inter_token_latency_avg": 0.053377431907042396, "input_tokens": 8190, "output_tokens": 656, "output_tps_per_user": 47.74748505574812, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.8119999999180436, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012445958219993045, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 1102, "output_tps_per_user": 80.34736918798357, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7029999999795109, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.023928917609290588, "chunk_inter_token_latency_avg": 0.057189189189771714, "input_tokens": 8190, "output_tokens": 620, "output_tps_per_user": 41.79044018320921, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.7969999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.026100952381356842, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 526, "output_tps_per_user": 38.31277822315293, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.7969999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.022537828947717668, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 609, "output_tps_per_user": 44.36984601843235, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.781999999890104, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02113713405255762, "chunk_inter_token_latency_avg": 0.053377431907042396, "input_tokens": 8190, "output_tokens": 650, "output_tps_per_user": 47.31010351325272, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.7969999997876585, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.021647709321030554, "chunk_inter_token_latency_avg": 0.05331906614868615, "input_tokens": 8190, "output_tokens": 634, "output_tps_per_user": 46.19426402905868, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9379999998491257, "time_to_second_token": 0.8589999999385327, "latency": 0.0, "inter_token_latency_avg": 0.029418181818486613, "chunk_inter_token_latency_avg": 0.056441860465701064, "input_tokens": 8190, "output_tokens": 496, "output_tps_per_user": 33.99258343598897, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 15039, "wall_time": 16.109999999869615, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1501.6475287223002, "server_utilization": 0.03548097860894073, "server_spec_accept_rate": 0.32083333333333336, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.079, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.563, "gpu_count": 1, "cpu_util_avg_pct": 16.52, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 3.0, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 33.5, "temp_max_c": 34.0, "power_total_avg_w": 54.87, "power_total_max_w": 64.6, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95030.25, "vram_used_max_mb": 95144.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.08, "vram_used_max_pct": 97.2, "pcie_rx_avg_mb_s": 3.5, "pcie_rx_max_mb_s": 10.0, "pcie_tx_avg_mb_s": 173.75, "pcie_tx_max_mb_s": 624.0 } }, { "concurrency": 64, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 24259, "server_output_tokens": 24445, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 2425.9, "per_request_avg_tps": 37.9046875, "ttft_avg": 3.602640624914784, "ttft_p50": 2.3589999999385327, "ttft_p90": 7.405999999959022, "ttft_p99": 7.411919999981765, "time_to_second_token_avg": 2.469359375016211, "time_to_second_token_p50": 0.75, "time_to_second_token_p90": 5.047000000020489, "time_to_second_token_p99": 5.047000000020489, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.03036844281975937, "inter_token_latency_p50": 0.03183018274944936, "inter_token_latency_p90": 0.0356510261847566, "inter_token_latency_p99": 0.03813688527784272, "output_tps_per_user_avg": 33.82388805925092, "output_tps_per_user_p50": 31.417721518987342, "output_tps_per_user_p90": 42.82119295387862, "output_tps_per_user_p99": 48.08106578872237, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0992531367636074, "chunk_inter_token_latency_p50": 0.10505319148936171, "chunk_inter_token_latency_p90": 0.11225633528265107, "chunk_inter_token_latency_p99": 0.11566374316719412, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 613.296875, "output_seq_len_p50": 611.0, "output_seq_len_p90": 663.5, "output_seq_len_p99": 709.3299999999999, "request_count": 64, "completed_request_count": 0, "request_samples": [ { "ttft": 7.422000000020489, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022619999999968478, "chunk_inter_token_latency_avg": 0.07947567567556492, "input_tokens": 8190, "output_tokens": 651, "output_tps_per_user": 44.20866489838168, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.375, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.02925925925925926, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 676, "output_tps_per_user": 34.177215189873415, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.375, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03313758389261745, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 597, "output_tps_per_user": 30.17721518987342, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.035357737104825294, "chunk_inter_token_latency_avg": 0.11303191489361702, "input_tokens": 8190, "output_tokens": 602, "output_tps_per_user": 28.28235294117647, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.031650641025641024, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 625, "output_tps_per_user": 31.59493670886076, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.035240464344941956, "chunk_inter_token_latency_avg": 0.1118421052631579, "input_tokens": 8190, "output_tokens": 604, "output_tps_per_user": 28.376470588235296, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023791262135889175, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 619, "output_tps_per_user": 42.03223831876904, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.03728070175438596, "chunk_inter_token_latency_avg": 0.1118421052631579, "input_tokens": 8190, "output_tokens": 571, "output_tps_per_user": 26.823529411764707, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03324915824915825, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 595, "output_tps_per_user": 30.075949367088608, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.037677304964539006, "chunk_inter_token_latency_avg": 0.11243386243386243, "input_tokens": 8190, "output_tokens": 565, "output_tps_per_user": 26.541176470588237, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.020855319148907106, "chunk_inter_token_latency_avg": 0.0816833333332195, "input_tokens": 8190, "output_tokens": 706, "output_tps_per_user": 47.9493980820909, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03399311531841653, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 582, "output_tps_per_user": 29.41772151898734, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03706168224295236, "chunk_inter_token_latency_avg": 0.10894505494494237, "input_tokens": 8190, "output_tokens": 536, "output_tps_per_user": 26.98204559211987, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.020914651493569716, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 704, "output_tps_per_user": 47.81337142086511, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.031200631911532384, "chunk_inter_token_latency_avg": 0.10792349726775956, "input_tokens": 8190, "output_tokens": 634, "output_tps_per_user": 32.050632911392405, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03206168831168831, "chunk_inter_token_latency_avg": 0.10561497326203209, "input_tokens": 8190, "output_tokens": 617, "output_tps_per_user": 31.189873417721518, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03364565587734242, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 588, "output_tps_per_user": 29.721518987341774, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022689814814783195, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 649, "output_tps_per_user": 44.072638237155886, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03269867549668874, "chunk_inter_token_latency_avg": 0.10618279569892473, "input_tokens": 8190, "output_tokens": 605, "output_tps_per_user": 30.582278481012658, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1559999999590218, "time_to_second_token": 1.202999999979511, "latency": 0.0, "inter_token_latency_avg": 0.03233487654317826, "chunk_inter_token_latency_avg": 0.11086243386232546, "input_tokens": 8190, "output_tokens": 649, "output_tps_per_user": 30.926358993969057, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02521955403083964, "chunk_inter_token_latency_avg": 0.07990760869554082, "input_tokens": 8190, "output_tokens": 584, "output_tps_per_user": 39.65177174731772, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03237704918032787, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 611, "output_tps_per_user": 30.88607594936709, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03353140916808149, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 590, "output_tps_per_user": 29.822784810126585, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02521955403083964, "chunk_inter_token_latency_avg": 0.07990760869554082, "input_tokens": 8190, "output_tokens": 584, "output_tps_per_user": 39.65177174731772, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.030907668231611892, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 640, "output_tps_per_user": 32.35443037974684, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.031399046104928455, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 630, "output_tps_per_user": 31.848101265822788, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023264240506296694, "chunk_inter_token_latency_avg": 0.08034426229497, "input_tokens": 8190, "output_tokens": 633, "output_tps_per_user": 42.98442494734957, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.03891941391941392, "chunk_inter_token_latency_avg": 0.11424731182795698, "input_tokens": 8190, "output_tokens": 547, "output_tps_per_user": 25.694117647058825, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.03577441077441077, "chunk_inter_token_latency_avg": 0.11675824175824176, "input_tokens": 8190, "output_tokens": 595, "output_tps_per_user": 27.952941176470592, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.030572755417956655, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 647, "output_tps_per_user": 32.708860759493675, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.030279503105590064, "chunk_inter_token_latency_avg": 0.08602941176470588, "input_tokens": 8190, "output_tokens": 484, "output_tps_per_user": 33.02564102564102, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025794736842069316, "chunk_inter_token_latency_avg": 0.08034426229497, "input_tokens": 8190, "output_tokens": 571, "output_tps_per_user": 38.76759844935009, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03024502297090352, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 654, "output_tps_per_user": 33.063291139240505, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.03547579298831385, "chunk_inter_token_latency_avg": 0.11303191489361702, "input_tokens": 8190, "output_tokens": 600, "output_tps_per_user": 28.18823529411765, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.020701680672211516, "chunk_inter_token_latency_avg": 0.07989729729707579, "input_tokens": 8190, "output_tokens": 715, "output_tps_per_user": 48.30525674866244, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.034169550173010384, "chunk_inter_token_latency_avg": 0.10561497326203209, "input_tokens": 8190, "output_tokens": 579, "output_tps_per_user": 29.265822784810123, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1559999999590218, "time_to_second_token": 1.202999999979511, "latency": 0.0, "inter_token_latency_avg": 0.03243498452009212, "chunk_inter_token_latency_avg": 0.11145212765946548, "input_tokens": 8190, "output_tokens": 647, "output_tps_per_user": 30.83090726867903, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.029044117647058824, "chunk_inter_token_latency_avg": 0.10911602209944751, "input_tokens": 8190, "output_tokens": 681, "output_tps_per_user": 34.43037974683544, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03319327731092437, "chunk_inter_token_latency_avg": 0.10675675675675676, "input_tokens": 8190, "output_tokens": 596, "output_tps_per_user": 30.126582278481013, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024794266441786696, "chunk_inter_token_latency_avg": 0.07904838709666404, "input_tokens": 8190, "output_tokens": 594, "output_tps_per_user": 40.33190505344667, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025393782383384304, "chunk_inter_token_latency_avg": 0.07904838709666404, "input_tokens": 8190, "output_tokens": 580, "output_tps_per_user": 39.37971842486614, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.031004709576138146, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 638, "output_tps_per_user": 32.25316455696203, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02255061349690109, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 653, "output_tps_per_user": 44.34469155960747, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3589999999385327, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.0350177304964539, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 565, "output_tps_per_user": 28.556962025316455, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.405999999959022, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02434271523175416, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 605, "output_tps_per_user": 41.08005169018851, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.033570300157977885, "chunk_inter_token_latency_avg": 0.11486486486486487, "input_tokens": 8190, "output_tokens": 634, "output_tps_per_user": 29.788235294117644, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.390999999828637, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.027430970149215507, "chunk_inter_token_latency_avg": 0.0821396648043548, "input_tokens": 8190, "output_tokens": 537, "output_tps_per_user": 36.45514520851166, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.390999999828637, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023562499999967165, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 625, "output_tps_per_user": 42.44031830244641, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.125, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.036984848484525504, "chunk_inter_token_latency_avg": 0.1150209424073725, "input_tokens": 8190, "output_tokens": 595, "output_tps_per_user": 27.038099139932964, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.0320097244732577, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 618, "output_tps_per_user": 31.240506329113924, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.028916544655929723, "chunk_inter_token_latency_avg": 0.10618279569892473, "input_tokens": 8190, "output_tokens": 684, "output_tps_per_user": 34.58227848101266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.032324058919803604, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 612, "output_tps_per_user": 30.93670886075949, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.390999999828637, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024231147540916428, "chunk_inter_token_latency_avg": 0.07862234042531395, "input_tokens": 8190, "output_tokens": 611, "output_tps_per_user": 41.26919694213458, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.390999999828637, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025090443685971863, "chunk_inter_token_latency_avg": 0.07862566844908829, "input_tokens": 8190, "output_tokens": 587, "output_tps_per_user": 39.8558117391564, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8439999998081475, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.032364188163853586, "chunk_inter_token_latency_avg": 0.11166492146586131, "input_tokens": 8190, "output_tokens": 660, "output_tps_per_user": 30.89834958742653, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.029743975903614456, "chunk_inter_token_latency_avg": 0.10618279569892473, "input_tokens": 8190, "output_tokens": 665, "output_tps_per_user": 33.620253164556964, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 7.390999999828637, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02838416988413033, "chunk_inter_token_latency_avg": 0.08401714285702577, "input_tokens": 8190, "output_tokens": 519, "output_tps_per_user": 35.230905257479556, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8439999998081475, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.03284389489953632, "chunk_inter_token_latency_avg": 0.1118421052631579, "input_tokens": 8190, "output_tokens": 648, "output_tps_per_user": 30.447058823529414, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03206168831168831, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 617, "output_tps_per_user": 31.189873417721518, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03297161936560935, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 600, "output_tps_per_user": 30.32911392405063, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8439999998081475, "time_to_second_token": 0.2970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.033676703645007924, "chunk_inter_token_latency_avg": 0.1118421052631579, "input_tokens": 8190, "output_tokens": 632, "output_tps_per_user": 29.694117647058825, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03271947194716091, "chunk_inter_token_latency_avg": 0.1049100529099445, "input_tokens": 8190, "output_tokens": 607, "output_tps_per_user": 30.56284042770961, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.035726126126089205, "chunk_inter_token_latency_avg": 0.10717837837826763, "input_tokens": 8190, "output_tokens": 556, "output_tps_per_user": 27.99072019369445, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3439999998081475, "time_to_second_token": 5.047000000020489, "latency": 0.0, "inter_token_latency_avg": 0.03062015503875969, "chunk_inter_token_latency_avg": 0.10505319148936171, "input_tokens": 8190, "output_tokens": 646, "output_tps_per_user": 32.65822784810127, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 24259, "wall_time": 22.92200000002049, "num_completed": 64, "num_errors": 0, "server_gen_throughput": 2436.9454689993486, "server_utilization": 0.0020494428077366678, "server_spec_accept_rate": 0.39557291666666666, "server_spec_accept_length": 0.0, "avg_running_reqs": 64, "max_running_reqs": 64, "effective_concurrency": 64, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 12.719, "ready_reason": "running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 5, "duration_seconds": 9.375, "gpu_count": 1, "cpu_util_avg_pct": 17.7, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.8, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 69.92, "power_total_max_w": 70.71, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94954.0, "vram_used_max_mb": 94981.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.0, "vram_used_max_pct": 97.03, "pcie_rx_avg_mb_s": 7.6, "pcie_rx_max_mb_s": 9.0, "pcie_tx_avg_mb_s": 64.6, "pcie_tx_max_mb_s": 210.0 } } ], "summary_table": { "0": { "1": 219.7, "2": 388.5, "4": 545.6, "8": 813.4014423127, "16": 1130.0080128274699, "32": 1460.8626197993415, "64": 2282.2 }, "8192": { "1": 396.3, "2": 382.5, "4": 710.9335995913434, "8": 947.1361219742389, "16": 1156.18417092513, "32": 1508.7279293573633, "64": 2425.9 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": true, "mode": "standalone_cold", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." }, "coding_peak": { "name": "Coding Peak", "present": true, "formula": "usage.completion_tokens / (last_stream_time - first_token_time)", "notes": "Sequential cc1 Sieve-of-Eratosthenes coding prompt, matching /mnt/test.py throughput semantics. Uses OpenAI stream usage with continuous_usage_stats when the server supports it." } }, "coding_peak": { "mode": "coding_peak", "prompt": "Write a Python script that implements the Sieve of Eratosthenes.", "runs_requested": 5, "runs_ok": 5, "max_tokens": 2000, "temperature": null, "summary": { "mean_generation_tok_s": 297.6971142704723, "median_generation_tok_s": 296.98548565507724, "max_generation_tok_s": 306.3333333333333, "min_generation_tok_s": 293.3612440359695, "cjk_runs": 0 }, "samples": [ { "run": 1, "ok": true, "finish_reason": "stop", "completion_tokens": 981, "ttft": 0.07800000021234155, "gen_elapsed": 3.3439999998081475, "total_elapsed": 3.422000000020489, "generation_tok_s": 293.3612440359695, "total_tok_s": 286.6744593787628, "content_chars": 2504, "reasoning_chars": 1409, "cjk_chars": 0, "content_preview": "Below is a Python script implementing the **Sieve of Eratosthenes** algorithm. It efficiently finds all prime numbers up to a given integer `n`.\n\n```python\n\"\"\"\nSieve of Eratosthenes - Python implementation\nFinds all prime numbers up to a given limit n.\n\"\"\"\n\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers <= n using the Sieve of Eratosthenes.\n \n Args:\n n (int): Upper bound (inclusive).\n\n Returns:\n list[int]: Sorted list of prim" }, { "run": 2, "ok": true, "finish_reason": "stop", "completion_tokens": 919, "ttft": 0.0940000000409782, "gen_elapsed": 3.0, "total_elapsed": 3.094000000040978, "generation_tok_s": 306.3333333333333, "total_tok_s": 297.0265029049219, "content_chars": 2412, "reasoning_chars": 1011, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the **Sieve of Eratosthenes**, a classic algorithm for finding all prime numbers up to a given limit `n`. The script includes a well-documented function, input handling, and a usage example.\n\n```python\nimport math\n\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n.\n\n Uses the Sieve of Eratosthenes algorithm.\n - Time complexity: O(n log log n)\n - Space complexity: O(n)\n \"\"\"\n if " }, { "run": 3, "ok": true, "finish_reason": "stop", "completion_tokens": 721, "ttft": 0.0779999999795109, "gen_elapsed": 2.422000000020489, "total_elapsed": 2.5, "generation_tok_s": 297.68786126915796, "total_tok_s": 288.4, "content_chars": 2018, "reasoning_chars": 668, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the Sieve of Eratosthenes, which efficiently finds all prime numbers up to a given limit `n`.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n.\n\n Uses the classic Sieve of Eratosthenes algorithm.\n \"\"\"\n if n < 2:\n return []\n\n # Step 1: Create a boolean array \"is_prime[0..n]\" and initialize\n # all entries as True. We'll later mark non-primes as False.\n is_pr" }, { "run": 4, "ok": true, "finish_reason": "stop", "completion_tokens": 798, "ttft": 0.0779999999795109, "gen_elapsed": 2.6870000001508743, "total_elapsed": 2.765000000130385, "generation_tok_s": 296.98548565507724, "total_tok_s": 288.6075949230994, "content_chars": 2497, "reasoning_chars": 327, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the **Sieve of Eratosthenes**, an efficient algorithm for finding all prime numbers up to a given limit `n`.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n.\n\n Parameters:\n n (int): Upper bound (inclusive). Must be >= 2.\n\n Returns:\n list[int]: Sorted list of primes <= n.\n \"\"\"\n if n < 2:\n return []\n\n # Step 1: Create a boolean array \"prime[0..n]" }, { "run": 5, "ok": true, "finish_reason": "stop", "completion_tokens": 625, "ttft": 0.09399999980814755, "gen_elapsed": 2.125, "total_elapsed": 2.2189999998081475, "generation_tok_s": 294.11764705882354, "total_tok_s": 281.65840471114774, "content_chars": 1858, "reasoning_chars": 622, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the **Sieve of Eratosthenes**, a classic algorithm for finding all prime numbers up to a given limit `n`. The script is efficient, has clear comments, and includes a simple test case.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Returns a list of all prime numbers up to and including n.\n\n Args:\n n (int): Upper bound (inclusive).\n\n Returns:\n list[int]: List of primes <= n.\n\n Raises:\n ValueError: If n is ne" } ] } }