{ "metadata": { "version": "0.4.29", "engine": "vllm", "model": "DeepSeek-V4-Flash", "server": "192.168.0.213:8000", "timestamp": "2026-07-11T00:44:08.672656", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 10.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "standalone_cold", "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 3.0, "decode_warmup_context": 8192, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 0.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 8192, "temperature": null, "ignore_eos": true, "max_total_tokens": 3287424, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1, 2, 4, 8, 16, 32, 64 ], "context_lengths": [ 0, 8192 ], "startup_diagnostics_available": true, "nvidia_p2p_override_effective": false, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.29", "server_url": "http://192.168.0.213:8000", "hostname": "DESKTOP-RMAJ6JK", "uname": "", "env": {}, "args": { "concurrency": "1,2,4,8,16,32,64", "contexts": "0,8k", "max_tokens": 8192, "duration": 10.0, "request_count": 0, "run_burst": false, "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "prefill_contexts": "8k,64k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 3287424 }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 595.79, 00000000:01:00.0, 5, 16, 450.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 255, "stdout": "ERROR: Option -m is missing its value. Please run 'nvidia-smi -h' for help.", "stderr": "" } }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": { "samples": 116, "duration_seconds": 289.578, "gpu_count": 1, "cpu_util_avg_pct": 17.7, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.28, "gpu_util_max_pct": 12.0, "mem_util_avg_pct": 0.98, "mem_util_max_pct": 2.0, "temp_avg_c": 34.2, "temp_max_c": 35.0, "power_total_avg_w": 67.88, "power_total_max_w": 82.06, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95170.87, "vram_used_max_mb": 95400.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.23, "vram_used_max_pct": 97.46, "pcie_rx_avg_mb_s": 49.84, "pcie_rx_max_mb_s": 1942.0, "pcie_tx_avg_mb_s": 188.87, "pcie_tx_max_mb_s": 1207.0 }, "event_log": [ "00:39:16 benchmark start engine=vllm", "00:39:16 startup server=http://192.168.0.213:8000 model=DeepSeek-V4-Flash", "00:39:16 startup decode concurrency=1,2,4,8,16,32,64 contexts=0,8k", "00:39:16 startup NVIDIA P2P override: unknown: /proc/driver/nvidia/params is not readable", "00:39:16 startup KV cache budget manual: 3,287,424 tokens", "00:39:16 startup engine vLLM 0.11.2.dev279+eldritch.enlightenment.v45c1582.b12xf3686b5.patchc1441b5.fi25dd814.cu132.20260704 models=['DeepSeek-V4-Flash']", "00:39:16 startup model context length: 524,288 tokens", "00:39:16 startup prefill tests: standalone cold profile ['8k', '64k']", "00:39:16 startup calibrating padding text run=uukhkohzxccp up_to=64k", "00:39:16 startup token targeting: estimate from 8k", "00:39:16 startup calibrated: 6.25 chars/token (cached, source=8k)", "00:39:16 startup context 8k: 51,188 chars (~8,191 tokens)", "00:39:16 startup context 64k: 409,507 chars (~65,535 tokens)", "00:39:16 startup startup preparation done", "00:39:16 hardware monitor interval=2s", "00:39:17 prefill warmup start", "00:39:18 prefill start ctx=8k", "00:39:28 prefill done ctx=8k 11,645 tok/s", "00:39:29 prefill start ctx=64k", "00:39:41 prefill done ctx=64k 10,580 tok/s", "00:39:42 decode warmup start C=1 ctx=8k 3s", "00:39:43 cell start C=1 ctx=8k", "00:39:49 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "00:39:52 cell done C=1 ctx=8k 211.7 tok/s", "00:39:52 decode warmup done C=1 ctx=8k", "00:39:54 cell start C=1 ctx=0", "00:39:59 ready C=1 ctx=0 running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "00:40:09 cell done C=1 ctx=0 213.2 tok/s", "00:40:11 cell start C=1 ctx=8k", "00:40:17 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "00:40:27 cell done C=1 ctx=8k 272.1 tok/s", "00:40:29 cell start C=2 ctx=0", "00:40:34 ready C=2 ctx=0 running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "00:40:44 cell done C=2 ctx=0 358.8 tok/s", "00:40:46 cell start C=4 ctx=0", "00:40:53 ready C=4 ctx=0 running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "00:41:03 cell done C=4 ctx=0 600.8 tok/s", "00:41:05 cell start C=8 ctx=0", "00:41:10 ready C=8 ctx=0 running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "00:41:20 cell done C=8 ctx=0 899.1 tok/s", "00:41:22 cell start C=16 ctx=0", "00:41:27 ready C=16 ctx=0 running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "00:41:37 cell done C=16 ctx=0 1284.0 tok/s", "00:41:39 cell start C=32 ctx=0", "00:41:46 ready C=32 ctx=0 running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "00:41:56 cell done C=32 ctx=0 1875.3 tok/s", "00:41:58 cell start C=64 ctx=0", "00:42:04 ready C=64 ctx=0 running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "00:42:14 cell done C=64 ctx=0 2689.5 tok/s", "00:42:17 cell start C=2 ctx=8k", "00:42:23 ready C=2 ctx=8k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "00:42:33 cell done C=2 ctx=8k 386.5 tok/s", "00:42:35 cell start C=4 ctx=8k", "00:42:41 ready C=4 ctx=8k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "00:42:51 cell done C=4 ctx=8k 581.9 tok/s", "00:42:53 cell start C=8 ctx=8k", "00:42:59 ready C=8 ctx=8k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "00:43:09 cell done C=8 ctx=8k 856.7 tok/s", "00:43:11 cell start C=16 ctx=8k", "00:43:18 ready C=16 ctx=8k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "00:43:28 cell done C=16 ctx=8k 1158.2 tok/s", "00:43:30 cell start C=32 ctx=8k", "00:43:37 ready C=32 ctx=8k running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "00:43:47 cell done C=32 ctx=8k 1696.1 tok/s", "00:43:49 cell start C=64 ctx=8k", "00:43:56 ready C=64 ctx=8k running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "00:44:06 cell done C=64 ctx=8k 2644.5 tok/s" ], "prefill": { "8192": { "ttft_seconds": 0.704, "prefill_seconds": 0.704, "tok_per_sec": 11645.0, "client_ttft_seconds": 0.704, "client_tok_per_sec": 11645.0, "prompt_tokens": 8192, "samples": 10, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 4, "duration_seconds": 7.359, "gpu_count": 1, "cpu_util_avg_pct": 16.23, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.25, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 33.25, "temp_max_c": 34.0, "power_total_avg_w": 66.92, "power_total_max_w": 67.13, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95265.5, "vram_used_max_mb": 95269.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.32, "vram_used_max_pct": 97.33, "pcie_rx_avg_mb_s": 8.25, "pcie_rx_max_mb_s": 10.0, "pcie_tx_avg_mb_s": 4.5, "pcie_tx_max_mb_s": 6.0 } }, "65536": { "ttft_seconds": 6.101, "prefill_seconds": 6.101, "tok_per_sec": 10580.0, "client_ttft_seconds": 6.101, "client_tok_per_sec": 10580.0, "prompt_tokens": 64556, "samples": 2, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": { "samples": 6, "duration_seconds": 11.625, "gpu_count": 1, "cpu_util_avg_pct": 16.2, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.33, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 66.62, "power_total_max_w": 67.16, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95267.67, "vram_used_max_mb": 95270.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.32, "vram_used_max_pct": 97.33, "pcie_rx_avg_mb_s": 12.17, "pcie_rx_max_mb_s": 34.0, "pcie_tx_avg_mb_s": 68.17, "pcie_tx_max_mb_s": 390.0 } } }, "results": [ { "concurrency": 1, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 2132, "server_output_tokens": 2132, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 213.2, "per_request_avg_tps": 213.2, "ttft_avg": 0.0779999999795109, "ttft_p50": 0.0779999999795109, "ttft_p90": 0.0779999999795109, "ttft_p99": 0.0779999999795109, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.004794987146536147, "inter_token_latency_p50": 0.004794987146536147, "inter_token_latency_p90": 0.004794987146536147, "inter_token_latency_p99": 0.004794987146536147, "output_tps_per_user_avg": 208.5511325556713, "output_tps_per_user_p50": 208.5511325556713, "output_tps_per_user_p90": 208.5511325556713, "output_tps_per_user_p99": 208.5511325556713, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.011020679468257378, "chunk_inter_token_latency_p50": 0.011020679468257378, "chunk_inter_token_latency_p90": 0.011020679468257378, "chunk_inter_token_latency_p99": 0.011020679468257378, "input_seq_len_avg": 65.0, "output_seq_len_avg": 3113.0, "output_seq_len_p50": 3113.0, "output_seq_len_p90": 3113.0, "output_seq_len_p99": 3113.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.004794987146536147, "chunk_inter_token_latency_avg": 0.011020679468257378, "input_tokens": 65, "output_tokens": 3113, "output_tps_per_user": 208.5511325556713, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2132, "wall_time": 15.016000000061467, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 212.8594249188215, "server_utilization": 0.001196521333099887, "server_spec_accept_rate": 0.6296296296296297, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.0, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 3, "duration_seconds": 5.047, "gpu_count": 1, "cpu_util_avg_pct": 21.13, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 7.33, "gpu_util_max_pct": 10.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 71.17, "power_total_max_w": 74.81, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95283.33, "vram_used_max_mb": 95400.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.34, "vram_used_max_pct": 97.46, "pcie_rx_avg_mb_s": 107.0, "pcie_rx_max_mb_s": 297.0, "pcie_tx_avg_mb_s": 72.33, "pcie_tx_max_mb_s": 190.0 } }, { "concurrency": 1, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 2721, "server_output_tokens": 2721, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 272.1, "per_request_avg_tps": 272.1, "ttft_avg": 0.09399999980814755, "ttft_p50": 0.09399999980814755, "ttft_p90": 0.09399999980814755, "ttft_p99": 0.09399999980814755, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.0038778645833493406, "inter_token_latency_p50": 0.0038778645833493406, "inter_token_latency_p90": 0.0038778645833493406, "inter_token_latency_p99": 0.0038778645833493406, "output_tps_per_user_avg": 257.87388355276, "output_tps_per_user_p50": 257.87388355276, "output_tps_per_user_p90": 257.87388355276, "output_tps_per_user_p99": 257.87388355276, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01098966789672433, "chunk_inter_token_latency_p50": 0.01098966789672433, "chunk_inter_token_latency_p90": 0.01098966789672433, "chunk_inter_token_latency_p99": 0.01098966789672433, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 3841.0, "output_seq_len_p50": 3841.0, "output_seq_len_p90": 3841.0, "output_seq_len_p99": 3841.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.09399999980814755, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.0038778645833493406, "chunk_inter_token_latency_avg": 0.01098966789672433, "input_tokens": 8190, "output_tokens": 3841, "output_tps_per_user": 257.87388355276, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2721, "wall_time": 16.015000000130385, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 272.1, "server_utilization": 0.0021887585361582973, "server_spec_accept_rate": 1.0, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.0, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 3, "duration_seconds": 7.281, "gpu_count": 1, "cpu_util_avg_pct": 21.83, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.33, "gpu_util_max_pct": 8.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 68.72, "power_total_max_w": 71.72, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95204.33, "vram_used_max_mb": 95234.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.26, "vram_used_max_pct": 97.29, "pcie_rx_avg_mb_s": 10.33, "pcie_rx_max_mb_s": 12.0, "pcie_tx_avg_mb_s": 757.33, "pcie_tx_max_mb_s": 1162.0 } }, { "concurrency": 2, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 3588, "server_output_tokens": 3588, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 358.8, "per_request_avg_tps": 179.4, "ttft_avg": 0.1404999999795109, "ttft_p50": 0.1404999999795109, "ttft_p90": 0.1904999999795109, "ttft_p99": 0.2017499999795109, "time_to_second_token_avg": 0.07050000003073364, "time_to_second_token_p50": 0.07050000003073364, "time_to_second_token_p90": 0.11410000000614673, "time_to_second_token_p99": 0.12391000000061467, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005703438134360188, "inter_token_latency_p50": 0.005703438134360188, "inter_token_latency_p90": 0.005748020702910784, "inter_token_latency_p99": 0.005758051780834668, "output_tps_per_user_avg": 175.34957997462368, "output_tps_per_user_p50": 175.34957997462368, "output_tps_per_user_p90": 176.72025052199763, "output_tps_per_user_p99": 177.02865139515677, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013074817749965382, "chunk_inter_token_latency_p50": 0.013074817749965382, "chunk_inter_token_latency_p90": 0.013123414254232859, "chunk_inter_token_latency_p99": 0.01313434846769304, "input_seq_len_avg": 65.0, "output_seq_len_avg": 2606.5, "output_seq_len_p50": 2606.5, "output_seq_len_p90": 2618.1, "output_seq_len_p99": 2620.71, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.2029999999795109, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.005647709923671943, "chunk_inter_token_latency_avg": 0.013014072119631038, "input_tokens": 65, "output_tokens": 2621, "output_tps_per_user": 177.06291815884111, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0779999999795109, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.005759166345048433, "chunk_inter_token_latency_avg": 0.013135563380299727, "input_tokens": 65, "output_tokens": 2592, "output_tps_per_user": 173.63624179040627, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3588, "wall_time": 15.016000000061467, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 358.2268370585045, "server_utilization": 0.0021887585361582973, "server_spec_accept_rate": 0.618421052631579, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.0, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.61, "gpu_count": 1, "cpu_util_avg_pct": 20.82, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.75, "gpu_util_max_pct": 8.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 68.79, "power_total_max_w": 73.79, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95228.0, "vram_used_max_mb": 95236.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.28, "vram_used_max_pct": 97.29, "pcie_rx_avg_mb_s": 63.0, "pcie_rx_max_mb_s": 194.0, "pcie_tx_avg_mb_s": 434.0, "pcie_tx_max_mb_s": 1174.0 } }, { "concurrency": 4, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 6008, "server_output_tokens": 6008, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 600.8, "per_request_avg_tps": 150.2, "ttft_avg": 0.20374999986961484, "ttft_p50": 0.17249999986961484, "ttft_p90": 0.32249999986961486, "ttft_p99": 0.3562499998696148, "time_to_second_token_avg": 0.125, "time_to_second_token_p50": 0.125, "time_to_second_token_p90": 0.125, "time_to_second_token_p99": 0.125, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006861438022417697, "inter_token_latency_p50": 0.006874025004954431, "inter_token_latency_p90": 0.006942881784422899, "inter_token_latency_p99": 0.006954908399342884, "output_tps_per_user_avg": 145.76267416824848, "output_tps_per_user_p50": 145.47954680977938, "output_tps_per_user_p90": 147.71819142689372, "output_tps_per_user_p99": 148.27410921902526, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.015726807441654805, "chunk_inter_token_latency_p50": 0.015753233823886, "chunk_inter_token_latency_p90": 0.015805984555984556, "chunk_inter_token_latency_p99": 0.015805984555984556, "input_seq_len_avg": 65.0, "output_seq_len_avg": 2374.25, "output_seq_len_p50": 2366.5, "output_seq_len_p90": 2414.4, "output_seq_len_p99": 2428.44, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 0.10999999986961484, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.006741457389872376, "chunk_inter_token_latency_avg": 0.015805984555984556, "input_tokens": 65, "output_tokens": 2430, "output_tps_per_user": 148.33587786259542, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10999999986961484, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.0069562446898895494, "chunk_inter_token_latency_avg": 0.015805984555984556, "input_tokens": 65, "output_tokens": 2355, "output_tps_per_user": 143.7557251908397, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.35999999986961484, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.006911701671667381, "chunk_inter_token_latency_avg": 0.015594777562862669, "input_tokens": 65, "output_tokens": 2334, "output_tps_per_user": 144.68217054263567, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.23499999986961484, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.006836348338241481, "chunk_inter_token_latency_avg": 0.01570048309178744, "input_tokens": 65, "output_tokens": 2378, "output_tps_per_user": 146.27692307692308, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 6008, "wall_time": 16.5, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 599.900149767527, "server_utilization": 0.004377517072316595, "server_spec_accept_rate": 0.718421052631579, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.485, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 8.578, "gpu_count": 1, "cpu_util_avg_pct": 19.73, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.25, "gpu_util_max_pct": 9.0, "mem_util_avg_pct": 0.75, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 68.52, "power_total_max_w": 72.54, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95251.0, "vram_used_max_mb": 95279.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.31, "vram_used_max_pct": 97.34, "pcie_rx_avg_mb_s": 31.25, "pcie_rx_max_mb_s": 88.0, "pcie_tx_avg_mb_s": 351.75, "pcie_tx_max_mb_s": 1170.0 } }, { "concurrency": 8, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.015, "measurement_wall_seconds": 10.015, "client_output_tokens": 9004, "server_output_tokens": 9004, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 899.051422874898, "per_request_avg_tps": 112.38142785936225, "ttft_avg": 0.3861250000772998, "ttft_p50": 0.46800000010989606, "ttft_p90": 0.46800000010989606, "ttft_p99": 0.46800000010989606, "time_to_second_token_avg": 0.06074999988777563, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.16700000003911553, "time_to_second_token_p99": 0.25520000012125815, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.008985711735563399, "inter_token_latency_p50": 0.008898972319498006, "inter_token_latency_p90": 0.00924323125758756, "inter_token_latency_p99": 0.009281385975801346, "output_tps_per_user_avg": 111.33081188050929, "output_tps_per_user_p50": 112.37269474348673, "output_tps_per_user_p90": 113.38351748935804, "output_tps_per_user_p99": 113.48312386838927, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.020458459498810765, "chunk_inter_token_latency_p50": 0.020345454545157564, "chunk_inter_token_latency_p90": 0.02070985542650297, "chunk_inter_token_latency_p99": 0.02080152947573041, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1628.125, "output_seq_len_p50": 1634.0, "output_seq_len_p90": 1649.2, "output_seq_len_p99": 1651.72, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.00928562538893621, "chunk_inter_token_latency_avg": 0.020811715481200123, "input_tokens": 65, "output_tokens": 1608, "output_tps_per_user": 107.6933386943971, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.00881102362191863, "chunk_inter_token_latency_avg": 0.020345454545157564, "input_tokens": 65, "output_tokens": 1652, "output_tps_per_user": 113.49419124383718, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2029999999795109, "time_to_second_token": 0.26500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.009225062344152424, "chunk_inter_token_latency_avg": 0.02066620111734705, "input_tokens": 65, "output_tokens": 1605, "output_tps_per_user": 108.40035142243556, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008888073394428198, "chunk_inter_token_latency_avg": 0.020324475524321824, "input_tokens": 65, "output_tokens": 1636, "output_tps_per_user": 112.51032204874514, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008909871244567814, "chunk_inter_token_latency_avg": 0.020324475524321824, "input_tokens": 65, "output_tokens": 1632, "output_tps_per_user": 112.23506743822834, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008843161094095841, "chunk_inter_token_latency_avg": 0.020345454545157564, "input_tokens": 65, "output_tokens": 1646, "output_tps_per_user": 113.08173506729992, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008823315118330361, "chunk_inter_token_latency_avg": 0.020324475524321824, "input_tokens": 65, "output_tokens": 1648, "output_tps_per_user": 113.33608588029556, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.009099561678077711, "chunk_inter_token_latency_avg": 0.02052542372865834, "input_tokens": 65, "output_tokens": 1598, "output_tps_per_user": 109.89540324883546, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 9004, "wall_time": 15.047000000020489, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 899.051422874898, "server_utilization": 0.0077919803887235295, "server_spec_accept_rate": 0.6970486111111112, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.016, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.859, "gpu_count": 1, "cpu_util_avg_pct": 20.7, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.5, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 0.75, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 70.18, "power_total_max_w": 72.68, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95166.25, "vram_used_max_mb": 95179.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.22, "vram_used_max_pct": 97.23, "pcie_rx_avg_mb_s": 17.25, "pcie_rx_max_mb_s": 38.0, "pcie_tx_avg_mb_s": 6.0, "pcie_tx_max_mb_s": 10.0 } }, { "concurrency": 16, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.015, "measurement_wall_seconds": 10.015, "client_output_tokens": 12859, "server_output_tokens": 12864, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1283.9740389547217, "per_request_avg_tps": 80.2483774346701, "ttft_avg": 0.7410624999756692, "ttft_p50": 0.8589999999385327, "ttft_p90": 0.875, "ttft_p99": 0.875, "time_to_second_token_avg": 0.10087500004738104, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.3904999999795109, "time_to_second_token_p99": 0.625, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.012372128144960788, "inter_token_latency_p50": 0.012266934592428927, "inter_token_latency_p90": 0.01291717914843247, "inter_token_latency_p99": 0.013330379757808888, "output_tps_per_user_avg": 80.91890422450615, "output_tps_per_user_p50": 81.52020344753748, "output_tps_per_user_p90": 84.06329471626482, "output_tps_per_user_p99": 85.06993501013606, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.027296089998999358, "chunk_inter_token_latency_p50": 0.027066921606040194, "chunk_inter_token_latency_p90": 0.02820801526709737, "chunk_inter_token_latency_p99": 0.0284149165756462, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1156.1875, "output_seq_len_p50": 1157.0, "output_seq_len_p90": 1191.0, "output_seq_len_p99": 1205.25, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.25, "time_to_second_token": 0.625, "latency": 0.0, "inter_token_latency_avg": 0.012742241379275019, "chunk_inter_token_latency_avg": 0.02820801526709737, "input_tokens": 65, "output_tokens": 1161, "output_tps_per_user": 78.4791286112723, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.1559999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013372426141376941, "chunk_inter_token_latency_avg": 0.028451428571272464, "input_tokens": 65, "output_tokens": 1118, "output_tps_per_user": 74.78074579943286, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011996610169456799, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1181, "output_tps_per_user": 83.35688047495167, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.625, "latency": 0.0, "inter_token_latency_avg": 0.013092116917589922, "chunk_inter_token_latency_avg": 0.02820801526709737, "input_tokens": 65, "output_tokens": 1130, "output_tps_per_user": 76.38184155355727, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012650580875745328, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1120, "output_tps_per_user": 79.04775360294146, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012151072961338216, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1166, "output_tps_per_user": 82.29725911298195, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012161512027456205, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1165, "output_tps_per_user": 82.22661768885062, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012395796847599843, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1143, "output_tps_per_user": 80.67250635796171, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011796666666632517, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1201, "output_tps_per_user": 84.76970895757798, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01250530035332069, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1133, "output_tps_per_user": 79.96609211664855, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011747717842289644, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1206, "output_tps_per_user": 85.12291607823455, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012245674740448981, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1157, "output_tps_per_user": 81.6614862958001, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012068201193485952, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1174, "output_tps_per_user": 82.86239050603247, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012494263018498696, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1134, "output_tps_per_user": 80.03673354077986, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012245674740448981, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1157, "output_tps_per_user": 81.6614862958001, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012288194444408873, "chunk_inter_token_latency_avg": 0.027066921606040194, "input_tokens": 65, "output_tokens": 1153, "output_tps_per_user": 81.37892059927485, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 12859, "wall_time": 15.062000000150874, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1282.424484104531, "server_utilization": 0.015029475281620241, "server_spec_accept_rate": 0.6633254716981132, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.016, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.531, "gpu_count": 1, "cpu_util_avg_pct": 17.9, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.25, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 67.0, "power_total_max_w": 69.91, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95216.25, "vram_used_max_mb": 95237.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.27, "vram_used_max_pct": 97.29, "pcie_rx_avg_mb_s": 10.75, "pcie_rx_max_mb_s": 17.0, "pcie_tx_avg_mb_s": 63.5, "pcie_tx_max_mb_s": 209.0 } }, { "concurrency": 32, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.985, "measurement_wall_seconds": 10.016, "client_output_tokens": 18725, "server_output_tokens": 18797, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1875.3129694786694, "per_request_avg_tps": 58.60353029620842, "ttft_avg": 1.24546874997759, "ttft_p50": 1.3589999999385327, "ttft_p90": 1.3589999999385327, "ttft_p99": 1.3589999999385327, "time_to_second_token_avg": 0.08821875005378388, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 1.1089999999385327, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.016773447417269736, "inter_token_latency_p50": 0.016669330552780325, "inter_token_latency_p90": 0.017129077273843094, "inter_token_latency_p99": 0.018192849634299547, "output_tps_per_user_avg": 59.66161054573456, "output_tps_per_user_p50": 59.990488484105015, "output_tps_per_user_p90": 61.26774916757181, "output_tps_per_user_p99": 61.787485562705896, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.036874830831975146, "chunk_inter_token_latency_p50": 0.03661442786079845, "chunk_inter_token_latency_p90": 0.03661442786079845, "chunk_inter_token_latency_p99": 0.03951684863529013, "input_seq_len_avg": 65.0, "output_seq_len_avg": 885.1875, "output_seq_len_p50": 885.5, "output_seq_len_p90": 902.8, "output_seq_len_p99": 910.45, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 0.10899999993853271, "time_to_second_token": 0.1410000000614673, "latency": 0.0, "inter_token_latency_avg": 0.018313073394542405, "chunk_inter_token_latency_avg": 0.03962531017379895, "input_tokens": 65, "output_tokens": 873, "output_tps_per_user": 54.60579873490904, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 1.1089999999385327, "latency": 0.0, "inter_token_latency_avg": 0.017925254813113828, "chunk_inter_token_latency_avg": 0.03927543424312534, "input_tokens": 65, "output_tokens": 884, "output_tps_per_user": 55.78721253482077, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 1.1089999999385327, "latency": 0.0, "inter_token_latency_avg": 0.017665178571405704, "chunk_inter_token_latency_avg": 0.03927543424312534, "input_tokens": 65, "output_tokens": 897, "output_tps_per_user": 56.60854182468788, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016519640853020177, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 892, "output_tps_per_user": 60.53400366855897, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.0166316384181254, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 886, "output_tps_per_user": 60.1263672802185, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01655680539937118, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 890, "output_tps_per_user": 60.39812487244548, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01642745535718859, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 897, "output_tps_per_user": 60.87370065884269, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01713504074510009, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 860, "output_tps_per_user": 58.35994293074316, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01686025200462884, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 874, "output_tps_per_user": 59.31109450353757, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016354444444489975, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 901, "output_tps_per_user": 61.14545825106966, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01665045248873414, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 885, "output_tps_per_user": 60.05842788216175, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016918390804644804, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 871, "output_tps_per_user": 59.107276309367336, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01659413754232354, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 888, "output_tps_per_user": 60.26224607633199, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016501121076279122, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 893, "output_tps_per_user": 60.60194306661572, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01642745535718859, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 897, "output_tps_per_user": 60.87370065884269, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016575450450496597, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 889, "output_tps_per_user": 60.330185474388735, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017075406032530136, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 863, "output_tps_per_user": 58.56376112491339, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016318181818227247, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 903, "output_tps_per_user": 61.28133704718316, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01624613686538739, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 907, "output_tps_per_user": 61.55309463941013, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016821714285761118, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 876, "output_tps_per_user": 59.44697329965106, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3589999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016802511415571892, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 877, "output_tps_per_user": 59.51491269770781, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016688208616826506, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 883, "output_tps_per_user": 59.92254908604827, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016918390804644804, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 871, "output_tps_per_user": 59.107276309367336, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016976931949297554, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 868, "output_tps_per_user": 58.903458115197104, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01615697036228428, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 912, "output_tps_per_user": 61.892791629693846, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016264088397835334, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 906, "output_tps_per_user": 61.48515524135339, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016688208616826506, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 883, "output_tps_per_user": 59.92254908604827, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016783352337560977, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 878, "output_tps_per_user": 59.58285209576456, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016840961098445056, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 875, "output_tps_per_user": 59.37903390159431, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016519640853020177, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 892, "output_tps_per_user": 60.53400366855897, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01705561993052257, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 864, "output_tps_per_user": 58.631700522970135, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.3440000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016538202247237056, "chunk_inter_token_latency_avg": 0.03661442786079845, "input_tokens": 65, "output_tokens": 891, "output_tps_per_user": 60.466064270502216, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 18725, "wall_time": 16.125, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1873.704146750988, "server_utilization": 0.0292418140430748, "server_spec_accept_rate": 0.6465371621621622, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.093, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.906, "gpu_count": 1, "cpu_util_avg_pct": 20.5, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.5, "gpu_util_max_pct": 8.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.25, "temp_max_c": 35.0, "power_total_avg_w": 67.42, "power_total_max_w": 70.66, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95218.25, "vram_used_max_mb": 95233.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.28, "vram_used_max_pct": 97.29, "pcie_rx_avg_mb_s": 27.25, "pcie_rx_max_mb_s": 49.0, "pcie_tx_avg_mb_s": 205.25, "pcie_tx_max_mb_s": 330.0 } }, { "concurrency": 64, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.953, "measurement_wall_seconds": 10.0, "client_output_tokens": 26769, "server_output_tokens": 26916, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 2689.540841899819, "per_request_avg_tps": 42.02407565468467, "ttft_avg": 2.1823281250763102, "ttft_p50": 2.218000000109896, "ttft_p90": 2.218000000109896, "ttft_p99": 2.2340000001713634, "time_to_second_token_avg": 0.04650000002220622, "time_to_second_token_p50": 0.0470000000204891, "time_to_second_token_p90": 0.0470000000204891, "time_to_second_token_p99": 0.0470000000204891, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.023083248412486417, "inter_token_latency_p50": 0.023059677419387886, "inter_token_latency_p90": 0.023856179465810482, "inter_token_latency_p99": 0.024749775731017252, "output_tps_per_user_avg": 43.35457495054904, "output_tps_per_user_p50": 43.36574106449685, "output_tps_per_user_p90": 44.97447016850238, "output_tps_per_user_p99": 45.47177729587104, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.05009763879791171, "chunk_inter_token_latency_p50": 0.04998951048958213, "chunk_inter_token_latency_p90": 0.04998951048958213, "chunk_inter_token_latency_p99": 0.052549988830826455, "input_seq_len_avg": 65.0, "output_seq_len_avg": 622.109375, "output_seq_len_p50": 621.5, "output_seq_len_p90": 644.0, "output_seq_len_p99": 651.11, "request_count": 64, "completed_request_count": 0, "request_samples": [ { "ttft": 2.2340000001713634, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023059677419387886, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 621, "output_tps_per_user": 43.36574106449685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.2340000001713634, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023553542009918434, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 608, "output_tps_per_user": 42.456459397015465, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.025770440251777334, "chunk_inter_token_latency_avg": 0.05690972222267495, "input_tokens": 65, "output_tokens": 637, "output_tps_per_user": 38.80414887095427, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.2340000001713634, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022693650793683316, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 631, "output_tps_per_user": 44.06518850102099, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.2340000001713634, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02302254428344684, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 622, "output_tps_per_user": 43.435685808149266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022409090909123023, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 639, "output_tps_per_user": 44.62474645024031, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023096930533151032, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 620, "output_tps_per_user": 43.29579632084444, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023323001631354794, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 614, "output_tps_per_user": 42.87612785892995, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024109612141687164, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 594, "output_tps_per_user": 41.47723298588167, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023209415584448848, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 617, "output_tps_per_user": 43.08596208988719, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02291185897439181, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 625, "output_tps_per_user": 43.64552003910651, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02378868552416055, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 602, "output_tps_per_user": 42.036790935100974, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023096930533151032, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 620, "output_tps_per_user": 43.29579632084444, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022479559748459887, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 637, "output_tps_per_user": 44.48485696293548, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023399345335549084, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 612, "output_tps_per_user": 42.73623837162512, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022234836702986763, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 644, "output_tps_per_user": 44.97447016850238, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02276592356691161, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 629, "output_tps_per_user": 43.92529901371616, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02336111111114459, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 613, "output_tps_per_user": 42.806183115277534, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023059677419387886, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 621, "output_tps_per_user": 43.36574106449685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023209415584448848, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 617, "output_tps_per_user": 43.08596208988719, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022838658146997586, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 627, "output_tps_per_user": 43.785409526411335, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022729729729762303, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 630, "output_tps_per_user": 43.99524375736858, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02287520000003278, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 626, "output_tps_per_user": 43.71546478275892, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02276592356691161, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 629, "output_tps_per_user": 43.92529901371616, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02302254428344684, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 622, "output_tps_per_user": 43.435685808149266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.0235924092409579, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 607, "output_tps_per_user": 42.38651465336305, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022131578947400136, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 647, "output_tps_per_user": 45.18430439945963, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023948073701876866, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 598, "output_tps_per_user": 41.757011960491326, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02347619047622412, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 610, "output_tps_per_user": 42.59634888432029, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022029275808968397, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 650, "output_tps_per_user": 45.39413863041686, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023323001631354794, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 614, "output_tps_per_user": 42.87612785892995, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02237402190926524, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 640, "output_tps_per_user": 44.69469119389272, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02336111111114459, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 613, "output_tps_per_user": 42.806183115277534, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.021927914110460873, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 653, "output_tps_per_user": 45.60397286137411, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022234836702986763, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 644, "output_tps_per_user": 44.97447016850238, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022479559748459887, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 637, "output_tps_per_user": 44.48485696293548, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023948073701876866, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 598, "output_tps_per_user": 41.757011960491326, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023868113522571767, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 600, "output_tps_per_user": 41.896901447796154, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024069023569058063, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 595, "output_tps_per_user": 41.54717772953408, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02347619047622412, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 610, "output_tps_per_user": 42.59634888432029, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023247154471578033, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 616, "output_tps_per_user": 43.016017346234776, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023553542009918434, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 608, "output_tps_per_user": 42.456459397015465, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02206327160496989, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 649, "output_tps_per_user": 45.324193886764455, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022234836702986763, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 644, "output_tps_per_user": 44.97447016850238, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023828333333367483, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 601, "output_tps_per_user": 41.96684619144856, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.218000000109896, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02291185897439181, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 625, "output_tps_per_user": 43.64552003910651, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022234836702986763, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 644, "output_tps_per_user": 44.97447016850238, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02347619047622412, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 610, "output_tps_per_user": 42.59634888432029, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02302254428344684, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 622, "output_tps_per_user": 43.435685808149266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023059677419387886, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 621, "output_tps_per_user": 43.36574106449685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022729729729762303, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 630, "output_tps_per_user": 43.99524375736858, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023437704918066377, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 611, "output_tps_per_user": 42.666293627972706, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02415033783787245, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 593, "output_tps_per_user": 41.40728824222925, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023059677419387886, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 621, "output_tps_per_user": 43.36574106449685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02226947040501634, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 643, "output_tps_per_user": 44.904525424849965, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022693650793683316, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 631, "output_tps_per_user": 44.06518850102099, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023247154471578033, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 616, "output_tps_per_user": 43.016017346234776, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02276592356691161, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 629, "output_tps_per_user": 43.92529901371616, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023323001631354794, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 614, "output_tps_per_user": 42.87612785892995, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02302254428344684, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 622, "output_tps_per_user": 43.435685808149266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022729729729762303, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 630, "output_tps_per_user": 43.99524375736858, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02302254428344684, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 622, "output_tps_per_user": 43.435685808149266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023631404958711553, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 606, "output_tps_per_user": 42.31656990971064, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.202999999979511, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022586097946319886, "chunk_inter_token_latency_avg": 0.04998951048958213, "input_tokens": 65, "output_tokens": 634, "output_tps_per_user": 44.27502273197824, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 26769, "wall_time": 16.625, "num_completed": 64, "num_errors": 0, "server_gen_throughput": 2683.281826287031, "server_utilization": 0.05626568610284244, "server_spec_accept_rate": 0.5958059210526315, "server_spec_accept_length": 0.0, "avg_running_reqs": 64, "max_running_reqs": 64, "effective_concurrency": 64, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.578, "ready_reason": "running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.907, "gpu_count": 1, "cpu_util_avg_pct": 19.25, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.25, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 65.35, "power_total_max_w": 65.9, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95214.75, "vram_used_max_mb": 95228.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.27, "vram_used_max_pct": 97.28, "pcie_rx_avg_mb_s": 9.75, "pcie_rx_max_mb_s": 14.0, "pcie_tx_avg_mb_s": 99.75, "pcie_tx_max_mb_s": 365.0 } }, { "concurrency": 2, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 3865, "server_output_tokens": 3865, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 386.5, "per_request_avg_tps": 193.25, "ttft_avg": 0.14850000001024455, "ttft_p50": 0.14850000001024455, "ttft_p90": 0.19209999998565763, "ttft_p99": 0.20190999998012557, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.005242296401846572, "inter_token_latency_p50": 0.005242296401846572, "inter_token_latency_p90": 0.00539576303464112, "inter_token_latency_p99": 0.005430293027019893, "output_tps_per_user_avg": 191.01187599546287, "output_tps_per_user_p50": 191.01187599546287, "output_tps_per_user_p90": 196.60369057169854, "output_tps_per_user_p99": 197.86184885135154, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.013384669598238742, "chunk_inter_token_latency_p50": 0.013384669598238742, "chunk_inter_token_latency_p90": 0.01342596345552656, "chunk_inter_token_latency_p99": 0.013435254573416319, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 3031.5, "output_seq_len_p50": 3031.5, "output_seq_len_p90": 3111.9, "output_seq_len_p99": 3129.99, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.0940000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.005434129692839757, "chunk_inter_token_latency_avg": 0.013436286919848514, "input_tokens": 8190, "output_tokens": 2931, "output_tps_per_user": 184.0221077751683, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2029999999795109, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.005050463110853388, "chunk_inter_token_latency_avg": 0.013333052276628968, "input_tokens": 8190, "output_tokens": 3132, "output_tps_per_user": 198.00164421575744, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3865, "wall_time": 16.54699999978766, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 385.9211183264639, "server_utilization": 0.003326912974960572, "server_spec_accept_rate": 0.6991150442477876, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.532, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 8.282, "gpu_count": 1, "cpu_util_avg_pct": 16.93, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.75, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 66.28, "power_total_max_w": 66.78, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95031.5, "vram_used_max_mb": 95091.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.08, "vram_used_max_pct": 97.14, "pcie_rx_avg_mb_s": 9.5, "pcie_rx_max_mb_s": 13.0, "pcie_tx_avg_mb_s": 55.25, "pcie_tx_max_mb_s": 204.0 } }, { "concurrency": 4, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 5819, "server_output_tokens": 5819, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 581.9, "per_request_avg_tps": 145.475, "ttft_avg": 0.3399999997927807, "ttft_p50": 0.42199999978765845, "ttft_p90": 0.42199999978765845, "ttft_p99": 0.42199999978765845, "time_to_second_token_avg": 0.01600000006146729, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.01600000006146729, "time_to_second_token_p99": 0.01600000006146729, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006943107330055141, "inter_token_latency_p50": 0.007100445562094038, "inter_token_latency_p90": 0.0071196271025286105, "inter_token_latency_p99": 0.007124355054776938, "output_tps_per_user_avg": 144.2862758251877, "output_tps_per_user_p50": 140.83636613255905, "output_tps_per_user_p90": 150.87550641953132, "output_tps_per_user_p99": 154.69477442747524, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.016590601206970104, "chunk_inter_token_latency_p50": 0.016492723343494317, "chunk_inter_token_latency_p90": 0.016804137472989814, "chunk_inter_token_latency_p99": 0.016909845565543664, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 2113.75, "output_seq_len_p50": 2072.5, "output_seq_len_p90": 2209.3, "output_seq_len_p99": 2254.93, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 0.09399999980814755, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.0071248803828045295, "chunk_inter_token_latency_avg": 0.016921590909160757, "input_tokens": 8190, "output_tokens": 2091, "output_tps_per_user": 140.35323349616365, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.42199999978765845, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.0070935216756366085, "chunk_inter_token_latency_avg": 0.016455367231731025, "input_tokens": 8190, "output_tokens": 2054, "output_tps_per_user": 140.9737004730101, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.42199999978765845, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.006446657813227958, "chunk_inter_token_latency_avg": 0.016455367231731025, "input_tokens": 8190, "output_tokens": 2260, "output_tps_per_user": 155.119137539469, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.42199999978765845, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.007107369448551467, "chunk_inter_token_latency_avg": 0.016530079455257612, "input_tokens": 8190, "output_tokens": 2050, "output_tps_per_user": 140.699031792108, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 5819, "wall_time": 15.515999999828637, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 581.0284573064646, "server_utilization": 0.005253020486779891, "server_spec_accept_rate": 0.7668539325842697, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.485, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 8.109, "gpu_count": 1, "cpu_util_avg_pct": 16.57, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 5.75, "gpu_util_max_pct": 6.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.0, "temp_max_c": 34.0, "power_total_avg_w": 66.24, "power_total_max_w": 66.62, "power_limit_total_w": 450.0, "vram_used_avg_mb": 94997.25, "vram_used_max_mb": 95002.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.05, "vram_used_max_pct": 97.05, "pcie_rx_avg_mb_s": 8.0, "pcie_rx_max_mb_s": 9.0, "pcie_tx_avg_mb_s": 54.75, "pcie_tx_max_mb_s": 209.0 } }, { "concurrency": 8, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.985, "measurement_wall_seconds": 10.0, "client_output_tokens": 8554, "server_output_tokens": 8554, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 856.6850275524987, "per_request_avg_tps": 107.08562844406234, "ttft_avg": 0.5213750000111759, "ttft_p50": 0.5779999999795109, "ttft_p90": 0.5940000000409782, "ttft_p99": 0.5940000000409782, "time_to_second_token_avg": 0.015999999828636646, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999828636646, "time_to_second_token_p99": 0.015999999828636646, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.009223329494434718, "inter_token_latency_p50": 0.009277010041523835, "inter_token_latency_p90": 0.009439680325791514, "inter_token_latency_p99": 0.009492535823806676, "output_tps_per_user_avg": 108.48028819110802, "output_tps_per_user_p50": 107.79386141253127, "output_tps_per_user_p90": 111.64608396552471, "output_tps_per_user_p99": 113.33245418166496, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.02195668293673807, "chunk_inter_token_latency_p50": 0.021879812436740333, "chunk_inter_token_latency_p90": 0.022192835271291157, "chunk_inter_token_latency_p99": 0.022373789340938845, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 1626.625, "output_seq_len_p50": 1609.5, "output_seq_len_p90": 1683.5, "output_seq_len_p99": 1743.35, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5940000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009379006913903512, "chunk_inter_token_latency_avg": 0.022041358936514755, "input_tokens": 8190, "output_tokens": 1592, "output_tps_per_user": 106.62109636763273, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5940000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009021765417182883, "chunk_inter_token_latency_avg": 0.021815789473714164, "input_tokens": 8190, "output_tokens": 1655, "output_tps_per_user": 110.84305052926746, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.00925682382135266, "chunk_inter_token_latency_avg": 0.02175218658895115, "input_tokens": 8190, "output_tokens": 1613, "output_tps_per_user": 108.02841442151097, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.00941451104102239, "chunk_inter_token_latency_avg": 0.02191189427315784, "input_tokens": 8190, "output_tokens": 1586, "output_tps_per_user": 106.21900549509607, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009297196261695008, "chunk_inter_token_latency_avg": 0.02210666666669702, "input_tokens": 8190, "output_tokens": 1606, "output_tps_per_user": 107.55930840355155, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.09300000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008809033733499201, "chunk_inter_token_latency_avg": 0.022393895348677476, "input_tokens": 8190, "output_tokens": 1750, "output_tps_per_user": 113.51982865012498, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009109890109902619, "chunk_inter_token_latency_avg": 0.021847730600322825, "input_tokens": 8190, "output_tokens": 1639, "output_tps_per_user": 109.77080820250308, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.009498408656919472, "chunk_inter_token_latency_avg": 0.021783941605869328, "input_tokens": 8190, "output_tokens": 1572, "output_tps_per_user": 105.28079345917725, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 8554, "wall_time": 16.07799999997951, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 855.4, "server_utilization": 0.00913441895756728, "server_spec_accept_rate": 0.6923076923076923, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.062, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 5, "duration_seconds": 9.11, "gpu_count": 1, "cpu_util_avg_pct": 17.56, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 8.0, "gpu_util_max_pct": 12.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 2.0, "temp_avg_c": 34.6, "temp_max_c": 35.0, "power_total_avg_w": 73.87, "power_total_max_w": 82.06, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95111.6, "vram_used_max_mb": 95167.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.16, "vram_used_max_pct": 97.22, "pcie_rx_avg_mb_s": 77.8, "pcie_rx_max_mb_s": 225.0, "pcie_tx_avg_mb_s": 194.0, "pcie_tx_max_mb_s": 379.0 } }, { "concurrency": 16, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.0, "measurement_wall_seconds": 10.0, "client_output_tokens": 11582, "server_output_tokens": 11607, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1158.2, "per_request_avg_tps": 72.3875, "ttft_avg": 0.7981250001030276, "ttft_p50": 0.6880000000819564, "ttft_p90": 1.2810000001918525, "ttft_p99": 1.2810000001918525, "time_to_second_token_avg": 0.21862499995040707, "time_to_second_token_p50": 0.2809999999590218, "time_to_second_token_p90": 0.29650000005494803, "time_to_second_token_p99": 0.31200000015087426, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01331078641585704, "inter_token_latency_p50": 0.013400430964481841, "inter_token_latency_p90": 0.013781711271126205, "inter_token_latency_p99": 0.014335987511564386, "output_tps_per_user_avg": 75.2717915256858, "output_tps_per_user_p50": 74.62483471843511, "output_tps_per_user_p90": 79.79589313295665, "output_tps_per_user_p99": 83.35626270941417, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03073247817592778, "chunk_inter_token_latency_p50": 0.03084104627762477, "chunk_inter_token_latency_p90": 0.03102649682385404, "chunk_inter_token_latency_p99": 0.03181391664201816, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 1144.9375, "output_seq_len_p50": 1139.5, "output_seq_len_p90": 1197.5, "output_seq_len_p99": 1269.85, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.012746666666741173, "chunk_inter_token_latency_avg": 0.03083870967759961, "input_tokens": 8190, "output_tokens": 1201, "output_tps_per_user": 78.45188284472972, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2810000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.012324392288331527, "chunk_inter_token_latency_avg": 0.02988414634142177, "input_tokens": 8190, "output_tokens": 1194, "output_tps_per_user": 81.13990342118359, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10900000017136335, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013654077253124553, "chunk_inter_token_latency_avg": 0.03194176706805242, "input_tokens": 8190, "output_tokens": 1166, "output_tps_per_user": 73.23819702068577, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.011940671350577211, "chunk_inter_token_latency_avg": 0.030963562753217422, "input_tokens": 8190, "output_tokens": 1282, "output_tps_per_user": 83.74738493674897, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.01311206159108598, "chunk_inter_token_latency_avg": 0.03084104627762477, "input_tokens": 8190, "output_tokens": 1170, "output_tps_per_user": 76.2656576201437, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013673505798376014, "chunk_inter_token_latency_avg": 0.03084104627762477, "input_tokens": 8190, "output_tokens": 1122, "output_tps_per_user": 73.13413361178878, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013524314765773128, "chunk_inter_token_latency_avg": 0.031089430894490663, "input_tokens": 8190, "output_tokens": 1132, "output_tps_per_user": 73.94089958115777, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.0135286849073076, "chunk_inter_token_latency_avg": 0.03084104627762477, "input_tokens": 8190, "output_tokens": 1134, "output_tps_per_user": 73.91701461387751, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013312445604951616, "chunk_inter_token_latency_avg": 0.03083870967759961, "input_tokens": 8190, "output_tokens": 1150, "output_tps_per_user": 75.11767782382871, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013370629370707524, "chunk_inter_token_latency_avg": 0.030901010101190722, "input_tokens": 8190, "output_tokens": 1145, "output_tps_per_user": 74.79079497864234, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013202411714021972, "chunk_inter_token_latency_avg": 0.030903225806410303, "input_tokens": 8190, "output_tokens": 1162, "output_tps_per_user": 75.74373695208455, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9690000000409782, "time_to_second_token": 0.31200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.013430232558256159, "chunk_inter_token_latency_avg": 0.03051829268319184, "input_tokens": 8190, "output_tokens": 1119, "output_tps_per_user": 74.45887445822788, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2810000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014414705882332854, "chunk_inter_token_latency_avg": 0.02994501018325766, "input_tokens": 8190, "output_tokens": 1021, "output_tps_per_user": 69.37359722515278, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9690000000409782, "time_to_second_token": 0.31200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.013889916743876397, "chunk_inter_token_latency_avg": 0.030768442623218, "input_tokens": 8190, "output_tokens": 1082, "output_tps_per_user": 71.99467199404683, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.2809999999590218, "latency": 0.0, "inter_token_latency_avg": 0.013661319073065517, "chunk_inter_token_latency_avg": 0.03084104627762477, "input_tokens": 8190, "output_tokens": 1123, "output_tps_per_user": 73.19937369529617, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.2810000001918525, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013186547085183417, "chunk_inter_token_latency_avg": 0.029763157894695365, "input_tokens": 8190, "output_tokens": 1116, "output_tps_per_user": 75.83486363337781, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 11582, "wall_time": 16.594000000040978, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1158.9615576484161, "server_utilization": 0.006712192844218756, "server_spec_accept_rate": 0.6533203125, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.547, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.078, "gpu_count": 1, "cpu_util_avg_pct": 15.2, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.25, "gpu_util_max_pct": 9.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 35.0, "temp_max_c": 35.0, "power_total_avg_w": 66.92, "power_total_max_w": 67.6, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95131.0, "vram_used_max_mb": 95153.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.18, "vram_used_max_pct": 97.21, "pcie_rx_avg_mb_s": 17.25, "pcie_rx_max_mb_s": 23.0, "pcie_tx_avg_mb_s": 337.0, "pcie_tx_max_mb_s": 1169.0 } }, { "concurrency": 32, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 10.015, "measurement_wall_seconds": 10.015, "client_output_tokens": 16986, "server_output_tokens": 17049, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1696.0559161037304, "per_request_avg_tps": 53.001747378241575, "ttft_avg": 1.2932187499682186, "ttft_p50": 1.6869999999180436, "ttft_p90": 1.702999999979511, "ttft_p99": 1.702999999979511, "time_to_second_token_avg": 0.0738125000934815, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.10900000017136335, "time_to_second_token_p99": 0.6161499999207454, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.018003901099039453, "inter_token_latency_p50": 0.018101411903806375, "inter_token_latency_p90": 0.018836194671251454, "inter_token_latency_p99": 0.019108024824023507, "output_tps_per_user_avg": 55.6282161701746, "output_tps_per_user_p50": 55.244482062017916, "output_tps_per_user_p90": 57.86679960095798, "output_tps_per_user_p99": 62.012007035239904, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.041323492267579504, "chunk_inter_token_latency_p50": 0.0405323033713542, "chunk_inter_token_latency_p90": 0.042891455618112896, "chunk_inter_token_latency_p99": 0.043905026071122524, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 824.90625, "output_seq_len_p50": 824.5, "output_seq_len_p90": 867.2, "output_seq_len_p99": 897.39, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 0.1410000000614673, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.018396313364181908, "chunk_inter_token_latency_avg": 0.04435555555586082, "input_tokens": 8190, "output_tokens": 869, "output_tps_per_user": 54.35871743449563, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01782920792102952, "chunk_inter_token_latency_avg": 0.04012813370526978, "input_tokens": 8190, "output_tokens": 809, "output_tps_per_user": 56.087741218189606, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.0181435768264381, "chunk_inter_token_latency_avg": 0.04012813370526978, "input_tokens": 8190, "output_tokens": 795, "output_tps_per_user": 55.11592391985464, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.0175531428573387, "chunk_inter_token_latency_avg": 0.04290223463735018, "input_tokens": 8190, "output_tokens": 876, "output_tps_per_user": 56.96985480762012, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.019131474103840442, "chunk_inter_token_latency_avg": 0.040240223464223054, "input_tokens": 8190, "output_tokens": 754, "output_tps_per_user": 52.26988754615937, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.018235443038217534, "chunk_inter_token_latency_avg": 0.04012813370526978, "input_tokens": 8190, "output_tokens": 791, "output_tps_per_user": 54.83826183461608, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.01821040189147973, "chunk_inter_token_latency_avg": 0.04279444444497737, "input_tokens": 8190, "output_tokens": 847, "output_tps_per_user": 54.91366999801796, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.018527141134102972, "chunk_inter_token_latency_avg": 0.0426638888893649, "input_tokens": 8190, "output_tokens": 830, "output_tps_per_user": 53.97486815487666, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.018416067146488444, "chunk_inter_token_latency_avg": 0.04290223463735018, "input_tokens": 8190, "output_tokens": 835, "output_tps_per_user": 54.300410182348784, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.015900662251867387, "chunk_inter_token_latency_avg": 0.040240223464223054, "input_tokens": 8190, "output_tokens": 907, "output_tps_per_user": 62.89046230653438, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.018845398773216396, "chunk_inter_token_latency_avg": 0.04278272980549126, "input_tokens": 8190, "output_tokens": 816, "output_tps_per_user": 53.06335047795474, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.019008663366548717, "chunk_inter_token_latency_avg": 0.042545706371665826, "input_tokens": 8190, "output_tokens": 809, "output_tps_per_user": 52.60759163949378, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017377563329543853, "chunk_inter_token_latency_avg": 0.04012813370526978, "input_tokens": 8190, "output_tokens": 830, "output_tps_per_user": 57.54546716569205, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.018069411764907485, "chunk_inter_token_latency_avg": 0.042545706371665826, "input_tokens": 8190, "output_tokens": 851, "output_tps_per_user": 55.34214467025955, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01784320987680536, "chunk_inter_token_latency_avg": 0.04059831460733804, "input_tokens": 8190, "output_tokens": 811, "output_tps_per_user": 56.04372794493182, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.016650921659230808, "chunk_inter_token_latency_avg": 0.04014722222281206, "input_tokens": 8190, "output_tokens": 869, "output_tps_per_user": 60.056735624939286, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.75, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.018133412042705268, "chunk_inter_token_latency_avg": 0.04278272980549126, "input_tokens": 8190, "output_tokens": 848, "output_tps_per_user": 55.14681945377628, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.702999999979511, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.018398467433195214, "chunk_inter_token_latency_avg": 0.04046629213537037, "input_tokens": 8190, "output_tokens": 784, "output_tps_per_user": 54.352353185448585, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01782920792102952, "chunk_inter_token_latency_avg": 0.04046629213537037, "input_tokens": 8190, "output_tokens": 809, "output_tps_per_user": 56.087741218189606, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017455314009918288, "chunk_inter_token_latency_avg": 0.04014722222281206, "input_tokens": 8190, "output_tokens": 829, "output_tps_per_user": 57.289144121485855, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.018154846335899956, "chunk_inter_token_latency_avg": 0.042545706371665826, "input_tokens": 8190, "output_tokens": 847, "output_tps_per_user": 55.08171104828186, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017252694611008205, "chunk_inter_token_latency_avg": 0.040240223464223054, "input_tokens": 8190, "output_tokens": 836, "output_tps_per_user": 57.96196029354991, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.018007500000239814, "chunk_inter_token_latency_avg": 0.040240223464223054, "input_tokens": 8190, "output_tokens": 801, "output_tps_per_user": 55.532417047712485, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017654411764940994, "chunk_inter_token_latency_avg": 0.04046629213537037, "input_tokens": 8190, "output_tokens": 817, "output_tps_per_user": 56.643065388666734, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8430000001098961, "time_to_second_token": 0.8439999998081475, "latency": 0.0, "inter_token_latency_avg": 0.018496977025417763, "chunk_inter_token_latency_avg": 0.04237396121889332, "input_tokens": 8190, "output_tokens": 828, "output_tps_per_user": 54.06288814793046, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017273381295194068, "chunk_inter_token_latency_avg": 0.04012813370526978, "input_tokens": 8190, "output_tokens": 835, "output_tps_per_user": 57.89254477224026, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01789565217415137, "chunk_inter_token_latency_avg": 0.04012813370526978, "input_tokens": 8190, "output_tokens": 806, "output_tps_per_user": 55.87949465426068, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017625609756356513, "chunk_inter_token_latency_avg": 0.040827683616419044, "input_tokens": 8190, "output_tokens": 821, "output_tps_per_user": 56.73562582079518, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017350540216341345, "chunk_inter_token_latency_avg": 0.04014722222281206, "input_tokens": 8190, "output_tokens": 834, "output_tps_per_user": 57.63509305941754, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.01875335775356699, "chunk_inter_token_latency_avg": 0.042545706371665826, "input_tokens": 8190, "output_tokens": 820, "output_tps_per_user": 53.32378409993243, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.019055831265721294, "chunk_inter_token_latency_avg": 0.04290223463735018, "input_tokens": 8190, "output_tokens": 807, "output_tps_per_user": 52.477374828504935, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.6869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.018649032258338506, "chunk_inter_token_latency_avg": 0.040712676056936174, "input_tokens": 8190, "output_tokens": 776, "output_tps_per_user": 53.62208537941007, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 16986, "wall_time": 16.70299999997951, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1699.6311434622319, "server_utilization": 0.008550750014591713, "server_spec_accept_rate": 0.6711956521739131, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.641, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 5, "duration_seconds": 8.687, "gpu_count": 1, "cpu_util_avg_pct": 16.34, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.6, "gpu_util_max_pct": 8.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 35.0, "temp_max_c": 35.0, "power_total_avg_w": 68.52, "power_total_max_w": 71.28, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95158.8, "vram_used_max_mb": 95208.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.21, "vram_used_max_pct": 97.26, "pcie_rx_avg_mb_s": 30.8, "pcie_rx_max_mb_s": 109.0, "pcie_tx_avg_mb_s": 377.4, "pcie_tx_max_mb_s": 1186.0 } }, { "concurrency": 64, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.985, "measurement_wall_seconds": 10.0, "client_output_tokens": 26405, "server_output_tokens": 26406, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 2644.466700084607, "per_request_avg_tps": 41.319792188821985, "ttft_avg": 2.2196718750383297, "ttft_p50": 2.344000000040978, "ttft_p90": 3.125, "ttft_p99": 3.1410000000614673, "time_to_second_token_avg": 0.48599999997895793, "time_to_second_token_p50": 0.7970000000204891, "time_to_second_token_p90": 0.7970000000204891, "time_to_second_token_p99": 1.281999999890104, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.0243103899300932, "inter_token_latency_p50": 0.02400655663778834, "inter_token_latency_p90": 0.026402305724653893, "inter_token_latency_p99": 0.02780999701558218, "output_tps_per_user_avg": 41.28728163779739, "output_tps_per_user_p50": 41.65530677957999, "output_tps_per_user_p90": 44.17912968510006, "output_tps_per_user_p99": 45.89310934371072, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.05642727389343491, "chunk_inter_token_latency_p50": 0.05588235294117647, "chunk_inter_token_latency_p90": 0.06140856031085644, "chunk_inter_token_latency_p99": 0.06292481259869774, "input_seq_len_avg": 8190.0, "output_seq_len_avg": 592.078125, "output_seq_len_p50": 592.0, "output_seq_len_p90": 614.8, "output_seq_len_p99": 635.0699999999999, "request_count": 64, "completed_request_count": 0, "request_samples": [ { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02435897435897436, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 586, "output_tps_per_user": 41.05263157894737, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8120000001508743, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.026364548494696716, "chunk_inter_token_latency_avg": 0.06134630350127874, "input_tokens": 8190, "output_tokens": 599, "output_tps_per_user": 37.92972218739692, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1559999999590218, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.027766891892030333, "chunk_inter_token_latency_avg": 0.06396108949448231, "input_tokens": 8190, "output_tokens": 593, "output_tps_per_user": 36.01411363894929, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02386934673366834, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 598, "output_tps_per_user": 41.89473684210527, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025675675675675677, "chunk_inter_token_latency_avg": 0.05677290836653386, "input_tokens": 8190, "output_tokens": 556, "output_tps_per_user": 38.94736842105263, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023989898989898988, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 595, "output_tps_per_user": 41.684210526315795, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024043918918815087, "chunk_inter_token_latency_avg": 0.055819607842896204, "input_tokens": 8190, "output_tokens": 593, "output_tps_per_user": 41.59055781948549, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02284751203842461, "chunk_inter_token_latency_avg": 0.055819607842896204, "input_tokens": 8190, "output_tokens": 624, "output_tps_per_user": 43.768441759357195, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8120000001508743, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02513057324823265, "chunk_inter_token_latency_avg": 0.06140856031085644, "input_tokens": 8190, "output_tokens": 629, "output_tps_per_user": 39.79216829326911, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023399014778325122, "chunk_inter_token_latency_avg": 0.05610236220472441, "input_tokens": 8190, "output_tokens": 610, "output_tps_per_user": 42.73684210526316, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02359271523178808, "chunk_inter_token_latency_avg": 0.05654761904761905, "input_tokens": 8190, "output_tokens": 605, "output_tps_per_user": 42.385964912280706, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.1410000000614673, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.023275086505154863, "chunk_inter_token_latency_avg": 0.053173913043397275, "input_tokens": 8190, "output_tokens": 579, "output_tps_per_user": 42.9643945588999, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.1410000000614673, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.023247404844148865, "chunk_inter_token_latency_avg": 0.052901574802826944, "input_tokens": 8190, "output_tokens": 579, "output_tps_per_user": 43.01555406739045, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8120000001508743, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02625956738750433, "chunk_inter_token_latency_avg": 0.061890196078000406, "input_tokens": 8190, "output_tokens": 602, "output_tps_per_user": 38.08135850995976, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023949579831932775, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 596, "output_tps_per_user": 41.75438596491228, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025327402135121944, "chunk_inter_token_latency_avg": 0.056709163346368656, "input_tokens": 8190, "output_tokens": 563, "output_tps_per_user": 39.482928200254804, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022230889235569422, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 642, "output_tps_per_user": 44.98245614035088, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02456896551724138, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 581, "output_tps_per_user": 40.70175438596491, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8120000001508743, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02617247097825888, "chunk_inter_token_latency_avg": 0.06140856031085644, "input_tokens": 8190, "output_tokens": 604, "output_tps_per_user": 38.208085160575266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.1410000000614673, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.023275086505154863, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 579, "output_tps_per_user": 42.9643945588999, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024912587412587412, "chunk_inter_token_latency_avg": 0.05632411067193676, "input_tokens": 8190, "output_tokens": 573, "output_tps_per_user": 40.140350877192986, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024626297577748327, "chunk_inter_token_latency_avg": 0.055819607842896204, "input_tokens": 8190, "output_tokens": 579, "output_tps_per_user": 40.60699733051117, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8120000001508743, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.026749152542186617, "chunk_inter_token_latency_avg": 0.06140856031085644, "input_tokens": 8190, "output_tokens": 591, "output_tps_per_user": 37.384361931574475, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.1410000000614673, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022126644736808407, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 609, "output_tps_per_user": 45.194380435659404, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.344000000040978, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024070945945945946, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 593, "output_tps_per_user": 41.54385964912281, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022054098360622148, "chunk_inter_token_latency_avg": 0.053173913043397275, "input_tokens": 8190, "output_tokens": 611, "output_tps_per_user": 45.34304616077671, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02295733788392408, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 587, "output_tps_per_user": 43.5590574593691, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022918228279351808, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 588, "output_tps_per_user": 43.63339032192775, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02375, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 601, "output_tps_per_user": 42.10526315789474, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025629496402877698, "chunk_inter_token_latency_avg": 0.05610236220472441, "input_tokens": 8190, "output_tokens": 557, "output_tps_per_user": 39.01754385964912, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023095623987034037, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 618, "output_tps_per_user": 43.29824561403508, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.796000000089407, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.025957236841924514, "chunk_inter_token_latency_avg": 0.06164843749957072, "input_tokens": 8190, "output_tokens": 609, "output_tps_per_user": 38.52490178711403, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.023560420315200545, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 572, "output_tps_per_user": 42.44406452098934, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02444253859348199, "chunk_inter_token_latency_avg": 0.05654761904761905, "input_tokens": 8190, "output_tokens": 584, "output_tps_per_user": 40.91228070175439, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02433162393151886, "chunk_inter_token_latency_avg": 0.05603937007849816, "input_tokens": 8190, "output_tokens": 586, "output_tps_per_user": 41.09877757499833, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.796000000089407, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.026085950413041493, "chunk_inter_token_latency_avg": 0.06140856031085644, "input_tokens": 8190, "output_tokens": 606, "output_tps_per_user": 38.334811811190775, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02351923076919495, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 573, "output_tps_per_user": 42.51839738354799, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022918228279351808, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 588, "output_tps_per_user": 43.63339032192775, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023909395973154363, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 597, "output_tps_per_user": 41.824561403508774, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.796000000089407, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02788339222595425, "chunk_inter_token_latency_avg": 0.06140856031085644, "input_tokens": 8190, "output_tokens": 567, "output_tps_per_user": 35.863642124188395, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023949579831932775, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 596, "output_tps_per_user": 41.75438596491228, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02319482758617157, "chunk_inter_token_latency_avg": 0.053173913043397275, "input_tokens": 8190, "output_tokens": 581, "output_tps_per_user": 43.113060284017195, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.024023214285677698, "chunk_inter_token_latency_avg": 0.05338492063483933, "input_tokens": 8190, "output_tokens": 561, "output_tps_per_user": 41.62640303284419, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022545302013285307, "chunk_inter_token_latency_avg": 0.052901574802826944, "input_tokens": 8190, "output_tokens": 597, "output_tps_per_user": 44.35513879613271, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024696707105719237, "chunk_inter_token_latency_avg": 0.05610236220472441, "input_tokens": 8190, "output_tokens": 578, "output_tps_per_user": 40.49122807017544, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.02423469387755102, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 589, "output_tps_per_user": 41.26315789473684, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023322422258592473, "chunk_inter_token_latency_avg": 0.05610236220472441, "input_tokens": 8190, "output_tokens": 612, "output_tps_per_user": 42.87719298245614, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.022995153473244802, "chunk_inter_token_latency_avg": 0.055819607842896204, "input_tokens": 8190, "output_tokens": 620, "output_tps_per_user": 43.48742447679311, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025104056437281363, "chunk_inter_token_latency_avg": 0.05603937007849816, "input_tokens": 8190, "output_tokens": 568, "output_tps_per_user": 39.83419980345992, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02319482758617157, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 581, "output_tps_per_user": 43.113060284017195, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.046000000089407, "time_to_second_token": 1.281999999890104, "latency": 0.0, "inter_token_latency_avg": 0.025843594009800505, "chunk_inter_token_latency_avg": 0.06067187499957072, "input_tokens": 8190, "output_tokens": 602, "output_tps_per_user": 38.69430852461063, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02219966996696289, "chunk_inter_token_latency_avg": 0.05338492063483933, "input_tokens": 8190, "output_tokens": 607, "output_tps_per_user": 45.04571471054211, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.796000000089407, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02695042735013442, "chunk_inter_token_latency_avg": 0.06134630350127874, "input_tokens": 8190, "output_tokens": 586, "output_tps_per_user": 37.10516300941003, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.02135396825393573, "chunk_inter_token_latency_avg": 0.05381199999991804, "input_tokens": 8190, "output_tokens": 631, "output_tps_per_user": 46.829703411949716, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024653979238754325, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 579, "output_tps_per_user": 40.56140350877193, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.327999999979511, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023170731707317073, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 616, "output_tps_per_user": 43.1578947368421, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022879251700645427, "chunk_inter_token_latency_avg": 0.053173913043397275, "input_tokens": 8190, "output_tokens": 589, "output_tps_per_user": 43.7077231844864, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.125, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.022347176079700184, "chunk_inter_token_latency_avg": 0.05296456692905319, "input_tokens": 8190, "output_tokens": 603, "output_tps_per_user": 44.748383260307506, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.796000000089407, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02713597246097872, "chunk_inter_token_latency_avg": 0.062316205532919514, "input_tokens": 8190, "output_tokens": 582, "output_tps_per_user": 36.851452493106365, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0310000001918525, "time_to_second_token": 1.281999999890104, "latency": 0.0, "inter_token_latency_avg": 0.02665979381413855, "chunk_inter_token_latency_avg": 0.06060937499933061, "input_tokens": 8190, "output_tokens": 583, "output_tps_per_user": 37.50966744047614, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3130000000819564, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.025491949910554562, "chunk_inter_token_latency_avg": 0.05610236220472441, "input_tokens": 8190, "output_tokens": 560, "output_tps_per_user": 39.228070175438596, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7810000001918525, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.02641848739463554, "chunk_inter_token_latency_avg": 0.0616431372541496, "input_tokens": 8190, "output_tokens": 596, "output_tps_per_user": 37.85228067989453, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3130000000819564, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.023829431438127092, "chunk_inter_token_latency_avg": 0.05588235294117647, "input_tokens": 8190, "output_tokens": 599, "output_tps_per_user": 41.96491228070175, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 2.3130000000819564, "time_to_second_token": 0.7970000000204891, "latency": 0.0, "inter_token_latency_avg": 0.024825783972125436, "chunk_inter_token_latency_avg": 0.05610236220472441, "input_tokens": 8190, "output_tokens": 575, "output_tps_per_user": 40.280701754385966, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 26405, "wall_time": 17.187999999849126, "num_completed": 64, "num_errors": 0, "server_gen_throughput": 2636.381789182486, "server_utilization": 0.05918403081772017, "server_spec_accept_rate": 0.6171875, "server_spec_accept_length": 0.0, "avg_running_reqs": 64, "max_running_reqs": 64, "effective_concurrency": 64, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 7.125, "ready_reason": "running_reqs=64/64, queue_reqs=0, active_streams=64/64, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 6.5, "gpu_count": 1, "cpu_util_avg_pct": 14.17, "cpu_temp_max_c": 0.0, "gpu_util_avg_pct": 6.5, "gpu_util_max_pct": 7.0, "mem_util_avg_pct": 1.0, "mem_util_max_pct": 1.0, "temp_avg_c": 34.75, "temp_max_c": 35.0, "power_total_avg_w": 66.06, "power_total_max_w": 66.72, "power_limit_total_w": 450.0, "vram_used_avg_mb": 95172.0, "vram_used_max_mb": 95176.0, "vram_total_mb": 97887.0, "vram_used_avg_pct": 97.23, "vram_used_max_pct": 97.23, "pcie_rx_avg_mb_s": 8.75, "pcie_rx_max_mb_s": 10.0, "pcie_tx_avg_mb_s": 106.0, "pcie_tx_max_mb_s": 209.0 } } ], "summary_table": { "0": { "1": 213.2, "2": 358.8, "4": 600.8, "8": 899.051422874898, "16": 1283.9740389547217, "32": 1875.3129694786694, "64": 2689.540841899819 }, "8192": { "1": 272.1, "2": 386.5, "4": 581.9, "8": 856.6850275524987, "16": 1158.2, "32": 1696.0559161037304, "64": 2644.466700084607 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": true, "mode": "standalone_cold", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." }, "coding_peak": { "name": "Coding Peak", "present": true, "formula": "usage.completion_tokens / (last_stream_time - first_token_time)", "notes": "Sequential cc1 Sieve-of-Eratosthenes coding prompt, matching /mnt/test.py throughput semantics. Uses OpenAI stream usage with continuous_usage_stats when the server supports it." } }, "coding_peak": { "mode": "coding_peak", "prompt": "Write a Python script that implements the Sieve of Eratosthenes.", "runs_requested": 5, "runs_ok": 5, "max_tokens": 2000, "temperature": null, "summary": { "mean_generation_tok_s": 228.61063082466228, "median_generation_tok_s": 228.46889953463887, "max_generation_tok_s": 234.09090907934774, "min_generation_tok_s": 223.55347749720036, "cjk_runs": 0 }, "samples": [ { "run": 1, "ok": true, "finish_reason": "stop", "completion_tokens": 618, "ttft": 0.0779999999795109, "gen_elapsed": 2.640000000130385, "total_elapsed": 2.718000000109896, "generation_tok_s": 234.09090907934774, "total_tok_s": 227.37306842347778, "content_chars": 1908, "reasoning_chars": 296, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the Sieve of Eratosthenes, a classic algorithm to find all prime numbers up to a given integer `n`. The script includes a function with clear comments and handles edge cases.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n using the\n Sieve of Eratosthenes algorithm.\n\n Parameters:\n n (int): Upper bound (inclusive).\n\n Returns:\n list[int]: Sorted list of prime" }, { "run": 2, "ok": true, "finish_reason": "stop", "completion_tokens": 726, "ttft": 0.0779999999795109, "gen_elapsed": 3.156999999890104, "total_elapsed": 3.234999999869615, "generation_tok_s": 229.96515680243024, "total_tok_s": 224.42040186375922, "content_chars": 2202, "reasoning_chars": 596, "cjk_chars": 0, "content_preview": "Below is a Python script implementing the Sieve of Eratosthenes, a classic algorithm for finding all prime numbers up to a given limit `n`.\n\n```python\ndef sieve_of_eratosthenes(n):\n \"\"\"\n Return a list of all prime numbers up to n (inclusive) using the\n Sieve of Eratosthenes algorithm.\n\n Parameters:\n n (int): Upper limit for prime search.\n\n Returns:\n list: Sorted list of primes <= n. Empty list if n < 2.\n \"\"\"\n if n < 2:\n return []\n\n # Initialize a bool" }, { "run": 3, "ok": true, "finish_reason": "stop", "completion_tokens": 765, "ttft": 0.09300000010989606, "gen_elapsed": 3.422000000020489, "total_elapsed": 3.515000000130385, "generation_tok_s": 223.55347749720036, "total_tok_s": 217.63869131482878, "content_chars": 2233, "reasoning_chars": 711, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the **Sieve of Eratosthenes**, a classic algorithm for finding all prime numbers up to a given limit `n`. The script includes:\n\n- A function `sieve_of_eratosthenes(n)` that returns a list of primes ≤ `n`.\n- Optimizations: start marking from `p*p` and only iterate up to `√n`.\n- A `main` block that demonstrates the function with a sample input.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or " }, { "run": 4, "ok": true, "finish_reason": "stop", "completion_tokens": 764, "ttft": 0.0779999999795109, "gen_elapsed": 3.3439999998081475, "total_elapsed": 3.4219999997876585, "generation_tok_s": 228.46889953463887, "total_tok_s": 223.2612507444207, "content_chars": 2481, "reasoning_chars": 505, "cjk_chars": 0, "content_preview": "Below is a Python script that implements the **Sieve of Eratosthenes** — an efficient algorithm for finding all prime numbers up to a given limit.\n\n```python\ndef sieve_of_eratosthenes(limit: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers up to the given limit (inclusive).\n\n Args:\n limit: A positive integer (>= 2) representing the upper bound.\n\n Returns:\n List of prime numbers from 2 up to `limit`.\n\n Raises:\n ValueError: If limit is less than 2.\n " }, { "run": 5, "ok": true, "finish_reason": "stop", "completion_tokens": 727, "ttft": 0.07800000021234155, "gen_elapsed": 3.202999999979511, "total_elapsed": 3.2810000001918525, "generation_tok_s": 226.9747112096942, "total_tok_s": 221.57878694224004, "content_chars": 1954, "reasoning_chars": 753, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the **Sieve of Eratosthenes**. It efficiently finds all prime numbers up to a given limit `n`. The script includes a function `sieve_of_eratosthenes(n)` and a simple demonstration in the `__main__` block.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list:\n \"\"\"\n Return a list of all prime numbers up to n (inclusive) using the\n Sieve of Eratosthenes algorithm.\n\n Args:\n n (int): Upper bound (must be >= 2 to return any primes).\n\n Returns:\n" } ] } }