{ "metadata": { "version": "0.4.30", "engine": "vllm", "model": "DeepSeek-V4-Flash", "server": "http://192.168.0.213", "timestamp": "2026-07-13T16:35:10.286792", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 15.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "standalone_cold", "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 30.0, "decode_warmup_context": 32768, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 0.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 2048, "temperature": null, "ignore_eos": true, "max_total_tokens": 1149077, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1, 4, 8, 16, 32 ], "context_lengths": [ 0, 4096, 8192, 16384, 32768 ], "unique_context_percent": 0.0, "shared_context_percent": 100.0, "context_sharing_mode": "fully_shared", "startup_diagnostics_available": true, "nvidia_p2p_override_effective": false, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.30", "server_url": "http://192.168.0.213:8000", "hostname": "DESKTOP-RMAJ6JK", "uname": "", "env": {}, "args": { "concurrency": "1,4,8,16,32", "contexts": "0,4k,8k,16k,32k\n", "max_tokens": 2048, "duration": 15.0, "request_count": 0, "run_burst": false, "standalone_prefill": true, "prefill_only": false, "skip_prefill": false, "prefill_contexts": "8k,16k,32k,64k,128k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 1149077 }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 595.79, 00000000:01:00.0, 1, 16, 450.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 255, "stdout": "ERROR: Option -m is missing its value. Please run 'nvidia-smi -h' for help.", "stderr": "" } }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": {}, "event_log": [ "16:24:52 decode warmup start C=1 ctx=32k 30s", "16:24:53 cell start C=1 ctx=32k unique_context=0%", "16:24:59 ready C=1 ctx=32k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "16:25:29 cell done C=1 ctx=32k 234.8 tok/s", "16:25:29 decode warmup done C=1 ctx=32k", "16:25:31 cell start C=1 ctx=0 unique_context=0%", "16:25:38 ready C=1 ctx=0 running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "16:25:53 cell done C=1 ctx=0 226.1 tok/s", "16:25:55 cell start C=1 ctx=4k unique_context=0%", "16:26:01 ready C=1 ctx=4k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "16:26:16 cell done C=1 ctx=4k 264.1 tok/s", "16:26:19 cell start C=1 ctx=8k unique_context=0%", "16:26:25 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "16:26:40 cell done C=1 ctx=8k 268.7 tok/s", "16:26:42 cell start C=1 ctx=16k unique_context=0%", "16:26:47 ready C=1 ctx=16k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "16:27:02 cell done C=1 ctx=16k 248.7 tok/s", "16:27:04 cell start C=1 ctx=32k unique_context=0%", "16:27:11 ready C=1 ctx=32k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "16:27:26 cell done C=1 ctx=32k 250.4 tok/s", "16:27:28 cell start C=4 ctx=0 unique_context=0%", "16:27:33 ready C=4 ctx=0 running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "16:27:48 cell done C=4 ctx=0 501.0 tok/s", "16:27:51 cell start C=8 ctx=0 unique_context=0%", "16:27:57 ready C=8 ctx=0 running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "16:28:12 cell done C=8 ctx=0 734.2 tok/s", "16:28:14 cell start C=16 ctx=0 unique_context=0%", "16:28:19 ready C=16 ctx=0 running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "16:28:34 cell done C=16 ctx=0 1105.5 tok/s", "16:28:36 cell start C=32 ctx=0 unique_context=0%", "16:28:41 ready C=32 ctx=0 running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "16:28:56 cell done C=32 ctx=0 1655.3 tok/s", "16:28:58 cell start C=4 ctx=4k unique_context=0%", "16:29:04 ready C=4 ctx=4k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "16:29:19 cell done C=4 ctx=4k 562.1 tok/s", "16:29:21 cell start C=8 ctx=4k unique_context=0%", "16:29:27 ready C=8 ctx=4k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "16:29:42 cell done C=8 ctx=4k 834.7 tok/s", "16:29:44 cell start C=16 ctx=4k unique_context=0%", "16:29:50 ready C=16 ctx=4k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "16:30:05 cell done C=16 ctx=4k 1192.8 tok/s", "16:30:08 cell start C=32 ctx=4k unique_context=0%", "16:30:14 ready C=32 ctx=4k running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "16:30:29 cell done C=32 ctx=4k 1833.7 tok/s", "16:30:31 cell start C=4 ctx=8k unique_context=0%", "16:30:36 ready C=4 ctx=8k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "16:30:52 cell done C=4 ctx=8k 525.2 tok/s", "16:30:54 cell start C=8 ctx=8k unique_context=0%", "16:31:00 ready C=8 ctx=8k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "16:31:15 cell done C=8 ctx=8k 875.0 tok/s", "16:31:17 cell start C=16 ctx=8k unique_context=0%", "16:31:22 ready C=16 ctx=8k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "16:31:37 cell done C=16 ctx=8k 1193.4 tok/s", "16:31:39 cell start C=32 ctx=8k unique_context=0%", "16:31:44 ready C=32 ctx=8k running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "16:32:00 cell done C=32 ctx=8k 1692.7 tok/s", "16:32:02 cell start C=4 ctx=16k unique_context=0%", "16:32:07 ready C=4 ctx=16k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "16:32:22 cell done C=4 ctx=16k 491.3 tok/s", "16:32:24 cell start C=8 ctx=16k unique_context=0%", "16:32:30 ready C=8 ctx=16k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "16:32:45 cell done C=8 ctx=16k 855.8 tok/s", "16:32:47 cell start C=16 ctx=16k unique_context=0%", "16:32:54 ready C=16 ctx=16k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "16:33:09 cell done C=16 ctx=16k 1330.9 tok/s", "16:33:11 cell start C=32 ctx=16k unique_context=0%", "16:33:17 ready C=32 ctx=16k running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "16:33:32 cell done C=32 ctx=16k 1861.5 tok/s", "16:33:35 cell start C=4 ctx=32k unique_context=0%", "16:33:42 ready C=4 ctx=32k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "16:33:57 cell done C=4 ctx=32k 504.8 tok/s", "16:33:59 cell start C=8 ctx=32k unique_context=0%", "16:34:05 ready C=8 ctx=32k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "16:34:21 cell done C=8 ctx=32k 776.7 tok/s", "16:34:23 cell start C=16 ctx=32k unique_context=0%", "16:34:29 ready C=16 ctx=32k running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "16:34:44 cell done C=16 ctx=32k 1225.2 tok/s", "16:34:46 cell start C=32 ctx=32k unique_context=0%", "16:34:53 ready C=32 ctx=32k running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "16:35:08 cell done C=32 ctx=32k 1673.4 tok/s" ], "prefill": { "8192": { "ttft_seconds": 0.703, "prefill_seconds": 0.703, "tok_per_sec": 11656.0, "client_ttft_seconds": 0.703, "client_tok_per_sec": 11656.0, "prompt_tokens": 8194, "samples": 20, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": {} }, "16384": { "ttft_seconds": 1.391, "prefill_seconds": 1.391, "tok_per_sec": 11780.0, "client_ttft_seconds": 1.391, "client_tok_per_sec": 11780.0, "prompt_tokens": 16386, "samples": 13, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": {} }, "32768": { "ttft_seconds": 2.859, "prefill_seconds": 2.859, "tok_per_sec": 11462.0, "client_ttft_seconds": 2.859, "client_tok_per_sec": 11462.0, "prompt_tokens": 32770, "samples": 7, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": {} }, "65536": { "ttft_seconds": 6.093, "prefill_seconds": 6.093, "tok_per_sec": 10755.0, "client_ttft_seconds": 6.093, "client_tok_per_sec": 10755.0, "prompt_tokens": 65538, "samples": 4, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": {} }, "131072": { "ttft_seconds": 13.531, "prefill_seconds": 13.531, "tok_per_sec": 9687.0, "client_ttft_seconds": 13.531, "client_tok_per_sec": 9687.0, "prompt_tokens": 131074, "samples": 2, "method": "client", "server_validation": { "method": "", "tok_per_sec": 0.0, "prefill_seconds": 0.0, "prompt_tokens": 0, "request_prompt_tokens": 0, "cached_tokens": 0, "token_source": "", "samples": 0, "invalid_reason": "" }, "hardware_summary": {} } }, "results": [ { "concurrency": 1, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 3391, "server_output_tokens": 3393, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 226.06666666666666, "per_request_avg_tps": 226.06666666666666, "ttft_avg": 0.07300000009126961, "ttft_p50": 0.0779999999795109, "ttft_p90": 0.07800000016577542, "ttft_p99": 0.07800000020768494, "time_to_second_token_avg": 0.015666666595886152, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999828636646, "time_to_second_token_p99": 0.015999999828636646, "request_latency_avg": 9.203499999945052, "request_latency_p50": 9.203499999945052, "request_latency_p90": 9.266299999901094, "request_latency_p99": 9.280429999891203, "inter_token_latency_avg": 0.004526696221636894, "inter_token_latency_p50": 0.004496336101568439, "inter_token_latency_p90": 0.004624696237311647, "inter_token_latency_p99": 0.004653577267853869, "output_tps_per_user_avg": 221.01066543588996, "output_tps_per_user_p50": 222.40330291393786, "output_tps_per_user_p90": 225.1913204828991, "output_tps_per_user_p99": 225.81862443591538, "e2e_output_tps_per_user_avg": 222.54022958102001, "e2e_output_tps_per_user_p50": 222.54022958102001, "e2e_output_tps_per_user_p90": 224.05873084771085, "e2e_output_tps_per_user_p99": 224.4003936327163, "chunk_inter_token_latency_avg": 0.012913819186089725, "chunk_inter_token_latency_p50": 0.012890469416668625, "chunk_inter_token_latency_p90": 0.012960702578532384, "chunk_inter_token_latency_p99": 0.01297650503995173, "input_seq_len_avg": 65.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 3, "completed_request_count": 2, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.015999999828636646, "latency": 9.281999999890104, "inter_token_latency_avg": 0.004496336101568439, "chunk_inter_token_latency_avg": 0.012872727272602228, "input_tokens": 65, "output_tokens": 2048, "output_tps_per_user": 222.40330291393786, "e2e_output_tps_per_user": 220.6421029976565, "completed": true }, { "ttft": 0.06300000008195639, "time_to_second_token": 0.01500000013038516, "latency": 9.125, "inter_token_latency_avg": 0.004426966292094794, "chunk_inter_token_latency_avg": 0.012890469416668625, "input_tokens": 65, "output_tokens": 2048, "output_tps_per_user": 225.8883248751394, "e2e_output_tps_per_user": 224.43835616438355, "completed": true }, { "ttft": 0.07800000021234155, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.004656786271247449, "chunk_inter_token_latency_avg": 0.012978260868998325, "input_tokens": 65, "output_tokens": 642, "output_tps_per_user": 214.7403685185926, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3391, "wall_time": 21.5, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 225.97402597206369, "server_utilization": 0.004199664026877881, "server_spec_accept_rate": 0.32241379310344825, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.485, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 4096, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 3962, "server_output_tokens": 3962, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 264.1333333333333, "per_request_avg_tps": 264.1333333333333, "ttft_avg": 0.0779999999795109, "ttft_p50": 0.0779999999795109, "ttft_p90": 0.0779999999795109, "ttft_p99": 0.0779999999795109, "time_to_second_token_avg": 0.02066666663934787, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.027999999932944775, "time_to_second_token_p99": 0.030699999956414103, "request_latency_avg": 8.327999999979511, "request_latency_p50": 8.327999999979511, "request_latency_p90": 9.74000000008382, "request_latency_p99": 10.057700000107289, "inter_token_latency_avg": 0.004008420156091125, "inter_token_latency_p50": 0.003964684014927014, "inter_token_latency_p90": 0.004706957320867332, "inter_token_latency_p99": 0.0048739688147039035, "output_tps_per_user_avg": 257.4239412698424, "output_tps_per_user_p50": 252.22691045112433, "output_tps_per_user_p90": 302.96658487093805, "output_tps_per_user_p99": 314.3830116153962, "e2e_output_tps_per_user_avg": 257.48266249831795, "e2e_output_tps_per_user_p50": 257.48266249831795, "e2e_output_tps_per_user_p90": 301.1384645486754, "e2e_output_tps_per_user_p99": 310.9610200100058, "chunk_inter_token_latency_avg": 0.013341908488142283, "chunk_inter_token_latency_p50": 0.013261758690939908, "chunk_inter_token_latency_p90": 0.013464767392849667, "chunk_inter_token_latency_p99": 0.013510444350779362, "input_seq_len_avg": 4096.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 3, "completed_request_count": 2, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.01500000013038516, "latency": 10.093000000109896, "inter_token_latency_avg": 0.004892525647352411, "chunk_inter_token_latency_avg": 0.013515519568327106, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 204.39340988251124, "e2e_output_tps_per_user": 202.91290993537112, "completed": true }, { "ttft": 0.0779999999795109, "time_to_second_token": 0.015999999828636646, "latency": 6.562999999849126, "inter_token_latency_avg": 0.0031680508059939495, "chunk_inter_token_latency_avg": 0.013261758690939908, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 315.6515034758915, "e2e_output_tps_per_user": 312.05241506126475, "completed": true }, { "ttft": 0.0779999999795109, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.003964684014927014, "chunk_inter_token_latency_avg": 0.013248447205159837, "input_tokens": 4096, "output_tokens": 1077, "output_tps_per_user": 252.22691045112433, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3962, "wall_time": 21.530999999959022, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 263.8694638712642, "server_utilization": 0.006599472042236654, "server_spec_accept_rate": 0.43157894736842106, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.516, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 4031, "server_output_tokens": 4031, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 268.73333333333335, "per_request_avg_tps": 268.73333333333335, "ttft_avg": 0.08899999999751647, "ttft_p50": 0.0940000000409782, "ttft_p90": 0.0940000000409782, "ttft_p99": 0.0940000000409782, "time_to_second_token_avg": 0.015000000052774945, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.01500000013038516, "time_to_second_token_p99": 0.01500000013038516, "request_latency_avg": 7.507999999914318, "request_latency_p50": 7.507999999914318, "request_latency_p90": 8.251999999862164, "request_latency_p99": 8.41939999985043, "inter_token_latency_avg": 0.003794151700553233, "inter_token_latency_p50": 0.004083536883213743, "inter_token_latency_p90": 0.004121792122405461, "inter_token_latency_p99": 0.004130399551223597, "output_tps_per_user_avg": 267.5457396704054, "output_tps_per_user_p50": 244.88575188599745, "output_tps_per_user_p90": 301.5372984363488, "output_tps_per_user_p99": 314.2838964101778, "e2e_output_tps_per_user_avg": 277.026188052473, "e2e_output_tps_per_user_p50": 277.026188052473, "e2e_output_tps_per_user_p90": 304.4779040752412, "e2e_output_tps_per_user_p99": 310.65454018036405, "chunk_inter_token_latency_avg": 0.013419303162198512, "chunk_inter_token_latency_p50": 0.013417335473416586, "chunk_inter_token_latency_p90": 0.013558100218269956, "chunk_inter_token_latency_p99": 0.013589772285861964, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 3, "completed_request_count": 2, "request_samples": [ { "ttft": 0.07899999991059303, "time_to_second_token": 0.014999999897554517, "latency": 8.437999999849126, "inter_token_latency_avg": 0.004083536883213743, "chunk_inter_token_latency_avg": 0.013417335473416586, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 244.88575188599745, "e2e_output_tps_per_user": 242.71154302401268, "completed": true }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.01500000013038516, "latency": 6.577999999979511, "inter_token_latency_avg": 0.003167562286242566, "chunk_inter_token_latency_avg": 0.013593291404483298, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 315.7001850739366, "e2e_output_tps_per_user": 311.3408330809333, "completed": true }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.00413135593220339, "chunk_inter_token_latency_avg": 0.013247282608695652, "input_tokens": 8192, "output_tokens": 1181, "output_tps_per_user": 242.05128205128204, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4031, "wall_time": 21.015000000130385, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 268.4648684625372, "server_utilization": 0.008279337652987717, "server_spec_accept_rate": 0.39152542372881355, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.0, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.985, "measurement_wall_seconds": 15.0, "client_output_tokens": 3727, "server_output_tokens": 3729, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 248.71538204701503, "per_request_avg_tps": 248.71538204701503, "ttft_avg": 0.10933333324889342, "ttft_p50": 0.10899999993853271, "ttft_p90": 0.10979999988339842, "ttft_p99": 0.1099799998709932, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 8.085500000044703, "request_latency_p50": 8.085500000044703, "request_latency_p90": 8.641899999976157, "request_latency_p99": 8.767089999960735, "inter_token_latency_avg": 0.004003995769937388, "inter_token_latency_p50": 0.0042186311787851295, "inter_token_latency_p90": 0.004232881096536895, "inter_token_latency_p99": 0.004236087328031042, "output_tps_per_user_avg": 251.4111538403636, "output_tps_per_user_p50": 237.0437133800299, "output_tps_per_user_p90": 272.32290281291705, "output_tps_per_user_p99": 280.26072043531667, "e2e_output_tps_per_user_avg": 255.18104888286246, "e2e_output_tps_per_user_p50": 255.18104888286246, "e2e_output_tps_per_user_p90": 272.7412165385607, "e2e_output_tps_per_user_p99": 276.6922542610928, "chunk_inter_token_latency_avg": 0.013564189329207254, "chunk_inter_token_latency_p50": 0.013609345794751126, "chunk_inter_token_latency_p90": 0.013747956115675582, "chunk_inter_token_latency_p99": 0.013779143437883586, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 3, "completed_request_count": 2, "request_samples": [ { "ttft": 0.10899999993853271, "time_to_second_token": 0.0, "latency": 8.780999999959022, "inter_token_latency_avg": 0.004236443575974836, "chunk_inter_token_latency_avg": 0.01330061349696394, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 236.047047969922, "e2e_output_tps_per_user": 233.23083931323964, "completed": true }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.0, "latency": 7.390000000130385, "inter_token_latency_avg": 0.0035569125550521997, "chunk_inter_token_latency_avg": 0.013609345794751126, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 281.14270017113887, "e2e_output_tps_per_user": 277.13125845248527, "completed": true }, { "ttft": 0.10999999986961484, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0042186311787851295, "chunk_inter_token_latency_avg": 0.013782608695906697, "input_tokens": 16384, "output_tokens": 527, "output_tps_per_user": 237.0437133800299, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3727, "wall_time": 20.07799999997951, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 248.6, "server_utilization": 0.011879049676025932, "server_spec_accept_rate": 0.34464285714285714, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.078, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 3756, "server_output_tokens": 3756, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 250.4, "per_request_avg_tps": 250.4, "ttft_avg": 0.13533333331967393, "ttft_p50": 0.13999999989755452, "ttft_p90": 0.14080000002868473, "ttft_p99": 0.14098000005818903, "time_to_second_token_avg": 0.015999999945051968, "time_to_second_token_p50": 0.015999999945051968, "time_to_second_token_p90": 0.016000000038184227, "time_to_second_token_p99": 0.016000000059138982, "request_latency_avg": 8.234000000054948, "request_latency_p50": 8.234000000054948, "request_latency_p90": 9.071600000164471, "request_latency_p99": 9.260060000189114, "inter_token_latency_avg": 0.004061151504605278, "inter_token_latency_p50": 0.004275784753116825, "inter_token_latency_p90": 0.004427213618969388, "inter_token_latency_p99": 0.0044612851137862146, "output_tps_per_user_avg": 249.43800900871017, "output_tps_per_user_p50": 233.87519665742104, "output_tps_per_user_p90": 279.1576134758348, "output_tps_per_user_p99": 289.34615725997793, "e2e_output_tps_per_user_avg": 252.81241509347825, "e2e_output_tps_per_user_p50": 252.81241509347825, "e2e_output_tps_per_user_p90": 278.52964595436885, "e2e_output_tps_per_user_p99": 284.3160228980692, "chunk_inter_token_latency_avg": 0.013966694247969615, "chunk_inter_token_latency_p50": 0.013899408284064082, "chunk_inter_token_latency_p90": 0.014204881656975797, "chunk_inter_token_latency_p99": 0.014273613165880933, "input_seq_len_avg": 32768.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 3, "completed_request_count": 2, "request_samples": [ { "ttft": 0.13999999989755452, "time_to_second_token": 0.01600000006146729, "latency": 7.186999999918044, "inter_token_latency_avg": 0.003442598925266482, "chunk_inter_token_latency_avg": 0.013899408284064082, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 290.47821768043826, "e2e_output_tps_per_user": 284.9589536695915, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.0, "latency": 9.281000000191852, "inter_token_latency_avg": 0.0044650708354325285, "chunk_inter_token_latency_avg": 0.014281250000203726, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 223.9606126882712, "e2e_output_tps_per_user": 220.665876517365, "completed": true }, { "ttft": 0.125, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.004275784753116825, "chunk_inter_token_latency_avg": 0.013719424459641036, "input_tokens": 32768, "output_tokens": 447, "output_tps_per_user": 233.87519665742104, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3756, "wall_time": 21.625, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 250.4, "server_utilization": 0.019678425725941917, "server_spec_accept_rate": 0.42745098039215684, "server_spec_accept_length": 0.0, "avg_running_reqs": 0.9, "max_running_reqs": 1, "effective_concurrency": 0.9, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.61, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.016, "measurement_wall_seconds": 15.016, "client_output_tokens": 7523, "server_output_tokens": 7523, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 500.9989344756162, "per_request_avg_tps": 125.24973361890405, "ttft_avg": 0.10549999997601844, "ttft_p50": 0.10149999998975545, "ttft_p90": 0.12790000012610106, "ttft_p99": 0.1675900000310503, "time_to_second_token_avg": 0.03337499999906868, "time_to_second_token_p50": 0.030999999959021807, "time_to_second_token_p90": 0.06300000008195639, "time_to_second_token_p99": 0.06300000008195639, "request_latency_avg": 16.601000000024214, "request_latency_p50": 16.49950000003446, "request_latency_p90": 17.207100000116043, "request_latency_p99": 17.32941000011051, "inter_token_latency_avg": 0.007925937945189752, "inter_token_latency_p50": 0.007947688199748944, "inter_token_latency_p90": 0.008264240351824966, "inter_token_latency_p99": 0.00840365901327445, "output_tps_per_user_avg": 126.36338441672603, "output_tps_per_user_p50": 125.86135421160046, "output_tps_per_user_p90": 132.99245902792146, "output_tps_per_user_p99": 134.0133018717275, "e2e_output_tps_per_user_avg": 123.49574621384625, "e2e_output_tps_per_user_p50": 124.19454044197592, "e2e_output_tps_per_user_p90": 127.39431029855334, "e2e_output_tps_per_user_p99": 127.49475415333964, "chunk_inter_token_latency_avg": 0.022407893261091657, "chunk_inter_token_latency_p50": 0.022440746989980075, "chunk_inter_token_latency_p90": 0.022632276665056173, "chunk_inter_token_latency_p99": 0.022664782070650695, "input_seq_len_avg": 65.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 8, "completed_request_count": 4, "request_samples": [ { "ttft": 0.0779999999795109, "time_to_second_token": 0.030999999959021807, "latency": 16.061999999918044, "inter_token_latency_avg": 0.007808500244229865, "chunk_inter_token_latency_avg": 0.02241795231407929, "input_tokens": 65, "output_tokens": 2048, "output_tps_per_user": 128.06556556605804, "e2e_output_tps_per_user": 127.50591458164924, "completed": true }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.008086876155268022, "chunk_inter_token_latency_avg": 0.02266839378238342, "input_tokens": 65, "output_tokens": 542, "output_tps_per_user": 123.65714285714286, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.06300000008195639, "latency": 17.343000000109896, "inter_token_latency_avg": 0.008419149975657726, "chunk_inter_token_latency_avg": 0.022616797900487352, "input_tokens": 65, "output_tokens": 2048, "output_tps_per_user": 118.77683648483497, "e2e_output_tps_per_user": 118.08798938978393, "completed": true }, { "ttft": 0.07899999991059303, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.007455635491459311, "chunk_inter_token_latency_avg": 0.022049645389634984, "input_tokens": 65, "output_tokens": 418, "output_tps_per_user": 134.1267288543726, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.06300000008195639, "latency": 16.890000000130385, "inter_token_latency_avg": 0.008197850513039498, "chunk_inter_token_latency_avg": 0.02249463806996227, "input_tokens": 65, "output_tokens": 2048, "output_tps_per_user": 121.98319527898202, "e2e_output_tps_per_user": 121.25518057928893, "completed": true }, { "ttft": 0.09399999980814755, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.007546808510681892, "chunk_inter_token_latency_avg": 0.022168750000128058, "input_tokens": 65, "output_tokens": 471, "output_tps_per_user": 132.50634338801382, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.014999999897554517, "latency": 16.108999999938533, "inter_token_latency_avg": 0.007785539814322445, "chunk_inter_token_latency_avg": 0.02238342696617703, "input_tokens": 65, "output_tokens": 2048, "output_tps_per_user": 128.44324527894375, "e2e_output_tps_per_user": 127.1339003046629, "completed": true }, { "ttft": 0.10900000017136335, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.008107142856859258, "chunk_inter_token_latency_avg": 0.022463541665880864, "input_tokens": 65, "output_tokens": 533, "output_tps_per_user": 123.34801762546026, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7523, "wall_time": 20.54700000002049, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 500.9989344756162, "server_utilization": 0.016438684905207612, "server_spec_accept_rate": 0.35289855072463766, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.515, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.984, "measurement_wall_seconds": 15.016, "client_output_tokens": 11002, "server_output_tokens": 11002, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 734.2498665158954, "per_request_avg_tps": 91.78123331448693, "ttft_avg": 0.15837499996996485, "ttft_p50": 0.1720000000204891, "ttft_p90": 0.18799999984912574, "ttft_p99": 0.18799999984912574, "time_to_second_token_avg": 0.02375000005122274, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.04060000008903443, "time_to_second_token_p99": 0.0607599998731166, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.010786946245965757, "inter_token_latency_p50": 0.010827407944580009, "inter_token_latency_p90": 0.010980424146248613, "inter_token_latency_p99": 0.011153264696442873, "output_tps_per_user_avg": 92.7440247843778, "output_tps_per_user_p50": 92.35821467875607, "output_tps_per_user_p90": 95.20228153733751, "output_tps_per_user_p99": 96.2885296056313, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.03017699408550904, "chunk_inter_token_latency_p50": 0.03014501882374638, "chunk_inter_token_latency_p90": 0.03028883362520564, "chunk_inter_token_latency_p99": 0.030319331987978027, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1937.375, "output_seq_len_p50": 1929.0, "output_seq_len_p90": 1991.0, "output_seq_len_p99": 2009.9, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.09399999980814755, "time_to_second_token": 0.03100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.01086210471758027, "chunk_inter_token_latency_avg": 0.03032272069495274, "input_tokens": 65, "output_tokens": 1930, "output_tps_per_user": 92.06318904120896, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18799999984912574, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.0108302180686248, "chunk_inter_token_latency_avg": 0.030099567099814376, "input_tokens": 65, "output_tokens": 1927, "output_tps_per_user": 92.33424421037333, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18799999984912574, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.010372451516743592, "chunk_inter_token_latency_avg": 0.030143063584062664, "input_tokens": 65, "output_tokens": 2012, "output_tps_per_user": 96.40922383544172, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.010824597820535218, "chunk_inter_token_latency_avg": 0.030274310595314026, "input_tokens": 65, "output_tokens": 1928, "output_tps_per_user": 92.38218514713883, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.010898119122346586, "chunk_inter_token_latency_avg": 0.030143063584062664, "input_tokens": 65, "output_tokens": 1915, "output_tps_per_user": 91.75895296918719, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10900000017136335, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.010561332660282932, "chunk_inter_token_latency_avg": 0.030146974063430098, "input_tokens": 65, "output_tokens": 1982, "output_tps_per_user": 94.68502055243572, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011172469202020013, "chunk_inter_token_latency_avg": 0.030099567099814376, "input_tokens": 65, "output_tokens": 1868, "output_tps_per_user": 89.5057289412082, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.010774276859592646, "chunk_inter_token_latency_avg": 0.030186685962621366, "input_tokens": 65, "output_tokens": 1937, "output_tps_per_user": 92.81365357802844, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 11002, "wall_time": 21.078999999910593, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 732.6851358520887, "server_utilization": 0.03875689944804417, "server_spec_accept_rate": 0.4453125, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.063, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 16, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.985, "measurement_wall_seconds": 15.0, "client_output_tokens": 16566, "server_output_tokens": 16566, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1105.5055055151245, "per_request_avg_tps": 69.09409409469528, "ttft_avg": 0.18118750014400575, "ttft_p50": 0.18700000015087426, "ttft_p90": 0.18700000015087426, "ttft_p99": 0.18700000015087426, "time_to_second_token_avg": 0.015999999828636646, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999828636646, "time_to_second_token_p99": 0.015999999828636646, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.013945337689816217, "inter_token_latency_p50": 0.013923648805040715, "inter_token_latency_p90": 0.014346917812419587, "inter_token_latency_p99": 0.014395847801304627, "output_tps_per_user_avg": 71.74660898324669, "output_tps_per_user_p50": 71.82153131836856, "output_tps_per_user_p90": 73.6889920764709, "output_tps_per_user_p99": 75.19557866104805, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.038168419479629116, "chunk_inter_token_latency_p50": 0.03817533718660718, "chunk_inter_token_latency_p90": 0.038323017407831964, "chunk_inter_token_latency_p99": 0.038323017407831964, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1422.9375, "output_seq_len_p50": 1424.0, "output_seq_len_p90": 1461.0, "output_seq_len_p99": 1490.85, "request_count": 16, "completed_request_count": 0, "request_samples": [ { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013683011049619562, "chunk_inter_token_latency_avg": 0.038323017407831964, "input_tokens": 65, "output_tokens": 1449, "output_tps_per_user": 73.08332912789716, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.013775778546684445, "chunk_inter_token_latency_avg": 0.038134099616779736, "input_tokens": 65, "output_tokens": 1446, "output_tps_per_user": 72.59117853928336, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014192693409634045, "chunk_inter_token_latency_avg": 0.03810192307663293, "input_tokens": 65, "output_tokens": 1397, "output_tps_per_user": 70.45878968407764, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013270596115103232, "chunk_inter_token_latency_avg": 0.03817533718660718, "input_tokens": 65, "output_tokens": 1494, "output_tps_per_user": 75.35456518504866, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013692467173358069, "chunk_inter_token_latency_avg": 0.03810192307663293, "input_tokens": 65, "output_tokens": 1448, "output_tps_per_user": 73.03285721551602, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013845562543570318, "chunk_inter_token_latency_avg": 0.03817533718660718, "input_tokens": 65, "output_tokens": 1432, "output_tps_per_user": 72.2253066174177, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.0140717329544383, "chunk_inter_token_latency_avg": 0.03817533718660718, "input_tokens": 65, "output_tokens": 1409, "output_tps_per_user": 71.06445263265138, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014315751444977692, "chunk_inter_token_latency_avg": 0.03802879078665859, "input_tokens": 65, "output_tokens": 1385, "output_tps_per_user": 69.85312673550392, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013982357092342362, "chunk_inter_token_latency_avg": 0.038249034748743485, "input_tokens": 65, "output_tokens": 1418, "output_tps_per_user": 71.51869984408168, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014398982558029887, "chunk_inter_token_latency_avg": 0.03817533718660718, "input_tokens": 65, "output_tokens": 1377, "output_tps_per_user": 69.44935143645476, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013459918478158374, "chunk_inter_token_latency_avg": 0.03802879078665859, "input_tokens": 65, "output_tokens": 1473, "output_tps_per_user": 74.29465502504462, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013864940517739066, "chunk_inter_token_latency_avg": 0.038323017407831964, "input_tokens": 65, "output_tokens": 1430, "output_tps_per_user": 72.12436279265542, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014378084179861484, "chunk_inter_token_latency_avg": 0.03810192307663293, "input_tokens": 65, "output_tokens": 1379, "output_tps_per_user": 69.55029526121704, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.01426421886238238, "chunk_inter_token_latency_avg": 0.038323017407831964, "input_tokens": 65, "output_tokens": 1390, "output_tps_per_user": 70.10548629740964, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.013797353760340617, "chunk_inter_token_latency_avg": 0.038249034748743485, "input_tokens": 65, "output_tokens": 1437, "output_tps_per_user": 72.47766617932342, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.014131954350819632, "chunk_inter_token_latency_avg": 0.03802879078665859, "input_tokens": 65, "output_tokens": 1403, "output_tps_per_user": 70.76162115836452, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 16566, "wall_time": 20.04700000002049, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1103.2232285687967, "server_utilization": 0.07343412526997839, "server_spec_accept_rate": 0.36578947368421055, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.015, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 32, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.969, "measurement_wall_seconds": 15.016, "client_output_tokens": 24778, "server_output_tokens": 24778, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1655.2875943571494, "per_request_avg_tps": 51.72773732366092, "ttft_avg": 0.2491250000166474, "ttft_p50": 0.25, "ttft_p90": 0.2660000000614673, "ttft_p99": 0.2660000000614673, "time_to_second_token_avg": 0.03049999996437691, "time_to_second_token_p50": 0.030999999959021807, "time_to_second_token_p90": 0.030999999959021807, "time_to_second_token_p99": 0.030999999959021807, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01840169790702485, "inter_token_latency_p50": 0.018323135573875104, "inter_token_latency_p90": 0.019269604230122906, "inter_token_latency_p99": 0.019854797160181566, "output_tps_per_user_avg": 54.43155770976817, "output_tps_per_user_p50": 54.57585892385501, "output_tps_per_user_p90": 57.20583764079844, "output_tps_per_user_p99": 59.547481504188724, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.049925928271572106, "chunk_inter_token_latency_p50": 0.04983333333317811, "chunk_inter_token_latency_p90": 0.05008629441608765, "chunk_inter_token_latency_p99": 0.05017423218490979, "input_seq_len_avg": 65.0, "output_seq_len_avg": 1075.4375, "output_seq_len_p50": 1078.0, "output_seq_len_p90": 1129.9, "output_seq_len_p99": 1176.1100000000001, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01798906107560486, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1098, "output_tps_per_user": 55.589338198207, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018938579654451566, "chunk_inter_token_latency_avg": 0.05008629441608765, "input_tokens": 65, "output_tokens": 1043, "output_tps_per_user": 52.80227019373901, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018121212121155677, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1090, "output_tps_per_user": 55.183946488466205, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01842577030806586, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1072, "output_tps_per_user": 54.271815141549396, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018204797047913775, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1085, "output_tps_per_user": 54.9305766698782, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.017956323930790295, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1100, "output_tps_per_user": 55.690686125642195, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.01885037878783998, "chunk_inter_token_latency_avg": 0.05001507537678146, "input_tokens": 65, "output_tokens": 1057, "output_tps_per_user": 53.04933185984999, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.016709568162522042, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1182, "output_tps_per_user": 59.84595115048539, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01763538873989145, "chunk_inter_token_latency_avg": 0.05008629441608765, "input_tokens": 65, "output_tokens": 1120, "output_tps_per_user": 56.7041653999942, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01813786764700233, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1089, "output_tps_per_user": 55.1332725247486, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018669820245921034, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1058, "output_tps_per_user": 53.56237964950301, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01813786764700233, "chunk_inter_token_latency_avg": 0.05008629441608765, "input_tokens": 65, "output_tokens": 1089, "output_tps_per_user": 55.1332725247486, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018794285714227175, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1051, "output_tps_per_user": 53.2076619034798, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01798906107560486, "chunk_inter_token_latency_avg": 0.05008629441608765, "input_tokens": 65, "output_tokens": 1098, "output_tps_per_user": 55.589338198207, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.019442364531959147, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1016, "output_tps_per_user": 51.434073173363814, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018616981132017485, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1061, "output_tps_per_user": 53.7144015406558, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.017463716814104895, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1131, "output_tps_per_user": 57.2615790008878, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018340148698827632, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1077, "output_tps_per_user": 54.52518496013741, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018687499999941792, "chunk_inter_token_latency_avg": 0.05008629441608765, "input_tokens": 65, "output_tokens": 1057, "output_tps_per_user": 53.51170568578541, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018306122448922574, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1079, "output_tps_per_user": 54.6265328875726, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.019271484374939973, "chunk_inter_token_latency_avg": 0.05021374045785886, "input_tokens": 65, "output_tokens": 1025, "output_tps_per_user": 51.89013884682221, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.02001419878289912, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 987, "output_tps_per_user": 49.964528225553416, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01949999999993926, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1013, "output_tps_per_user": 51.282051282211015, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.019085106382919277, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1035, "output_tps_per_user": 52.39687848399821, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.018993262752587614, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1040, "output_tps_per_user": 52.65024830258621, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01790744101627816, "chunk_inter_token_latency_avg": 0.05008629441608765, "input_tokens": 65, "output_tokens": 1103, "output_tps_per_user": 55.842708016795, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01698278829598841, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1163, "output_tps_per_user": 58.88314583985099, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.0192526829267693, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1026, "output_tps_per_user": 51.94081281053981, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01744827586201462, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1132, "output_tps_per_user": 57.31225296460539, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.017972677595572435, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1099, "output_tps_per_user": 55.640012161924595, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01921518987335787, "chunk_inter_token_latency_avg": 0.04983333333317811, "input_tokens": 65, "output_tokens": 1028, "output_tps_per_user": 52.04216073797502, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01779440937776243, "chunk_inter_token_latency_avg": 0.049959493670730465, "input_tokens": 65, "output_tokens": 1110, "output_tps_per_user": 56.19742576281819, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 24778, "wall_time": 20.062000000150874, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1650.106553003368, "server_utilization": 0.14218862491000717, "server_spec_accept_rate": 0.3694078947368421, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.031, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 4096, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 8432, "server_output_tokens": 8432, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 562.1333333333333, "per_request_avg_tps": 140.53333333333333, "ttft_avg": 0.13449999995646067, "ttft_p50": 0.1095000000204891, "ttft_p90": 0.1869999999180436, "ttft_p99": 0.1869999999180436, "time_to_second_token_avg": 0.017625000065891072, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.020500000100582835, "time_to_second_token_p99": 0.029950000182725485, "request_latency_avg": 15.437249999959022, "request_latency_p50": 16.241999999969266, "request_latency_p90": 17.784299999987706, "request_latency_p99": 18.31592999999877, "inter_token_latency_avg": 0.007061956272756975, "inter_token_latency_p50": 0.007466382150324089, "inter_token_latency_p90": 0.00822784562776494, "inter_token_latency_p99": 0.008867386419160687, "output_tps_per_user_avg": 146.72102219186402, "output_tps_per_user_p50": 133.97357814634427, "output_tps_per_user_p90": 192.91329362861765, "output_tps_per_user_p99": 196.39615465605573, "e2e_output_tps_per_user_avg": 137.93240764085172, "e2e_output_tps_per_user_p50": 126.10570282040771, "e2e_output_tps_per_user_p90": 169.8574179994734, "e2e_output_tps_per_user_p99": 186.24194014864702, "chunk_inter_token_latency_avg": 0.024518965900162008, "chunk_inter_token_latency_p50": 0.024602451108890463, "chunk_inter_token_latency_p90": 0.02511635706905822, "chunk_inter_token_latency_p99": 0.025319530443773466, "input_seq_len_avg": 4096.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 8, "completed_request_count": 4, "request_samples": [ { "ttft": 0.1869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 16.07799999997951, "inter_token_latency_avg": 0.00776306790428015, "chunk_inter_token_latency_avg": 0.023896240601596192, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 128.81505254496773, "e2e_output_tps_per_user": 127.37902724235663, "completed": true }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.007595238095197443, "chunk_inter_token_latency_avg": 0.02501960784300334, "input_tokens": 4096, "output_tokens": 505, "output_tps_per_user": 131.66144200697428, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 10.889999999897555, "inter_token_latency_avg": 0.005228627259393996, "chunk_inter_token_latency_avg": 0.023731707317027742, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 191.25478837745666, "e2e_output_tps_per_user": 188.0624426096663, "completed": true }, { "ttft": 0.11000000010244548, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.005081736189344732, "chunk_inter_token_latency_avg": 0.024903314916844072, "input_tokens": 4096, "output_tokens": 1775, "output_tps_per_user": 196.78313921465997, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0779999999795109, "time_to_second_token": 0.01500000013038516, "latency": 18.375, "inter_token_latency_avg": 0.008938446507093547, "chunk_inter_token_latency_avg": 0.02534210526318627, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 111.87626386826844, "e2e_output_tps_per_user": 111.45578231292517, "completed": true }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.006627705627528233, "chunk_inter_token_latency_avg": 0.024301587300936855, "input_tokens": 4096, "output_tokens": 232, "output_tps_per_user": 150.88177662062893, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1869999999180436, "time_to_second_token": 0.01600000006146729, "latency": 16.405999999959022, "inter_token_latency_avg": 0.007923302393766965, "chunk_inter_token_latency_avg": 0.023957163958701594, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 126.21000061624194, "e2e_output_tps_per_user": 124.83237839845881, "completed": true }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.03100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.007337526205450734, "chunk_inter_token_latency_avg": 0.025, "input_tokens": 4096, "output_tokens": 478, "output_tps_per_user": 136.28571428571428, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 8432, "wall_time": 20.562000000150874, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 561.5343633434659, "server_utilization": 0.019198464122870185, "server_spec_accept_rate": 0.6253164556962025, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.531, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 4096, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 12520, "server_output_tokens": 12518, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 834.6666666666666, "per_request_avg_tps": 104.33333333333333, "ttft_avg": 0.19730769235712403, "ttft_p50": 0.25, "ttft_p90": 0.2660000000614673, "ttft_p99": 0.2660000000614673, "time_to_second_token_avg": 0.023499999986961485, "time_to_second_token_p50": 0.02350000001024455, "time_to_second_token_p90": 0.030999999959021807, "time_to_second_token_p99": 0.030999999959021807, "request_latency_avg": 17.409400000004098, "request_latency_p50": 16.875, "request_latency_p90": 18.75, "request_latency_p99": 18.75, "inter_token_latency_avg": 0.009680256581124081, "inter_token_latency_p50": 0.00903761602344895, "inter_token_latency_p90": 0.011714786842528642, "inter_token_latency_p99": 0.012128655864917981, "output_tps_per_user_avg": 105.74340522703913, "output_tps_per_user_p50": 110.64864864864865, "output_tps_per_user_p90": 125.88700343475126, "output_tps_per_user_p99": 127.8869136268776, "e2e_output_tps_per_user_avg": 118.11156387873481, "e2e_output_tps_per_user_p50": 121.36296296296297, "e2e_output_tps_per_user_p90": 125.50276308510495, "e2e_output_tps_per_user_p99": 125.9779686162028, "chunk_inter_token_latency_avg": 0.03490650033459652, "chunk_inter_token_latency_p50": 0.03435687732330582, "chunk_inter_token_latency_p90": 0.03584041994573701, "chunk_inter_token_latency_p99": 0.03738039412455718, "input_seq_len_avg": 4096.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 13, "completed_request_count": 5, "request_samples": [ { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 18.75, "inter_token_latency_avg": 0.009029799706858101, "chunk_inter_token_latency_avg": 0.03435687732330582, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 110.74442761343903, "e2e_output_tps_per_user": 109.22666666666667, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.008487309644774056, "chunk_inter_token_latency_avg": 0.03412244898000998, "input_tokens": 4096, "output_tokens": 198, "output_tps_per_user": 117.82296650573322, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011766376811606081, "chunk_inter_token_latency_avg": 0.03417003367006816, "input_tokens": 4096, "output_tokens": 1726, "output_tps_per_user": 84.98792925054238, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 16.25, "inter_token_latency_avg": 0.007808500244229865, "chunk_inter_token_latency_avg": 0.03415384615371481, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 128.06556556605804, "e2e_output_tps_per_user": 126.03076923076924, "completed": true }, { "ttft": 0.125, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.011306233062386149, "chunk_inter_token_latency_avg": 0.03758558558577017, "input_tokens": 4096, "output_tokens": 370, "output_tps_per_user": 88.44678811078327, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012178057554005969, "chunk_inter_token_latency_avg": 0.03569947275937075, "input_tokens": 4096, "output_tokens": 1669, "output_tps_per_user": 82.11490178670164, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.07800000021234155, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.011508426966218885, "chunk_inter_token_latency_avg": 0.03587565674232857, "input_tokens": 4096, "output_tokens": 1781, "output_tps_per_user": 86.89284842623039, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.01600000006146729, "latency": 16.875, "inter_token_latency_avg": 0.008121641426477772, "chunk_inter_token_latency_avg": 0.03514799154334038, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 123.12781954887218, "e2e_output_tps_per_user": 121.36296296296297, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01010541310547148, "chunk_inter_token_latency_avg": 0.034774509804122444, "input_tokens": 4096, "output_tokens": 352, "output_tps_per_user": 98.95686495572947, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.01600000006146729, "latency": 16.42200000002049, "inter_token_latency_avg": 0.007900341963859546, "chunk_inter_token_latency_avg": 0.03411814345995884, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 126.57679940622103, "e2e_output_tps_per_user": 124.7107538666085, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.00997506234413965, "chunk_inter_token_latency_avg": 0.035398230088495575, "input_tokens": 4096, "output_tokens": 402, "output_tps_per_user": 100.25, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.01600000006146729, "latency": 18.75, "inter_token_latency_avg": 0.00903761602344895, "chunk_inter_token_latency_avg": 0.03425925925925926, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 110.64864864864865, "e2e_output_tps_per_user": 109.22666666666667, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.008618556701136542, "chunk_inter_token_latency_avg": 0.03412244898000998, "input_tokens": 4096, "output_tokens": 195, "output_tps_per_user": 116.02870813254944, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 12520, "wall_time": 21.108999999938533, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 833.6996336923942, "server_utilization": 0.03755699544036473, "server_spec_accept_rate": 0.4386904761904762, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.078, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 16, "context_tokens": 4096, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 17892, "server_output_tokens": 17888, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1192.8, "per_request_avg_tps": 74.55, "ttft_avg": 0.3959411764380467, "ttft_p50": 0.4220000000204891, "ttft_p90": 0.4369999999180436, "ttft_p99": 0.45043999996967615, "time_to_second_token_avg": 0.02443750001839362, "time_to_second_token_p50": 0.02350000001024455, "time_to_second_token_p90": 0.030999999959021807, "time_to_second_token_p99": 0.044600000011269, "request_latency_avg": 18.515000000130385, "request_latency_p50": 18.515000000130385, "request_latency_p90": 18.515000000130385, "request_latency_p99": 18.515000000130385, "inter_token_latency_avg": 0.013278463177496579, "inter_token_latency_p50": 0.01326618705065452, "inter_token_latency_p90": 0.016453211879894417, "inter_token_latency_p99": 0.01708722638794243, "output_tps_per_user_avg": 78.2140840330648, "output_tps_per_user_p50": 75.37960954279343, "output_tps_per_user_p90": 98.52426807815995, "output_tps_per_user_p99": 111.05951786744858, "e2e_output_tps_per_user_avg": 110.61301647235095, "e2e_output_tps_per_user_p50": 110.61301647235095, "e2e_output_tps_per_user_p90": 110.61301647235095, "e2e_output_tps_per_user_p99": 110.61301647235095, "chunk_inter_token_latency_avg": 0.04546927038939903, "chunk_inter_token_latency_p50": 0.04508744394605052, "chunk_inter_token_latency_p90": 0.04664410437931036, "chunk_inter_token_latency_p99": 0.04827388096501362, "input_seq_len_avg": 4096.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 17, "completed_request_count": 1, "request_samples": [ { "ttft": 0.09300000010989606, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.016629780309071706, "chunk_inter_token_latency_avg": 0.04551893095734772, "input_tokens": 4096, "output_tokens": 1230, "output_tps_per_user": 60.133085429546554, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.010262621111677715, "chunk_inter_token_latency_avg": 0.044921875, "input_tokens": 4096, "output_tokens": 1962, "output_tps_per_user": 97.44099378881988, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.016335499593776226, "chunk_inter_token_latency_avg": 0.04559863945564293, "input_tokens": 4096, "output_tokens": 1232, "output_tps_per_user": 61.21637077944019, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.015833858267668137, "chunk_inter_token_latency_avg": 0.04644110854489269, "input_tokens": 4096, "output_tokens": 1271, "output_tps_per_user": 63.15580088536883, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.013642469470785979, "chunk_inter_token_latency_avg": 0.0443907284766855, "input_tokens": 4096, "output_tokens": 1475, "output_tps_per_user": 73.30051220868793, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.009985104270160102, "chunk_inter_token_latency_avg": 0.0447884187084687, "input_tokens": 4096, "output_tokens": 2015, "output_tps_per_user": 100.14917951217006, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.01526899696051746, "chunk_inter_token_latency_avg": 0.04475278396445652, "input_tokens": 4096, "output_tokens": 1317, "output_tps_per_user": 65.49218672227114, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011384702549598289, "chunk_inter_token_latency_avg": 0.04515505617986737, "input_tokens": 4096, "output_tokens": 1766, "output_tps_per_user": 87.83716532280285, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 18.515000000130385, "inter_token_latency_avg": 0.008838788470986758, "chunk_inter_token_latency_avg": 0.04445454545481547, "input_tokens": 4096, "output_tokens": 2048, "output_tps_per_user": 113.13767755416829, "e2e_output_tps_per_user": 110.61301647235095, "completed": true }, { "ttft": 0.17199999978765845, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.01326618705065452, "chunk_inter_token_latency_avg": 0.048526315790552055, "input_tokens": 4096, "output_tokens": 140, "output_tps_per_user": 75.37960954279343, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.012224316109385127, "chunk_inter_token_latency_avg": 0.04633410138234685, "input_tokens": 4096, "output_tokens": 1646, "output_tps_per_user": 81.80416728852893, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4220000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.012647169811282097, "chunk_inter_token_latency_avg": 0.04508744394605052, "input_tokens": 4096, "output_tokens": 1591, "output_tps_per_user": 79.06907354939878, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.017174358974393997, "chunk_inter_token_latency_avg": 0.04455432372514629, "input_tokens": 4096, "output_tokens": 1171, "output_tps_per_user": 58.22633621964836, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011150943396249155, "chunk_inter_token_latency_avg": 0.04694859813093687, "input_tokens": 4096, "output_tokens": 1803, "output_tps_per_user": 89.67851099812508, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.014785871964710064, "chunk_inter_token_latency_avg": 0.044357615894130197, "input_tokens": 4096, "output_tokens": 1360, "output_tps_per_user": 67.63212899359155, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4529999999795109, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.011068357221598407, "chunk_inter_token_latency_avg": 0.04491722595073716, "input_tokens": 4096, "output_tokens": 1815, "output_tps_per_user": 90.3476441877603, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.015234848484926095, "chunk_inter_token_latency_avg": 0.04622988505770677, "input_tokens": 4096, "output_tokens": 1321, "output_tps_per_user": 65.6389855789794, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 17892, "wall_time": 21.156000000191852, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1192.5333333333333, "server_utilization": 0.006599472042236654, "server_spec_accept_rate": 0.62, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.063, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 32, "context_tokens": 4096, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.984, "measurement_wall_seconds": 15.0, "client_output_tokens": 27476, "server_output_tokens": 27477, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1833.6892685606456, "per_request_avg_tps": 57.302789642520175, "ttft_avg": 0.5830625001326553, "ttft_p50": 0.5780000002123415, "ttft_p90": 0.8280000002123415, "ttft_p99": 0.8430000001098961, "time_to_second_token_avg": 0.1300000000234333, "time_to_second_token_p50": 0.14000000013038516, "time_to_second_token_p90": 0.25, "time_to_second_token_p99": 0.25, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01692652695882891, "inter_token_latency_p50": 0.017437767126807047, "inter_token_latency_p90": 0.01999895197646254, "inter_token_latency_p99": 0.022317462350113, "output_tps_per_user_avg": 60.94672112770046, "output_tps_per_user_p50": 57.34925525712542, "output_tps_per_user_p90": 75.36167746540647, "output_tps_per_user_p99": 86.2671227032221, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.05913671820154512, "chunk_inter_token_latency_p50": 0.05877761631680368, "chunk_inter_token_latency_p90": 0.060721132280644706, "chunk_inter_token_latency_p99": 0.06363657001073449, "input_seq_len_avg": 4096.0, "output_seq_len_avg": 1214.375, "output_seq_len_p50": 1141.0, "output_seq_len_p90": 1497.9, "output_seq_len_p99": 1707.93, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5930000001098961, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.018497678737035894, "chunk_inter_token_latency_avg": 0.05859411764643429, "input_tokens": 4096, "output_tokens": 1078, "output_tps_per_user": 54.06083726591102, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8430000001098961, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.019438735177655788, "chunk_inter_token_latency_avg": 0.05872238805906764, "input_tokens": 4096, "output_tokens": 1013, "output_tps_per_user": 51.44367629173057, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8430000001098961, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.013149732620178916, "chunk_inter_token_latency_avg": 0.058898203592178615, "input_tokens": 4096, "output_tokens": 1497, "output_tps_per_user": 76.04717364864518, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.014541605839261064, "chunk_inter_token_latency_avg": 0.05825146198768321, "input_tokens": 4096, "output_tokens": 1371, "output_tps_per_user": 68.7681959649936, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5930000001098961, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.014746113989480131, "chunk_inter_token_latency_avg": 0.05842228738940662, "input_tokens": 4096, "output_tokens": 1352, "output_tps_per_user": 67.81447645890974, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8430000001098961, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.01730167106401729, "chunk_inter_token_latency_avg": 0.05837388723972599, "input_tokens": 4096, "output_tokens": 1138, "output_tps_per_user": 57.79788531985934, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.011325273459866239, "chunk_inter_token_latency_avg": 0.05785882352878723, "input_tokens": 4096, "output_tokens": 1738, "output_tps_per_user": 88.29808865487746, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.11000000010244548, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.02240659340648083, "chunk_inter_token_latency_avg": 0.06432176656119103, "input_tokens": 4096, "output_tokens": 911, "output_tps_per_user": 44.6297204514258, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5780000002123415, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.01732347826068492, "chunk_inter_token_latency_avg": 0.05911572700233726, "input_tokens": 4096, "output_tokens": 1151, "output_tps_per_user": 57.72512799981214, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5780000002123415, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.013254823685820132, "chunk_inter_token_latency_avg": 0.05859411764643429, "input_tokens": 4096, "output_tokens": 1504, "output_tps_per_user": 75.44423250758057, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.014348650619830532, "chunk_inter_token_latency_avg": 0.0576891495594946, "input_tokens": 4096, "output_tokens": 1372, "output_tps_per_user": 69.6929646205164, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.02211907386981041, "chunk_inter_token_latency_avg": 0.061539877300362096, "input_tokens": 4096, "output_tokens": 908, "output_tps_per_user": 45.209849466838065, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.015941653160281732, "chunk_inter_token_latency_avg": 0.05802949852444737, "input_tokens": 4096, "output_tokens": 1235, "output_tps_per_user": 62.72875152568727, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.015998405103602904, "chunk_inter_token_latency_avg": 0.05970833333308941, "input_tokens": 4096, "output_tokens": 1255, "output_tps_per_user": 62.50623068513224, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.018282527880843547, "chunk_inter_token_latency_avg": 0.05820118343132443, "input_tokens": 4096, "output_tokens": 1077, "output_tps_per_user": 54.6970313141325, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.018890772127983093, "chunk_inter_token_latency_avg": 0.05953115726978648, "input_tokens": 4096, "output_tokens": 1063, "output_tps_per_user": 52.93589871420289, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.012232926829218319, "chunk_inter_token_latency_avg": 0.05900588235270013, "input_tokens": 4096, "output_tokens": 1641, "output_tps_per_user": 81.74658558502142, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.013401469605823677, "chunk_inter_token_latency_avg": 0.058660818713210654, "input_tokens": 4096, "output_tokens": 1498, "output_tps_per_user": 74.61868208583967, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.01955467196797978, "chunk_inter_token_latency_avg": 0.05802949852444737, "input_tokens": 4096, "output_tokens": 1007, "output_tps_per_user": 51.138674258380384, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.01770697263893914, "chunk_inter_token_latency_avg": 0.05900588235270013, "input_tokens": 4096, "output_tokens": 1134, "output_tps_per_user": 56.47492772428613, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.019198086124323485, "chunk_inter_token_latency_avg": 0.06079393939369104, "input_tokens": 4096, "output_tokens": 1046, "output_tps_per_user": 52.08852557094353, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.01353711201074092, "chunk_inter_token_latency_avg": 0.058660818713210654, "input_tokens": 4096, "output_tokens": 1483, "output_tps_per_user": 73.87099990061081, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.02000199401786445, "chunk_inter_token_latency_avg": 0.062111455108105396, "input_tokens": 4096, "output_tokens": 1004, "output_tps_per_user": 49.99501545230273, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.01943992248054074, "chunk_inter_token_latency_avg": 0.05988656716393446, "input_tokens": 4096, "output_tokens": 1033, "output_tps_per_user": 51.44053434374518, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.02049233912146889, "chunk_inter_token_latency_avg": 0.06006586826322768, "input_tokens": 4096, "output_tokens": 980, "output_tps_per_user": 48.798723955936566, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.019971573603845338, "chunk_inter_token_latency_avg": 0.05907507507443741, "input_tokens": 4096, "output_tokens": 986, "output_tps_per_user": 50.07116714165475, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5780000002123415, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.016252045826407213, "chunk_inter_token_latency_avg": 0.05963963963924809, "input_tokens": 4096, "output_tokens": 1223, "output_tps_per_user": 61.53071500543921, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8280000002123415, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.013473972602594286, "chunk_inter_token_latency_avg": 0.05752046783563643, "input_tokens": 4096, "output_tokens": 1461, "output_tps_per_user": 74.21716144854409, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.01822161671200549, "chunk_inter_token_latency_avg": 0.05831976744162222, "input_tokens": 4096, "output_tokens": 1102, "output_tps_per_user": 54.87987239579792, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.017552055992929173, "chunk_inter_token_latency_avg": 0.05883284457453972, "input_tokens": 4096, "output_tokens": 1144, "output_tps_per_user": 56.973382514438704, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4380000000819564, "time_to_second_token": 0.14000000013038516, "latency": 0.0, "inter_token_latency_avg": 0.014633114514892812, "chunk_inter_token_latency_avg": 0.05883284457453972, "input_tokens": 4096, "output_tokens": 1372, "output_tps_per_user": 68.33815172991729, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5780000002123415, "time_to_second_token": 0.25, "latency": 0.0, "inter_token_latency_avg": 0.018412199630117983, "chunk_inter_token_latency_avg": 0.05808163265244215, "input_tokens": 4096, "output_tokens": 1083, "output_tps_per_user": 54.31181608330151, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 27476, "wall_time": 21.219000000040978, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1829.9700299541391, "server_utilization": 0.010559155267578557, "server_spec_accept_rate": 0.634375, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.047, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.984, "measurement_wall_seconds": 15.0, "client_output_tokens": 7870, "server_output_tokens": 7870, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 525.2269087047707, "per_request_avg_tps": 131.30672717619268, "ttft_avg": 0.15024999997694977, "ttft_p50": 0.14099999982863665, "ttft_p90": 0.2029999999795109, "ttft_p99": 0.2029999999795109, "time_to_second_token_avg": 0.018285714289439575, "time_to_second_token_p50": 0.01600000006146729, "time_to_second_token_p90": 0.022399999992921953, "time_to_second_token_p99": 0.03103999990038573, "request_latency_avg": 15.359000000054948, "request_latency_p50": 16.062000000150874, "request_latency_p90": 17.002800000016578, "request_latency_p99": 17.365679999964776, "inter_token_latency_avg": 0.007321540002783339, "inter_token_latency_p50": 0.0077474352712121955, "inter_token_latency_p90": 0.008128617012844739, "inter_token_latency_p99": 0.008376466977289145, "output_tps_per_user_avg": 138.63170019555284, "output_tps_per_user_p50": 129.07497319994206, "output_tps_per_user_p90": 161.6562421116112, "output_tps_per_user_p99": 172.33244628429838, "e2e_output_tps_per_user_avg": 136.17162912639222, "e2e_output_tps_per_user_p50": 127.50591457980093, "e2e_output_tps_per_user_p90": 158.66165174710795, "e2e_output_tps_per_user_p99": 170.67886465449777, "chunk_inter_token_latency_avg": 0.02409761352473045, "chunk_inter_token_latency_p50": 0.02402783765333741, "chunk_inter_token_latency_p90": 0.024623801716509054, "chunk_inter_token_latency_p99": 0.024998141760810638, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 8, "completed_request_count": 4, "request_samples": [ { "ttft": 0.2029999999795109, "time_to_second_token": 0.01600000006146729, "latency": 16.062000000150874, "inter_token_latency_avg": 0.0077474352712121955, "chunk_inter_token_latency_avg": 0.02346005917185113, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 129.07497319994206, "e2e_output_tps_per_user": 127.50591457980093, "completed": true }, { "ttft": 0.14099999982863665, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.006386824324255105, "chunk_inter_token_latency_avg": 0.0243935483868324, "input_tokens": 8192, "output_tokens": 593, "output_tps_per_user": 156.57233536271252, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.0, "latency": 11.905999999959022, "inter_token_latency_avg": 0.005763067904260131, "chunk_inter_token_latency_avg": 0.02412474437632002, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 173.51869119237475, "e2e_output_tps_per_user": 172.01411053309667, "completed": true }, { "ttft": 0.09300000010989606, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.007767056530027434, "chunk_inter_token_latency_avg": 0.023930930930354796, "input_tokens": 8192, "output_tokens": 1027, "output_tps_per_user": 128.74890199833115, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2029999999795109, "time_to_second_token": 0.01600000006146729, "latency": 17.405999999959022, "inter_token_latency_avg": 0.008404005862227412, "chunk_inter_token_latency_avg": 0.023926286509011838, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 118.99087368496413, "e2e_output_tps_per_user": 117.66057681287036, "completed": true }, { "ttft": 0.10900000017136335, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.006745901638820075, "chunk_inter_token_latency_avg": 0.024445544552555917, "input_tokens": 8192, "output_tokens": 367, "output_tps_per_user": 148.2381531099392, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2029999999795109, "time_to_second_token": 0.01600000006146729, "latency": 16.062000000150874, "inter_token_latency_avg": 0.0077474352712121955, "chunk_inter_token_latency_avg": 0.02346005917185113, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 129.07497319994206, "e2e_output_tps_per_user": 127.50591457980093, "completed": true }, { "ttft": 0.14099999982863665, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.008010593220252165, "chunk_inter_token_latency_avg": 0.02503973509906637, "input_tokens": 8192, "output_tokens": 473, "output_tps_per_user": 124.83469981621674, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7870, "wall_time": 20.530999999959022, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 524.6666666666666, "server_utilization": 0.020998320134389292, "server_spec_accept_rate": 0.41885245901639345, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.531, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 13125, "server_output_tokens": 13125, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 875.0, "per_request_avg_tps": 109.375, "ttft_avg": 0.2150769230658905, "ttft_p50": 0.2809999999590218, "ttft_p90": 0.2809999999590218, "ttft_p99": 0.2809999999590218, "time_to_second_token_avg": 0.028666666670081515, "time_to_second_token_p50": 0.03100000007543713, "time_to_second_token_p90": 0.031999999890103936, "time_to_second_token_p99": 0.031999999890103936, "request_latency_avg": 16.12480000006035, "request_latency_p50": 15.562000000150874, "request_latency_p90": 18.431199999991804, "request_latency_p99": 19.7585199999623, "inter_token_latency_avg": 0.009534447782862667, "inter_token_latency_p50": 0.010019138755619918, "inter_token_latency_p90": 0.011478256316289797, "inter_token_latency_p99": 0.011949334349863533, "output_tps_per_user_avg": 109.28308371732484, "output_tps_per_user_p50": 99.80897803606939, "output_tps_per_user_p90": 134.39377953142284, "output_tps_per_user_p99": 153.0634277982297, "e2e_output_tps_per_user_avg": 129.06036740059508, "e2e_output_tps_per_user_p50": 131.6026217697047, "e2e_output_tps_per_user_p90": 144.30059705044636, "e2e_output_tps_per_user_p99": 151.6035359258755, "chunk_inter_token_latency_avg": 0.03472701442863864, "chunk_inter_token_latency_p50": 0.03461157024668699, "chunk_inter_token_latency_p90": 0.03604335961168719, "chunk_inter_token_latency_p99": 0.03703906743167931, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 13, "completed_request_count": 5, "request_samples": [ { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 15.5, "inter_token_latency_avg": 0.007434782608715671, "chunk_inter_token_latency_avg": 0.03389532293995764, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 134.50292397624602, "e2e_output_tps_per_user": 132.1290322580645, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.011435897435801916, "chunk_inter_token_latency_avg": 0.0360735294114634, "input_tokens": 8192, "output_tokens": 430, "output_tps_per_user": 87.4439461890712, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.0940000000409782, "time_to_second_token": 0.01500000013038516, "latency": 19.905999999959022, "inter_token_latency_avg": 0.00967855398139621, "chunk_inter_token_latency_avg": 0.037170731707163306, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 103.32121946337917, "e2e_output_tps_per_user": 102.88355269789089, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.007692307692307693, "chunk_inter_token_latency_avg": 0.03333333333333333, "input_tokens": 8192, "output_tokens": 66, "output_tps_per_user": 130.0, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 15.562000000150874, "inter_token_latency_avg": 0.007465070835462556, "chunk_inter_token_latency_avg": 0.033882483370713645, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 133.9572017521301, "e2e_output_tps_per_user": 131.6026217697047, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.010395248379803728, "chunk_inter_token_latency_avg": 0.034876811593109604, "input_tokens": 8192, "output_tokens": 464, "output_tps_per_user": 96.19779763443044, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011462669683292685, "chunk_inter_token_latency_avg": 0.03406050420178398, "input_tokens": 8192, "output_tokens": 1769, "output_tps_per_user": 87.23971183236148, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.012013040901044141, "chunk_inter_token_latency_avg": 0.03482130584203001, "input_tokens": 8192, "output_tokens": 1688, "output_tps_per_user": 83.24286983099198, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 13.437000000150874, "inter_token_latency_avg": 0.006426966292228555, "chunk_inter_token_latency_avg": 0.03364705882402008, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 155.59440559213658, "e2e_output_tps_per_user": 152.41497357870094, "completed": true }, { "ttft": 0.14099999982863665, "time_to_second_token": 0.03100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.010655963302814951, "chunk_inter_token_latency_avg": 0.03592268041258236, "input_tokens": 8192, "output_tokens": 655, "output_tps_per_user": 93.84416702484639, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011482152974539075, "chunk_inter_token_latency_avg": 0.035245217391411246, "input_tokens": 8192, "output_tokens": 1766, "output_tps_per_user": 87.09168064712557, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 16.219000000040978, "inter_token_latency_avg": 0.00778602833418757, "chunk_inter_token_latency_avg": 0.03391063829804672, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 128.43518634643456, "e2e_output_tps_per_user": 126.27165669861432, "completed": true }, { "ttft": 0.14000000013038516, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.010019138755619918, "chunk_inter_token_latency_avg": 0.03461157024668699, "input_tokens": 8192, "output_tokens": 419, "output_tps_per_user": 99.80897803606939, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 13125, "wall_time": 21.140999999828637, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 874.1258741182835, "server_utilization": 0.007919366450683918, "server_spec_accept_rate": 0.42707423580786025, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.063, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 16, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.985, "measurement_wall_seconds": 15.0, "client_output_tokens": 17883, "server_output_tokens": 17883, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1193.393393403777, "per_request_avg_tps": 74.58708708773607, "ttft_avg": 0.40531250000640284, "ttft_p50": 0.2970000000204891, "ttft_p90": 0.5940000000409782, "ttft_p99": 0.6466999999713152, "time_to_second_token_avg": 0.04126666674079994, "time_to_second_token_p50": 0.01500000013038516, "time_to_second_token_p90": 0.09080000002868471, "time_to_second_token_p99": 0.11000000010244548, "request_latency_avg": 19.5, "request_latency_p50": 19.5, "request_latency_p90": 19.5, "request_latency_p99": 19.5, "inter_token_latency_avg": 0.013263772964743997, "inter_token_latency_p50": 0.013895453379527077, "inter_token_latency_p90": 0.01627544631934902, "inter_token_latency_p99": 0.01704756700615684, "output_tps_per_user_avg": 78.24606537963123, "output_tps_per_user_p50": 71.96847280574178, "output_tps_per_user_p90": 102.7792207792208, "output_tps_per_user_p99": 107.84201978090539, "e2e_output_tps_per_user_avg": 105.02564102564102, "e2e_output_tps_per_user_p50": 105.02564102564102, "e2e_output_tps_per_user_p90": 105.02564102564102, "e2e_output_tps_per_user_p99": 105.02564102564102, "chunk_inter_token_latency_avg": 0.04493177562091291, "chunk_inter_token_latency_p50": 0.04415137614678899, "chunk_inter_token_latency_p90": 0.046720715099963825, "chunk_inter_token_latency_p99": 0.04901808318261825, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 16, "completed_request_count": 1, "request_samples": [ { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.00948743223262691, "chunk_inter_token_latency_avg": 0.04415137614678899, "input_tokens": 8192, "output_tokens": 2030, "output_tps_per_user": 105.40259740259741, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6559999999590218, "time_to_second_token": 0.03100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.01645557491294553, "chunk_inter_token_latency_avg": 0.047825316455851814, "input_tokens": 8192, "output_tokens": 1149, "output_tps_per_user": 60.769678682773, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.009984439834024895, "chunk_inter_token_latency_avg": 0.04415137614678899, "input_tokens": 8192, "output_tokens": 1929, "output_tps_per_user": 100.15584415584416, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.015027322404371584, "chunk_inter_token_latency_avg": 0.043849658314350795, "input_tokens": 8192, "output_tokens": 1282, "output_tps_per_user": 66.54545454545455, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4839999999385327, "time_to_second_token": 0.11000000010244548, "latency": 0.0, "inter_token_latency_avg": 0.012674867021331087, "chunk_inter_token_latency_avg": 0.044025404157233154, "input_tokens": 8192, "output_tokens": 1505, "output_tps_per_user": 78.89629124448062, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.014694656488549618, "chunk_inter_token_latency_avg": 0.043949771689497714, "input_tokens": 8192, "output_tokens": 1311, "output_tps_per_user": 68.05194805194806, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.10899999993853271, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010816917084074545, "chunk_inter_token_latency_avg": 0.04448054919927221, "input_tokens": 8192, "output_tokens": 1798, "output_tps_per_user": 92.44778269330298, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.014627659574468085, "chunk_inter_token_latency_avg": 0.045294117647058825, "input_tokens": 8192, "output_tokens": 1317, "output_tps_per_user": 68.36363636363636, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4839999999385327, "time_to_second_token": 0.11000000010244548, "latency": 0.0, "inter_token_latency_avg": 0.013813768116001417, "chunk_inter_token_latency_avg": 0.04506619385362164, "input_tokens": 8192, "output_tokens": 1381, "output_tps_per_user": 72.39154382804738, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5940000000409782, "time_to_second_token": 0.06199999991804361, "latency": 19.5, "inter_token_latency_avg": 0.009235955056159756, "chunk_inter_token_latency_avg": 0.04406993006983455, "input_tokens": 8192, "output_tokens": 2048, "output_tps_per_user": 108.27250608295974, "e2e_output_tps_per_user": 105.02564102564102, "completed": true }, { "ttft": 0.5940000000409782, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.010898792409418925, "chunk_inter_token_latency_avg": 0.043872685185137754, "input_tokens": 8192, "output_tokens": 1740, "output_tps_per_user": 91.75328444055717, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.01305084745762712, "chunk_inter_token_latency_avg": 0.04540094339622641, "input_tokens": 8192, "output_tokens": 1476, "output_tps_per_user": 76.62337662337661, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.016095317725752508, "chunk_inter_token_latency_avg": 0.04561611374407583, "input_tokens": 8192, "output_tokens": 1197, "output_tps_per_user": 62.129870129870135, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.014227642276422764, "chunk_inter_token_latency_avg": 0.043849658314350795, "input_tokens": 8192, "output_tokens": 1354, "output_tps_per_user": 70.28571428571429, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5940000000409782, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.01715203619907648, "chunk_inter_token_latency_avg": 0.04922857142851821, "input_tokens": 8192, "output_tokens": 1106, "output_tps_per_user": 58.30211576010102, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5940000000409782, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.013977138643052736, "chunk_inter_token_latency_avg": 0.04407674418599886, "input_tokens": 8192, "output_tokens": 1357, "output_tps_per_user": 71.54540178343618, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 17883, "wall_time": 20.17200000002049, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1190.92967502691, "server_utilization": 0.019918406527477783, "server_spec_accept_rate": 0.571875, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.062, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 32, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.953, "measurement_wall_seconds": 15.0, "client_output_tokens": 25311, "server_output_tokens": 25311, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1692.7038052324328, "per_request_avg_tps": 52.896993913513526, "ttft_avg": 0.609531250091095, "ttft_p50": 0.6880000000819564, "ttft_p90": 0.9060000001918525, "ttft_p99": 0.9494700000877493, "time_to_second_token_avg": 0.12696774192004195, "time_to_second_token_p50": 0.0779999999795109, "time_to_second_token_p90": 0.21800000010989606, "time_to_second_token_p99": 0.21800000010989606, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.018098392717571122, "inter_token_latency_p50": 0.018947012072565387, "inter_token_latency_p90": 0.02084732800551874, "inter_token_latency_p99": 0.021368580859299643, "output_tps_per_user_avg": 56.44513739709138, "output_tps_per_user_p50": 52.77924827914867, "output_tps_per_user_p90": 69.17416622503869, "output_tps_per_user_p99": 81.47502722806747, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.060290917233151706, "chunk_inter_token_latency_p50": 0.059910459272907446, "chunk_inter_token_latency_p90": 0.062361252427800384, "chunk_inter_token_latency_p99": 0.0637083293787713, "input_seq_len_avg": 8192.0, "output_seq_len_avg": 1071.625, "output_seq_len_p50": 1001.0, "output_seq_len_p90": 1307.7, "output_seq_len_p99": 1571.9200000000003, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.017036496350383658, "chunk_inter_token_latency_avg": 0.0585329153605658, "input_tokens": 8192, "output_tokens": 1097, "output_tps_per_user": 58.6975149956511, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.01187985212581137, "chunk_inter_token_latency_avg": 0.05969349845260635, "input_tokens": 8192, "output_tokens": 1624, "output_tps_per_user": 84.17613194252635, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.125, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.018317325800576593, "chunk_inter_token_latency_avg": 0.05967177914175565, "input_tokens": 8192, "output_tokens": 1063, "output_tps_per_user": 54.59312188291819, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.019138804458085495, "chunk_inter_token_latency_avg": 0.060544871795289695, "input_tokens": 8192, "output_tokens": 988, "output_tps_per_user": 52.24986765448319, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.020872928176939653, "chunk_inter_token_latency_avg": 0.06193442622993569, "input_tokens": 8192, "output_tokens": 906, "output_tps_per_user": 47.908946532226224, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.017782857142876658, "chunk_inter_token_latency_avg": 0.06162376237630524, "input_tokens": 8192, "output_tokens": 1051, "output_tps_per_user": 56.23393316189202, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.020821814255066795, "chunk_inter_token_latency_avg": 0.06239805825304807, "input_tokens": 8192, "output_tokens": 927, "output_tps_per_user": 48.02655463880431, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.015168155970772127, "chunk_inter_token_latency_avg": 0.05816822429912925, "input_tokens": 8192, "output_tokens": 1232, "output_tps_per_user": 65.92759211646579, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.01392393736019425, "chunk_inter_token_latency_avg": 0.058716981132139905, "input_tokens": 8192, "output_tokens": 1342, "output_tps_per_user": 71.81876606675924, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.021591265397751233, "chunk_inter_token_latency_avg": 0.06260064935127224, "input_tokens": 8192, "output_tokens": 894, "output_tps_per_user": 46.31502515383613, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.015355263157911586, "chunk_inter_token_latency_avg": 0.0585329153605658, "input_tokens": 8192, "output_tokens": 1217, "output_tps_per_user": 65.12425021415305, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.020862569832425126, "chunk_inter_token_latency_avg": 0.06351020408170234, "input_tokens": 8192, "output_tokens": 896, "output_tps_per_user": 47.93273350466035, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.01955183246075444, "chunk_inter_token_latency_avg": 0.05816822429912925, "input_tokens": 8192, "output_tokens": 956, "output_tps_per_user": 51.146101113911314, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.02049615806807957, "chunk_inter_token_latency_avg": 0.060038585209069094, "input_tokens": 8192, "output_tokens": 912, "output_tps_per_user": 48.78963153379394, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.019281000000191853, "chunk_inter_token_latency_avg": 0.06006542056134533, "input_tokens": 8192, "output_tokens": 1001, "output_tps_per_user": 51.864529847520856, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.11000000010244548, "latency": 0.0, "inter_token_latency_avg": 0.017457272727465764, "chunk_inter_token_latency_avg": 0.06076898734244412, "input_tokens": 8192, "output_tokens": 1101, "output_tps_per_user": 57.2827162416204, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4850000001024455, "time_to_second_token": 0.2029999999795109, "latency": 0.0, "inter_token_latency_avg": 0.018591041869629888, "chunk_inter_token_latency_avg": 0.05929503105624191, "input_tokens": 8192, "output_tokens": 1028, "output_tps_per_user": 53.78934688074629, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4850000001024455, "time_to_second_token": 0.2029999999795109, "latency": 0.0, "inter_token_latency_avg": 0.01932489878553633, "chunk_inter_token_latency_avg": 0.05911145510869937, "input_tokens": 8192, "output_tokens": 989, "output_tps_per_user": 51.746713454895165, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.019004024145000388, "chunk_inter_token_latency_avg": 0.05866459627369685, "input_tokens": 8192, "output_tokens": 995, "output_tps_per_user": 52.620434091749026, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.017820754717104138, "chunk_inter_token_latency_avg": 0.05958990536318733, "input_tokens": 8192, "output_tokens": 1061, "output_tps_per_user": 56.114346214541214, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.019694586312759808, "chunk_inter_token_latency_avg": 0.05932615384674416, "input_tokens": 8192, "output_tokens": 980, "output_tps_per_user": 50.77537472072292, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9690000000409782, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.01998818474776731, "chunk_inter_token_latency_avg": 0.06203000000057121, "input_tokens": 8192, "output_tokens": 932, "output_tps_per_user": 50.029555590919806, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.019654434250960095, "chunk_inter_token_latency_avg": 0.06179807692369183, "input_tokens": 8192, "output_tokens": 982, "output_tps_per_user": 50.87910378041796, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.013251546391884434, "chunk_inter_token_latency_avg": 0.061209523810132865, "input_tokens": 8192, "output_tokens": 1456, "output_tps_per_user": 75.46289092814285, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.014430863254492273, "chunk_inter_token_latency_avg": 0.05977848101307084, "input_tokens": 8192, "output_tokens": 1310, "output_tps_per_user": 69.29592376871175, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4850000001024455, "time_to_second_token": 0.2029999999795109, "latency": 0.0, "inter_token_latency_avg": 0.01706255585353878, "chunk_inter_token_latency_avg": 0.061990259740616546, "input_tokens": 8192, "output_tokens": 1120, "output_tps_per_user": 58.60786675711304, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9060000001918525, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.019269349845222383, "chunk_inter_token_latency_avg": 0.060232258064582225, "input_tokens": 8192, "output_tokens": 970, "output_tps_per_user": 51.89588688940321, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4850000001024455, "time_to_second_token": 0.2029999999795109, "latency": 0.0, "inter_token_latency_avg": 0.019150451354172413, "chunk_inter_token_latency_avg": 0.05985266457714701, "input_tokens": 8192, "output_tokens": 998, "output_tps_per_user": 52.21809039932234, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.11000000010244548, "latency": 0.0, "inter_token_latency_avg": 0.020850162866680066, "chunk_inter_token_latency_avg": 0.06379734219339649, "input_tokens": 8192, "output_tokens": 922, "output_tps_per_user": 47.96125605321126, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.014688958009432648, "chunk_inter_token_latency_avg": 0.05903125000040745, "input_tokens": 8192, "output_tokens": 1287, "output_tps_per_user": 68.07834833198113, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6880000000819564, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.018890000000130386, "chunk_inter_token_latency_avg": 0.05866459627369685, "input_tokens": 8192, "output_tokens": 1001, "output_tps_per_user": 52.93806246654831, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6720000000204891, "time_to_second_token": 0.21800000010989606, "latency": 0.0, "inter_token_latency_avg": 0.01793922127267843, "chunk_inter_token_latency_avg": 0.05996825396866789, "input_tokens": 8192, "output_tokens": 1054, "output_tps_per_user": 55.74377977727537, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 25311, "wall_time": 20.187000000150874, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1685.6020245002924, "server_utilization": 0.1469882409407247, "server_spec_accept_rate": 0.5283333333333333, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.156, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.015, "measurement_wall_seconds": 15.015, "client_output_tokens": 7377, "server_output_tokens": 7377, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 491.30869131204344, "per_request_avg_tps": 122.82717282801086, "ttft_avg": 0.16600000005564652, "ttft_p50": 0.15600000007543713, "ttft_p90": 0.2190000000409782, "ttft_p99": 0.2190000000409782, "time_to_second_token_avg": 0.021499999973457307, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.030099999997764817, "time_to_second_token_p99": 0.059709999863989636, "request_latency_avg": 16.16775000008056, "request_latency_p50": 16.874500000150874, "request_latency_p90": 17.19820000003092, "request_latency_p99": 17.202519999984652, "inter_token_latency_avg": 0.007569913358671816, "inter_token_latency_p50": 0.007702641764389797, "inter_token_latency_p90": 0.008358971698164538, "inter_token_latency_p99": 0.008376222132606546, "output_tps_per_user_avg": 133.38497915946462, "output_tps_per_user_p50": 129.99889122640832, "output_tps_per_user_p90": 151.91996152083163, "output_tps_per_user_p99": 152.52965849235582, "e2e_output_tps_per_user_avg": 127.78685350982558, "e2e_output_tps_per_user_p50": 121.40819665948938, "e2e_output_tps_per_user_p90": 141.59438131228953, "e2e_output_tps_per_user_p99": 148.51325400662031, "chunk_inter_token_latency_avg": 0.024000726916443428, "chunk_inter_token_latency_p50": 0.024207675870764972, "chunk_inter_token_latency_p90": 0.02465936916090476, "chunk_inter_token_latency_p99": 0.024718684168837726, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 8, "completed_request_count": 4, "request_samples": [ { "ttft": 0.10900000017136335, "time_to_second_token": 0.015999999828636646, "latency": 17.20299999997951, "inter_token_latency_avg": 0.008350757205573106, "chunk_inter_token_latency_avg": 0.024631123919031914, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 119.74961975096375, "e2e_output_tps_per_user": 119.04900308099978, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.006553191489237148, "chunk_inter_token_latency_avg": 0.02293617021233002, "input_tokens": 16384, "output_tokens": 330, "output_tps_per_user": 152.59740260030296, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2190000000409782, "time_to_second_token": 0.01500000013038516, "latency": 13.719000000040978, "inter_token_latency_avg": 0.006595017098192477, "chunk_inter_token_latency_avg": 0.024725274725274724, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 151.62962962962965, "e2e_output_tps_per_user": 149.28201763932375, "completed": true }, { "ttft": 0.14000000013038516, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.008378138847544547, "chunk_inter_token_latency_avg": 0.024239316238408797, "input_tokens": 16384, "output_tokens": 678, "output_tps_per_user": 119.35825106229632, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2190000000409782, "time_to_second_token": 0.01500000013038516, "latency": 17.187000000150874, "inter_token_latency_avg": 0.008289203712804052, "chunk_inter_token_latency_avg": 0.024240000000156994, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 120.63884959846432, "e2e_output_tps_per_user": 119.15983010310245, "completed": true }, { "ttft": 0.125, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.0074214046816326006, "chunk_inter_token_latency_avg": 0.023357894734822606, "input_tokens": 16384, "output_tokens": 300, "output_tps_per_user": 134.74538081381309, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2190000000409782, "time_to_second_token": 0.01500000013038516, "latency": 16.562000000150874, "inter_token_latency_avg": 0.007983878847146993, "chunk_inter_token_latency_avg": 0.024176035503121147, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 125.25240163900358, "e2e_output_tps_per_user": 123.6565632158763, "completed": true }, { "ttft": 0.125, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.006987714987243605, "chunk_inter_token_latency_avg": 0.02369999999840123, "input_tokens": 16384, "output_tokens": 408, "output_tps_per_user": 143.1082981812432, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7377, "wall_time": 20.093000000109896, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 490.7857095349685, "server_utilization": 0.024118070554355664, "server_spec_accept_rate": 0.3921875, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.047, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.969, "measurement_wall_seconds": 15.0, "client_output_tokens": 12810, "server_output_tokens": 12837, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 855.7685884270279, "per_request_avg_tps": 106.97107355337849, "ttft_avg": 0.23907692309540624, "ttft_p50": 0.25, "ttft_p90": 0.4119999999180437, "ttft_p99": 0.5056400001049041, "time_to_second_token_avg": 0.03946153846426079, "time_to_second_token_p50": 0.03100000019185245, "time_to_second_token_p90": 0.06199999991804361, "time_to_second_token_p99": 0.07608000017702578, "request_latency_avg": 16.505999999959023, "request_latency_p50": 16.233999999938533, "request_latency_p90": 18.090399999963118, "request_latency_p99": 18.894639999940992, "inter_token_latency_avg": 0.009687744396844956, "inter_token_latency_p50": 0.009645539905958997, "inter_token_latency_p90": 0.011637678525424622, "inter_token_latency_p99": 0.011854495616218371, "output_tps_per_user_avg": 106.37187360403185, "output_tps_per_user_p50": 103.67486006482663, "output_tps_per_user_p90": 134.4713705009501, "output_tps_per_user_p99": 137.58878171602692, "e2e_output_tps_per_user_avg": 124.87664183422172, "e2e_output_tps_per_user_p50": 126.15498336871717, "e2e_output_tps_per_user_p90": 134.34136925975122, "e2e_output_tps_per_user_p99": 135.42765072619343, "chunk_inter_token_latency_avg": 0.03486169288564047, "chunk_inter_token_latency_p50": 0.034599078341155456, "chunk_inter_token_latency_p90": 0.035446695600268203, "chunk_inter_token_latency_p99": 0.036052344108295865, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 13, "completed_request_count": 5, "request_samples": [ { "ttft": 0.125, "time_to_second_token": 0.01500000013038516, "latency": 16.233999999938533, "inter_token_latency_avg": 0.007869565217361277, "chunk_inter_token_latency_avg": 0.03501956521725768, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 127.07182320490475, "e2e_output_tps_per_user": 126.15498336871717, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.009645539905958997, "chunk_inter_token_latency_avg": 0.034822033897784176, "input_tokens": 16384, "output_tokens": 427, "output_tps_per_user": 103.67486006482663, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.011874413145503833, "chunk_inter_token_latency_avg": 0.03447018739342169, "input_tokens": 16384, "output_tokens": 1705, "output_tps_per_user": 84.21468814891651, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.011160507446187828, "chunk_inter_token_latency_avg": 0.03447018739342169, "input_tokens": 16384, "output_tokens": 1814, "output_tps_per_user": 89.6016605715878, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.3119999999180436, "time_to_second_token": 0.01600000006146729, "latency": 16.75, "inter_token_latency_avg": 0.008030288226713218, "chunk_inter_token_latency_avg": 0.03438912133908359, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 124.52853145089391, "e2e_output_tps_per_user": 122.26865671641791, "completed": true }, { "ttft": 0.14000000013038516, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.01050877192926359, "chunk_inter_token_latency_avg": 0.03455769230584757, "input_tokens": 16384, "output_tokens": 343, "output_tps_per_user": 95.15859766785097, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4369999999180436, "time_to_second_token": 0.07800000021234155, "latency": 15.452999999979511, "inter_token_latency_avg": 0.007335613092360267, "chunk_inter_token_latency_avg": 0.034599078341155456, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 136.3212573249614, "e2e_output_tps_per_user": 132.53090014901414, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.011708433734791645, "chunk_inter_token_latency_avg": 0.03546715328422287, "input_tokens": 16384, "output_tokens": 416, "output_tps_per_user": 85.40852027274127, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.011354657687956527, "chunk_inter_token_latency_avg": 0.036132142857033096, "input_tokens": 16384, "output_tokens": 1783, "output_tps_per_user": 88.06958584587395, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5150000001303852, "time_to_second_token": 0.015999999828636646, "latency": 18.983999999938533, "inter_token_latency_avg": 0.009022471910018636, "chunk_inter_token_latency_avg": 0.03497916666630331, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 110.83437110949504, "e2e_output_tps_per_user": 107.8803202700501, "completed": true }, { "ttft": 0.125, "time_to_second_token": 0.03100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.009105960264900662, "chunk_inter_token_latency_avg": 0.034375, "input_tokens": 16384, "output_tokens": 152, "output_tps_per_user": 109.81818181818183, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.25, "time_to_second_token": 0.06199999991804361, "latency": 15.108999999938533, "inter_token_latency_avg": 0.007258915486047158, "chunk_inter_token_latency_avg": 0.03455581395334543, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 137.76162595117222, "e2e_output_tps_per_user": 135.54834866690925, "completed": true }, { "ttft": 0.1410000000614673, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.011065539111920788, "chunk_inter_token_latency_avg": 0.03536486486444954, "input_tokens": 16384, "output_tokens": 474, "output_tps_per_user": 90.3706534210078, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 12810, "wall_time": 21.031000000191852, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 855.8, "server_utilization": 0.042836573074154116, "server_spec_accept_rate": 0.4777292576419214, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.031, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 16, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 19963, "server_output_tokens": 19962, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1330.8666666666666, "per_request_avg_tps": 83.17916666666666, "ttft_avg": 0.4183157894294709, "ttft_p50": 0.375, "ttft_p90": 0.6715999999549239, "ttft_p99": 0.7339999999385327, "time_to_second_token_avg": 0.05038461535094449, "time_to_second_token_p50": 0.0470000000204891, "time_to_second_token_p90": 0.0779999999795109, "time_to_second_token_p99": 0.11935999999754127, "request_latency_avg": 19.457666666635003, "request_latency_p50": 18.889999999897555, "request_latency_p90": 20.427599999913944, "request_latency_p99": 20.773559999917634, "inter_token_latency_avg": 0.012141193815917726, "inter_token_latency_p50": 0.012114596035242078, "inter_token_latency_p90": 0.014766852449237393, "inter_token_latency_p99": 0.01626251280709081, "output_tps_per_user_avg": 84.62658613814129, "output_tps_per_user_p50": 82.58693031085963, "output_tps_per_user_p90": 103.41651632399365, "output_tps_per_user_p99": 112.74564720799944, "e2e_output_tps_per_user_avg": 105.50358021708692, "e2e_output_tps_per_user_p50": 108.41715193282725, "e2e_output_tps_per_user_p90": 109.43448817630112, "e2e_output_tps_per_user_p99": 109.66338883108273, "chunk_inter_token_latency_avg": 0.046673622921616496, "chunk_inter_token_latency_p50": 0.0461606702442061, "chunk_inter_token_latency_p90": 0.04860340315625013, "chunk_inter_token_latency_p99": 0.05044494492113117, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 19, "completed_request_count": 3, "request_samples": [ { "ttft": 0.2649999998975545, "time_to_second_token": 0.0, "latency": 20.811999999918044, "inter_token_latency_avg": 0.01003761602345896, "chunk_inter_token_latency_avg": 0.04566000000004553, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 99.62524942804102, "e2e_output_tps_per_user": 98.40476648126393, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.0, "chunk_inter_token_latency_avg": 0.0, "input_tokens": 16384, "output_tokens": 1, "output_tps_per_user": 0.0, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.014621933621977971, "chunk_inter_token_latency_avg": 0.045236607142994344, "input_tokens": 16384, "output_tokens": 1387, "output_tps_per_user": 68.39040757898925, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.01400545702584591, "chunk_inter_token_latency_avg": 0.04808430913323209, "input_tokens": 16384, "output_tokens": 1467, "output_tps_per_user": 71.40074030819437, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6559999999590218, "time_to_second_token": 0.0779999999795109, "latency": 18.889999999897555, "inter_token_latency_avg": 0.008907669760595278, "chunk_inter_token_latency_avg": 0.04513366336618449, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 112.26280574788309, "e2e_output_tps_per_user": 108.41715193282725, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.011719512195246884, "chunk_inter_token_latency_avg": 0.050578947368960236, "input_tokens": 16384, "output_tokens": 165, "output_tps_per_user": 85.32778355788331, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.012387387387387387, "chunk_inter_token_latency_avg": 0.04614093959731544, "input_tokens": 16384, "output_tokens": 1666, "output_tps_per_user": 80.72727272727272, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2649999998975545, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011392857142913432, "chunk_inter_token_latency_avg": 0.04567180616762653, "input_tokens": 16384, "output_tokens": 1821, "output_tps_per_user": 87.77429467041273, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6559999999590218, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.012420024420049438, "chunk_inter_token_latency_avg": 0.045716853932676355, "input_tokens": 16384, "output_tokens": 1639, "output_tps_per_user": 80.51513959873675, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.13999999989755452, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015104996379509374, "chunk_inter_token_latency_avg": 0.047194570135978386, "input_tokens": 16384, "output_tokens": 1382, "output_tps_per_user": 66.20325982709576, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.01261467889908257, "chunk_inter_token_latency_avg": 0.04573170731707317, "input_tokens": 16384, "output_tokens": 1636, "output_tps_per_user": 79.27272727272727, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2649999998975545, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010936181434653188, "chunk_inter_token_latency_avg": 0.04618040089109676, "input_tokens": 16384, "output_tokens": 1897, "output_tps_per_user": 91.43959488741898, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.012949520766812439, "chunk_inter_token_latency_avg": 0.046375286041330586, "input_tokens": 16384, "output_tokens": 1566, "output_tps_per_user": 77.22293496473173, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2649999998975545, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.011841804683096771, "chunk_inter_token_latency_avg": 0.04712500000023283, "input_tokens": 16384, "output_tokens": 1752, "output_tps_per_user": 84.44658789444652, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6559999999590218, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.012746867167945476, "chunk_inter_token_latency_avg": 0.04809456264785101, "input_tokens": 16384, "output_tokens": 1597, "output_tps_per_user": 78.4506488397948, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6559999999590218, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.01649959448502918, "chunk_inter_token_latency_avg": 0.04655377574380087, "input_tokens": 16384, "output_tokens": 1234, "output_tps_per_user": 60.607550137510636, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5309999999590218, "time_to_second_token": 0.125, "latency": 18.671000000089407, "inter_token_latency_avg": 0.00886174890089418, "chunk_inter_token_latency_avg": 0.045124378109777076, "input_tokens": 16384, "output_tokens": 2048, "output_tps_per_user": 112.84454244681845, "e2e_output_tps_per_user": 109.68882223716957, "completed": true }, { "ttft": 0.18799999984912574, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.01138829787266738, "chunk_inter_token_latency_avg": 0.049790697675848074, "input_tokens": 16384, "output_tokens": 189, "output_tps_per_user": 87.80943484101009, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.010105340519353257, "chunk_inter_token_latency_avg": 0.04573170731707317, "input_tokens": 16384, "output_tokens": 2042, "output_tps_per_user": 98.95757575757577, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 19963, "wall_time": 21.562000000150874, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1330.8, "server_utilization": 0.0773938084953204, "server_spec_accept_rate": 0.6808988764044944, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.516, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 32, "context_tokens": 16384, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.984, "measurement_wall_seconds": 15.0, "client_output_tokens": 27893, "server_output_tokens": 27897, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1861.5189535291647, "per_request_avg_tps": 58.1724672977864, "ttft_avg": 0.6862500000424916, "ttft_p50": 0.7260000001406297, "ttft_p90": 1.0620000001508743, "ttft_p99": 1.077999999979511, "time_to_second_token_avg": 0.10318749996076804, "time_to_second_token_p50": 0.09399999980814755, "time_to_second_token_p90": 0.18700000015087426, "time_to_second_token_p99": 0.18700000015087426, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.017072978724558586, "inter_token_latency_p50": 0.01751385554635535, "inter_token_latency_p90": 0.019967881451605777, "inter_token_latency_p99": 0.02261073804512651, "output_tps_per_user_avg": 60.16226694603271, "output_tps_per_user_p50": 57.136702689265235, "output_tps_per_user_p90": 76.48143532535671, "output_tps_per_user_p99": 81.6377833515595, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0600449180450446, "chunk_inter_token_latency_p50": 0.059926381535693275, "chunk_inter_token_latency_p90": 0.06192018126894556, "chunk_inter_token_latency_p99": 0.06269113267377531, "input_seq_len_avg": 16384.0, "output_seq_len_avg": 1222.25, "output_seq_len_p50": 1156.5, "output_seq_len_p90": 1546.0000000000002, "output_seq_len_p99": 1672.99, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5, "time_to_second_token": 0.09399999980814755, "latency": 0.0, "inter_token_latency_avg": 0.018141592920353982, "chunk_inter_token_latency_avg": 0.06083086053412463, "input_tokens": 16384, "output_tokens": 1131, "output_tps_per_user": 55.1219512195122, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.077999999979511, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.016423742786496692, "chunk_inter_token_latency_avg": 0.05859411764711909, "input_tokens": 16384, "output_tokens": 1214, "output_tps_per_user": 60.88746109822069, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.077999999979511, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.022645381983887818, "chunk_inter_token_latency_avg": 0.062257053291127316, "input_tokens": 16384, "output_tokens": 878, "output_tps_per_user": 44.159113796865945, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5, "time_to_second_token": 0.09399999980814755, "latency": 0.0, "inter_token_latency_avg": 0.016748366013071895, "chunk_inter_token_latency_avg": 0.059941520467836254, "input_tokens": 16384, "output_tokens": 1225, "output_tps_per_user": 59.707317073170735, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.15699999989010394, "time_to_second_token": 0.014999999897554517, "latency": 0.0, "inter_token_latency_avg": 0.015874333587288573, "chunk_inter_token_latency_avg": 0.060066282421065986, "input_tokens": 16384, "output_tokens": 1314, "output_tps_per_user": 62.99477042618995, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.29699999978765845, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.013004396985058004, "chunk_inter_token_latency_avg": 0.06000869565278939, "input_tokens": 16384, "output_tokens": 1593, "output_tps_per_user": 76.8970680569807, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.019276536313046874, "chunk_inter_token_latency_avg": 0.06000869565278939, "input_tokens": 16384, "output_tokens": 1075, "output_tps_per_user": 51.87653963140532, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.18700000015087426, "latency": 0.0, "inter_token_latency_avg": 0.01705597964391125, "chunk_inter_token_latency_avg": 0.060026865672153325, "input_tokens": 16384, "output_tokens": 1180, "output_tps_per_user": 58.630463970856475, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0620000001508743, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.01863610851264779, "chunk_inter_token_latency_avg": 0.05842228739008941, "input_tokens": 16384, "output_tokens": 1070, "output_tps_per_user": 53.65927115745912, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.01994508670540688, "chunk_inter_token_latency_avg": 0.061800000000633855, "input_tokens": 16384, "output_tokens": 1039, "output_tps_per_user": 50.137661208006264, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.48400000017136335, "time_to_second_token": 0.09399999980814755, "latency": 0.0, "inter_token_latency_avg": 0.014385964912280702, "chunk_inter_token_latency_avg": 0.0597667638483965, "input_tokens": 16384, "output_tokens": 1426, "output_tps_per_user": 69.51219512195122, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0620000001508743, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.01908237547894683, "chunk_inter_token_latency_avg": 0.05894082840242748, "input_tokens": 16384, "output_tokens": 1045, "output_tps_per_user": 52.40437707052135, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.15600000019185245, "latency": 0.0, "inter_token_latency_avg": 0.01797173144879945, "chunk_inter_token_latency_avg": 0.06001179941015038, "input_tokens": 16384, "output_tokens": 1133, "output_tps_per_user": 55.642941407674, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.48400000017136335, "time_to_second_token": 0.09399999980814755, "latency": 0.0, "inter_token_latency_avg": 0.020117762512266928, "chunk_inter_token_latency_avg": 0.061933534743202415, "input_tokens": 16384, "output_tokens": 1020, "output_tps_per_user": 49.707317073170735, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.18700000015087426, "latency": 0.0, "inter_token_latency_avg": 0.01816531165327133, "chunk_inter_token_latency_avg": 0.05879824561453615, "input_tokens": 16384, "output_tokens": 1108, "output_tps_per_user": 55.0499776214912, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0620000001508743, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.014746113989652471, "chunk_inter_token_latency_avg": 0.05911572700302816, "input_tokens": 16384, "output_tokens": 1352, "output_tps_per_user": 67.81447645811718, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.18700000015087426, "latency": 0.0, "inter_token_latency_avg": 0.01858502772659091, "chunk_inter_token_latency_avg": 0.05967062314590909, "input_tokens": 16384, "output_tokens": 1083, "output_tps_per_user": 53.80675319462825, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.18700000015087426, "latency": 0.0, "inter_token_latency_avg": 0.014928730512376661, "chunk_inter_token_latency_avg": 0.05949408284074368, "input_tokens": 16384, "output_tokens": 1348, "output_tps_per_user": 66.98493211937547, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.18700000015087426, "latency": 0.0, "inter_token_latency_avg": 0.019206303725092036, "chunk_inter_token_latency_avg": 0.05897067448730605, "input_tokens": 16384, "output_tokens": 1048, "output_tps_per_user": 52.06623899702013, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.48400000017136335, "time_to_second_token": 0.09399999980814755, "latency": 0.0, "inter_token_latency_avg": 0.0225336273427223, "chunk_inter_token_latency_avg": 0.06288615384568962, "input_tokens": 16384, "output_tokens": 908, "output_tps_per_user": 44.3781191900722, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5779999999795109, "time_to_second_token": 0.15600000019185245, "latency": 0.0, "inter_token_latency_avg": 0.014905770635640506, "chunk_inter_token_latency_avg": 0.061649546828374174, "input_tokens": 16384, "output_tokens": 1370, "output_tps_per_user": 67.08811133917128, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.0620000001508743, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.012828074694153566, "chunk_inter_token_latency_avg": 0.05894082840242748, "input_tokens": 16384, "output_tokens": 1554, "output_tps_per_user": 77.95402068057439, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7340000001713634, "time_to_second_token": 0.14099999982863665, "latency": 0.0, "inter_token_latency_avg": 0.014160839160839161, "chunk_inter_token_latency_avg": 0.059911242603550297, "input_tokens": 16384, "output_tokens": 1431, "output_tps_per_user": 70.61728395061728, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7340000001713634, "time_to_second_token": 0.14099999982863665, "latency": 0.0, "inter_token_latency_avg": 0.018112701252236137, "chunk_inter_token_latency_avg": 0.05921052631578947, "input_tokens": 16384, "output_tokens": 1119, "output_tps_per_user": 55.20987654320987, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7340000001713634, "time_to_second_token": 0.14099999982863665, "latency": 0.0, "inter_token_latency_avg": 0.01374745417515275, "chunk_inter_token_latency_avg": 0.06062874251497006, "input_tokens": 16384, "output_tokens": 1474, "output_tps_per_user": 72.74074074074075, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7340000001713634, "time_to_second_token": 0.14099999982863665, "latency": 0.0, "inter_token_latency_avg": 0.019970414201183433, "chunk_inter_token_latency_avg": 0.062116564417177916, "input_tokens": 16384, "output_tokens": 1015, "output_tps_per_user": 50.07407407407407, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7180000001098961, "time_to_second_token": 0.14099999982863665, "latency": 0.0, "inter_token_latency_avg": 0.016007905138339922, "chunk_inter_token_latency_avg": 0.05955882352941176, "input_tokens": 16384, "output_tokens": 1266, "output_tps_per_user": 62.46913580246913, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2649999998975545, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.016845402766649586, "chunk_inter_token_latency_avg": 0.06000869565278939, "input_tokens": 16384, "output_tokens": 1230, "output_tps_per_user": 59.36337728770684, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5619999999180436, "time_to_second_token": 0.15600000019185245, "latency": 0.0, "inter_token_latency_avg": 0.012046044864339937, "chunk_inter_token_latency_avg": 0.059319767442418174, "input_tokens": 16384, "output_tokens": 1695, "output_tps_per_user": 83.0147995679738, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7180000001098961, "time_to_second_token": 0.14099999982863665, "latency": 0.0, "inter_token_latency_avg": 0.018925233644859814, "chunk_inter_token_latency_avg": 0.05921052631578947, "input_tokens": 16384, "output_tokens": 1071, "output_tps_per_user": 52.8395061728395, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.3119999999180436, "time_to_second_token": 0.15600000019185245, "latency": 0.0, "inter_token_latency_avg": 0.012727048675410876, "chunk_inter_token_latency_avg": 0.06022157434458266, "input_tokens": 16384, "output_tokens": 1624, "output_tps_per_user": 78.57281177308897, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.046000000089407, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.01858395522389971, "chunk_inter_token_latency_avg": 0.05911572700302816, "input_tokens": 16384, "output_tokens": 1073, "output_tps_per_user": 53.80985844789165, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 27893, "wall_time": 21.656000000191852, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1857.8183271101361, "server_utilization": 0.011999040076793865, "server_spec_accept_rate": 0.6803571428571429, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.547, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.953, "measurement_wall_seconds": 15.015, "client_output_tokens": 7549, "server_output_tokens": 7575, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 504.84852538021426, "per_request_avg_tps": 126.21213134505356, "ttft_avg": 0.28342857148631345, "ttft_p50": 0.2660000000614673, "ttft_p90": 0.47440000013448297, "ttft_p99": 0.4830400001676753, "time_to_second_token_avg": 0.015749999962281436, "time_to_second_token_p50": 0.015999999828636646, "time_to_second_token_p90": 0.015999999991618098, "time_to_second_token_p99": 0.01600000005448237, "request_latency_avg": 16.320500000030734, "request_latency_p50": 16.125, "request_latency_p90": 18.608000000030735, "request_latency_p99": 19.312700000058392, "inter_token_latency_avg": 0.007469172422278906, "inter_token_latency_p50": 0.007426966292124822, "inter_token_latency_p90": 0.00856287249629661, "inter_token_latency_p99": 0.009169086467949688, "output_tps_per_user_avg": 135.8015143842685, "output_tps_per_user_p50": 134.64447806372155, "output_tps_per_user_p90": 154.9440946153399, "output_tps_per_user_p99": 157.50553823389924, "e2e_output_tps_per_user_avg": 127.54705859082858, "e2e_output_tps_per_user_p50": 127.21830319126121, "e2e_output_tps_per_user_p90": 144.81308035775373, "e2e_output_tps_per_user_p99": 149.60336653527648, "chunk_inter_token_latency_avg": 0.02528489610123603, "chunk_inter_token_latency_p50": 0.02532142857152004, "chunk_inter_token_latency_p90": 0.02584354864520323, "chunk_inter_token_latency_p99": 0.025957884275557775, "input_seq_len_avg": 32768.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 7, "completed_request_count": 4, "request_samples": [ { "ttft": 0.1720000000204891, "time_to_second_token": 0.0, "latency": 16.780999999959022, "inter_token_latency_avg": 0.008113825109886924, "chunk_inter_token_latency_avg": 0.025396024464737817, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 123.2464326574493, "e2e_output_tps_per_user": 122.04278648501288, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.006337531486300925, "chunk_inter_token_latency_avg": 0.025160000000614673, "input_tokens": 32768, "output_tokens": 398, "output_tps_per_user": 157.79014308040584, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 13.641000000061467, "inter_token_latency_avg": 0.006533952125061065, "chunk_inter_token_latency_avg": 0.02458639705882353, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 153.0467289719626, "e2e_output_tps_per_user": 150.13562055500122, "completed": true }, { "ttft": 0.1559999999590218, "time_to_second_token": 0.01500000013038516, "latency": 0.0, "inter_token_latency_avg": 0.00707231920202056, "chunk_inter_token_latency_avg": 0.02532142857152004, "input_tokens": 32768, "output_tokens": 803, "output_tps_per_user": 141.39633286267681, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.48400000017136335, "time_to_second_token": 0.015999999828636646, "latency": 19.391000000061467, "inter_token_latency_avg": 0.00923644357591114, "chunk_inter_token_latency_avg": 0.02575885558568134, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 108.26677950028551, "e2e_output_tps_per_user": 105.61600742579073, "completed": true }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 15.469000000040978, "inter_token_latency_avg": 0.007426966292124822, "chunk_inter_token_latency_avg": 0.024800978792788763, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 134.64447806372155, "e2e_output_tps_per_user": 132.39381989750953, "completed": true }, { "ttft": 0.46800000010989606, "time_to_second_token": 0.015999999828636646, "latency": 0.0, "inter_token_latency_avg": 0.0075631691646469035, "chunk_inter_token_latency_avg": 0.02597058823448606, "input_tokens": 32768, "output_tokens": 468, "output_tps_per_user": 132.21970555337782, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 7549, "wall_time": 22.625, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 503.9584857973955, "server_utilization": 0.04415646748260138, "server_spec_accept_rate": 0.4491525423728814, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 7.594, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 15.0, "measurement_wall_seconds": 15.0, "client_output_tokens": 11651, "server_output_tokens": 11680, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 776.7333333333333, "per_request_avg_tps": 97.09166666666667, "ttft_avg": 0.35033333336468786, "ttft_p50": 0.2660000000614673, "ttft_p90": 0.6090000001713634, "ttft_p99": 0.6650700000370853, "time_to_second_token_avg": 0.031222222243539162, "time_to_second_token_p50": 0.030999999959021807, "time_to_second_token_p90": 0.06299999984912574, "time_to_second_token_p99": 0.06299999984912574, "request_latency_avg": 17.257750000047963, "request_latency_p50": 17.42950000008568, "request_latency_p90": 19.812800000049172, "request_latency_p99": 20.108180000060237, "inter_token_latency_avg": 0.009916263701564761, "inter_token_latency_p50": 0.010396852834383196, "inter_token_latency_p90": 0.012535065706723095, "inter_token_latency_p99": 0.013203409985787949, "output_tps_per_user_avg": 107.25376418103514, "output_tps_per_user_p50": 96.36952152612565, "output_tps_per_user_p90": 147.69041254329846, "output_tps_per_user_p99": 174.28388955055885, "e2e_output_tps_per_user_avg": 121.17275552866167, "e2e_output_tps_per_user_p50": 118.52268831353483, "e2e_output_tps_per_user_p90": 141.03032274143118, "e2e_output_tps_per_user_p99": 145.46929269787506, "chunk_inter_token_latency_avg": 0.03583857266250421, "chunk_inter_token_latency_p50": 0.03610552563202439, "chunk_inter_token_latency_p90": 0.037133657880344104, "chunk_inter_token_latency_p99": 0.03741541493438755, "input_seq_len_avg": 32768.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 12, "completed_request_count": 4, "request_samples": [ { "ttft": 0.5, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.013271400132714002, "chunk_inter_token_latency_avg": 0.03656307129798903, "input_tokens": 32768, "output_tokens": 1508, "output_tps_per_user": 75.35, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 20.141000000061467, "inter_token_latency_avg": 0.00970933072789448, "chunk_inter_token_latency_avg": 0.03646788990825688, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 102.9937106918239, "e2e_output_tps_per_user": 101.68313390565264, "completed": true }, { "ttft": 0.1869999999180436, "time_to_second_token": 0.03100000019185245, "latency": 0.0, "inter_token_latency_avg": 0.0056388888883197475, "chunk_inter_token_latency_avg": 0.03383333332991848, "input_tokens": 32768, "output_tokens": 37, "output_tps_per_user": 177.33990149573168, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2660000000614673, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010854311729993333, "chunk_inter_token_latency_avg": 0.03684545454526828, "input_tokens": 32768, "output_tokens": 1868, "output_tps_per_user": 92.12928694840555, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6720000000204891, "time_to_second_token": 0.01500000013038516, "latency": 19.04700000002049, "inter_token_latency_avg": 0.008976551050317538, "chunk_inter_token_latency_avg": 0.03533653846153846, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 111.4013605442177, "e2e_output_tps_per_user": 107.52349451345603, "completed": true }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.009939393938773057, "chunk_inter_token_latency_avg": 0.03644444444216788, "input_tokens": 32768, "output_tokens": 133, "output_tps_per_user": 100.60975610384575, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6090000001713634, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.01095819581946516, "chunk_inter_token_latency_avg": 0.035766606821880896, "input_tokens": 32768, "output_tokens": 1819, "output_tps_per_user": 91.25589800318127, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.3440000000409782, "time_to_second_token": 0.01500000013038516, "latency": 14.030999999959022, "inter_token_latency_avg": 0.006686370297957032, "chunk_inter_token_latency_avg": 0.034217499999795106, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 149.55797472143325, "e2e_output_tps_per_user": 145.9625115819244, "completed": true }, { "ttft": 0.2190000000409782, "time_to_second_token": 0.01600000006146729, "latency": 0.0, "inter_token_latency_avg": 0.011196078431299505, "chunk_inter_token_latency_avg": 0.037165680473130305, "input_tokens": 32768, "output_tokens": 562, "output_tps_per_user": 89.31698774138832, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.1720000000204891, "time_to_second_token": 0.0, "latency": 15.812000000150874, "inter_token_latency_avg": 0.007640449438265943, "chunk_inter_token_latency_avg": 0.03483296213837502, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 130.88235294008535, "e2e_output_tps_per_user": 129.52188211361363, "completed": true }, { "ttft": 0.18799999984912574, "time_to_second_token": 0.030999999959021807, "latency": 0.0, "inter_token_latency_avg": 0.011470886075845625, "chunk_inter_token_latency_avg": 0.03744628099139687, "input_tokens": 32768, "output_tokens": 396, "output_tps_per_user": 87.17722357174407, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6090000001713634, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.012653307887931703, "chunk_inter_token_latency_avg": 0.03514310954033328, "input_tokens": 32768, "output_tokens": 1573, "output_tps_per_user": 79.03071741056472, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 11651, "wall_time": 21.093999999808148, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 777.8369738913285, "server_utilization": 0.04175665946724261, "server_spec_accept_rate": 0.45938864628820963, "server_spec_accept_length": 0.0, "avg_running_reqs": 7.9, "max_running_reqs": 8, "effective_concurrency": 7.9, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.047, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 16, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.985, "measurement_wall_seconds": 15.016, "client_output_tokens": 18359, "server_output_tokens": 18359, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1225.1584918167825, "per_request_avg_tps": 76.57240573854891, "ttft_avg": 0.5578235294532907, "ttft_p50": 0.5470000000204891, "ttft_p90": 0.875, "ttft_p99": 0.9270800001267344, "time_to_second_token_avg": 0.06641666664897154, "time_to_second_token_p50": 0.0625, "time_to_second_token_p90": 0.09240000003483147, "time_to_second_token_p99": 0.0940000000409782, "request_latency_avg": 19.45299999997951, "request_latency_p50": 19.45299999997951, "request_latency_p90": 19.45299999997951, "request_latency_p99": 19.45299999997951, "inter_token_latency_avg": 0.012795821361620932, "inter_token_latency_p50": 0.013127129750910946, "inter_token_latency_p90": 0.015231237079764667, "inter_token_latency_p99": 0.015326397562859138, "output_tps_per_user_avg": 80.16336762610558, "output_tps_per_user_p50": 76.17811501639235, "output_tps_per_user_p90": 97.15990711868767, "output_tps_per_user_p99": 108.10106181762679, "e2e_output_tps_per_user_avg": 105.27939135362962, "e2e_output_tps_per_user_p50": 105.27939135362962, "e2e_output_tps_per_user_p90": 105.27939135362962, "e2e_output_tps_per_user_p99": 105.27939135362962, "chunk_inter_token_latency_avg": 0.04612066757407402, "chunk_inter_token_latency_p50": 0.04579493087557604, "chunk_inter_token_latency_p90": 0.047243460086088915, "chunk_inter_token_latency_p99": 0.04896887411882773, "input_seq_len_avg": 32768.0, "output_seq_len_avg": 2048.0, "output_seq_len_p50": 2048.0, "output_seq_len_p90": 2048.0, "output_seq_len_p99": 2048.0, "request_count": 17, "completed_request_count": 1, "request_samples": [ { "ttft": 0.7970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.010291753653466063, "chunk_inter_token_latency_avg": 0.045227064220277474, "input_tokens": 32768, "output_tokens": 1917, "output_tps_per_user": 97.16517064739685, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.015339680121905154, "chunk_inter_token_latency_avg": 0.045362612612751055, "input_tokens": 32768, "output_tokens": 1314, "output_tps_per_user": 65.19040762603609, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5470000000204891, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.011886309523833915, "chunk_inter_token_latency_avg": 0.045800458715690315, "input_tokens": 32768, "output_tokens": 1681, "output_tps_per_user": 84.13040212311846, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.18700000015087426, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014296061884606606, "chunk_inter_token_latency_avg": 0.04705787037016341, "input_tokens": 32768, "output_tokens": 1423, "output_tps_per_user": 69.94933346481646, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.010522620904889467, "chunk_inter_token_latency_avg": 0.04578054298665712, "input_tokens": 32768, "output_tokens": 1924, "output_tps_per_user": 95.03335804251368, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.375, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.013861665519656894, "chunk_inter_token_latency_avg": 0.0492444987776564, "input_tokens": 32768, "output_tokens": 1454, "output_tps_per_user": 72.14140310786782, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.014484513274381614, "chunk_inter_token_latency_avg": 0.04494508009167384, "input_tokens": 32768, "output_tokens": 1357, "output_tps_per_user": 69.03925462022077, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.48400000017136335, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.015256664127867558, "chunk_inter_token_latency_avg": 0.04532126696807716, "input_tokens": 32768, "output_tokens": 1314, "output_tps_per_user": 65.54512779588674, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7970000000204891, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.014247832369971805, "chunk_inter_token_latency_avg": 0.046950000000097566, "input_tokens": 32768, "output_tokens": 1385, "output_tps_per_user": 70.18611491440356, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01521428571436274, "chunk_inter_token_latency_avg": 0.045574324324555056, "input_tokens": 32768, "output_tokens": 1331, "output_tps_per_user": 65.72769953018367, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9370000001508743, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.013935231316662343, "chunk_inter_token_latency_avg": 0.04752184465997717, "input_tokens": 32768, "output_tokens": 1406, "output_tps_per_user": 71.76055978376914, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6410000000614673, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.010795763172189029, "chunk_inter_token_latency_avg": 0.04579493087557604, "input_tokens": 32768, "output_tokens": 1842, "output_tps_per_user": 92.62893081761005, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.48400000017136335, "time_to_second_token": 0.06299999984912574, "latency": 0.0, "inter_token_latency_avg": 0.013127129750910946, "chunk_inter_token_latency_avg": 0.046156682027396555, "input_tokens": 32768, "output_tokens": 1527, "output_tps_per_user": 76.17811501639235, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.875, "time_to_second_token": 0.06200000015087426, "latency": 19.45299999997951, "inter_token_latency_avg": 0.009075720566672942, "chunk_inter_token_latency_avg": 0.044766265060191594, "input_tokens": 32768, "output_tokens": 2048, "output_tps_per_user": 110.18408870719439, "e2e_output_tps_per_user": 105.27939135362962, "completed": true }, { "ttft": 0.2190000000409782, "time_to_second_token": 0.0470000000204891, "latency": 0.0, "inter_token_latency_avg": 0.010292682927329003, "chunk_inter_token_latency_avg": 0.04688888889116546, "input_tokens": 32768, "output_tokens": 83, "output_tps_per_user": 97.15639809954823, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6410000000614673, "time_to_second_token": 0.06199999991804361, "latency": 0.0, "inter_token_latency_avg": 0.012797810688989053, "chunk_inter_token_latency_avg": 0.04600694444444445, "input_tokens": 32768, "output_tokens": 1554, "output_tps_per_user": 78.13836477987422, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.6869999999180436, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.012103237629860694, "chunk_inter_token_latency_avg": 0.045652073732907736, "input_tokens": 32768, "output_tokens": 1638, "output_tps_per_user": 82.62252056696252, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 18359, "wall_time": 21.20299999997951, "num_completed": 16, "num_errors": 0, "server_gen_throughput": 1222.6291955197687, "server_utilization": 0.019918406527477783, "server_spec_accept_rate": 0.6073170731707317, "server_spec_accept_length": 0.0, "avg_running_reqs": 16, "max_running_reqs": 16, "effective_concurrency": 16, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.062, "ready_reason": "running_reqs=16/16, queue_reqs=0, active_streams=16/16, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 32, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 14.969, "measurement_wall_seconds": 15.0, "client_output_tokens": 25049, "server_output_tokens": 25104, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 1673.3916761260891, "per_request_avg_tps": 52.293489878940285, "ttft_avg": 0.9377499999827705, "ttft_p50": 0.9449999999487773, "ttft_p90": 1.4777999999467284, "ttft_p99": 1.593000000109896, "time_to_second_token_avg": 0.08796296297158632, "time_to_second_token_p50": 0.0940000000409782, "time_to_second_token_p90": 0.10900000017136335, "time_to_second_token_p99": 0.125, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01849961250681022, "inter_token_latency_p50": 0.018564626982565467, "inter_token_latency_p90": 0.02222254896768293, "inter_token_latency_p99": 0.023536365391272237, "output_tps_per_user_avg": 55.2961316652554, "output_tps_per_user_p50": 53.866012606722705, "output_tps_per_user_p90": 68.32358357522082, "output_tps_per_user_p99": 73.09107211030027, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0625582038971581, "chunk_inter_token_latency_p50": 0.06203155572425161, "chunk_inter_token_latency_p90": 0.06450321581944107, "chunk_inter_token_latency_p99": 0.0653246651331803, "input_seq_len_avg": 32768.0, "output_seq_len_avg": 1115.625, "output_seq_len_p50": 1077.5, "output_seq_len_p90": 1374.6, "output_seq_len_p99": 1502.5900000000001, "request_count": 32, "completed_request_count": 0, "request_samples": [ { "ttft": 1.297000000020489, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.01727486910996552, "chunk_inter_token_latency_avg": 0.061865625000064026, "input_tokens": 32768, "output_tokens": 1147, "output_tps_per_user": 57.88755872095843, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8279999999795109, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.014792700729971874, "chunk_inter_token_latency_avg": 0.06178658536604106, "input_tokens": 32768, "output_tokens": 1371, "output_tps_per_user": 67.60090792439775, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.01961637080873912, "chunk_inter_token_latency_avg": 0.061203076923266056, "input_tokens": 32768, "output_tokens": 1015, "output_tps_per_user": 50.977829168813365, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.202999999979511, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.02255215419508103, "chunk_inter_token_latency_avg": 0.06479153094482562, "input_tokens": 32768, "output_tokens": 883, "output_tps_per_user": 44.34166205808026, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.5, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.01871442215858737, "chunk_inter_token_latency_avg": 0.06200632911405373, "input_tokens": 32768, "output_tokens": 1048, "output_tps_per_user": 53.4347249156788, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2970000000204891, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.022314377682425418, "chunk_inter_token_latency_avg": 0.06458695652180275, "input_tokens": 32768, "output_tokens": 933, "output_tps_per_user": 44.81415588782429, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.297000000020489, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.017997272727291354, "chunk_inter_token_latency_avg": 0.06129102167189006, "input_tokens": 32768, "output_tokens": 1101, "output_tps_per_user": 55.56397433948889, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2029999999795109, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014743119266098424, "chunk_inter_token_latency_avg": 0.06199109792303106, "input_tokens": 32768, "output_tokens": 1418, "output_tps_per_user": 67.8282513999249, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.013407552083360011, "chunk_inter_token_latency_avg": 0.0618438438439669, "input_tokens": 32768, "output_tokens": 1537, "output_tps_per_user": 74.5848305330166, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.5, "time_to_second_token": 0.06200000015087426, "latency": 0.0, "inter_token_latency_avg": 0.020072124756375223, "chunk_inter_token_latency_avg": 0.061658682634853225, "input_tokens": 32768, "output_tokens": 1027, "output_tps_per_user": 49.820336020100925, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.4059999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.01879019073576926, "chunk_inter_token_latency_avg": 0.0634601226996379, "input_tokens": 32768, "output_tokens": 1102, "output_tps_per_user": 53.21925754039242, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.422000000020489, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.018593572778847343, "chunk_inter_token_latency_avg": 0.06071604938277929, "input_tokens": 32768, "output_tokens": 1059, "output_tps_per_user": 53.78202521344541, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.422000000020489, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.015928744939287847, "chunk_inter_token_latency_avg": 0.062056782334449494, "input_tokens": 32768, "output_tokens": 1236, "output_tps_per_user": 62.779585197169254, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9059999999590218, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.020228456913909777, "chunk_inter_token_latency_avg": 0.06368454258700933, "input_tokens": 32768, "output_tokens": 999, "output_tps_per_user": 49.43530810362336, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.1869999999180436, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.018520484171379394, "chunk_inter_token_latency_avg": 0.0619657320874189, "input_tokens": 32768, "output_tokens": 1075, "output_tps_per_user": 53.99426876460113, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4059999999590218, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.02072918861962117, "chunk_inter_token_latency_avg": 0.06325401929267038, "input_tokens": 32768, "output_tokens": 950, "output_tps_per_user": 48.241154940982696, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.3899999998975545, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.014517894736899618, "chunk_inter_token_latency_avg": 0.06175522388084166, "input_tokens": 32768, "output_tokens": 1426, "output_tps_per_user": 68.8805104405624, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.01667540983609916, "chunk_inter_token_latency_avg": 0.06221406727841278, "input_tokens": 32768, "output_tokens": 1221, "output_tps_per_user": 59.968541093076226, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.10899999993853271, "latency": 0.0, "inter_token_latency_avg": 0.020827902240305, "chunk_inter_token_latency_avg": 0.06371651090336296, "input_tokens": 32768, "output_tokens": 983, "output_tps_per_user": 48.01251650129486, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4839999999385327, "time_to_second_token": 0.10900000017136335, "latency": 0.0, "inter_token_latency_avg": 0.01433357717632844, "chunk_inter_token_latency_avg": 0.06104049844249526, "input_tokens": 32768, "output_tokens": 1368, "output_tps_per_user": 69.76625497586716, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9839999999385327, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.01462445414850144, "chunk_inter_token_latency_avg": 0.06163803680994165, "input_tokens": 32768, "output_tokens": 1375, "output_tps_per_user": 68.37862048358704, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.546000000089407, "time_to_second_token": 0.07899999991059303, "latency": 0.0, "inter_token_latency_avg": 0.01786945169703229, "chunk_inter_token_latency_avg": 0.0625975609752747, "input_tokens": 32768, "output_tokens": 1150, "output_tps_per_user": 55.96142606692724, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.625, "time_to_second_token": 0.10899999993853271, "latency": 0.0, "inter_token_latency_avg": 0.020473473473452963, "chunk_inter_token_latency_avg": 0.06160542168668527, "input_tokens": 32768, "output_tokens": 1000, "output_tps_per_user": 48.843690412213405, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.7339999999385327, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.017052808046974836, "chunk_inter_token_latency_avg": 0.06318012422372975, "input_tokens": 32768, "output_tokens": 1194, "output_tps_per_user": 58.641368462327804, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.9839999999385327, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.019895049504991068, "chunk_inter_token_latency_avg": 0.06182769230781839, "input_tokens": 32768, "output_tokens": 1011, "output_tps_per_user": 50.263760326363105, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.593000000109896, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.021019417475587502, "chunk_inter_token_latency_avg": 0.060512422359843526, "input_tokens": 32768, "output_tokens": 928, "output_tps_per_user": 47.575057737038904, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8119999999180436, "time_to_second_token": 0.0779999999795109, "latency": 0.0, "inter_token_latency_avg": 0.016343548387146346, "chunk_inter_token_latency_avg": 0.06235692307711221, "input_tokens": 32768, "output_tokens": 1241, "output_tps_per_user": 61.186223230841755, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.077999999979511, "time_to_second_token": 0.10899999993853271, "latency": 0.0, "inter_token_latency_avg": 0.018535681186283594, "chunk_inter_token_latency_avg": 0.06389776357827476, "input_tokens": 32768, "output_tokens": 1080, "output_tps_per_user": 53.95, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.8899999998975545, "time_to_second_token": 0.0940000000409782, "latency": 0.0, "inter_token_latency_avg": 0.017448573898082936, "chunk_inter_token_latency_avg": 0.06289096573234254, "input_tokens": 32768, "output_tokens": 1158, "output_tps_per_user": 57.31127402394011, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.077999999979511, "time_to_second_token": 0.10899999993853271, "latency": 0.0, "inter_token_latency_avg": 0.022857142857142857, "chunk_inter_token_latency_avg": 0.06451612903225806, "input_tokens": 32768, "output_tokens": 876, "output_tps_per_user": 43.75, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 0.2809999999590218, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.021396090535000502, "chunk_inter_token_latency_avg": 0.0643869969040882, "input_tokens": 32768, "output_tokens": 973, "output_tps_per_user": 46.73751021777384, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.593000000109896, "time_to_second_token": 0.031999999890103936, "latency": 0.0, "inter_token_latency_avg": 0.023841523341388334, "chunk_inter_token_latency_avg": 0.06556418918881791, "input_tokens": 32768, "output_tokens": 815, "output_tps_per_user": 41.94362858786054, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 25049, "wall_time": 21.687999999849126, "num_completed": 32, "num_errors": 0, "server_gen_throughput": 1673.6, "server_utilization": 0.16102711783057355, "server_spec_accept_rate": 0.5493421052631579, "server_spec_accept_length": 0.0, "avg_running_reqs": 32, "max_running_reqs": 32, "effective_concurrency": 32, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 6.641, "ready_reason": "running_reqs=32/32, queue_reqs=0, active_streams=32/32, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} } ], "summary_table": { "0": { "1": 226.06666666666666, "4": 500.9989344756162, "8": 734.2498665158954, "16": 1105.5055055151245, "32": 1655.2875943571494 }, "4096": { "1": 264.1333333333333, "4": 562.1333333333333, "8": 834.6666666666666, "16": 1192.8, "32": 1833.6892685606456 }, "8192": { "1": 268.73333333333335, "4": 525.2269087047707, "8": 875.0, "16": 1193.393393403777, "32": 1692.7038052324328 }, "16384": { "1": 248.71538204701503, "4": 491.30869131204344, "8": 855.7685884270279, "16": 1330.8666666666666, "32": 1861.5189535291647 }, "32768": { "1": 250.4, "4": 504.84852538021426, "8": 776.7333333333333, "16": 1225.1584918167825, "32": 1673.3916761260891 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": true, "mode": "standalone_cold", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline. Context sharing is controlled by --unique-context-percent: one scout warms stream 0, then each worker diverges at the requested suffix boundary." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." }, "coding_peak": { "name": "Coding Peak", "present": true, "formula": "usage.completion_tokens / (last_stream_time - first_token_time)", "notes": "Sequential cc1 Sieve-of-Eratosthenes coding prompt, matching /mnt/test.py throughput semantics. Uses OpenAI stream usage with continuous_usage_stats when the server supports it." } }, "coding_peak": { "mode": "coding_peak", "prompt": "Write a Python script that implements the Sieve of Eratosthenes.", "runs_requested": 10, "runs_ok": 10, "max_tokens": 2000, "temperature": null, "summary": { "mean_generation_tok_s": 309.5797739798034, "median_generation_tok_s": 308.65663696004685, "max_generation_tok_s": 327.21202002965816, "min_generation_tok_s": 296.5925925925926, "cjk_runs": 0 }, "samples": [ { "run": 1, "ok": true, "finish_reason": "stop", "completion_tokens": 1001, "ttft": 0.030999999959021807, "gen_elapsed": 3.375, "total_elapsed": 3.405999999959022, "generation_tok_s": 296.5925925925926, "total_tok_s": 293.89312977452823, "content_chars": 2858, "reasoning_chars": 787, "cjk_chars": 0, "content_preview": "Below is a Python script implementing the **Sieve of Eratosthenes** to find all prime numbers up to a given limit. The algorithm is optimized to mark multiples starting from `p²` and only iterating up to `√n`.\n\n```python\n#!/usr/bin/env python3\n\"\"\"\nSieve of Eratosthenes – find all prime numbers up to a given integer n.\n\"\"\"\n\nimport math\n\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n.\n\n Args:\n n: Upper bound (inclusi" }, { "run": 2, "ok": true, "finish_reason": "stop", "completion_tokens": 714, "ttft": 0.03100000019185245, "gen_elapsed": 2.359999999869615, "total_elapsed": 2.3910000000614673, "generation_tok_s": 302.54237289807077, "total_tok_s": 298.61982433360293, "content_chars": 2090, "reasoning_chars": 639, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the Sieve of Eratosthenes. The script defines a function `sieve(n)` that returns a list of all prime numbers up to `n` (inclusive). It also includes a simple demonstration when run directly.\n\n```python\ndef sieve(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n using\n the Sieve of Eratosthenes algorithm.\n\n Args:\n n (int): Upper limit (inclusive).\n\n Returns:\n list[int]: Sorted list of primes in [" }, { "run": 3, "ok": true, "finish_reason": "stop", "completion_tokens": 425, "ttft": 0.030999999959021807, "gen_elapsed": 1.3440000000409782, "total_elapsed": 1.375, "generation_tok_s": 316.22023808559663, "total_tok_s": 309.09090909090907, "content_chars": 1299, "reasoning_chars": 384, "cjk_chars": 0, "content_preview": "```python\ndef sieve_of_eratosthenes(limit):\n \"\"\"\n Return a list of all prime numbers up to the given limit (inclusive)\n using the Sieve of Eratosthenes algorithm.\n\n Parameters:\n limit (int): The upper bound (inclusive) for finding primes.\n\n Returns:\n list: A list of prime numbers less than or equal to limit.\n \"\"\"\n if limit < 2:\n return []\n\n # Initialize a boolean list; index i represents whether i is prime.\n is_prime = [True] * (limit + 1)\n is_prime[0] " }, { "run": 4, "ok": true, "finish_reason": "stop", "completion_tokens": 717, "ttft": 0.0470000000204891, "gen_elapsed": 2.327999999979511, "total_elapsed": 2.375, "generation_tok_s": 307.9896907243602, "total_tok_s": 301.89473684210526, "content_chars": 2231, "reasoning_chars": 587, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the Sieve of Eratosthenes. The function `sieve_of_eratosthenes(n)` returns a list of all prime numbers up to (and including) `n`. The algorithm uses a boolean array to mark composites, starting from 2. The main optimization is to only iterate up to the square root of `n` for marking multiples.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers up to n (inclusive).\n \n Implements the Sieve of Eratosthe" }, { "run": 5, "ok": true, "finish_reason": "stop", "completion_tokens": 812, "ttft": 0.0470000000204891, "gen_elapsed": 2.718000000109896, "total_elapsed": 2.765000000130385, "generation_tok_s": 298.7490801939546, "total_tok_s": 293.67088606210115, "content_chars": 2237, "reasoning_chars": 767, "cjk_chars": 0, "content_preview": "Below is a Python script implementing the Sieve of Eratosthenes. It defines a function that returns a list of all prime numbers up to a given limit `n`. The script includes a docstring, input validation, and an example usage block.\n\n```python\nimport math\n\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers <= n using the Sieve of Eratosthenes.\n\n Args:\n n: A positive integer (n >= 2). If n < 2, an empty list is returned.\n\n Returns:\n A" }, { "run": 6, "ok": true, "finish_reason": "stop", "completion_tokens": 846, "ttft": 0.0470000000204891, "gen_elapsed": 2.734999999869615, "total_elapsed": 2.781999999890104, "generation_tok_s": 309.32358319573353, "total_tok_s": 304.09777139950364, "content_chars": 2810, "reasoning_chars": 517, "cjk_chars": 0, "content_preview": "Below is a Python script that implements the **Sieve of Eratosthenes** to find all prime numbers up to a given limit `n`. The code includes a clear function, explanatory comments, and a simple demonstration.\n\n```python\n#!/usr/bin/env python3\n\"\"\"\nSieve of Eratosthenes – Find all prime numbers up to a given integer n.\n\"\"\"\n\nimport sys\n\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n.\n Uses the classic Sieve of Eratosthenes al" }, { "run": 7, "ok": true, "finish_reason": "stop", "completion_tokens": 711, "ttft": 0.0470000000204891, "gen_elapsed": 2.296000000089407, "total_elapsed": 2.343000000109896, "generation_tok_s": 309.66898953497974, "total_tok_s": 303.4571062597743, "content_chars": 2138, "reasoning_chars": 574, "cjk_chars": 0, "content_preview": "Below is a Python implementation of the Sieve of Eratosthenes. The function `sieve_of_eratosthenes(n)` returns a list of all prime numbers less than or equal to `n`.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Returns a list of all prime numbers up to n (inclusive) using the\n Sieve of Eratosthenes algorithm.\n\n Args:\n n (int): Upper bound for primes (must be >= 2).\n\n Returns:\n list[int]: Sorted list of primes <= n.\n \"\"\"\n if n < 2:\n re" }, { "run": 8, "ok": true, "finish_reason": "stop", "completion_tokens": 720, "ttft": 0.0470000000204891, "gen_elapsed": 2.234999999869615, "total_elapsed": 2.281999999890104, "generation_tok_s": 322.14765102550484, "total_tok_s": 315.51270816593933, "content_chars": 2101, "reasoning_chars": 525, "cjk_chars": 0, "content_preview": "Below is a Python script implementing the Sieve of Eratosthenes. It includes a function `sieve_of_eratosthenes(n)` that returns a list of all prime numbers up to (and including) `n`, along with a simple demonstration.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers less than or equal to n.\n\n Implements the Sieve of Eratosthenes algorithm.\n\n Args:\n n (int): Upper bound (inclusive).\n\n Returns:\n list[int]: List of prim" }, { "run": 9, "ok": true, "finish_reason": "stop", "completion_tokens": 873, "ttft": 0.0470000000204891, "gen_elapsed": 2.8589999999385327, "total_elapsed": 2.905999999959022, "generation_tok_s": 305.35152151758274, "total_tok_s": 300.4129387516553, "content_chars": 2326, "reasoning_chars": 816, "cjk_chars": 0, "content_preview": "Below is a Python script that implements the Sieve of Eratosthenes, including a function that returns all prime numbers up to a given limit `n`.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list:\n \"\"\"\n Return a list of all prime numbers less than or equal to n\n using the Sieve of Eratosthenes algorithm.\n\n Parameters:\n n (int): Upper bound (inclusive) for the prime search.\n\n Returns:\n list: Sorted list of primes <= n.\n \"\"\"\n if n < 2:\n return []\n\n # " }, { "run": 10, "ok": true, "finish_reason": "stop", "completion_tokens": 588, "ttft": 0.0470000000204891, "gen_elapsed": 1.797000000020489, "total_elapsed": 1.8440000000409782, "generation_tok_s": 327.21202002965816, "total_tok_s": 318.87201734649307, "content_chars": 1715, "reasoning_chars": 323, "cjk_chars": 0, "content_preview": "Below is a clean and efficient Python implementation of the **Sieve of Eratosthenes**. It returns all prime numbers up to a given limit `n`.\n\n```python\ndef sieve_of_eratosthenes(n: int) -> list[int]:\n \"\"\"\n Return a list of all prime numbers <= n using the Sieve of Eratosthenes.\n\n Args:\n n: Upper bound (inclusive).\n\n Returns:\n List of primes from 2 up to n (or empty if n < 2).\n \"\"\"\n if n < 2:\n return []\n\n # Initially mark all numbers as prime\n is_prime" } ] } }