{ "metadata": { "version": "0.4.30", "engine": "vllm", "model": "deepseek-v4-flash-dspark", "server": "http://[private-ip]", "timestamp": "2026-07-16T20:37:47.855187", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 30.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "skipped", "standalone_prefill": false, "prefill_only": false, "skip_prefill": true, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 10.0, "decode_warmup_context": 131072, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 300.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 2048, "temperature": null, "ignore_eos": true, "max_total_tokens": 1515055, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1, 2, 4, 8, 12 ], "context_lengths": [ 8192, 32768, 65536, 131072 ], "unique_context_percent": 25.0, "shared_context_percent": 75.0, "context_sharing_mode": "shared_prefix_unique_tail", "startup_diagnostics_available": true, "nvidia_p2p_override_effective": false, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.30", "server_url": "http://[private-ip]:8000", "hostname": "DESKTOP-RMAJ6JK", "uname": "", "env": {}, "args": { "concurrency": "1,2,4,8,12", "contexts": "8192,32768,65536,131072", "max_tokens": 2048, "duration": 30.0, "request_count": 0, "run_burst": false, "standalone_prefill": false, "prefill_only": false, "skip_prefill": true, "prefill_contexts": "8k,64k,128k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 1515055 }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA GeForce RTX 5090, 595.79, 00000000:01:00.0, 2, 16, 600.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 255, "stdout": "ERROR: Option -m is missing its value. Please run 'nvidia-smi -h' for help.", "stderr": "" } }, "nvidia_p2p_override": { "effective": false, "configured": false, "params_path": "/proc/driver/nvidia/params", "params_available": false, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": false, "runtime": { "ForceP2P": "", "RMForceP2PType": "", "RMPcieP2PType": "", "GrdmaPciTopoCheckOverride": "", "EnableResizableBar": "", "DmaRemapPeerMmio": "" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [ "ForceP2P", "RMForceP2PType", "RMPcieP2PType", "GrdmaPciTopoCheckOverride", "EnableResizableBar" ], "mismatched": {}, "registry_dwords": "", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": {}, "event_log": [ "19:47:02 startup NVIDIA P2P override: unknown: /proc/driver/nvidia/params is not readable", "19:47:02 startup KV cache budget manual: 1,515,055 tokens", "19:47:02 startup engine vLLM 0.21.1rc1.dev339+g1967a5627bc3 models=['deepseek-v4-flash-dspark']", "19:47:02 startup model context length: 1,048,576 tokens", "19:47:02 startup prefill tests: skipped", "19:47:02 startup calibrating padding text run=ncojsdakfvzi up_to=128k", "19:47:02 startup token targeting: estimate from 8k", "19:47:02 startup calibrated: 6.25 chars/token (cached, source=8k)", "19:47:02 startup context 8k: 51,188 chars (~8,191 tokens)", "19:47:02 startup context 32k: 204,753 chars (~32,767 tokens)", "19:47:02 startup context 64k: 409,507 chars (~65,535 tokens)", "19:47:02 startup context 128k: 819,014 chars (~131,071 tokens)", "19:47:02 startup startup preparation done", "19:47:02 startup hardware monitor disabled", "19:47:02 decode warmup start", "19:47:03 decode warmup start C=1 ctx=128k 10s", "19:47:03 cell start C=1 ctx=128k unique_context=25%", "19:48:19 ready C=1 ctx=128k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:48:29 cell done C=1 ctx=128k 60.9 tok/s", "19:48:29 decode warmup done C=1 ctx=128k", "19:48:31 cell start C=1 ctx=8k unique_context=25%", "19:48:41 ready C=1 ctx=8k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:49:11 cell done C=1 ctx=8k 45.4 tok/s", "19:49:13 cell start C=1 ctx=32k unique_context=25%", "19:49:33 ready C=1 ctx=32k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:50:03 cell done C=1 ctx=32k 43.8 tok/s", "19:50:06 cell start C=1 ctx=64k unique_context=25%", "19:50:44 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:51:14 cell done C=1 ctx=64k 52.6 tok/s", "19:51:16 cell start C=1 ctx=128k unique_context=25%", "19:52:33 ready C=1 ctx=128k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "19:53:03 cell done C=1 ctx=128k 38.3 tok/s", "19:53:05 cell start C=2 ctx=8k unique_context=25%", "19:53:15 ready C=2 ctx=8k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:53:45 cell done C=2 ctx=8k 70.2 tok/s", "19:53:47 cell start C=4 ctx=8k unique_context=25%", "19:53:57 ready C=4 ctx=8k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "19:54:27 cell done C=4 ctx=8k 101.3 tok/s", "19:54:29 cell start C=8 ctx=8k unique_context=25%", "19:54:42 ready C=8 ctx=8k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:55:12 cell done C=8 ctx=8k 134.3 tok/s", "19:55:15 cell start C=12 ctx=8k unique_context=25%", "19:55:34 ready C=12 ctx=8k running_reqs=12/12, queue_reqs=0, active_streams=12/12, stable=3.0s", "19:56:04 cell done C=12 ctx=8k 144.2 tok/s", "19:56:06 cell start C=2 ctx=32k unique_context=25%", "19:56:44 ready C=2 ctx=32k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "19:57:14 cell done C=2 ctx=32k 67.5 tok/s", "19:57:17 cell start C=4 ctx=32k unique_context=25%", "19:58:15 ready C=4 ctx=32k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "19:58:45 cell done C=4 ctx=32k 89.1 tok/s", "19:58:48 cell start C=8 ctx=32k unique_context=25%", "19:59:27 ready C=8 ctx=32k running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "19:59:57 cell done C=8 ctx=32k 117.3 tok/s", "19:59:59 cell start C=12 ctx=32k unique_context=25%", "20:00:56 ready C=12 ctx=32k running_reqs=12/12, queue_reqs=0, active_streams=12/12, stable=3.0s", "20:01:26 cell done C=12 ctx=32k 145.5 tok/s", "20:01:29 cell start C=2 ctx=64k unique_context=25%", "20:02:42 ready C=2 ctx=64k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "20:03:12 cell done C=2 ctx=64k 69.1 tok/s", "20:03:14 cell start C=4 ctx=64k unique_context=25%", "20:05:34 ready C=4 ctx=64k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "20:06:04 cell done C=4 ctx=64k 83.1 tok/s", "20:06:06 cell start C=8 ctx=64k unique_context=25%", "20:10:46 ready C=8 ctx=64k running_reqs=9/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "20:11:16 cell done C=8 ctx=64k 47.3 tok/s", "20:11:18 cell start C=12 ctx=64k unique_context=25%", "20:16:19 warmup timeout C=12 ctx=64k running_reqs=9/12", "20:17:19 cell done C=12 ctx=64k 4.8 tok/s", "20:17:21 cell start C=2 ctx=128k unique_context=25%", "20:19:48 ready C=2 ctx=128k running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "20:20:19 cell done C=2 ctx=128k 62.6 tok/s", "20:20:21 cell start C=4 ctx=128k unique_context=25%", "20:25:10 ready C=4 ctx=128k running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "20:25:40 cell done C=4 ctx=128k 80.0 tok/s", "20:25:42 cell start C=8 ctx=128k unique_context=25%", "20:30:42 warmup timeout C=8 ctx=128k running_reqs=5/8", "20:31:42 cell done C=8 ctx=128k 2.4 tok/s", "20:31:44 cell start C=12 ctx=128k unique_context=25%", "20:36:45 warmup timeout C=12 ctx=128k running_reqs=5/12", "20:37:45 cell done C=12 ctx=128k 2.2 tok/s" ], "prefill": {}, "results": [ { "concurrency": 1, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 30.0, "measurement_wall_seconds": 30.0, "client_output_tokens": 1363, "server_output_tokens": 1363, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 45.43333333333333, "per_request_avg_tps": 45.43333333333333, "ttft_avg": 0.18700000000171713, "ttft_p50": 0.18700000000171713, "ttft_p90": 0.18700000000171713, "ttft_p99": 0.18700000000171713, "time_to_second_token_avg": 0.046999999998661224, "time_to_second_token_p50": 0.046999999998661224, "time_to_second_token_p90": 0.046999999998661224, "time_to_second_token_p99": 0.046999999998661224, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.022850299401195824, "inter_token_latency_p50": 0.022850299401195824, "inter_token_latency_p90": 0.022850299401195824, "inter_token_latency_p99": 0.022850299401195824, "output_tps_per_user_avg": 43.76310272537029, "output_tps_per_user_p50": 43.76310272537029, "output_tps_per_user_p90": 43.76310272537029, "output_tps_per_user_p99": 43.76310272537029, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.06057142857142385, "chunk_inter_token_latency_p50": 0.06057142857142385, "chunk_inter_token_latency_p90": 0.06057142857142385, "chunk_inter_token_latency_p99": 0.06057142857142385, "input_seq_len_avg": 8223.0, "output_seq_len_avg": 1504.0, "output_seq_len_p50": 1504.0, "output_seq_len_p90": 1504.0, "output_seq_len_p99": 1504.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.18700000000171713, "time_to_second_token": 0.046999999998661224, "latency": 0.0, "inter_token_latency_avg": 0.022850299401195824, "chunk_inter_token_latency_avg": 0.06057142857142385, "input_tokens": 8223, "output_tokens": 1504, "output_tps_per_user": 43.76310272537029, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1363, "wall_time": 39.28099999999904, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 45.410628019324555, "server_utilization": 0.46115410992456163, "server_spec_accept_rate": 0.3137254901960784, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 9.219, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.953, "measurement_wall_seconds": 30.0, "client_output_tokens": 1313, "server_output_tokens": 1313, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 43.83534203585954, "per_request_avg_tps": 43.83534203585954, "ttft_avg": 0.4530000000013388, "ttft_p50": 0.4530000000013388, "ttft_p90": 0.4530000000013388, "ttft_p99": 0.4530000000013388, "time_to_second_token_avg": 0.046999999998661224, "time_to_second_token_p50": 0.046999999998661224, "time_to_second_token_p90": 0.046999999998661224, "time_to_second_token_p99": 0.046999999998661224, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.02319635343618352, "inter_token_latency_p50": 0.02319635343618352, "inter_token_latency_p90": 0.02319635343618352, "inter_token_latency_p99": 0.02319635343618352, "output_tps_per_user_avg": 43.110224318281006, "output_tps_per_user_p50": 43.110224318281006, "output_tps_per_user_p90": 43.110224318281006, "output_tps_per_user_p99": 43.110224318281006, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.060251366120214395, "chunk_inter_token_latency_p50": 0.060251366120214395, "chunk_inter_token_latency_p90": 0.060251366120214395, "chunk_inter_token_latency_p99": 0.060251366120214395, "input_seq_len_avg": 32372.0, "output_seq_len_avg": 1427.0, "output_seq_len_p50": 1427.0, "output_seq_len_p90": 1427.0, "output_seq_len_p99": 1427.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.4530000000013388, "time_to_second_token": 0.046999999998661224, "latency": 0.0, "inter_token_latency_avg": 0.02319635343618352, "chunk_inter_token_latency_avg": 0.060251366120214395, "input_tokens": 32372, "output_tokens": 1427, "output_tps_per_user": 43.110224318281006, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1313, "wall_time": 50.59300000000076, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 43.74479426953275, "server_utilization": 0.4811403938473596, "server_spec_accept_rate": 0.75, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 20.578, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.953, "measurement_wall_seconds": 30.015, "client_output_tokens": 1575, "server_output_tokens": 1575, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 52.5823790605322, "per_request_avg_tps": 52.5823790605322, "ttft_avg": 0.39099999999962165, "ttft_p50": 0.39099999999962165, "ttft_p90": 0.39099999999962165, "ttft_p99": 0.39099999999962165, "time_to_second_token_avg": 0.014999999999417923, "time_to_second_token_p50": 0.014999999999417923, "time_to_second_token_p90": 0.014999999999417923, "time_to_second_token_p99": 0.014999999999417923, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.01969826589595243, "inter_token_latency_p50": 0.01969826589595243, "inter_token_latency_p90": 0.01969826589595243, "inter_token_latency_p99": 0.01969826589595243, "output_tps_per_user_avg": 50.765890017023196, "output_tps_per_user_p50": 50.765890017023196, "output_tps_per_user_p90": 50.765890017023196, "output_tps_per_user_p99": 50.765890017023196, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.06063701067615249, "chunk_inter_token_latency_p50": 0.06063701067615249, "chunk_inter_token_latency_p90": 0.06063701067615249, "chunk_inter_token_latency_p99": 0.06063701067615249, "input_seq_len_avg": 64591.0, "output_seq_len_avg": 1731.0, "output_seq_len_p50": 1731.0, "output_seq_len_p90": 1731.0, "output_seq_len_p99": 1731.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.39099999999962165, "time_to_second_token": 0.014999999999417923, "latency": 0.0, "inter_token_latency_avg": 0.01969826589595243, "chunk_inter_token_latency_avg": 0.06063701067615249, "input_tokens": 64591, "output_tokens": 1731, "output_tps_per_user": 50.765890017023196, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1575, "wall_time": 68.46800000000076, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 52.473763118441795, "server_utilization": 0.511707651611639, "server_spec_accept_rate": 0.4722222222222222, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 38.453, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 1, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.969, "measurement_wall_seconds": 30.0, "client_output_tokens": 1148, "server_output_tokens": 1148, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 38.306249791449936, "per_request_avg_tps": 38.306249791449936, "ttft_avg": 0.7659999999996217, "ttft_p50": 0.7659999999996217, "ttft_p90": 0.7659999999996217, "ttft_p99": 0.7659999999996217, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.026669224865694553, "inter_token_latency_p50": 0.026669224865694553, "inter_token_latency_p90": 0.026669224865694553, "inter_token_latency_p99": 0.026669224865694553, "output_tps_per_user_avg": 37.49640287769784, "output_tps_per_user_p50": 37.49640287769784, "output_tps_per_user_p90": 37.49640287769784, "output_tps_per_user_p99": 37.49640287769784, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.06238779174147217, "chunk_inter_token_latency_p50": 0.06238779174147217, "chunk_inter_token_latency_p90": 0.06238779174147217, "chunk_inter_token_latency_p99": 0.06238779174147217, "input_seq_len_avg": 129014.0, "output_seq_len_avg": 1304.0, "output_seq_len_p50": 1304.0, "output_seq_len_p90": 1304.0, "output_seq_len_p99": 1304.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.7659999999996217, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.026669224865694553, "chunk_inter_token_latency_avg": 0.06238779174147217, "input_tokens": 129014, "output_tokens": 1304, "output_tps_per_user": 37.49640287769784, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1148, "wall_time": 106.95300000000134, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 38.2475428952198, "server_utilization": 0.5780346820809248, "server_spec_accept_rate": 0.4533333333333333, "server_spec_accept_length": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 76.922, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 2, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.953, "measurement_wall_seconds": 30.016, "client_output_tokens": 2103, "server_output_tokens": 2103, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 70.20999565986399, "per_request_avg_tps": 35.104997829931996, "ttft_avg": 1.4059999999990396, "ttft_p50": 1.4059999999990396, "ttft_p90": 1.4059999999990396, "ttft_p99": 1.4059999999990396, "time_to_second_token_avg": 0.06299999999828287, "time_to_second_token_p50": 0.06299999999828287, "time_to_second_token_p90": 0.06299999999828287, "time_to_second_token_p99": 0.06299999999828287, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.028844666660788575, "inter_token_latency_p50": 0.028844666660788575, "inter_token_latency_p90": 0.029743977895437573, "inter_token_latency_p99": 0.029946322923233598, "output_tps_per_user_avg": 34.721189591078065, "output_tps_per_user_p50": 34.721189591078065, "output_tps_per_user_p90": 35.803717472118954, "output_tps_per_user_p99": 36.047286245353156, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.07875095985081176, "chunk_inter_token_latency_p50": 0.07875095985081176, "chunk_inter_token_latency_p90": 0.07919358819657744, "chunk_inter_token_latency_p99": 0.07929317957437472, "input_seq_len_avg": 8220.0, "output_seq_len_avg": 1168.5, "output_seq_len_p50": 1168.5, "output_seq_len_p90": 1204.9, "output_seq_len_p99": 1213.09, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 1.4059999999990396, "time_to_second_token": 0.06299999999828287, "latency": 0.0, "inter_token_latency_avg": 0.02996880570409982, "chunk_inter_token_latency_avg": 0.07930424528301887, "input_tokens": 8221, "output_tokens": 1123, "output_tps_per_user": 33.36802973977695, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 1.4059999999990396, "time_to_second_token": 0.06299999999828287, "latency": 0.0, "inter_token_latency_avg": 0.027720527617477328, "chunk_inter_token_latency_avg": 0.07819767441860465, "input_tokens": 8219, "output_tokens": 1214, "output_tps_per_user": 36.07434944237918, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2103, "wall_time": 39.71900000000096, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 70.06263326226102, "server_utilization": 0.591456843342804, "server_spec_accept_rate": 0.5641025641025641, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 9.672, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.938, "measurement_wall_seconds": 30.016, "client_output_tokens": 3033, "server_output_tokens": 3033, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 101.30937270358092, "per_request_avg_tps": 25.32734317589523, "ttft_avg": 3.7180000000007567, "ttft_p50": 3.7180000000007567, "ttft_p90": 3.7180000000007567, "ttft_p99": 3.7180000000007567, "time_to_second_token_avg": 0.046999999998661224, "time_to_second_token_p50": 0.046999999998661224, "time_to_second_token_p90": 0.046999999998661224, "time_to_second_token_p99": 0.046999999998661224, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.040359552632449516, "inter_token_latency_p50": 0.037716840177367136, "inter_token_latency_p90": 0.04733346055142543, "inter_token_latency_p99": 0.051026759228796015, "output_tps_per_user_avg": 25.349243306169534, "output_tps_per_user_p50": 26.513387660069398, "output_tps_per_user_p90": 28.213038416763197, "output_tps_per_user_p99": 28.857392316646774, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.10790329503926482, "chunk_inter_token_latency_p50": 0.10789371980676511, "chunk_inter_token_latency_p90": 0.1095672600030446, "chunk_inter_token_latency_p99": 0.10975544804503456, "input_seq_len_avg": 8224.5, "output_seq_len_avg": 872.0, "output_seq_len_p50": 912.0, "output_seq_len_p90": 970.4, "output_seq_len_p99": 992.54, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 3.7180000000007567, "time_to_second_token": 0.046999999998661224, "latency": 0.0, "inter_token_latency_avg": 0.03456740442655994, "chunk_inter_token_latency_avg": 0.10604938271605117, "input_tokens": 8223, "output_tokens": 995, "output_tps_per_user": 28.928987194411615, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.7180000000007567, "time_to_second_token": 0.046999999998661224, "latency": 0.0, "inter_token_latency_avg": 0.0377582417582424, "chunk_inter_token_latency_avg": 0.1067080745341633, "input_tokens": 8222, "output_tokens": 911, "output_tps_per_user": 26.484284051221902, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.7180000000007567, "time_to_second_token": 0.046999999998661224, "latency": 0.0, "inter_token_latency_avg": 0.05143712574850386, "chunk_inter_token_latency_avg": 0.1097763578274779, "input_tokens": 8226, "output_tokens": 669, "output_tps_per_user": 19.441210710127727, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 3.7180000000007567, "time_to_second_token": 0.046999999998661224, "latency": 0.0, "inter_token_latency_avg": 0.03767543859649187, "chunk_inter_token_latency_avg": 0.10907936507936693, "input_tokens": 8227, "output_tokens": 913, "output_tps_per_user": 26.542491268916894, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3033, "wall_time": 39.75, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 100.99563784090097, "server_utilization": 0.6132066229058489, "server_spec_accept_rate": 0.7013888888888888, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 9.687, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.89, "measurement_wall_seconds": 30.015, "client_output_tokens": 4015, "server_output_tokens": 4017, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 134.32586149214046, "per_request_avg_tps": 16.790732686517558, "ttft_avg": 6.479000000002998, "ttft_p50": 6.4615000000030705, "ttft_p90": 8.574000000002707, "ttft_p99": 8.66220000000234, "time_to_second_token_avg": 2.1367499999996653, "time_to_second_token_p50": 2.14049999999952, "time_to_second_token_p90": 4.140999999999622, "time_to_second_token_p99": 4.140999999999622, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.06540167179097196, "inter_token_latency_p50": 0.05643772552671352, "inter_token_latency_p90": 0.08689618351249252, "inter_token_latency_p99": 0.09768935309925757, "output_tps_per_user_avg": 16.20996258866847, "output_tps_per_user_p50": 17.7340026700318, "output_tps_per_user_p90": 19.49367604835329, "output_tps_per_user_p99": 21.421304840804286, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.16650890176933994, "chunk_inter_token_latency_p50": 0.16479493433395748, "chunk_inter_token_latency_p90": 0.18202984189723173, "chunk_inter_token_latency_p99": 0.1876620750988097, "input_seq_len_avg": 8222.625, "output_seq_len_avg": 567.25, "output_seq_len_p50": 610.0, "output_seq_len_p90": 694.9, "output_seq_len_p99": 715.6899999999999, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 4.39100000000326, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.054504385964910876, "chunk_inter_token_latency_avg": 0.16869230769230334, "input_tokens": 8224, "output_tokens": 685, "output_tps_per_user": 18.34714734046881, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.39100000000326, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.08175657894736632, "chunk_inter_token_latency_avg": 0.17585377358490112, "input_tokens": 8224, "output_tokens": 457, "output_tps_per_user": 12.231431560312538, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.39100000000326, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.058098591549295774, "chunk_inter_token_latency_avg": 0.1793478260869565, "input_tokens": 8222, "output_tokens": 640, "output_tps_per_user": 17.21212121212121, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.532000000002881, "time_to_second_token": 0.13999999999941792, "latency": 0.0, "inter_token_latency_avg": 0.05477685950413127, "chunk_inter_token_latency_avg": 0.15342592592592322, "input_tokens": 8222, "output_tokens": 606, "output_tps_per_user": 18.255884127942384, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.532000000002881, "time_to_second_token": 0.13999999999941792, "latency": 0.0, "inter_token_latency_avg": 0.07513439635535393, "chunk_inter_token_latency_avg": 0.16089756097561161, "input_tokens": 8223, "output_tokens": 440, "output_tps_per_user": 13.309483385883913, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.6720000000023, "time_to_second_token": 0.11000000000058208, "latency": 0.0, "inter_token_latency_avg": 0.053833605220228384, "chunk_inter_token_latency_avg": 0.15492957746478872, "input_tokens": 8225, "output_tokens": 614, "output_tps_per_user": 18.575757575757578, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.532000000002881, "time_to_second_token": 0.13999999999941792, "latency": 0.0, "inter_token_latency_avg": 0.04622036262203545, "chunk_inter_token_latency_avg": 0.150636363636361, "input_tokens": 8220, "output_tokens": 718, "output_tps_per_user": 21.635485817743287, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.39100000000326, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.09888859416445368, "chunk_inter_token_latency_avg": 0.18828787878787392, "input_tokens": 8221, "output_tokens": 378, "output_tps_per_user": 10.112389689118041, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4015, "wall_time": 43.39099999999962, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 133.83308345827345, "server_utilization": 0.6553345743117469, "server_spec_accept_rate": 0.6388888888888888, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 13.344, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 12, "context_tokens": 8192, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.969, "measurement_wall_seconds": 30.016, "client_output_tokens": 4323, "server_output_tokens": 4324, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 144.24905735926663, "per_request_avg_tps": 12.020754779938885, "ttft_avg": 9.303416666665422, "ttft_p50": 8.546999999998661, "ttft_p90": 12.671999999998661, "ttft_p99": 13.450749999998662, "time_to_second_token_avg": 2.7135833333331902, "time_to_second_token_p50": 4.125, "time_to_second_token_p90": 4.140999999999622, "time_to_second_token_p99": 4.140999999999622, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.09207434822896154, "inter_token_latency_p50": 0.0953347298898545, "inter_token_latency_p90": 0.1054655812625005, "inter_token_latency_p99": 0.12359276557565266, "output_tps_per_user_avg": 11.263250753416411, "output_tps_per_user_p50": 10.495220238318117, "output_tps_per_user_p90": 15.007243700820382, "output_tps_per_user_p99": 15.977135531503613, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.2161000266208309, "chunk_inter_token_latency_p50": 0.21551044919529921, "chunk_inter_token_latency_p90": 0.23989775080760123, "chunk_inter_token_latency_p99": 0.2583859375624029, "input_seq_len_avg": 8224.416666666666, "output_seq_len_avg": 432.5, "output_seq_len_p50": 401.5, "output_seq_len_p90": 530.1, "output_seq_len_p99": 622.44, "request_count": 12, "completed_request_count": 0, "request_samples": [ { "ttft": 8.546999999998661, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 0.09758808933002813, "chunk_inter_token_latency_avg": 0.2172817679558085, "input_tokens": 8223, "output_tokens": 404, "output_tps_per_user": 10.247152156224224, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 13.546999999998661, "time_to_second_token": 0.13999999999941792, "latency": 0.0, "inter_token_latency_avg": 0.10595061728395475, "chunk_inter_token_latency_avg": 0.199581395348845, "input_tokens": 8224, "output_tokens": 325, "output_tps_per_user": 9.43835935679292, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.546999999998661, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 0.09881407035176215, "chunk_inter_token_latency_avg": 0.2221920903954878, "input_tokens": 8229, "output_tokens": 399, "output_tps_per_user": 10.12001627339266, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.40599999999904, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.12577325581395238, "chunk_inter_token_latency_avg": 0.2606385542168652, "input_tokens": 8227, "output_tokens": 345, "output_tps_per_user": 7.950815883141567, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.546999999998661, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 0.10110025706941218, "chunk_inter_token_latency_avg": 0.2197094972067114, "input_tokens": 8227, "output_tokens": 390, "output_tps_per_user": 9.89117168429584, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 12.671999999998661, "time_to_second_token": 0.875, "latency": 0.0, "inter_token_latency_avg": 0.08889646464646803, "chunk_inter_token_latency_avg": 0.1944917127071897, "input_tokens": 8224, "output_tokens": 397, "output_tps_per_user": 11.249041274890917, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.40599999999904, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.0981241534988735, "chunk_inter_token_latency_avg": 0.24016022099448045, "input_tokens": 8222, "output_tokens": 444, "output_tps_per_user": 10.191170719362999, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 12.671999999998661, "time_to_second_token": 0.875, "latency": 0.0, "inter_token_latency_avg": 0.07669498910675673, "chunk_inter_token_latency_avg": 0.19888700564972508, "input_tokens": 8224, "output_tokens": 460, "output_tps_per_user": 13.03866147771447, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 8.546999999998661, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 0.06222784810126794, "chunk_inter_token_latency_avg": 0.2137391304347899, "input_tokens": 8221, "output_tokens": 633, "output_tps_per_user": 16.069975589909948, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.40599999999904, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 0.09308137044968086, "chunk_inter_token_latency_avg": 0.23753551912568832, "input_tokens": 8223, "output_tokens": 468, "output_tps_per_user": 10.74328832041201, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 12.671999999998661, "time_to_second_token": 0.875, "latency": 0.0, "inter_token_latency_avg": 0.09096382428940915, "chunk_inter_token_latency_avg": 0.1944917127071897, "input_tokens": 8223, "output_tokens": 388, "output_tps_per_user": 10.993381245916122, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 12.671999999998661, "time_to_second_token": 0.875, "latency": 0.0, "inter_token_latency_avg": 0.06567723880597265, "chunk_inter_token_latency_avg": 0.1944917127071897, "input_tokens": 8226, "output_tokens": 537, "output_tps_per_user": 15.225975058943261, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4323, "wall_time": 49.625, "num_completed": 12, "num_errors": 0, "server_gen_throughput": 144.05650319829607, "server_utilization": 0.6632703046928579, "server_spec_accept_rate": 0.5166666666666667, "server_spec_accept_length": 0.0, "avg_running_reqs": 12, "max_running_reqs": 12, "effective_concurrency": 12, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 19.453, "ready_reason": "running_reqs=12/12, queue_reqs=0, active_streams=12/12, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 2, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.984, "measurement_wall_seconds": 30.016, "client_output_tokens": 2023, "server_output_tokens": 2023, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 67.4693169690575, "per_request_avg_tps": 33.73465848452875, "ttft_avg": 8.811999999999898, "ttft_p50": 8.811999999999898, "ttft_p90": 15.549600000000282, "ttft_p99": 17.065560000000367, "time_to_second_token_avg": 0.06300000000192085, "time_to_second_token_p50": 0.06300000000192085, "time_to_second_token_p90": 0.06300000000192085, "time_to_second_token_p99": 0.06300000000192085, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.037755936852043336, "inter_token_latency_p50": 0.037755936852043336, "inter_token_latency_p90": 0.04367964410546665, "inter_token_latency_p99": 0.0450124782374869, "output_tps_per_user_avg": 27.54536237588493, "output_tps_per_user_p50": 27.54536237588493, "output_tps_per_user_p90": 31.86708437535831, "output_tps_per_user_p99": 32.83947182523982, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.10053259782472694, "chunk_inter_token_latency_p50": 0.10053259782472694, "chunk_inter_token_latency_p90": 0.11630604450556296, "chunk_inter_token_latency_p99": 0.11985507000875106, "input_seq_len_avg": 32374.5, "output_seq_len_avg": 1118.0, "output_seq_len_p50": 1118.0, "output_seq_len_p90": 1121.2, "output_seq_len_p99": 1121.92, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.3899999999994179, "time_to_second_token": 0.06300000000192085, "latency": 0.0, "inter_token_latency_avg": 0.04516057091882248, "chunk_inter_token_latency_avg": 0.12024940617577197, "input_tokens": 32375, "output_tokens": 1122, "output_tps_per_user": 22.14320987654321, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 17.23400000000038, "time_to_second_token": 0.06300000000192085, "latency": 0.0, "inter_token_latency_avg": 0.030351302785264187, "chunk_inter_token_latency_avg": 0.08081578947368191, "input_tokens": 32374, "output_tokens": 1114, "output_tps_per_user": 32.94751487522665, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2023, "wall_time": 68.09400000000096, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 67.36372415171205, "server_utilization": 0.6717938669540511, "server_spec_accept_rate": 0.6666666666666666, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 38.016, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.906, "measurement_wall_seconds": 30.0, "client_output_tokens": 2664, "server_output_tokens": 2665, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 89.07911455895425, "per_request_avg_tps": 22.26977863973856, "ttft_avg": 31.538499999999658, "ttft_p50": 35.74150000000009, "ttft_p90": 37.76499999999942, "ttft_p99": 37.76499999999942, "time_to_second_token_avg": 2.066750000000866, "time_to_second_token_p50": 2.055000000000291, "time_to_second_token_p90": 4.07990000000027, "time_to_second_token_p99": 4.092590000000891, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.054256556883851455, "inter_token_latency_p50": 0.05057774947675882, "inter_token_latency_p90": 0.07204605505689946, "inter_token_latency_p99": 0.07983328197627818, "output_tps_per_user_avg": 20.09793412034369, "output_tps_per_user_p50": 19.784195448952843, "output_tps_per_user_p90": 25.9874538171751, "output_tps_per_user_p99": 28.187136816862456, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.13440199783986895, "chunk_inter_token_latency_p50": 0.11810061488672702, "chunk_inter_token_latency_p90": 0.169869593845532, "chunk_inter_token_latency_p99": 0.18787796284476077, "input_seq_len_avg": 32375.0, "output_seq_len_avg": 768.0, "output_seq_len_p50": 713.0, "output_seq_len_p90": 896.0, "output_seq_len_p99": 958.0999999999999, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 37.76499999999942, "time_to_second_token": 0.06300000000192085, "latency": 0.0, "inter_token_latency_avg": 0.035172199170123486, "chunk_inter_token_latency_avg": 0.1130199999999968, "input_tokens": 32378, "output_tokens": 965, "output_tps_per_user": 28.43154603904994, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 16.90599999999904, "time_to_second_token": 4.09400000000096, "latency": 0.0, "inter_token_latency_avg": 0.08069852941176471, "chunk_inter_token_latency_avg": 0.189878892733564, "input_tokens": 32371, "output_tokens": 681, "output_tps_per_user": 12.391799544419134, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 37.76499999999942, "time_to_second_token": 0.06300000000192085, "latency": 0.0, "inter_token_latency_avg": 0.04929855072463713, "chunk_inter_token_latency_avg": 0.11152786885245777, "input_tokens": 32376, "output_tokens": 691, "output_tps_per_user": 20.28457196613381, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 33.71800000000076, "time_to_second_token": 4.046999999998661, "latency": 0.0, "inter_token_latency_avg": 0.0518569482288805, "chunk_inter_token_latency_avg": 0.12318122977345723, "input_tokens": 32375, "output_tokens": 735, "output_tps_per_user": 19.28381893177188, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2664, "wall_time": 88.85900000000038, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 88.83333333333333, "server_utilization": 0.3920838640148917, "server_spec_accept_rate": 0.5370370370370371, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 58.844, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.937, "measurement_wall_seconds": 30.015, "client_output_tokens": 3512, "server_output_tokens": 3511, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 117.31302401711011, "per_request_avg_tps": 14.664128002138764, "ttft_avg": 19.839874999999665, "ttft_p50": 19.73450000000048, "ttft_p90": 30.729799999999887, "ttft_p99": 31.212379999999648, "time_to_second_token_avg": 3.326125000000502, "time_to_second_token_p50": 4.2810000000026776, "time_to_second_token_p90": 4.312300000000687, "time_to_second_token_p99": 4.3129299999985236, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.09231704019472667, "inter_token_latency_p50": 0.08128328908554126, "inter_token_latency_p90": 0.12926178855326087, "inter_token_latency_p99": 0.13724536804451617, "output_tps_per_user_avg": 11.770946177053697, "output_tps_per_user_p50": 12.32218928486073, "output_tps_per_user_p90": 15.576482771834657, "output_tps_per_user_p99": 16.947510678689028, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.21392661431644325, "chunk_inter_token_latency_p50": 0.2103673311184871, "chunk_inter_token_latency_p90": 0.2669747470188634, "chunk_inter_token_latency_p99": 0.274379963181153, "input_seq_len_avg": 32374.75, "output_seq_len_avg": 504.75, "output_seq_len_p50": 467.0, "output_seq_len_p90": 639.1, "output_seq_len_p99": 719.1099999999999, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 13.296999999998661, "time_to_second_token": 4.2810000000026776, "latency": 0.0, "inter_token_latency_avg": 0.13813243243243345, "chunk_inter_token_latency_avg": 0.26344845360824937, "input_tokens": 32371, "output_tokens": 371, "output_tps_per_user": 7.239429454694815, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 17.57800000000134, "time_to_second_token": 4.312999999998283, "latency": 0.0, "inter_token_latency_avg": 0.07804666666666284, "chunk_inter_token_latency_avg": 0.22299047619046525, "input_tokens": 32374, "output_tokens": 601, "output_tps_per_user": 12.812847014607275, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 31.26599999999962, "time_to_second_token": 0.10900000000037835, "latency": 0.0, "inter_token_latency_avg": 0.0748081264108339, "chunk_inter_token_latency_avg": 0.16487562189054436, "input_tokens": 32377, "output_tokens": 444, "output_tps_per_user": 13.367531683766078, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.686999999998079, "time_to_second_token": 4.25, "latency": 0.0, "inter_token_latency_avg": 0.12546008403361547, "chunk_inter_token_latency_avg": 0.27520276497696294, "input_tokens": 32373, "output_tokens": 477, "output_tps_per_user": 7.970662603191485, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 13.296999999998661, "time_to_second_token": 4.2810000000026776, "latency": 0.0, "inter_token_latency_avg": 0.11208114035087802, "chunk_inter_token_latency_avg": 0.24222274881516767, "input_tokens": 32376, "output_tokens": 457, "output_tps_per_user": 8.922107652272528, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 30.5, "time_to_second_token": 0.7659999999996217, "latency": 0.0, "inter_token_latency_avg": 0.06700790513833803, "chunk_inter_token_latency_avg": 0.16300961538461076, "input_tokens": 32376, "output_tokens": 507, "output_tps_per_user": 14.923612339999242, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 26.20300000000134, "time_to_second_token": 4.296999999998661, "latency": 0.0, "inter_token_latency_avg": 0.08451991150441969, "chunk_inter_token_latency_avg": 0.18191904761903668, "input_tokens": 32372, "output_tokens": 453, "output_tps_per_user": 11.831531555114186, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 21.89099999999962, "time_to_second_token": 4.312000000001717, "latency": 0.0, "inter_token_latency_avg": 0.05848005502063194, "chunk_inter_token_latency_avg": 0.19774418604650892, "input_tokens": 32379, "output_tokens": 728, "output_tps_per_user": 17.09984711278396, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 3512, "wall_time": 69.21799999999712, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 116.97484591038041, "server_utilization": 0.4719310277260703, "server_spec_accept_rate": 0.49074074074074076, "server_spec_accept_length": 0.0, "avg_running_reqs": 8, "max_running_reqs": 8, "effective_concurrency": 8, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 39.125, "ready_reason": "running_reqs=8/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 12, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.969, "measurement_wall_seconds": 30.0, "client_output_tokens": 4359, "server_output_tokens": 4359, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 145.45029864192534, "per_request_avg_tps": 12.120858220160445, "ttft_avg": 31.269916666668298, "ttft_p50": 32.6100000000024, "ttft_p90": 47.75, "ttft_p99": 48.59906000000137, "time_to_second_token_avg": 3.4037500000004, "time_to_second_token_p50": 4.312000000001717, "time_to_second_token_p90": 4.328000000000975, "time_to_second_token_p99": 4.328000000001339, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.12158257022524924, "inter_token_latency_p50": 0.1163019074908898, "inter_token_latency_p90": 0.17647041263345717, "inter_token_latency_p99": 0.1879257830022673, "output_tps_per_user_avg": 9.21664199094326, "output_tps_per_user_p50": 8.70225239010449, "output_tps_per_user_p90": 13.46104849232694, "output_tps_per_user_p99": 15.094762256425382, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.28941031876114853, "chunk_inter_token_latency_p50": 0.2786049300886885, "chunk_inter_token_latency_p90": 0.387511556240371, "chunk_inter_token_latency_p99": 0.4328774576271286, "input_seq_len_avg": 32374.333333333332, "output_seq_len_avg": 430.75, "output_seq_len_p50": 410.5, "output_seq_len_p90": 526.6, "output_seq_len_p99": 529.67, "request_count": 12, "completed_request_count": 0, "request_samples": [ { "ttft": 47.75, "time_to_second_token": 0.9540000000015425, "latency": 0.0, "inter_token_latency_avg": 0.09619553072626234, "chunk_inter_token_latency_avg": 0.20498809523810668, "input_tokens": 32372, "output_tokens": 359, "output_tps_per_user": 10.3954933503682, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 4.625, "time_to_second_token": 4.26600000000326, "latency": 0.0, "inter_token_latency_avg": 0.1891780487804925, "chunk_inter_token_latency_avg": 0.4382090395480335, "input_tokens": 32372, "output_tokens": 411, "output_tps_per_user": 5.286025553420959, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 13.204000000001543, "time_to_second_token": 4.312000000001717, "latency": 0.0, "inter_token_latency_avg": 0.17779381443299067, "chunk_inter_token_latency_avg": 0.3897401129943524, "input_tokens": 32375, "output_tokens": 389, "output_tps_per_user": 5.624492635973528, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 21.81300000000192, "time_to_second_token": 4.328000000001339, "latency": 0.0, "inter_token_latency_avg": 0.1472560975609756, "chunk_inter_token_latency_avg": 0.34898843930635837, "input_tokens": 32372, "output_tokens": 411, "output_tps_per_user": 6.790890269151139, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 43.4220000000023, "time_to_second_token": 4.327999999997701, "latency": 0.0, "inter_token_latency_avg": 0.07328166351606734, "chunk_inter_token_latency_avg": 0.21901694915254025, "input_tokens": 32376, "output_tokens": 530, "output_tps_per_user": 13.645978434710962, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 39.09400000000096, "time_to_second_token": 4.328000000001339, "latency": 0.0, "inter_token_latency_avg": 0.10359134615384846, "chunk_inter_token_latency_avg": 0.242101123595511, "input_tokens": 32374, "output_tokens": 417, "output_tps_per_user": 9.65331600686849, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 17.51600000000326, "time_to_second_token": 4.296999999998661, "latency": 0.0, "inter_token_latency_avg": 0.16455979643765561, "chunk_inter_token_latency_avg": 0.3674545454545379, "input_tokens": 32376, "output_tokens": 394, "output_tps_per_user": 6.076818406729468, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 47.75, "time_to_second_token": 0.9540000000015425, "latency": 0.0, "inter_token_latency_avg": 0.06547148288973749, "chunk_inter_token_latency_avg": 0.19567045454546547, "input_tokens": 32374, "output_tokens": 527, "output_tps_per_user": 15.27382542540132, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 30.454000000001543, "time_to_second_token": 4.312000000001717, "latency": 0.0, "inter_token_latency_avg": 0.12901246882793113, "chunk_inter_token_latency_avg": 0.2922824858757084, "input_tokens": 32377, "output_tokens": 402, "output_tps_per_user": 7.751188773340492, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 34.76600000000326, "time_to_second_token": 4.327999999997701, "latency": 0.0, "inter_token_latency_avg": 0.09084674329501659, "chunk_inter_token_latency_avg": 0.2649273743016685, "input_tokens": 32373, "output_tokens": 523, "output_tps_per_user": 11.007549238750258, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 48.70400000000154, "time_to_second_token": 0.125, "latency": 0.0, "inter_token_latency_avg": 0.08476962025316552, "chunk_inter_token_latency_avg": 0.19467441860465337, "input_tokens": 32376, "output_tokens": 396, "output_tps_per_user": 11.79667901087073, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 26.14100000000326, "time_to_second_token": 4.312999999998283, "latency": 0.0, "inter_token_latency_avg": 0.13703422982884758, "chunk_inter_token_latency_avg": 0.3148707865168464, "input_tokens": 32375, "output_tokens": 410, "output_tps_per_user": 7.297446785733577, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 4359, "wall_time": 87.5, "num_completed": 12, "num_errors": 0, "server_gen_throughput": 145.22254797441548, "server_utilization": 0.5840109728617615, "server_spec_accept_rate": 0.5034722222222222, "server_spec_accept_length": 0.0, "avg_running_reqs": 12, "max_running_reqs": 12, "effective_concurrency": 12, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 57.328, "ready_reason": "running_reqs=12/12, queue_reqs=0, active_streams=12/12, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 2, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.937, "measurement_wall_seconds": 30.015, "client_output_tokens": 2070, "server_output_tokens": 2068, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 69.14520493034978, "per_request_avg_tps": 34.57260246517489, "ttft_avg": 17.55449999999837, "ttft_p50": 17.55449999999837, "ttft_p90": 31.160499999998137, "ttft_p99": 34.221849999998085, "time_to_second_token_avg": 2.062500000001819, "time_to_second_token_p50": 2.062500000001819, "time_to_second_token_p90": 3.674900000001435, "time_to_second_token_p99": 4.037690000001348, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.04296942721699623, "inter_token_latency_p50": 0.04296942721699623, "inter_token_latency_p90": 0.04956293306244766, "inter_token_latency_p99": 0.051046471877674235, "output_tps_per_user_avg": 24.161261269082697, "output_tps_per_user_p50": 24.161261269082697, "output_tps_per_user_p90": 27.86872091490649, "output_tps_per_user_p99": 28.70289933521684, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.1224902501023655, "chunk_inter_token_latency_p50": 0.1224902501023655, "chunk_inter_token_latency_p90": 0.15225303841954752, "chunk_inter_token_latency_p99": 0.15894966579091346, "input_seq_len_avg": 64587.5, "output_seq_len_avg": 1174.0, "output_seq_len_p50": 1174.0, "output_seq_len_p90": 1310.8, "output_seq_len_p99": 1341.58, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5469999999986612, "time_to_second_token": 4.078000000001339, "latency": 0.0, "inter_token_latency_avg": 0.05121130952381052, "chunk_inter_token_latency_avg": 0.15969373549884303, "input_tokens": 64586, "output_tokens": 1345, "output_tps_per_user": 19.526936711802957, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 34.56199999999808, "time_to_second_token": 0.0470000000022992, "latency": 0.0, "inter_token_latency_avg": 0.034727544910181936, "chunk_inter_token_latency_avg": 0.08528676470588799, "input_tokens": 64589, "output_tokens": 1003, "output_tps_per_user": 28.795585826362437, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2070, "wall_time": 103.34299999999712, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 68.89888389138898, "server_utilization": 0.6134025668658764, "server_spec_accept_rate": 0.6282051282051282, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 73.328, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.954, "measurement_wall_seconds": 30.016, "client_output_tokens": 2489, "server_output_tokens": 2498, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 83.09407758562702, "per_request_avg_tps": 20.773519396406755, "ttft_avg": 59.78124999999909, "ttft_p50": 51.47649999999885, "ttft_p90": 91.95629999999838, "ttft_p99": 100.92542999999863, "time_to_second_token_avg": 3.0664999999999054, "time_to_second_token_p50": 4.046999999998661, "time_to_second_token_p90": 4.07990000000027, "time_to_second_token_p99": 4.092590000000891, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.10225627932839396, "inter_token_latency_p50": 0.10555494841496835, "inter_token_latency_p90": 0.13345718166973652, "inter_token_latency_p99": 0.14304941660224882, "output_tps_per_user_avg": 11.122347967942074, "output_tps_per_user_p50": 9.481569466721593, "output_tps_per_user_p90": 15.93736868109264, "output_tps_per_user_p99": 18.322361403425248, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.24287200601452041, "chunk_inter_token_latency_p50": 0.2659702235126868, "chunk_inter_token_latency_p90": 0.31770108446481565, "chunk_inter_token_latency_p99": 0.31850501410685406, "input_seq_len_avg": 64586.5, "output_seq_len_avg": 729.25, "output_seq_len_p50": 678.5, "output_seq_len_p90": 865.0, "output_seq_len_p99": 927.0999999999999, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 34.25, "time_to_second_token": 4.046999999998661, "latency": 0.0, "inter_token_latency_avg": 0.1085884244372972, "chunk_inter_token_latency_avg": 0.3156168224299012, "input_tokens": 64587, "output_tokens": 934, "output_tps_per_user": 9.209084717657317, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 34.25, "time_to_second_token": 4.046999999998661, "latency": 0.0, "inter_token_latency_avg": 0.1441152204836391, "chunk_inter_token_latency_avg": 0.3185943396226361, "input_tokens": 64587, "output_tokens": 704, "output_tps_per_user": 6.938892343529576, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 68.7029999999977, "time_to_second_token": 4.09400000000096, "latency": 0.0, "inter_token_latency_avg": 0.1025214723926395, "chunk_inter_token_latency_avg": 0.21632362459547236, "input_tokens": 64589, "output_tokens": 653, "output_tps_per_user": 9.75405421578587, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 101.92199999999866, "time_to_second_token": 0.07800000000133878, "latency": 0.0, "inter_token_latency_avg": 0.0538, "chunk_inter_token_latency_avg": 0.12095323741007194, "input_tokens": 64583, "output_tokens": 626, "output_tps_per_user": 18.587360594795538, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2489, "wall_time": 169.56299999999828, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 83.17794352689666, "server_utilization": 0.427549720779857, "server_spec_accept_rate": 0.5208333333333334, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 139.484, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.875, "measurement_wall_seconds": 30.015, "client_output_tokens": 1413, "server_output_tokens": 1435, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 47.29707112970711, "per_request_avg_tps": 5.912133891213389, "ttft_avg": 121.53900000000112, "ttft_p50": 121.48400000000038, "ttft_p90": 218.35650000000095, "ttft_p99": 240.17025000000095, "time_to_second_token_avg": 4.111714285713658, "time_to_second_token_p50": 4.110000000000582, "time_to_second_token_p90": 4.137399999999616, "time_to_second_token_p99": 4.1541399999990976, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.6146631769128976, "inter_token_latency_p50": 0.6617645740596256, "inter_token_latency_p90": 0.918661380480514, "inter_token_latency_p99": 1.027580855029182, "output_tps_per_user_avg": 2.15463673932857, "output_tps_per_user_p50": 1.527145099183599, "output_tps_per_user_p90": 3.808897520943435, "output_tps_per_user_p99": 4.938193122880859, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 1.2170933748650326, "chunk_inter_token_latency_p50": 1.317128146788991, "chunk_inter_token_latency_p90": 1.7682431509825554, "chunk_inter_token_latency_p99": 1.8410149191250988, "input_seq_len_avg": 64589.875, "output_seq_len_avg": 241.25, "output_seq_len_p50": 242.0, "output_seq_len_p90": 280.5, "output_seq_len_p99": 283.65, "request_count": 8, "completed_request_count": 0, "request_samples": [ { "ttft": 0.625, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 1.0396830188679231, "chunk_inter_token_latency_avg": 1.849100671140937, "input_tokens": 64590, "output_tokens": 266, "output_tps_per_user": 0.9618316177644868, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 69.6720000000023, "time_to_second_token": 4.0939999999973224, "latency": 0.0, "inter_token_latency_avg": 0.7295724381625347, "chunk_inter_token_latency_avg": 1.5882230769230563, "input_tokens": 64594, "output_tokens": 284, "output_tps_per_user": 1.3706658142384749, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 104.20300000000134, "time_to_second_token": 4.110000000000582, "latency": 0.0, "inter_token_latency_avg": 0.7744954954954878, "chunk_inter_token_latency_avg": 1.3755039999999863, "input_tokens": 64592, "output_tokens": 223, "output_tps_per_user": 1.2911630936733136, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 35.1720000000023, "time_to_second_token": 4.077999999997701, "latency": 0.0, "inter_token_latency_avg": 0.8667949640287673, "chunk_inter_token_latency_avg": 1.7335899280575346, "input_tokens": 64589, "output_tokens": 279, "output_tps_per_user": 1.153675369030884, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 207.96900000000096, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 0.30570403587443346, "chunk_inter_token_latency_avg": 0.6492571428571301, "input_tokens": 64587, "output_tokens": 224, "output_tps_per_user": 3.2711377104970425, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 173.31200000000172, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 0.4096135458167262, "chunk_inter_token_latency_avg": 0.9346636363636207, "input_tokens": 64592, "output_tokens": 252, "output_tps_per_user": 2.4413255133106144, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 242.59400000000096, "time_to_second_token": 4.15599999999904, "latency": 0.0, "inter_token_latency_avg": 0.19748520710059173, "chunk_inter_token_latency_avg": 0.34765625, "input_tokens": 64589, "output_tokens": 170, "output_tps_per_user": 5.063670411985019, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 138.76499999999942, "time_to_second_token": 4.09400000000096, "latency": 0.0, "inter_token_latency_avg": 0.5939567099567167, "chunk_inter_token_latency_avg": 1.2587522935779958, "input_tokens": 64586, "output_tokens": 232, "output_tps_per_user": 1.6836243841287235, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1413, "wall_time": 310.46900000000096, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 47.80942861902475, "server_utilization": 0.5505045556970707, "server_spec_accept_rate": 0.2638888888888889, "server_spec_accept_length": 0.0, "avg_running_reqs": 8.6, "max_running_reqs": 9, "effective_concurrency": 8.6, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 280.235, "ready_reason": "running_reqs=9/8, queue_reqs=0, active_streams=8/8, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 12, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 25.875, "measurement_wall_seconds": 30.0, "client_output_tokens": 125, "server_output_tokens": 125, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 4.830917874396135, "per_request_avg_tps": 0.4025764895330113, "ttft_avg": 140.5659999999997, "ttft_p50": 140.73400000000038, "ttft_p90": 251.48139999999913, "ttft_p99": 276.44524000000075, "time_to_second_token_avg": 4.117124999999305, "time_to_second_token_p50": 4.125, "time_to_second_token_p90": 4.145499999999447, "time_to_second_token_p99": 4.15494999999908, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 1.6022365607058078, "inter_token_latency_p50": 1.596972602739747, "inter_token_latency_p90": 1.823081216931219, "inter_token_latency_p99": 2.018116455026444, "output_tps_per_user_avg": 0.6374646378329573, "output_tps_per_user_p50": 0.6261848188781773, "output_tps_per_user_p90": 0.7335560469066646, "output_tps_per_user_p99": 0.8207638687337421, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 4.182804091418899, "chunk_inter_token_latency_p50": 4.317740740740798, "chunk_inter_token_latency_p90": 4.4139108235293865, "chunk_inter_token_latency_p99": 4.442605788235271, "input_seq_len_avg": 64591.333333333336, "output_seq_len_avg": 97.22222222222223, "output_seq_len_p50": 104.0, "output_seq_len_p90": 166.20000000000002, "output_seq_len_p99": 235.32, "request_count": 9, "completed_request_count": 0, "request_samples": [ { "ttft": 0.5, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 1.2041611570247903, "chunk_inter_token_latency_avg": 2.9140699999999926, "input_tokens": 64593, "output_tokens": 243, "output_tps_per_user": 0.8304536267145285, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 37.09400000000096, "time_to_second_token": 4.077999999997701, "latency": 0.0, "inter_token_latency_avg": 1.7452945205479335, "chunk_inter_token_latency_avg": 4.393327586206867, "input_tokens": 64591, "output_tokens": 147, "output_tps_per_user": 0.5729691970189942, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 244.54699999999866, "time_to_second_token": 4.15599999999904, "latency": 0.0, "inter_token_latency_avg": 1.4800000000000182, "chunk_inter_token_latency_avg": 4.305454545454598, "input_tokens": 64593, "output_tokens": 33, "output_tps_per_user": 0.6756756756756673, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 71.61000000000058, "time_to_second_token": 4.092999999997119, "latency": 0.0, "inter_token_latency_avg": 2.0397870370370246, "chunk_inter_token_latency_avg": 4.4059399999999735, "input_tokens": 64589, "output_tokens": 109, "output_tps_per_user": 0.49024725711199274, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 279.21900000000096, "time_to_second_token": 4.140999999999622, "latency": 0.0, "inter_token_latency_avg": 1.409777777777587, "chunk_inter_token_latency_avg": 4.229333333332761, "input_tokens": 64594, "output_tokens": 10, "output_tps_per_user": 0.7093316519546987, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 106.15599999999904, "time_to_second_token": 4.09400000000096, "latency": 0.0, "inter_token_latency_avg": 1.7689047619047675, "chunk_inter_token_latency_avg": 4.319418604651176, "input_tokens": 64593, "output_tokens": 106, "output_tps_per_user": 0.5653215602875046, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 209.92199999999866, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 1.70768750000002, "chunk_inter_token_latency_avg": 4.314157894736892, "input_tokens": 64590, "output_tokens": 49, "output_tps_per_user": 0.5855872341982876, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 175.31199999999808, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 1.596972602739747, "chunk_inter_token_latency_avg": 4.317740740740798, "input_tokens": 64591, "output_tokens": 74, "output_tps_per_user": 0.6261848188781773, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 140.73400000000038, "time_to_second_token": 4.125, "latency": 0.0, "inter_token_latency_avg": 1.467543689320381, "chunk_inter_token_latency_avg": 4.445794117647036, "input_tokens": 64588, "output_tokens": 104, "output_tps_per_user": 0.6814107186567643, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 125, "wall_time": 360.125, "num_completed": 12, "num_errors": 0, "server_gen_throughput": 4.1645843744795075, "server_utilization": 0.9256392671695894, "server_spec_accept_rate": 0.37037037037037035, "server_spec_accept_length": 0.0, "avg_running_reqs": 9.7, "max_running_reqs": 10, "effective_concurrency": 9.7, "avg_queue_reqs": 2.3, "max_queue_reqs": 3, "queue_fraction": 1.0, "underfilled": true, "warmup_timed_out": true, "warmup_duration": 300.094, "ready_reason": "warmup_timeout", "timeout_reason": "running_reqs=9/12", "capacity_limited": true, "hardware_summary": {} }, { "concurrency": 2, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.968, "measurement_wall_seconds": 30.0, "client_output_tokens": 1876, "server_output_tokens": 1876, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 62.60010678056435, "per_request_avg_tps": 31.300053390282176, "ttft_avg": 71.04699999999866, "ttft_p50": 71.04699999999866, "ttft_p90": 71.04699999999866, "ttft_p99": 71.04699999999866, "time_to_second_token_avg": 0.0, "time_to_second_token_p50": 0.0, "time_to_second_token_p90": 0.0, "time_to_second_token_p99": 0.0, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.03261250522535607, "inter_token_latency_p50": 0.03261250522535607, "inter_token_latency_p90": 0.0347792807060888, "inter_token_latency_p99": 0.03526680518925367, "output_tps_per_user_avg": 30.876045946674765, "output_tps_per_user_p50": 30.876045946674765, "output_tps_per_user_p90": 32.92745103919789, "output_tps_per_user_p99": 33.38901718501559, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.0830492971289629, "chunk_inter_token_latency_p50": 0.0830492971289629, "chunk_inter_token_latency_p90": 0.0834629922578743, "chunk_inter_token_latency_p99": 0.08355607366187937, "input_seq_len_avg": 129009.5, "output_seq_len_avg": 1030.5, "output_seq_len_p50": 1030.5, "output_seq_len_p90": 1098.9, "output_seq_len_p99": 1114.29, "request_count": 2, "completed_request_count": 0, "request_samples": [ { "ttft": 71.04699999999866, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.02990403587444014, "chunk_inter_token_latency_avg": 0.08356641604010215, "input_tokens": 129009, "output_tokens": 1116, "output_tps_per_user": 33.44030231232867, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 71.04699999999866, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.035320974576271987, "chunk_inter_token_latency_avg": 0.08253217821782366, "input_tokens": 129010, "output_tokens": 945, "output_tps_per_user": 28.31178958102086, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1876, "wall_time": 177.85900000000038, "num_completed": 2, "num_errors": 0, "server_gen_throughput": 62.502082292188454, "server_utilization": 0.6949152542372881, "server_spec_accept_rate": 0.625, "server_spec_accept_length": 0.0, "avg_running_reqs": 2, "max_running_reqs": 2, "effective_concurrency": 2, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 147.797, "ready_reason": "running_reqs=2/2, queue_reqs=0, active_streams=2/2, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 4, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.922, "measurement_wall_seconds": 30.0, "client_output_tokens": 2394, "server_output_tokens": 2394, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 80.00802085421482, "per_request_avg_tps": 20.002005213553705, "ttft_avg": 125.89449999999943, "ttft_p50": 108.375, "ttft_p90": 193.8045999999984, "ttft_p99": 212.72565999999773, "time_to_second_token_avg": 3.039249999999811, "time_to_second_token_p50": 4.015999999999622, "time_to_second_token_p90": 4.059399999998277, "time_to_second_token_p99": 4.076139999997759, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.1876591997808691, "inter_token_latency_p50": 0.20779008868376997, "inter_token_latency_p90": 0.2837283899693062, "inter_token_latency_p99": 0.29548919650114647, "output_tps_per_user_avg": 9.903622194197364, "output_tps_per_user_p50": 5.0543349658815755, "output_tps_per_user_p90": 20.143492352865543, "output_tps_per_user_p99": 25.537199832173858, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.40677714912469004, "chunk_inter_token_latency_p50": 0.45705272227554317, "chunk_inter_token_latency_p90": 0.585779984886106, "chunk_inter_token_latency_p99": 0.5955271510309835, "input_seq_len_avg": 129010.0, "output_seq_len_avg": 698.75, "output_seq_len_p50": 666.5, "output_seq_len_p90": 816.4000000000001, "output_seq_len_p99": 862.8399999999999, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 72.0, "time_to_second_token": 4.015999999999622, "latency": 0.0, "inter_token_latency_avg": 0.25323741007194245, "chunk_inter_token_latency_avg": 0.5605095541401274, "input_tokens": 129006, "output_tokens": 696, "output_tps_per_user": 3.9488636363636362, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 144.75, "time_to_second_token": 4.077999999997701, "latency": 0.0, "inter_token_latency_avg": 0.1623427672955975, "chunk_inter_token_latency_avg": 0.3535958904109589, "input_tokens": 129012, "output_tokens": 637, "output_tps_per_user": 6.159806295399515, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 214.8279999999977, "time_to_second_token": 0.0470000000022992, "latency": 0.0, "inter_token_latency_avg": 0.038260668973474396, "chunk_inter_token_latency_avg": 0.11639298245614842, "input_tokens": 129011, "output_tokens": 868, "output_tps_per_user": 26.13650066320812, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 72.0, "time_to_second_token": 4.015999999999622, "latency": 0.0, "inter_token_latency_avg": 0.29679595278246207, "chunk_inter_token_latency_avg": 0.5966101694915255, "input_tokens": 129011, "output_tokens": 594, "output_tps_per_user": 3.3693181818181817, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 2394, "wall_time": 319.03099999999904, "num_completed": 4, "num_errors": 0, "server_gen_throughput": 79.7574626865585, "server_utilization": 0.5121975115117077, "server_spec_accept_rate": 0.5185185185185185, "server_spec_accept_length": 0.0, "avg_running_reqs": 4, "max_running_reqs": 4, "effective_concurrency": 4, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 288.984, "ready_reason": "running_reqs=4/4, queue_reqs=0, active_streams=4/4, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": {} }, { "concurrency": 8, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 28.375, "measurement_wall_seconds": 30.016, "client_output_tokens": 67, "server_output_tokens": 67, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 2.3612334801762116, "per_request_avg_tps": 0.29515418502202645, "ttft_avg": 112.4807499999988, "ttft_p50": 113.24249999999847, "ttft_p90": 200.67519999999897, "ttft_p99": 220.35981999999868, "time_to_second_token_avg": 4.07300000000032, "time_to_second_token_p50": 4.077999999997701, "time_to_second_token_p90": 4.090800000000309, "time_to_second_token_p99": 4.093680000000895, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 1.6202312366723683, "inter_token_latency_p50": 1.6566221264367451, "inter_token_latency_p90": 1.9623849425287287, "inter_token_latency_p99": 2.0023284942528767, "output_tps_per_user_avg": 0.6463096634993024, "output_tps_per_user_p50": 0.6127671936142225, "output_tps_per_user_p90": 0.8092414238194522, "output_tps_per_user_p99": 0.8561753487180412, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 3.9535223214285193, "chunk_inter_token_latency_p50": 4.427708333333234, "chunk_inter_token_latency_p90": 4.508307499999983, "chunk_inter_token_latency_p99": 4.5145332500000075, "input_seq_len_avg": 129011.0, "output_seq_len_avg": 99.25, "output_seq_len_p50": 75.0, "output_seq_len_p90": 182.70000000000002, "output_seq_len_p99": 218.06999999999996, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 0.8909999999996217, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 1.1609140271493126, "chunk_inter_token_latency_avg": 2.4434476190476007, "input_tokens": 129011, "output_tokens": 222, "output_tps_per_user": 0.861390229262329, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 222.54699999999866, "time_to_second_token": 4.09400000000096, "latency": 0.0, "inter_token_latency_avg": 1.4544166666666267, "chunk_inter_token_latency_avg": 4.36324999999988, "input_tokens": 129012, "output_tokens": 25, "output_tps_per_user": 0.6875608777860729, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 76.84399999999732, "time_to_second_token": 4.047000000002299, "latency": 0.0, "inter_token_latency_avg": 2.006766666666671, "chunk_inter_token_latency_avg": 4.51522500000001, "input_tokens": 129011, "output_tokens": 91, "output_tps_per_user": 0.4983140375064355, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 149.64099999999962, "time_to_second_token": 4.077999999997701, "latency": 0.0, "inter_token_latency_avg": 1.8588275862068635, "chunk_inter_token_latency_avg": 4.492166666666587, "input_tokens": 129010, "output_tokens": 59, "output_tps_per_user": 0.5379735094423722, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 67, "wall_time": 360.25, "num_completed": 8, "num_errors": 0, "server_gen_throughput": 2.2310279377976805, "server_utilization": 0.8748897815224845, "server_spec_accept_rate": 0.4166666666666667, "server_spec_accept_length": 0.0, "avg_running_reqs": 5, "max_running_reqs": 5, "effective_concurrency": 5, "avg_queue_reqs": 3, "max_queue_reqs": 3, "queue_fraction": 1.0, "underfilled": true, "warmup_timed_out": true, "warmup_duration": 300.219, "ready_reason": "warmup_timeout", "timeout_reason": "running_reqs=5/8", "capacity_limited": true, "hardware_summary": {} }, { "concurrency": 12, "context_tokens": 131072, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 29.234, "measurement_wall_seconds": 30.016, "client_output_tokens": 64, "server_output_tokens": 64, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 2.189231716494737, "per_request_avg_tps": 0.1824359763745614, "ttft_avg": 113.34800000000087, "ttft_p50": 114.38300000000163, "ttft_p90": 201.8536000000011, "ttft_p99": 221.5468600000006, "time_to_second_token_avg": 4.077999999998913, "time_to_second_token_p50": 4.077999999997701, "time_to_second_token_p90": 4.102799999999843, "time_to_second_token_p99": 4.1083800000003246, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 1.6402959767477978, "inter_token_latency_p50": 1.8599260638297497, "inter_token_latency_p90": 1.9346995271866951, "inter_token_latency_p99": 1.9395199527185925, "output_tps_per_user_avg": 0.6753763446565264, "output_tps_per_user_p50": 0.5382592323493268, "output_tps_per_user_p90": 0.9435609737915138, "output_tps_per_user_p99": 1.0929400789127208, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 3.9012694672130195, "chunk_inter_token_latency_p50": 4.429624999999836, "chunk_inter_token_latency_p90": 4.591337499999925, "chunk_inter_token_latency_p99": 4.628833749999949, "input_seq_len_avg": 129009.75, "output_seq_len_avg": 115.5, "output_seq_len_p50": 78.0, "output_seq_len_p90": 229.40000000000003, "output_seq_len_p99": 281.23999999999995, "request_count": 4, "completed_request_count": 0, "request_samples": [ { "ttft": 0.8909999999996217, "time_to_second_token": 0.0, "latency": 0.0, "inter_token_latency_avg": 0.9012762237762217, "chunk_inter_token_latency_avg": 2.112827868852454, "input_tokens": 129007, "output_tokens": 287, "output_tps_per_user": 1.1095377572595218, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 223.73500000000058, "time_to_second_token": 4.109000000000378, "latency": 0.0, "inter_token_latency_avg": 1.94005555555547, "chunk_inter_token_latency_avg": 4.365124999999807, "input_tokens": 129010, "output_tokens": 19, "output_tps_per_user": 0.5154491566679303, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 77.96900000000096, "time_to_second_token": 4.046999999998661, "latency": 0.0, "inter_token_latency_avg": 1.922202127659554, "chunk_inter_token_latency_avg": 4.632999999999951, "input_tokens": 129007, "output_tokens": 95, "output_tps_per_user": 0.5202366523324921, "e2e_output_tps_per_user": 0.0, "completed": false }, { "ttft": 150.7970000000023, "time_to_second_token": 4.077999999997701, "latency": 0.0, "inter_token_latency_avg": 1.7976499999999456, "chunk_inter_token_latency_avg": 4.4941249999998645, "input_tokens": 129015, "output_tokens": 61, "output_tps_per_user": 0.5562818123661616, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 64, "wall_time": 360.9850000000006, "num_completed": 12, "num_errors": 0, "server_gen_throughput": 2.1321961620469354, "server_utilization": 0.8749877535024982, "server_spec_accept_rate": 0.5833333333333334, "server_spec_accept_length": 0.0, "avg_running_reqs": 5, "max_running_reqs": 5, "effective_concurrency": 5, "avg_queue_reqs": 7, "max_queue_reqs": 7, "queue_fraction": 1.0, "underfilled": true, "warmup_timed_out": true, "warmup_duration": 300.969, "ready_reason": "warmup_timeout", "timeout_reason": "running_reqs=5/12", "capacity_limited": true, "hardware_summary": {} } ], "summary_table": { "8192": { "1": 45.43333333333333, "2": 70.20999565986399, "4": 101.30937270358092, "8": 134.32586149214046, "12": 144.24905735926663 }, "32768": { "1": 43.83534203585954, "2": 67.4693169690575, "4": 89.07911455895425, "8": 117.31302401711011, "12": 145.45029864192534 }, "65536": { "1": 52.5823790605322, "2": 69.14520493034978, "4": 83.09407758562702, "8": 47.29707112970711, "12": 4.830917874396135 }, "131072": { "1": 38.306249791449936, "2": 62.60010678056435, "4": 80.00802085421482, "8": 2.3612334801762116, "12": 2.189231716494737 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": false, "mode": "skipped", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline. Context sharing is controlled by --unique-context-percent: one scout warms stream 0, then each worker diverges at the requested suffix boundary." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." } } }