[ { "round": 0, "format": "FP16", "warmup_discard": true, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:23Z", "avg_ns": 31384500, "stddev_ns": 0, "avg_ts": 16313.785467, "stddev_ts": 0.0, "samples_ns": [ 31384500 ], "samples_ts": [ 16313.8 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:23Z", "avg_ns": 1167113000, "stddev_ns": 0, "avg_ts": 109.672328, "stddev_ts": 0.0, "samples_ns": [ 1167113000 ], "samples_ts": [ 109.672 ] } ] }, { "round": 0, "format": "Q8_0", "warmup_discard": true, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:28Z", "avg_ns": 31944900, "stddev_ns": 0, "avg_ts": 16027.597519, "stddev_ts": 0.0, "samples_ns": [ 31944900 ], "samples_ts": [ 16027.6 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:28Z", "avg_ns": 677945200, "stddev_ns": 0, "avg_ts": 188.805821, "stddev_ts": 0.0, "samples_ns": [ 677945200 ], "samples_ts": [ 188.806 ] } ] }, { "round": 0, "format": "Q6_K", "warmup_discard": true, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:31Z", "avg_ns": 38600100, "stddev_ns": 0, "avg_ts": 13264.214341, "stddev_ts": 0.0, "samples_ns": [ 38600100 ], "samples_ts": [ 13264.2 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:31Z", "avg_ns": 552765400, "stddev_ns": 0, "avg_ts": 231.562974, "stddev_ts": 0.0, "samples_ns": [ 552765400 ], "samples_ts": [ 231.563 ] } ] }, { "round": 0, "format": "Q4_K_M", "warmup_discard": true, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:34Z", "avg_ns": 33413800, "stddev_ns": 0, "avg_ts": 15323.010253, "stddev_ts": 0.0, "samples_ns": [ 33413800 ], "samples_ts": [ 15323 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:34Z", "avg_ns": 449889900, "stddev_ns": 0, "avg_ts": 284.514056, "stddev_ts": 0.0, "samples_ns": [ 449889900 ], "samples_ts": [ 284.514 ] } ] }, { "round": 1, "format": "Q4_K_M", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:36Z", "avg_ns": 33412000, "stddev_ns": 0, "avg_ts": 15323.835748, "stddev_ts": 0.0, "samples_ns": [ 33412000 ], "samples_ts": [ 15323.8 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:36Z", "avg_ns": 449526400, "stddev_ns": 0, "avg_ts": 284.744122, "stddev_ts": 0.0, "samples_ns": [ 449526400 ], "samples_ts": [ 284.744 ] } ] }, { "round": 1, "format": "Q6_K", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:39Z", "avg_ns": 39297000, "stddev_ns": 0, "avg_ts": 13028.984401, "stddev_ts": 0.0, "samples_ns": [ 39297000 ], "samples_ts": [ 13029 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:39Z", "avg_ns": 552400200, "stddev_ns": 0, "avg_ts": 231.716064, "stddev_ts": 0.0, "samples_ns": [ 552400200 ], "samples_ts": [ 231.716 ] } ] }, { "round": 1, "format": "Q8_0", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:43Z", "avg_ns": 32072800, "stddev_ns": 0, "avg_ts": 15963.682622, "stddev_ts": 0.0, "samples_ns": [ 32072800 ], "samples_ts": [ 15963.7 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:43Z", "avg_ns": 678122400, "stddev_ns": 0, "avg_ts": 188.756484, "stddev_ts": 0.0, "samples_ns": [ 678122400 ], "samples_ts": [ 188.756 ] } ] }, { "round": 1, "format": "FP16", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:49Z", "avg_ns": 31129900, "stddev_ns": 0, "avg_ts": 16447.209917, "stddev_ts": 0.0, "samples_ns": [ 31129900 ], "samples_ts": [ 16447.2 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:49Z", "avg_ns": 1166828000, "stddev_ns": 0, "avg_ts": 109.699116, "stddev_ts": 0.0, "samples_ns": [ 1166828000 ], "samples_ts": [ 109.699 ] } ] }, { "round": 2, "format": "FP16", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:28:55Z", "avg_ns": 31306000, "stddev_ns": 0, "avg_ts": 16354.692391, "stddev_ts": 0.0, "samples_ns": [ 31306000 ], "samples_ts": [ 16354.7 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:28:55Z", "avg_ns": 1167177600, "stddev_ns": 0, "avg_ts": 109.666258, "stddev_ts": 0.0, "samples_ns": [ 1167177600 ], "samples_ts": [ 109.666 ] } ] }, { "round": 2, "format": "Q8_0", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:00Z", "avg_ns": 32093100, "stddev_ns": 0, "avg_ts": 15953.585039, "stddev_ts": 0.0, "samples_ns": [ 32093100 ], "samples_ts": [ 15953.6 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:00Z", "avg_ns": 678144800, "stddev_ns": 0, "avg_ts": 188.750249, "stddev_ts": 0.0, "samples_ns": [ 678144800 ], "samples_ts": [ 188.75 ] } ] }, { "round": 2, "format": "Q6_K", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:03Z", "avg_ns": 38861600, "stddev_ns": 0, "avg_ts": 13174.959343, "stddev_ts": 0.0, "samples_ns": [ 38861600 ], "samples_ts": [ 13175 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:04Z", "avg_ns": 552933200, "stddev_ns": 0, "avg_ts": 231.492701, "stddev_ts": 0.0, "samples_ns": [ 552933200 ], "samples_ts": [ 231.493 ] } ] }, { "round": 2, "format": "Q4_K_M", "warmup_discard": false, "baseline_mib": "2403", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:06Z", "avg_ns": 33231200, "stddev_ns": 0, "avg_ts": 15407.207684, "stddev_ts": 0.0, "samples_ns": [ 33231200 ], "samples_ts": [ 15407.2 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:06Z", "avg_ns": 449600900, "stddev_ns": 0, "avg_ts": 284.696939, "stddev_ts": 0.0, "samples_ns": [ 449600900 ], "samples_ts": [ 284.697 ] } ] }, { "round": 3, "format": "Q4_K_M", "warmup_discard": false, "baseline_mib": "2379", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:39Z", "avg_ns": 33112200, "stddev_ns": 0, "avg_ts": 15462.578747, "stddev_ts": 0.0, "samples_ns": [ 33112200 ], "samples_ts": [ 15462.6 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:39Z", "avg_ns": 449585000, "stddev_ns": 0, "avg_ts": 284.707008, "stddev_ts": 0.0, "samples_ns": [ 449585000 ], "samples_ts": [ 284.707 ] } ] }, { "round": 3, "format": "Q6_K", "warmup_discard": false, "baseline_mib": "2379", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:42Z", "avg_ns": 38777900, "stddev_ns": 0, "avg_ts": 13203.39678, "stddev_ts": 0.0, "samples_ns": [ 38777900 ], "samples_ts": [ 13203.4 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q6_K.gguf", "model_type": "qwen2 7B Q6_K", "model_size": 6248245248, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:42Z", "avg_ns": 553193100, "stddev_ns": 0, "avg_ts": 231.383942, "stddev_ts": 0.0, "samples_ns": [ 553193100 ], "samples_ts": [ 231.384 ] } ] }, { "round": 3, "format": "Q8_0", "warmup_discard": false, "baseline_mib": "2379", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:46Z", "avg_ns": 32099000, "stddev_ns": 0, "avg_ts": 15950.652668, "stddev_ts": 0.0, "samples_ns": [ 32099000 ], "samples_ts": [ 15950.7 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-Q8_0.gguf", "model_type": "qwen2 7B Q8_0", "model_size": 8092571648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:46Z", "avg_ns": 678633300, "stddev_ns": 0, "avg_ts": 188.614381, "stddev_ts": 0.0, "samples_ns": [ 678633300 ], "samples_ts": [ 188.614 ] } ] }, { "round": 3, "format": "FP16", "warmup_discard": false, "baseline_mib": "2379", "data": [ { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 512, "test_time": "2026-11-10T17:29:52Z", "avg_ns": 30930300, "stddev_ns": 0, "avg_ts": 16553.347365, "stddev_ts": 0.0, "samples_ns": [ 30930300 ], "samples_ts": [ 16553.3 ] }, { "build_commit": "ae4893a09", "build_number": 12250, "cpu_info": "AMD Ryzen 9 9950X 16-Core Processor ", "gpu_info": "NVIDIA GeForce RTX 5090", "backends": "CUDA", "model_filename": "C:\\Users\\coolt\\fp16-q8-bench-20261111\\model-f16.gguf", "model_type": "qwen2 7B F16", "model_size": 15231899648, "model_n_params": 7615616512, "n_batch": 512, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": 99, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": 1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "load_mode": "auto", "lazy_mode": "auto", "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 512, "test_time": "2026-11-10T17:29:52Z", "avg_ns": 1167835700, "stddev_ns": 0, "avg_ts": 109.604459, "stddev_ts": 0.0, "samples_ns": [ 1167835700 ], "samples_ts": [ 109.604 ] } ] } ]