{ "model_delay_seconds": 30.0, "models": [ { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_down": { "allow_approximate": false, "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.0009982450865209103, "max_logit_error": 0.34375, "mean_logit_error": 0.04991571605205536, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "format": "affine8", "group_size": 64, "group_tuning": { "cached": false, "group_size": 64, "median_nanoseconds": { "128": 243958, "32": 246625, "64": 234625 } }, "layer_indices": [ 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ], "projections": 19, "repack_bytes": 87980032, "selection": "suffix:19" }, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.0008934258366934955, "max_logit_error": 0.25, "mean_logit_error": 0.04244478419423103, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 64, "group_tuning": { "cached": false, "group_size": 64, "median_nanoseconds": { "128": 491917, "32": 496333, "64": 458666 } }, "layer_indices": [ 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 ], "layers": 19, "projections": 38, "repack_bytes": 175960064, "selection": "suffix:19" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00030488523771055043, "max_logit_error": 0.1328125, "mean_logit_error": 0.020555006340146065, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 64, "group_tuning": { "cached": false, "group_size": 64, "median_nanoseconds": { "128": 1340583, "32": 1446875, "64": 1342459 } }, "projections": 1, "repack_bytes": 144643072, "tied": true }, "dense_mlp_implementation": null, "features": { "affine_prefill": false, "attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": false, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.5074417887220768, "end_to_end_speedup": 1.4361341306216238, "metile_decode_tokens_per_second": 174.37889291632803, "metile_elapsed_seconds": 0.850463040987961, "metile_prefill_tokens_per_second": 4021.8584130308855, "metile_time_to_first_token_seconds": 0.10864716593641788, "mlx_decode_tokens_per_second": 115.77944130313642, "mlx_elapsed_seconds": 1.237486333004199, "mlx_prefill_tokens_per_second": 3691.429167821772, "mlx_time_to_first_token_seconds": 0.11154079099651426, "prefill_speedup": 1.112888538254143, "ttft_speedup": 1.0249883670020903 }, "memory": { "active_bytes": 1398255376, "cache_bytes": 5341318, "peak_bytes": 1650259472 }, "model": "mlx-community/Qwen2.5-0.5B-Instruct-bf16", "model_config": { "hidden_size": 896, "model_type": "qwen2", "num_attention_heads": 14, "num_hidden_layers": 24, "num_key_value_heads": 2, "vocab_size": 151936 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.4782832709349327, "end_to_end_speedup": 1.397232348671763, "prefill_speedup": 1.0794575371374096, "ttft_speedup": 1.1731901111582568 }, "pairs": [ { "decode_speedup": 1.4770831455180933, "end_to_end_speedup": 1.3846229207567726, "prefill_speedup": 0.6813835243209839, "ttft_speedup": 0.8553959736128174 }, { "decode_speedup": 1.4782832709349327, "end_to_end_speedup": 1.4759532727259819, "prefill_speedup": 2.0247460130175035, "ttft_speedup": 1.500723487521709 }, { "decode_speedup": 1.518295910998758, "end_to_end_speedup": 1.4574777191345818, "prefill_speedup": 1.1041346187928625, "ttft_speedup": 1.050656457803032 }, { "decode_speedup": 1.489619631357902, "end_to_end_speedup": 1.397232348671763, "prefill_speedup": 0.6587091919684884, "ttft_speedup": 0.8734133227225112 }, { "decode_speedup": 1.402794577442338, "end_to_end_speedup": 1.38336000199842, "prefill_speedup": 1.0794575371374096, "ttft_speedup": 1.2394837152934763 }, { "decode_speedup": 1.5011454036594185, "end_to_end_speedup": 1.4761128316471668, "prefill_speedup": 2.2332569677743446, "ttft_speedup": 1.3277518875157852 }, { "decode_speedup": 1.413954844113323, "end_to_end_speedup": 1.380479467559534, "prefill_speedup": 1.0313208863603598, "ttft_speedup": 1.1731901111582568 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T03:11:12.445075+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 113.23672845810192, "elapsed_seconds": 1.2507689580088481, "prefill_tokens_per_second": 3517.5263102698923, "time_to_first_token_seconds": 0.10765041701961309 }, { "decode_tokens_per_second": 117.16994302311102, "elapsed_seconds": 1.2168642079923302, "prefill_tokens_per_second": 3726.2153909199174, "time_to_first_token_seconds": 0.11385049996897578 }, { "decode_tokens_per_second": 107.68382257322206, "elapsed_seconds": 1.311814415967092, "prefill_tokens_per_second": 3691.429167821772, "time_to_first_token_seconds": 0.11154079099651426 }, { "decode_tokens_per_second": 115.77944130313642, "elapsed_seconds": 1.2213789999950677, "prefill_tokens_per_second": 3647.8365954230203, "time_to_first_token_seconds": 0.10304324992466718 }, { "decode_tokens_per_second": 116.41667209808139, "elapsed_seconds": 1.2375224590068683, "prefill_tokens_per_second": 1870.5599153467201, "time_to_first_token_seconds": 0.12457924999762326 }, { "decode_tokens_per_second": 117.40263193761942, "elapsed_seconds": 1.2134291250258684, "prefill_tokens_per_second": 3694.66129735065, "time_to_first_token_seconds": 0.1117369580315426 }, { "decode_tokens_per_second": 115.1891563106332, "elapsed_seconds": 1.237486333004199, "prefill_tokens_per_second": 3699.7828527232195, "time_to_first_token_seconds": 0.11144591704942286 } ], "MLX + meTile": [ { "decode_tokens_per_second": 175.74184728266488, "elapsed_seconds": 0.8435599999502301, "prefill_tokens_per_second": 4021.8584130308855, "time_to_first_token_seconds": 0.1050039998954162 }, { "decode_tokens_per_second": 175.3542762199362, "elapsed_seconds": 0.8615997080923989, "prefill_tokens_per_second": 2073.217252363328, "time_to_first_token_seconds": 0.12193566607311368 }, { "decode_tokens_per_second": 172.1152556576516, "elapsed_seconds": 0.8398184579564258, "prefill_tokens_per_second": 4108.149210645879, "time_to_first_token_seconds": 0.08623395790345967 }, { "decode_tokens_per_second": 174.53076809524268, "elapsed_seconds": 0.850463040987961, "prefill_tokens_per_second": 3825.0345483625088, "time_to_first_token_seconds": 0.10864716593641788 }, { "decode_tokens_per_second": 174.37889291632803, "elapsed_seconds": 0.8704824580345303, "prefill_tokens_per_second": 2457.167537051236, "time_to_first_token_seconds": 0.1279067499563098 }, { "decode_tokens_per_second": 173.61019864936446, "elapsed_seconds": 0.8493834160035476, "prefill_tokens_per_second": 4105.009213327863, "time_to_first_token_seconds": 0.10234908305574208 }, { "decode_tokens_per_second": 173.81355126352105, "elapsed_seconds": 0.8544277909677476, "prefill_tokens_per_second": 4142.026198402177, "time_to_first_token_seconds": 0.10872895794454962 } ] }, "schema_version": 15, "selected_dispatches": { "add_rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 896, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 896, "row_bucket": 128 } ], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [ { "algorithm": "mlx", "batch": 1, "block": 0, "dimension": 64, "dtype": "mlx.core.bfloat16", "key_value_heads": 2, "query_heads": 14, "token_bucket": 128 }, { "algorithm": "mlx", "batch": 1, "block": 0, "dimension": 64, "dtype": "mlx.core.bfloat16", "key_value_heads": 2, "query_heads": 14, "token_bucket": 256 } ], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [ { "algorithm": "metile", "block": 256, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 896, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 896, "row_bucket": 128 } ] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 198, "kl_divergence": 0.0009042357560247183, "max_logit_error": 0.21875, "mean_logit_error": 0.03293633088469505, "next_token": 198, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } }, { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_down": false, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_down": { "allow_approximate": false, "calibration_fidelity": { "actual_next_token": 98615, "kl_divergence": 0.0009546054061502218, "max_logit_error": 0.2421875, "mean_logit_error": 0.038803428411483765, "next_token": 98615, "reference_dtype": "mlx.core.bfloat16" }, "format": "affine8", "group_size": 64, "group_tuning": { "cached": false, "group_size": 64, "median_nanoseconds": { "128": 345031, "32": 358146, "64": 348156 } }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ], "projections": 16, "repack_bytes": 285212672, "selection": "all" }, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 98615, "kl_divergence": 0.0007754171965643764, "max_logit_error": 0.19921875, "mean_logit_error": 0.029853135347366333, "next_token": 98615, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": false, "group_size": 128, "median_nanoseconds": { "128": 340635, "32": 354302, "64": 350802 } }, "layer_indices": [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ], "layers": 14, "projections": 28, "repack_bytes": 484442112, "selection": "suffix:14" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 98615, "kl_divergence": 0.00015854770026635379, "max_logit_error": 0.09375, "mean_logit_error": 0.012944665737450123, "next_token": 98615, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": false, "group_size": 128, "median_nanoseconds": { "128": 2256438, "32": 2483177, "64": 2334010 } }, "projections": 1, "repack_bytes": 270876672, "tied": true }, "dense_mlp_implementation": null, "features": { "affine_prefill": false, "attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": false, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.373252165935847, "end_to_end_speedup": 1.346148647803066, "metile_decode_tokens_per_second": 69.40063762679853, "metile_elapsed_seconds": 1.9893563330406323, "metile_prefill_tokens_per_second": 2313.683374114228, "metile_time_to_first_token_seconds": 0.11980025004595518, "mlx_decode_tokens_per_second": 50.59015008922538, "mlx_elapsed_seconds": 2.6763057080097497, "mlx_prefill_tokens_per_second": 1914.3896605358843, "mlx_time_to_first_token_seconds": 0.13342766708228737, "prefill_speedup": 1.2009342371589768, "ttft_speedup": 1.0457250431027174 }, "memory": { "active_bytes": 3512440848, "cache_bytes": 10293990, "peak_bytes": 3871286032 }, "model": "mlx-community/Llama-3.2-1B-Instruct-bf16", "model_config": { "hidden_size": 2048, "model_type": "llama", "num_attention_heads": 32, "num_hidden_layers": 16, "num_key_value_heads": 8, "vocab_size": 128256 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.3777437871539056, "end_to_end_speedup": 1.3641985506502448, "prefill_speedup": 1.3891083988691764, "ttft_speedup": 1.140595130676748 }, "pairs": [ { "decode_speedup": 1.3831000343531865, "end_to_end_speedup": 1.3630316919764633, "prefill_speedup": 1.193474928216936, "ttft_speedup": 1.0953961418841411 }, { "decode_speedup": 1.3769254228549797, "end_to_end_speedup": 1.3733860572904801, "prefill_speedup": 1.825269289928973, "ttft_speedup": 1.3638966445873189 }, { "decode_speedup": 1.3757595228251178, "end_to_end_speedup": 1.3783405088923464, "prefill_speedup": 1.660443109839641, "ttft_speedup": 1.4724095237850738 }, { "decode_speedup": 1.3821917334041955, "end_to_end_speedup": 1.355149006630058, "prefill_speedup": 1.3891083988691764, "ttft_speedup": 0.997353240223563 }, { "decode_speedup": 1.3777437871539056, "end_to_end_speedup": 1.3698170741413886, "prefill_speedup": 1.703503040395663, "ttft_speedup": 1.2944337684431457 }, { "decode_speedup": 1.3814923881680836, "end_to_end_speedup": 1.3641985506502448, "prefill_speedup": 1.251915183462157, "ttft_speedup": 1.140595130676748 }, { "decode_speedup": 1.3587489931377934, "end_to_end_speedup": 1.2703163751402615, "prefill_speedup": 0.9637291635143407, "ttft_speedup": 0.6813243137777591 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T03:28:22.912583+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 50.7267624260782, "elapsed_seconds": 2.6694086670177057, "prefill_tokens_per_second": 1926.7133916726516, "time_to_first_token_seconds": 0.13342766708228737 }, { "decode_tokens_per_second": 50.46035050718972, "elapsed_seconds": 2.7028084159828722, "prefill_tokens_per_second": 1397.142215753444, "time_to_first_token_seconds": 0.15305395796895027 }, { "decode_tokens_per_second": 50.65521164647389, "elapsed_seconds": 2.664633332984522, "prefill_tokens_per_second": 1914.3896605358843, "time_to_first_token_seconds": 0.1251394160790369 }, { "decode_tokens_per_second": 49.94802122154764, "elapsed_seconds": 2.6802090000128374, "prefill_tokens_per_second": 1913.8600888936226, "time_to_first_token_seconds": 0.10482683300506324 }, { "decode_tokens_per_second": 50.59015008922538, "elapsed_seconds": 2.6763057080097497, "prefill_tokens_per_second": 1926.569584349312, "time_to_first_token_seconds": 0.13350666605401784 }, { "decode_tokens_per_second": 50.55521209083708, "elapsed_seconds": 2.6754984580911696, "prefill_tokens_per_second": 1928.2335575809293, "time_to_first_token_seconds": 0.13099641702137887 }, { "decode_tokens_per_second": 50.82363572935208, "elapsed_seconds": 2.7199119590222836, "prefill_tokens_per_second": 1048.066045127466, "time_to_first_token_seconds": 0.18873441708274186 } ], "MLX + meTile": [ { "decode_tokens_per_second": 69.66063637252503, "elapsed_seconds": 2.000647290959023, "prefill_tokens_per_second": 1533.1169692558806, "time_to_first_token_seconds": 0.1498310830211267 }, { "decode_tokens_per_second": 69.40063762679853, "elapsed_seconds": 2.0078082910040393, "prefill_tokens_per_second": 1574.6610844254112, "time_to_first_token_seconds": 0.14878854097332805 }, { "decode_tokens_per_second": 69.25975926602014, "elapsed_seconds": 1.9807394159724936, "prefill_tokens_per_second": 2315.0398428587364, "time_to_first_token_seconds": 0.11980025004595518 }, { "decode_tokens_per_second": 68.19481359125584, "elapsed_seconds": 1.9904965410241857, "prefill_tokens_per_second": 2332.3526322572825, "time_to_first_token_seconds": 0.10024320799857378 }, { "decode_tokens_per_second": 68.85738654333046, "elapsed_seconds": 1.9893563330406323, "prefill_tokens_per_second": 2313.683374114228, "time_to_first_token_seconds": 0.1175601250724867 }, { "decode_tokens_per_second": 69.7414456106633, "elapsed_seconds": 1.9586304579861462, "prefill_tokens_per_second": 2300.0777800385267, "time_to_first_token_seconds": 0.1099759159842506 }, { "decode_tokens_per_second": 69.84190741822623, "elapsed_seconds": 1.9659246249357238, "prefill_tokens_per_second": 2314.9160018841762, "time_to_first_token_seconds": 0.11980549991130829 } ] }, "schema_version": 15, "selected_dispatches": { "add_rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 2048, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 2048, "row_bucket": 128 } ], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [ { "algorithm": "mlx", "batch": 1, "block": 0, "dimension": 64, "dtype": "mlx.core.bfloat16", "key_value_heads": 8, "query_heads": 32, "token_bucket": 256 } ], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 2048, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 2048, "row_bucket": 128 } ] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_down": false, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 128000, "kl_divergence": 0.0006141751073300838, "max_logit_error": 0.1875, "mean_logit_error": 0.04136433079838753, "next_token": 128000, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } }, { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_down": { "allow_approximate": false, "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.0002719586482271552, "max_logit_error": 0.3125, "mean_logit_error": 0.048866260796785355, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "format": "affine8", "group_size": 128, "group_tuning": { "cached": false, "group_size": 128, "median_nanoseconds": { "128": 313365, "32": 329865, "64": 326000 } }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "projections": 28, "repack_bytes": 397393920, "selection": "all" }, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00043298560194671154, "max_logit_error": 0.3125, "mean_logit_error": 0.04793743044137955, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": false, "group_size": 128, "median_nanoseconds": { "128": 315344, "32": 326584, "64": 324021 } }, "layer_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "layers": 27, "projections": 54, "repack_bytes": 766402560, "selection": "suffix:27" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 8.163825441442896e-06, "max_logit_error": 0.1875, "mean_logit_error": 0.021962115541100502, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": false, "group_size": 128, "median_nanoseconds": { "128": 2050771, "32": 2248062, "64": 2121229 } }, "projections": 1, "repack_bytes": 240666624, "tied": true }, "dense_mlp_implementation": null, "features": { "affine_prefill": false, "attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": false, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.6836147747899823, "end_to_end_speedup": 1.6399845025365962, "metile_decode_tokens_per_second": 67.4926089478873, "metile_elapsed_seconds": 2.0551806250587106, "metile_prefill_tokens_per_second": 1953.9933770863515, "metile_time_to_first_token_seconds": 0.13040120794903487, "mlx_decode_tokens_per_second": 40.13549702329926, "mlx_elapsed_seconds": 3.3704643750097603, "mlx_prefill_tokens_per_second": 1455.742721337882, "mlx_time_to_first_token_seconds": 0.1641231660032645, "prefill_speedup": 1.3433109897029032, "ttft_speedup": 1.2192028014421314 }, "memory": { "active_bytes": 4494568464, "cache_bytes": 10870694, "peak_bytes": 4730334856 }, "model": "mlx-community/Qwen2.5-1.5B-Instruct-bf16", "model_config": { "hidden_size": 1536, "model_type": "qwen2", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 2, "vocab_size": 151936 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.6803223475106719, "end_to_end_speedup": 1.6221658670504568, "prefill_speedup": 1.337177827793721, "ttft_speedup": 1.1505738152030835 }, "pairs": [ { "decode_speedup": 1.6803223475106719, "end_to_end_speedup": 1.6012035321522566, "prefill_speedup": 0.7975829484551535, "ttft_speedup": 0.8758916593953096 }, { "decode_speedup": 1.6594284279023754, "end_to_end_speedup": 1.612304920719387, "prefill_speedup": 1.353214515256007, "ttft_speedup": 1.1505738152030835 }, { "decode_speedup": 1.690372649719705, "end_to_end_speedup": 1.650356041054562, "prefill_speedup": 1.3508037173605023, "ttft_speedup": 1.1883119730254663 }, { "decode_speedup": 1.7794923069553952, "end_to_end_speedup": 1.7238581508859243, "prefill_speedup": 1.08320233471107, "ttft_speedup": 1.2120320043362849 }, { "decode_speedup": 1.658092681095283, "end_to_end_speedup": 1.5740048880267763, "prefill_speedup": 0.7558731424455213, "ttft_speedup": 0.8411355866939247 }, { "decode_speedup": 1.6380955556783465, "end_to_end_speedup": 1.6221658670504568, "prefill_speedup": 2.174984309746919, "ttft_speedup": 1.5567638387743827 }, { "decode_speedup": 1.690290172301787, "end_to_end_speedup": 1.6439036516925, "prefill_speedup": 1.337177827793721, "ttft_speedup": 1.1275850932784601 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T03:25:52.351506+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 40.13549702329926, "elapsed_seconds": 3.3654257090529427, "prefill_tokens_per_second": 1456.6593978687351, "time_to_first_token_seconds": 0.1632409170269966 }, { "decode_tokens_per_second": 40.135613950631324, "elapsed_seconds": 3.3650104580447078, "prefill_tokens_per_second": 1452.7268053415542, "time_to_first_token_seconds": 0.16314900002907962 }, { "decode_tokens_per_second": 40.25826991114089, "elapsed_seconds": 3.4146415839204565, "prefill_tokens_per_second": 887.4306373558549, "time_to_first_token_seconds": 0.21779466699808836 }, { "decode_tokens_per_second": 40.087916760119, "elapsed_seconds": 3.3704643750097603, "prefill_tokens_per_second": 1461.0565686152295, "time_to_first_token_seconds": 0.1641231660032645 }, { "decode_tokens_per_second": 39.4348457682262, "elapsed_seconds": 3.428626541979611, "prefill_tokens_per_second": 1455.9365846864657, "time_to_first_token_seconds": 0.16464741702657193 }, { "decode_tokens_per_second": 39.89088336154197, "elapsed_seconds": 3.3880550420144573, "prefill_tokens_per_second": 1411.3696984163594, "time_to_first_token_seconds": 0.16562745801638812 }, { "decode_tokens_per_second": 40.15572893644715, "elapsed_seconds": 3.349748875014484, "prefill_tokens_per_second": 1455.742721337882, "time_to_first_token_seconds": 0.14944591699168086 } ], "MLX + meTile": [ { "decode_tokens_per_second": 66.99208564028504, "elapsed_seconds": 2.0625005409820005, "prefill_tokens_per_second": 1983.5389508006776, "time_to_first_token_seconds": 0.13040120794903487 }, { "decode_tokens_per_second": 67.29714657163186, "elapsed_seconds": 2.1186745839659125, "prefill_tokens_per_second": 1043.6069358642046, "time_to_first_token_seconds": 0.1978759589837864 }, { "decode_tokens_per_second": 67.91299693031466, "elapsed_seconds": 2.0276468329830095, "prefill_tokens_per_second": 1953.9933770863515, "time_to_first_token_seconds": 0.1275225409772247 }, { "decode_tokens_per_second": 67.4926089478873, "elapsed_seconds": 2.0551806250587106, "prefill_tokens_per_second": 1962.6533451984515, "time_to_first_token_seconds": 0.1404756250558421 }, { "decode_tokens_per_second": 67.04782218373599, "elapsed_seconds": 2.0500461669871584, "prefill_tokens_per_second": 1906.8581723806983, "time_to_first_token_seconds": 0.11738679208792746 }, { "decode_tokens_per_second": 67.5131494150717, "elapsed_seconds": 2.1101315829437226, "prefill_tokens_per_second": 1025.7543767121317, "time_to_first_token_seconds": 0.19834579096641392 }, { "decode_tokens_per_second": 67.56198764099327, "elapsed_seconds": 2.037762082996778, "prefill_tokens_per_second": 1972.5879620837331, "time_to_first_token_seconds": 0.12257675000000745 } ] }, "schema_version": 15, "selected_dispatches": { "add_rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 1536, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 1536, "row_bucket": 128 } ], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [ { "algorithm": "mlx", "batch": 1, "block": 0, "dimension": 128, "dtype": "mlx.core.bfloat16", "key_value_heads": 2, "query_heads": 12, "token_bucket": 256 } ], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 1536, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 1536, "row_bucket": 128 } ] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 11, "kl_divergence": 0.0007453290745615959, "max_logit_error": 0.1953125, "mean_logit_error": 0.037820808589458466, "next_token": 11, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } }, { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_attention": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00031251367181539536, "max_logit_error": 0.25, "mean_logit_error": 0.04357561096549034, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 32, "group_tuning": { "cached": true, "group_size": 32, "mean_absolute_error": { "128": 0.08762161433696747, "32": 0.03383176866918802, "64": 0.053859811276197433 }, "median_nanoseconds": { "128": 224823, "32": 217432, "64": 216953 }, "objective": "balanced" }, "layer_indices": [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35 ], "layers": 34, "projections": 136, "repack_bytes": 360972288, "selection": "suffix:34" }, "compressed_down": { "allow_approximate": false, "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.0003888432402163744, "max_logit_error": 0.25, "mean_logit_error": 0.047711484134197235, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "format": "affine8", "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.12181396782398224, "32": 0.04617849737405777, "64": 0.07283391058444977 }, "median_nanoseconds": { "128": 392010, "32": 411031, "64": 408208 }, "objective": "throughput" }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33 ], "projections": 34, "repack_bytes": 790462464, "selection": "prefix:34" }, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00035893521271646023, "max_logit_error": 0.28125, "mean_logit_error": 0.04805271700024605, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.056058041751384735, "32": 0.020828066393733025, "64": 0.034691959619522095 }, "median_nanoseconds": { "128": 390792, "32": 420844, "64": 397812 }, "objective": "throughput" }, "layer_indices": [ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35 ], "layers": 33, "projections": 66, "repack_bytes": 1534427136, "selection": "suffix:33" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 9.516619320493191e-05, "max_logit_error": 0.125, "mean_logit_error": 0.022052185609936714, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.04765083268284798, "32": 0.018215592950582504, "64": 0.029296765103936195 }, "median_nanoseconds": { "128": 2802833, "32": 3055667, "64": 2872469 }, "objective": "throughput" }, "projections": 1, "repack_bytes": 320888832, "tied": true }, "dense_mlp_implementation": null, "features": { "affine_prefill": true, "attention": true, "compressed_attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": true, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.748080788365956, "end_to_end_speedup": 1.689993906626431, "metile_decode_tokens_per_second": 34.45859490260217, "metile_elapsed_seconds": 4.007684667012654, "metile_prefill_tokens_per_second": 608.6093249003184, "metile_time_to_first_token_seconds": 0.29063462500926107, "mlx_decode_tokens_per_second": 19.791975259583623, "mlx_elapsed_seconds": 6.772962666931562, "mlx_prefill_tokens_per_second": 730.6233202688168, "mlx_time_to_first_token_seconds": 0.26153387501835823, "prefill_speedup": 0.9932041284816389, "ttft_speedup": 1.0948498461450402 }, "memory": { "active_bytes": 9180586000, "cache_bytes": 12771262, "peak_bytes": 9387631120 }, "model": "mlx-community/Qwen2.5-3B-Instruct-bf16", "model_config": { "hidden_size": 2048, "model_type": "qwen2", "num_attention_heads": 16, "num_hidden_layers": 36, "num_key_value_heads": 2, "vocab_size": 151936 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.730338441669643, "end_to_end_speedup": 1.6826763599513823, "prefill_speedup": 1.3558964789485228, "ttft_speedup": 1.1500114625524664 }, "pairs": [ { "decode_speedup": 1.8111936959166246, "end_to_end_speedup": 1.7862408542393156, "prefill_speedup": 1.3407039246135188, "ttft_speedup": 1.3818567340519838 }, { "decode_speedup": 1.7116835698493411, "end_to_end_speedup": 1.6843850607914945, "prefill_speedup": 1.3558964789485228, "ttft_speedup": 1.2930742712725034 }, { "decode_speedup": 1.7302779680477567, "end_to_end_speedup": 1.6598090700977657, "prefill_speedup": 1.3568022116727676, "ttft_speedup": 0.8532352368731679 }, { "decode_speedup": 1.730338441669643, "end_to_end_speedup": 1.6826763599513823, "prefill_speedup": 1.4853036122581202, "ttft_speedup": 1.1500114625524664 }, { "decode_speedup": 1.7430842512735776, "end_to_end_speedup": 1.712651484284716, "prefill_speedup": 1.3687580245575295, "ttft_speedup": 1.2439100813067554 }, { "decode_speedup": 1.605731295021281, "end_to_end_speedup": 1.5169003838176078, "prefill_speedup": 0.658761629277089, "ttft_speedup": 0.6272980746120975 }, { "decode_speedup": 1.742334415012509, "end_to_end_speedup": 1.6615934231647036, "prefill_speedup": 0.7362517377408957, "ttft_speedup": 0.8043088201544283 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T04:59:28.151230+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 19.84297322660732, "elapsed_seconds": 6.809501540963538, "prefill_tokens_per_second": 444.1550250395938, "time_to_first_token_seconds": 0.3438126660184935 }, { "decode_tokens_per_second": 19.852598719516376, "elapsed_seconds": 6.724710167036392, "prefill_tokens_per_second": 730.6233202688168, "time_to_first_token_seconds": 0.26153387501835823 }, { "decode_tokens_per_second": 19.88929960181012, "elapsed_seconds": 6.797417083987966, "prefill_tokens_per_second": 494.0248238589743, "time_to_first_token_seconds": 0.3440082089509815 }, { "decode_tokens_per_second": 19.65177615796776, "elapsed_seconds": 6.896784459007904, "prefill_tokens_per_second": 417.21891818791386, "time_to_first_token_seconds": 0.36823666701093316 }, { "decode_tokens_per_second": 19.791975259583623, "elapsed_seconds": 6.735102916951291, "prefill_tokens_per_second": 739.8886696185031, "time_to_first_token_seconds": 0.2509197499603033 }, { "decode_tokens_per_second": 19.73869045053857, "elapsed_seconds": 6.750560958054848, "prefill_tokens_per_second": 742.4642417106854, "time_to_first_token_seconds": 0.24856187496334314 }, { "decode_tokens_per_second": 19.6678035001235, "elapsed_seconds": 6.772962666931562, "prefill_tokens_per_second": 733.7549266657712, "time_to_first_token_seconds": 0.24799850000999868 } ], "MLX + meTile": [ { "decode_tokens_per_second": 34.078784939511245, "elapsed_seconds": 3.9862345410510898, "prefill_tokens_per_second": 1003.8605500864638, "time_to_first_token_seconds": 0.20616058306768537 }, { "decode_tokens_per_second": 34.70394642072515, "elapsed_seconds": 4.05579629202839, "prefill_tokens_per_second": 505.20349747005423, "time_to_first_token_seconds": 0.33768187498208135 }, { "decode_tokens_per_second": 31.855356112644753, "elapsed_seconds": 4.358205708907917, "prefill_tokens_per_second": 490.6674946291477, "time_to_first_token_seconds": 0.3142058339435607 }, { "decode_tokens_per_second": 34.36122576989642, "elapsed_seconds": 3.953860832960345, "prefill_tokens_per_second": 983.2300366322241, "time_to_first_token_seconds": 0.20234341698233038 }, { "decode_tokens_per_second": 34.45859490260217, "elapsed_seconds": 3.9435429580044, "prefill_tokens_per_second": 998.2585695304268, "time_to_first_token_seconds": 0.2064773750025779 }, { "decode_tokens_per_second": 34.67127238152593, "elapsed_seconds": 4.0727591660106555, "prefill_tokens_per_second": 463.8711978572326, "time_to_first_token_seconds": 0.36148037505336106 }, { "decode_tokens_per_second": 34.618137267525796, "elapsed_seconds": 4.007684667012654, "prefill_tokens_per_second": 608.6093249003184, "time_to_first_token_seconds": 0.29063462500926107 } ] }, "schema_version": 16, "selected_dispatches": { "add_rms_norm": [], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 8, "kl_divergence": 0.0005453577032312751, "max_logit_error": 0.140625, "mean_logit_error": 0.02448210120201111, "next_token": 8, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } }, { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_attention": true, "compressed_down": false, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_attention": { "calibration_fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0009432577062398195, "max_logit_error": 0.21875, "mean_logit_error": 0.03235931694507599, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 64, "group_tuning": { "cached": true, "group_size": 64, "mean_absolute_error": { "128": 0.09903422743082047, "32": 0.03818049468100071, "64": 0.06190771982073784 }, "median_nanoseconds": { "128": 252531, "32": 263833, "64": 259188 }, "micro_group_size": 128, "model_calibrated": true, "model_candidates": { "128": { "fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0008800143841654062, "max_logit_error": 0.171875, "mean_logit_error": 0.026725323870778084, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "layers": 20, "selection": "suffix:20" }, "32": { "fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0007797088474035263, "max_logit_error": 0.1875, "mean_logit_error": 0.04653530195355415, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "layers": 21, "selection": "suffix:21" }, "64": { "fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0009432577062398195, "max_logit_error": 0.21875, "mean_logit_error": 0.03235931694507599, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "layers": 26, "selection": "suffix:26" } }, "native_median_nanoseconds": 308672, "objective": "balanced", "predicted_nanoseconds": { "128": 7519996, "32": 7701197, "64": 7356232 } }, "layer_indices": [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "layers": 26, "projections": 104, "repack_bytes": 695205888, "selection": "suffix:26" }, "compressed_down": { "allow_approximate": false, "calibration_fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0006371359340846539, "max_logit_error": 0.1640625, "mean_logit_error": 0.022092485800385475, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "format": "affine8", "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.06563148647546768, "32": 0.029370075091719627, "64": 0.04589439556002617 }, "median_nanoseconds": { "128": 405958, "32": 429531, "64": 417479 }, "native_median_nanoseconds": 614781, "objective": "throughput" }, "layer_indices": [ 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "projections": 14, "repack_bytes": 363331584, "selection": "suffix:14" }, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 93674, "kl_divergence": 0.000821854337118566, "max_logit_error": 0.2265625, "mean_logit_error": 0.03303111344575882, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 32, "group_tuning": { "cached": true, "group_size": 32, "mean_absolute_error": { "128": 0.03958287835121155, "32": 0.019027817994356155, "64": 0.02783217653632164 }, "median_nanoseconds": { "128": 412771, "32": 433219, "64": 420208 }, "micro_group_size": 128, "model_calibrated": true, "model_candidates": { "128": { "fidelity": { "actual_next_token": 93674, "kl_divergence": 0.00013981404481455684, "max_logit_error": 0.0625, "mean_logit_error": 0.008413385599851608, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "layers": 1, "selection": "suffix:1" }, "32": { "fidelity": { "actual_next_token": 93674, "kl_divergence": 0.000821854337118566, "max_logit_error": 0.2265625, "mean_logit_error": 0.03303111344575882, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "layers": 26, "selection": "suffix:26" }, "64": { "fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0009805983863770962, "max_logit_error": 0.173828125, "mean_logit_error": 0.027849840000271797, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "layers": 23, "selection": "suffix:23" } }, "native_median_nanoseconds": 616188, "objective": "throughput", "predicted_nanoseconds": { "128": 17049847, "32": 12496070, "64": 12745724 } }, "layer_indices": [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "layers": 26, "projections": 52, "repack_bytes": 1472200704, "selection": "suffix:26" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 93674, "kl_divergence": 0.0002570429351180792, "max_logit_error": 0.06640625, "mean_logit_error": 0.01097534317523241, "next_token": 93674, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.0515950545668602, "32": 0.020427098497748375, "64": 0.031868159770965576 }, "median_nanoseconds": { "128": 3438198, "32": 3770146, "64": 3532833 }, "native_median_nanoseconds": 6497864, "objective": "throughput" }, "projections": 1, "repack_bytes": 406315008, "tied": true }, "dense_mlp_implementation": null, "features": { "affine_prefill": false, "attention": true, "compressed_attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": false, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.4911812070159896, "end_to_end_speedup": 1.4787552322386737, "metile_decode_tokens_per_second": 28.62849438213398, "metile_elapsed_seconds": 4.836764832958579, "metile_prefill_tokens_per_second": 809.6252657505161, "metile_time_to_first_token_seconds": 0.23882916697766632, "mlx_decode_tokens_per_second": 18.586020231040113, "mlx_elapsed_seconds": 7.155081457924098, "mlx_prefill_tokens_per_second": 685.1394133517647, "mlx_time_to_first_token_seconds": 0.2539288749685511, "prefill_speedup": 1.2042109795578817, "ttft_speedup": 1.1786419214981951 }, "memory": { "active_bytes": 9362830352, "cache_bytes": 31232814, "peak_bytes": 9738477392 }, "model": "mlx-community/Llama-3.2-3B-Instruct-bf16", "model_config": { "hidden_size": 3072, "model_type": "llama", "num_attention_heads": 24, "num_hidden_layers": 28, "num_key_value_heads": 8, "vocab_size": 128256 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.4796593917686705, "end_to_end_speedup": 1.4600249385500135, "prefill_speedup": 1.2179661161880793, "ttft_speedup": 1.1568282778912609 }, "pairs": [ { "decode_speedup": 1.4818177366789635, "end_to_end_speedup": 1.4657576875479355, "prefill_speedup": 1.2179661161880793, "ttft_speedup": 1.1568282778912609 }, { "decode_speedup": 1.4796593917686705, "end_to_end_speedup": 1.4600249385500135, "prefill_speedup": 1.1876202150673139, "ttft_speedup": 1.0862282500020597 }, { "decode_speedup": 1.5421109680204834, "end_to_end_speedup": 1.5534690343445137, "prefill_speedup": 2.1993821879513322, "ttft_speedup": 1.8314433412841895 }, { "decode_speedup": 1.4810796797543533, "end_to_end_speedup": 1.4305252907329593, "prefill_speedup": 0.7158038415429898, "ttft_speedup": 0.7325097426402275 }, { "decode_speedup": 1.456436839274532, "end_to_end_speedup": 1.418938662829616, "prefill_speedup": 1.0905668271499536, "ttft_speedup": 1.0056026387656447 }, { "decode_speedup": 1.4745922359593666, "end_to_end_speedup": 1.460035062594113, "prefill_speedup": 1.262318870484579, "ttft_speedup": 1.1865829114329076 }, { "decode_speedup": 1.3558312375041244, "end_to_end_speedup": 1.3617935010578215, "prefill_speedup": 1.6663752895823842, "ttft_speedup": 1.5024314306419666 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T06:43:17.077007+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 18.498655726158297, "elapsed_seconds": 7.200084750074893, "prefill_tokens_per_second": 667.5064108674538, "time_to_first_token_seconds": 0.2641802500002086 }, { "decode_tokens_per_second": 18.37427797764012, "elapsed_seconds": 7.235886957962066, "prefill_tokens_per_second": 685.1394133517647, "time_to_first_token_seconds": 0.2523813749430701 }, { "decode_tokens_per_second": 18.22154613481161, "elapsed_seconds": 7.448662333074026, "prefill_tokens_per_second": 394.38540006786945, "time_to_first_token_seconds": 0.39613375009503216 }, { "decode_tokens_per_second": 18.586020231040113, "elapsed_seconds": 7.155081457924098, "prefill_tokens_per_second": 702.3937875085975, "time_to_first_token_seconds": 0.24763891589827836 }, { "decode_tokens_per_second": 19.249374161630005, "elapsed_seconds": 7.0487396670505404, "prefill_tokens_per_second": 417.4442790281558, "time_to_first_token_seconds": 0.3832257919711992 }, { "decode_tokens_per_second": 19.3347344594593, "elapsed_seconds": 6.881790791056119, "prefill_tokens_per_second": 710.298541473735, "time_to_first_token_seconds": 0.24582366598770022 }, { "decode_tokens_per_second": 19.297695790951273, "elapsed_seconds": 6.902908292016946, "prefill_tokens_per_second": 712.1442147162246, "time_to_first_token_seconds": 0.2539288749685511 } ], "MLX + meTile": [ { "decode_tokens_per_second": 27.009503071582415, "elapsed_seconds": 4.966932957991958, "prefill_tokens_per_second": 817.0527852146134, "time_to_first_token_seconds": 0.21109491598326713 }, { "decode_tokens_per_second": 27.121081327537986, "elapsed_seconds": 5.107245708932169, "prefill_tokens_per_second": 422.94469425983164, "time_to_first_token_seconds": 0.3709350839490071 }, { "decode_tokens_per_second": 27.935088712767076, "elapsed_seconds": 4.836764832958579, "prefill_tokens_per_second": 809.6252657505161, "time_to_first_token_seconds": 0.23882916697766632 }, { "decode_tokens_per_second": 28.62849438213398, "elapsed_seconds": 4.809419249999337, "prefill_tokens_per_second": 507.39999686582865, "time_to_first_token_seconds": 0.3229411250213161 }, { "decode_tokens_per_second": 28.694876858633318, "elapsed_seconds": 4.691915249917656, "prefill_tokens_per_second": 865.0463023909742, "time_to_first_token_seconds": 0.2157735408982262 }, { "decode_tokens_per_second": 28.83159266859016, "elapsed_seconds": 4.846342000062577, "prefill_tokens_per_second": 451.78483746652506, "time_to_first_token_seconds": 0.3912739170482382 }, { "decode_tokens_per_second": 28.85192445026611, "elapsed_seconds": 4.668053333996795, "prefill_tokens_per_second": 857.5718823899033, "time_to_first_token_seconds": 0.21544191695284098 } ] }, "schema_version": 16, "selected_dispatches": { "add_rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 3072, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 3072, "row_bucket": 128 } ], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [ { "algorithm": "mlx", "batch": 1, "block": 0, "dimension": 128, "dtype": "mlx.core.bfloat16", "key_value_heads": 8, "query_heads": 24, "token_bucket": 256 } ], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 3072, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-05, "hidden": 3072, "row_bucket": 128 } ] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_attention": true, "compressed_down": false, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 271, "kl_divergence": 0.00014810037100687623, "max_logit_error": 0.078125, "mean_logit_error": 0.014282147400081158, "next_token": 271, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } }, { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_attention": false, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": false, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_attention": { "calibration_fidelity": { "actual_next_token": 134776, "kl_divergence": 0.00042994937393814325, "max_logit_error": 0.1875, "mean_logit_error": 0.028089888393878937, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 64, "group_tuning": { "cached": true, "group_size": 64, "mean_absolute_error": { "128": 0.05840393155813217, "32": 0.023561671376228333, "64": 0.03737928345799446 }, "median_nanoseconds": { "128": 268677, "32": 266432, "64": 260365 }, "micro_group_size": 32, "model_calibrated": true, "model_candidates": { "128": { "fidelity": { "actual_next_token": 134776, "kl_divergence": 0.00034887040965259075, "max_logit_error": 0.21875, "mean_logit_error": 0.03179774060845375, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "layers": 26, "selection": "suffix:26" }, "32": { "fidelity": { "actual_next_token": 134776, "kl_divergence": 0.00038791727274656296, "max_logit_error": 0.203125, "mean_logit_error": 0.031097128987312317, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "layers": 28, "selection": "all" }, "64": { "fidelity": { "actual_next_token": 134776, "kl_divergence": 0.00042994937393814325, "max_logit_error": 0.1875, "mean_logit_error": 0.028089888393878937, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "layers": 28, "selection": "all" } }, "native_median_nanoseconds": 327380, "objective": "balanced", "predicted_nanoseconds": { "128": 7640362, "32": 7460096, "64": 7290220 } }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 ], "layers": 28, "projections": 112, "repack_bytes": 873463808, "selection": "all" }, "compressed_down": { "allow_approximate": false, "calibration_fidelity": { "actual_next_token": 134776, "kl_divergence": 0.00023831427097320557, "max_logit_error": 0.203125, "mean_logit_error": 0.03134966269135475, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "format": "affine8", "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.09709563106298447, "32": 0.03775295615196228, "64": 0.06350421160459518 }, "median_nanoseconds": { "128": 782386, "32": 837198, "64": 800917 }, "native_median_nanoseconds": 1307521, "objective": "throughput" }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 ], "projections": 26, "repack_bytes": 1820442624, "selection": "prefix:26" }, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 134776, "kl_divergence": 0.000369828543625772, "max_logit_error": 0.1875, "mean_logit_error": 0.03128433972597122, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.0547160841524601, "32": 0.01950751431286335, "64": 0.03323354199528694 }, "median_nanoseconds": { "128": 762427, "32": 810521, "64": 790448 }, "micro_group_size": 128, "model_calibrated": true, "model_candidates": { "128": { "fidelity": { "actual_next_token": 134776, "kl_divergence": 0.000369828543625772, "max_logit_error": 0.1875, "mean_logit_error": 0.03128433972597122, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "layers": 26, "selection": "prefix:26" }, "32": { "fidelity": { "actual_next_token": 134776, "kl_divergence": 0.00035901949740946293, "max_logit_error": 0.19140625, "mean_logit_error": 0.027555027976632118, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "layers": 28, "selection": "all" }, "64": { "fidelity": { "actual_next_token": 134776, "kl_divergence": 0.000455568078905344, "max_logit_error": 0.2109375, "mean_logit_error": 0.037082452327013016, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "layers": 26, "selection": "suffix:26" } }, "native_median_nanoseconds": 1267406, "objective": "throughput", "predicted_nanoseconds": { "128": 22357914, "32": 22694588, "64": 23086460 } }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 ], "layers": 26, "projections": 52, "repack_bytes": 3640885248, "selection": "prefix:26" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 134776, "kl_divergence": 0.0003565381048247218, "max_logit_error": 0.203125, "mean_logit_error": 0.026369886472821236, "next_token": 134776, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.036514174193143845, "32": 0.013186595402657986, "64": 0.021706661209464073 }, "median_nanoseconds": { "128": 4691979, "32": 5162636, "64": 4829125 }, "native_median_nanoseconds": 8834708, "objective": "throughput" }, "projections": 1, "repack_bytes": 562028544, "tied": false }, "dense_mlp_implementation": null, "features": { "affine_prefill": false, "attention": true, "compressed_attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": false, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.5537568636040562, "end_to_end_speedup": 1.5091851646337313, "metile_decode_tokens_per_second": 13.954030218891587, "metile_elapsed_seconds": 9.778857417055406, "metile_prefill_tokens_per_second": 421.8449201859129, "metile_time_to_first_token_seconds": 0.3912674579769373, "mlx_decode_tokens_per_second": 8.97279417264423, "mlx_elapsed_seconds": 14.76707070809789, "mlx_prefill_tokens_per_second": 335.5414406391085, "mlx_time_to_first_token_seconds": 0.4670498750638217, "prefill_speedup": 1.2553965911331373, "ttft_speedup": 1.1830660600841234 }, "memory": { "active_bytes": 22130846736, "cache_bytes": 16936702, "peak_bytes": 22600232464 }, "model": "mlx-community/Qwen2.5-7B-Instruct-bf16", "model_config": { "hidden_size": 3584, "model_type": "qwen2", "num_attention_heads": 28, "num_hidden_layers": 28, "num_key_value_heads": 4, "vocab_size": 152064 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.5492577593109198, "end_to_end_speedup": 1.5326296698083979, "prefill_speedup": 1.2458849950508282, "ttft_speedup": 1.1754070614565473 }, "pairs": [ { "decode_speedup": 1.5570612242367354, "end_to_end_speedup": 1.491568724292829, "prefill_speedup": 1.1914139982167629, "ttft_speedup": 0.6590603223935698 }, { "decode_speedup": 1.5904817088869965, "end_to_end_speedup": 1.5958276121449513, "prefill_speedup": 1.266277315112051, "ttft_speedup": 1.7649274886992703 }, { "decode_speedup": 1.5180947652432841, "end_to_end_speedup": 1.476619670030612, "prefill_speedup": 1.206336777276984, "ttft_speedup": 0.8194711681378483 }, { "decode_speedup": 1.5622257140744271, "end_to_end_speedup": 1.5461659855573775, "prefill_speedup": 1.2415472675961754, "ttft_speedup": 1.1971491364833782 }, { "decode_speedup": 1.5149469868728949, "end_to_end_speedup": 1.4754149473184726, "prefill_speedup": 1.2458849950508282, "ttft_speedup": 0.8312051898918337 }, { "decode_speedup": 1.5371544363916239, "end_to_end_speedup": 1.5460563592731056, "prefill_speedup": 1.2997465972430198, "ttft_speedup": 1.8063753971873289 }, { "decode_speedup": 1.5492577593109198, "end_to_end_speedup": 1.5326296698083979, "prefill_speedup": 1.2574668035732834, "ttft_speedup": 1.1754070614565473 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T09:19:35.623632+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 8.694919573750795, "elapsed_seconds": 15.239660416962579, "prefill_tokens_per_second": 333.75741044745314, "time_to_first_token_seconds": 0.4944819170050323 }, { "decode_tokens_per_second": 9.000098359287136, "elapsed_seconds": 14.708855374949053, "prefill_tokens_per_second": 342.00541133373576, "time_to_first_token_seconds": 0.46289524994790554 }, { "decode_tokens_per_second": 8.815912899191355, "elapsed_seconds": 15.031272624968551, "prefill_tokens_per_second": 334.9455492245664, "time_to_first_token_seconds": 0.4769364999374375 }, { "decode_tokens_per_second": 8.983623039285604, "elapsed_seconds": 14.758106540888548, "prefill_tokens_per_second": 335.5414406391085, "time_to_first_token_seconds": 0.4789911248954013 }, { "decode_tokens_per_second": 8.989191252459111, "elapsed_seconds": 14.734727500006557, "prefill_tokens_per_second": 339.5924313289316, "time_to_first_token_seconds": 0.4632256249897182 }, { "decode_tokens_per_second": 8.907216078515386, "elapsed_seconds": 14.854576458921656, "prefill_tokens_per_second": 337.5429145016441, "time_to_first_token_seconds": 0.4593356669647619 }, { "decode_tokens_per_second": 8.97279417264423, "elapsed_seconds": 14.76707070809789, "prefill_tokens_per_second": 332.6347500961745, "time_to_first_token_seconds": 0.4670498750638217 } ], "MLX + meTile": [ { "decode_tokens_per_second": 13.910604522840389, "elapsed_seconds": 9.601710250019096, "prefill_tokens_per_second": 428.20630531053774, "time_to_first_token_seconds": 0.3755007090512663 }, { "decode_tokens_per_second": 13.983964598853992, "elapsed_seconds": 9.569264790974557, "prefill_tokens_per_second": 423.6595090014076, "time_to_first_token_seconds": 0.3912674579769373 }, { "decode_tokens_per_second": 13.954933306002488, "elapsed_seconds": 10.142859749961644, "prefill_tokens_per_second": 420.48950071173704, "time_to_first_token_seconds": 0.9398624579189345 }, { "decode_tokens_per_second": 13.954030218891587, "elapsed_seconds": 9.778857417055406, "prefill_tokens_per_second": 426.2029934098546, "time_to_first_token_seconds": 0.5822069579735398 }, { "decode_tokens_per_second": 13.286005112173676, "elapsed_seconds": 10.038093083072454, "prefill_tokens_per_second": 416.94478866206634, "time_to_first_token_seconds": 0.37783404206857085 }, { "decode_tokens_per_second": 13.992705827828976, "elapsed_seconds": 9.629370750044473, "prefill_tokens_per_second": 416.8111096900932, "time_to_first_token_seconds": 0.39386883296538144 }, { "decode_tokens_per_second": 13.590684567066946, "elapsed_seconds": 9.843015332939103, "prefill_tokens_per_second": 421.8449201859129, "time_to_first_token_seconds": 0.38904299994464964 } ] }, "schema_version": 16, "selected_dispatches": { "add_rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 3584, "row_bucket": 1 }, { "algorithm": "metile", "block": 512, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 3584, "row_bucket": 128 } ], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [ { "algorithm": "mlx", "batch": 1, "block": 0, "dimension": 128, "dtype": "mlx.core.bfloat16", "key_value_heads": 4, "query_heads": 28, "token_bucket": 256 } ], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [ { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 3584, "row_bucket": 1 }, { "algorithm": "mlx", "block": 0, "dtype": "mlx.core.bfloat16", "eps": 1e-06, "hidden": 3584, "row_bucket": 128 } ] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_attention": false, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": false, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 220, "kl_divergence": 0.0007977870991453528, "max_logit_error": 0.21875, "mean_logit_error": 0.03492638096213341, "next_token": 220, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } }, { "candidate_plan": { "affine_prefill": false, "attention": false, "compressed_attention": true, "compressed_down": false, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "comparison_mode": "alternating", "compressed_attention": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00011329457629472017, "max_logit_error": 0.203125, "mean_logit_error": 0.034100089222192764, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.07698841392993927, "32": 0.02966196369379759, "64": 0.04873207397758961 }, "median_nanoseconds": { "128": 281865, "32": 297912, "64": 291802 }, "micro_group_size": 128, "model_calibrated": true, "model_candidates": { "128": { "fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00011329457629472017, "max_logit_error": 0.203125, "mean_logit_error": 0.034100089222192764, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "layers": 36, "selection": "all" }, "32": { "fidelity": { "actual_next_token": 134663, "kl_divergence": 0.0002918398240581155, "max_logit_error": 0.25, "mean_logit_error": 0.03791539743542671, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "layers": 36, "selection": "all" }, "64": { "fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00015095301205292344, "max_logit_error": 0.1875, "mean_logit_error": 0.03061234951019287, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "layers": 36, "selection": "all" } }, "native_median_nanoseconds": 360781, "objective": "balanced", "predicted_nanoseconds": { "128": 10147140, "32": 10724832, "64": 10504872 } }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35 ], "layers": 36, "projections": 144, "repack_bytes": 1557135360, "selection": "all" }, "compressed_down": null, "compressed_gate_up": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00030978675931692123, "max_logit_error": 0.203125, "mean_logit_error": 0.03332380950450897, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.0741543397307396, "32": 0.0276801697909832, "64": 0.04490674287080765 }, "median_nanoseconds": { "128": 639948, "32": 708031, "64": 672823 }, "micro_group_size": 128, "model_calibrated": true, "model_candidates": { "128": { "fidelity": { "actual_next_token": 134663, "kl_divergence": 0.00030978675931692123, "max_logit_error": 0.203125, "mean_logit_error": 0.03332380950450897, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "layers": 36, "selection": "all" }, "32": { "fidelity": { "actual_next_token": 134663, "kl_divergence": 9.270112786907703e-05, "max_logit_error": 0.21875, "mean_logit_error": 0.047910142689943314, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "layers": 36, "selection": "all" }, "64": { "fidelity": { "actual_next_token": 134663, "kl_divergence": 0.0006070660892874002, "max_logit_error": 0.1875, "mean_logit_error": 0.031088201329112053, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "layers": 36, "selection": "all" } }, "native_median_nanoseconds": 1089229, "objective": "throughput", "predicted_nanoseconds": { "128": 23038128, "32": 25489116, "64": 24221628 } }, "layer_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35 ], "layers": 36, "projections": 72, "repack_bytes": 3737124864, "selection": "all" }, "compressed_vocab": { "calibration_fidelity": { "actual_next_token": 134663, "kl_divergence": 3.870832733809948e-05, "max_logit_error": 0.125, "mean_logit_error": 0.01646265760064125, "next_token": 134663, "reference_dtype": "mlx.core.bfloat16" }, "group_size": 128, "group_tuning": { "cached": true, "group_size": 128, "mean_absolute_error": { "128": 0.0733746588230133, "32": 0.028101591393351555, "64": 0.04607035964727402 }, "median_nanoseconds": { "128": 5330938, "32": 5836396, "64": 5501833 }, "native_median_nanoseconds": 10317844, "objective": "throughput" }, "projections": 1, "repack_bytes": 641777664, "tied": false }, "dense_mlp_implementation": null, "features": { "affine_prefill": false, "attention": true, "compressed_attention": true, "compressed_down": true, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": true, "model_autotune": true, "quantized_mlp": false, "rms_norm": true }, "hardware": { "chip": "Apple M5", "machine": "arm64", "memory": "32 GB", "model_name": "Mac17,2", "processor": "arm", "release": "25.2.0", "system": "Darwin" }, "medians": { "decode_speedup": 1.6007557506909442, "end_to_end_speedup": 1.6041217643179775, "metile_decode_tokens_per_second": 12.57256358216595, "metile_elapsed_seconds": 10.832087291055359, "metile_prefill_tokens_per_second": 220.6354948776881, "metile_time_to_first_token_seconds": 0.6754547499585897, "mlx_decode_tokens_per_second": 7.836637995384678, "mlx_elapsed_seconds": 17.072171584004536, "mlx_prefill_tokens_per_second": 190.73699918590424, "mlx_time_to_first_token_seconds": 0.7876501668943092, "prefill_speedup": 1.3490730983920824, "ttft_speedup": 1.2530839812008805 }, "memory": { "active_bytes": 22317836304, "cache_bytes": 40317806, "peak_bytes": 22531631656 }, "model": "mlx-community/Qwen3-8B-bf16", "model_config": { "hidden_size": 4096, "model_type": "qwen3", "num_attention_heads": 32, "num_hidden_layers": 36, "num_key_value_heads": 8, "vocab_size": 151936 }, "plan_confirmation": { "accepted": true, "decode_speedup_floor": 0.995, "decode_speedup_win": 1.01, "medians": { "decode_speedup": 1.58546600564999, "end_to_end_speedup": 1.5210534644122398, "prefill_speedup": 1.8372559750400084, "ttft_speedup": 1.8132787413395048 }, "pairs": [ { "decode_speedup": 1.5959712709526133, "end_to_end_speedup": 1.6026720781786212, "prefill_speedup": 1.953567935020379, "ttft_speedup": 1.8132787413395048 }, { "decode_speedup": 1.465120021945495, "end_to_end_speedup": 1.4793966080925345, "prefill_speedup": 1.9774660850327797, "ttft_speedup": 2.041197134513136 }, { "decode_speedup": 1.58546600564999, "end_to_end_speedup": 1.535292997653035, "prefill_speedup": 0.7574437567896694, "ttft_speedup": 0.8347961813941425 }, { "decode_speedup": 1.4402092423763764, "end_to_end_speedup": 1.4500511769823847, "prefill_speedup": 1.8372559750400084, "ttft_speedup": 1.8405199624690765 }, { "decode_speedup": 1.5689526303119032, "end_to_end_speedup": 1.7821168911034728, "prefill_speedup": 8.794413698471102, "ttft_speedup": 7.015364244367445 }, { "decode_speedup": 1.6098368863167065, "end_to_end_speedup": 1.5210534644122398, "prefill_speedup": 0.7364647910431256, "ttft_speedup": 0.6950625076273916 }, { "decode_speedup": 1.5921508662520834, "end_to_end_speedup": 1.5044118325879985, "prefill_speedup": 0.6536898444557484, "ttft_speedup": 0.675494466791114 } ], "required_wins": 5, "strong_decode_speedup_win": 1.05, "ttft_speedup_floor": 0.995 }, "precision_comparison": { "baseline_weights": "model_native", "class": "mixed_precision_affine_int8_decode", "derived_from": "selected_plan", "native_weights_preserved": true, "optimized_decode_weights": [ "affine8" ], "prefill_weights": "model_native", "same_weight_representation": false }, "recorded_at": "2026-07-19T07:57:10.536048+00:00", "revision": "f87f11bf087c5b20f591170c806d125dfa1f1c62", "samples": { "MLX": [ { "decode_tokens_per_second": 8.090982699983309, "elapsed_seconds": 16.684998041018844, "prefill_tokens_per_second": 190.73930852224348, "time_to_first_token_seconds": 0.8372797910124063 }, { "decode_tokens_per_second": 7.799237056088666, "elapsed_seconds": 16.96006266598124, "prefill_tokens_per_second": 292.0362792097215, "time_to_first_token_seconds": 0.52363241603598 }, { "decode_tokens_per_second": 8.007649140190365, "elapsed_seconds": 19.074480958050117, "prefill_tokens_per_second": 44.96383857353224, "time_to_first_token_seconds": 3.0613868329674006 }, { "decode_tokens_per_second": 7.698900087877222, "elapsed_seconds": 18.32896237494424, "prefill_tokens_per_second": 82.35945026380458, "time_to_first_token_seconds": 1.67630041600205 }, { "decode_tokens_per_second": 7.883104381074709, "elapsed_seconds": 17.06994591699913, "prefill_tokens_per_second": 190.1675926357502, "time_to_first_token_seconds": 0.7876501668943092 }, { "decode_tokens_per_second": 7.6209334975218415, "elapsed_seconds": 17.575205457978882, "prefill_tokens_per_second": 190.73699918590424, "time_to_first_token_seconds": 0.7543238330399618 }, { "decode_tokens_per_second": 7.836637995384678, "elapsed_seconds": 17.072171584004536, "prefill_tokens_per_second": 206.36644515991546, "time_to_first_token_seconds": 0.7115010000998154 } ], "MLX + meTile": [ { "decode_tokens_per_second": 12.689836777097627, "elapsed_seconds": 10.832087291055359, "prefill_tokens_per_second": 214.55621437261243, "time_to_first_token_seconds": 0.7167512500891462 }, { "decode_tokens_per_second": 12.642110261466648, "elapsed_seconds": 10.572802541079, "prefill_tokens_per_second": 393.9782880363543, "time_to_first_token_seconds": 0.41787495801690966 }, { "decode_tokens_per_second": 12.69913190636048, "elapsed_seconds": 10.802112333010882, "prefill_tokens_per_second": 220.6354948776881, "time_to_first_token_seconds": 0.6758191249100491 }, { "decode_tokens_per_second": 12.57256358216595, "elapsed_seconds": 10.88736512500327, "prefill_tokens_per_second": 219.06789179307026, "time_to_first_token_seconds": 0.6754547499585897 }, { "decode_tokens_per_second": 12.485176342681122, "elapsed_seconds": 12.477131459047087, "prefill_tokens_per_second": 63.78562051451341, "time_to_first_token_seconds": 2.124595334054902 }, { "decode_tokens_per_second": 12.566649930169733, "elapsed_seconds": 10.64139270898886, "prefill_tokens_per_second": 393.0759257754017, "time_to_first_token_seconds": 0.4206489169737324 }, { "decode_tokens_per_second": 12.544543337195178, "elapsed_seconds": 10.861099500092678, "prefill_tokens_per_second": 236.20303607320957, "time_to_first_token_seconds": 0.6260320410365239 } ] }, "schema_version": 16, "selected_dispatches": { "add_rms_norm": [], "affine_matmul": [], "affine_residual_qmv": [], "affine_swiglu": [], "attention": [], "block_scaled": [], "dense_matmul": [], "dense_residual": [], "dense_swiglu": [], "rms_norm": [] }, "selected_plan": { "affine_prefill": false, "attention": false, "compressed_attention": true, "compressed_down": false, "compressed_gate_up": true, "compressed_vocab": true, "dense_mlp": false, "dense_residual": false, "graph_fusion": false, "quantized_mlp": false, "rms_norm": false }, "software": { "mlx": "0.32.0", "mlx_lm": "0.31.3", "python": "3.12.12" }, "verification": { "actual_next_token": 198, "kl_divergence": 0.0002541199210099876, "max_logit_error": 0.111328125, "mean_logit_error": 0.021686704829335213, "next_token": 198, "reference_dtype": "mlx.core.bfloat16" }, "workload": { "confirmation_trials": 7, "delay_seconds": 1.0, "generation_tokens": 128, "plan_decode_steps": 8, "plan_trials": 7, "prefill_step_size": 2048, "prompt_tokens": 128, "seed": 0, "trials": 7 } } ], "recorded_at": "2026-07-19T04:59:58.011431+00:00", "schema_version": 5, "suite": "bf16" }