{
  "schema_version": 1,
  "status": "validation_failed",
  "created_at": "2026-10-04T22:52:24.342476+00:00",
  "model": "Qwen/Qwen3-0.6B",
  "checkpoint_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
  "checkpoint_sha256": {
    "config.json": "660db3b73d788119c04535e48cf9be5f55bc3100841a718637ae695b442f27dd",
    "model.safetensors": "f47f71177f32bcd101b7573ec9171e6a57f4f4d31148d38e382306f42996874b"
  },
  "hardware": {
    "device_name": "Apple M5",
    "max_recommended_working_set_size": 26800603136,
    "memory_size": 34359738368,
    "architecture": "applegpu_g17g",
    "max_buffer_length": 20100448256,
    "resource_limit": 499000
  },
  "software": {
    "mlx": "0.32.0",
    "mlx_lm": "0.31.3",
    "platform": "macOS-26.2-arm64-arm-64bit",
    "MLX_ENABLE_TF32": "0"
  },
  "metal_compiler": "Apple metal version 32023.864 (metalfe-32023.864)\nTarget: air64-apple-darwin25.2.0\nThread model: posix\nInstalledDir: /private/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v17.3.7003.10.GzPOjD/Metal.xctoolchain/usr/metal/current/bin",
  "source_sha256": {
    "benchmarks/megakernels/qwen3_end_to_end.py": "20f597b3a39e30b157d38f9d3594ad9642ae34eafc862381c024750b05d1b6ba",
    "benchmarks/megakernels/qwen3.py": "d54bd5fd5f9973c8fdd98a6c656c0ca79d424457db242061ce88b8601b9d91f4",
    "benchmarks/megakernels/qwen3_runtime.py": "7076d3413e6df90e577288874de55d5f1af0c22dd5d8c9e6706be238f01e5304",
    "benchmarks/megakernels/qwen3_staged_runtime.py": "a8cf552b5f294e0fa02032a7a55a672cb9836e7126ee07a20a46480b4d53cef0",
    "benchmarks/common/checkout.py": "521c2d03a27b1a46575deab789b83b4e5bb71df030d01eeebd11ba777eaaf0b2"
  },
  "compiler_and_kernels_sha256": "711da1f97c6bf9f9ce4762466ccb10613b3f34c3d8554e5ecd8752208265e8fa",
  "geometry": {
    "model_type": "qwen3",
    "hidden_size": 1024,
    "num_hidden_layers": 28,
    "intermediate_size": 3072,
    "num_attention_heads": 16,
    "rms_norm_eps": 1e-06,
    "vocab_size": 151936,
    "num_key_value_heads": 8,
    "max_position_embeddings": 40960,
    "rope_theta": 1000000,
    "head_dim": 128,
    "tie_word_embeddings": true,
    "rope_scaling": null,
    "capacity": 47,
    "threads": 256,
    "storage_dtype": "float16",
    "dtype": "float16"
  },
  "parameter_count": 596049920,
  "precision_comparison": {
    "class": "same_storage_precision",
    "same_weight_representation": true,
    "bitwise_exact": false,
    "storage_dtype": "float16",
    "accumulation_dtype": "float32",
    "policy": "Identical converted checkpoint weights; complete logits and all valid KV entries pass recorded dtype-specific tolerances, and all greedy output IDs agree exactly."
  },
  "weights": {
    "source_dtype": "bfloat16",
    "both_backends": "float16",
    "same_values": true,
    "quantized": false
  },
  "workload": {
    "batch": 1,
    "prompt_lengths": [
      8,
      32
    ],
    "output_lengths": [
      8,
      16
    ],
    "capacity": 47,
    "seed": 0,
    "prompt_kind": "fixed synthetic token IDs; shorter prompts share the longer prefix",
    "trials": 5,
    "warmup_generations_per_backend_per_workload": 2,
    "sampling": "greedy GPU argmax; EOS ignored; exact requested output count",
    "tokenizer": "not used; benchmark starts at token IDs and ends at token IDs"
  },
  "execution": {
    "candidate_backend": "GPU-wide DSL kernels separated by dispatch boundaries",
    "native_backend": "unmodified MLX-LM Qwen3 with batched prefill",
    "candidate_prefill": "sequential tokens; vocabulary projection only on the last prompt token",
    "native_prefill": "batched prompt except final token; evaluate only KV state, allowing MLX to prune unused final-layer work after its KV update; candidate sequential prefill computes the full decoder body",
    "scope": "embedding, every decoder layer, final RMSNorm, full vocabulary logits, GPU argmax",
    "intergroup_ordering": "dispatch boundaries; no unsupported grid barrier or spin waiting",
    "single_dispatch": false,
    "maximum_projection_threadgroups": 18992,
    "candidate_dispatches_per_prefill_token": 225,
    "candidate_dispatches_per_generated_token": 229
  },
  "timing": {
    "metric": "synchronized token-ID-to-token-ID generation wall seconds",
    "included": [
      "fresh native cache construction or reusable candidate cache reset",
      "all prefill and autoregressive decode model work and KV updates",
      "input token allocations/updates, graph/dispatch submission and synchronization",
      "full vocabulary projection for each returned output token",
      "GPU greedy selection and host token retrieval for each returned token"
    ],
    "excluded": [
      "checkpoint loading, weight conversion/packing, rotary-table preparation and compilation",
      "candidate persistent buffer allocation and pipeline preparation",
      "tokenization, text decoding and correctness comparisons",
      "two complete warmup generations per backend per workload"
    ],
    "ttft": "start of cache reset/construction through first returned token ID, including prefill",
    "decode": "first returned token ID through final returned token ID; output_count - 1 forwards",
    "allocation_difference": "candidate reuses persistent buffers and zeros KV inside timing; native MLX manages intermediate, output and growing KV allocations inside timing",
    "aggregation": "median of within-trial MLX/meTile ratios, separately for each metric",
    "correctness_gate": "all workloads must pass complete logits/KV checks before any timing; every timed and warmup generated sequence must exactly match validation",
    "limitations": "synthetic fixed prompts, batch one, short contexts, steady-state warmed pipelines; not text-serving or cold-start latency"
  },
  "results": [
    {
      "prompt_tokens": 8,
      "prompt_token_ids": [
        129240,
        96777,
        77660,
        40991,
        46771,
        6226,
        11432,
        2512
      ],
      "requested_output_tokens": 8,
      "status": "validation_failed",
      "correctness": {
        "checks": [
          {
            "position": 0,
            "input_token": 129240,
            "logits": {
              "passed": true,
              "max_absolute_error": 0.0087890625,
              "mean_absolute_error": 0.001302817021496594,
              "rtol": 0.02,
              "atol": 0.05,
              "mlx_next_token": 40864,
              "metile_next_token": 40864,
              "greedy_token_equal": true
            },
            "cache": {
              "passed": true,
              "max_absolute_error": 0.015625,
              "layers_checked": 28,
              "valid_tokens_checked": 1,
              "keys_and_values_checked": true,
              "rtol": 0.02,
              "atol": 0.02
            },
            "gpu_greedy_selection": {
              "passed": true,
              "token": 40864
            },
            "passed": true
          },
          {
            "position": 1,
            "input_token": 96777,
            "logits": {
              "passed": true,
              "max_absolute_error": 0.03515625,
              "mean_absolute_error": 0.005822177045047283,
              "rtol": 0.02,
              "atol": 0.05,
              "mlx_next_token": 11,
              "metile_next_token": 11,
              "greedy_token_equal": true
            },
            "cache": {
              "passed": false,
              "max_absolute_error": 0.125,
              "layers_checked": 28,
              "valid_tokens_checked": 2,
              "keys_and_values_checked": true,
              "rtol": 0.02,
              "atol": 0.02
            },
            "gpu_greedy_selection": {
              "passed": true,
              "token": 11
            },
            "passed": false
          }
        ]
      },
      "candidate_dispatches_per_generation": 3407
    }
  ]
}