{
  "schema_version": 1,
  "status": "validation_failed",
  "created_at": "2026-10-04T22:32:47.432274+00:00",
  "model": "Qwen/Qwen3-0.6B",
  "checkpoint_revision": "c1899de289a04d12100db370d81485cdf75e47ca",
  "checkpoint_sha256": {
    "config.json": "660db3b73d788119c04535e48cf9be5f55bc3100841a718637ae695b442f27dd",
    "model.safetensors": "f47f71177f32bcd101b7573ec9171e6a57f4f4d31148d38e382306f42996874b"
  },
  "hardware": {
    "device_name": "Apple M5",
    "max_recommended_working_set_size": 26800603136,
    "memory_size": 34359738368,
    "architecture": "applegpu_g17g",
    "max_buffer_length": 20100448256,
    "resource_limit": 499000
  },
  "software": {
    "mlx": "0.32.0",
    "mlx_lm": "0.31.3",
    "platform": "macOS-26.2-arm64-arm-64bit",
    "MLX_ENABLE_TF32": "0"
  },
  "metal_compiler": "Apple metal version 32023.864 (metalfe-32023.864)\nTarget: air64-apple-darwin25.2.0\nThread model: posix\nInstalledDir: /private/var/run/com.apple.security.cryptexd/mnt/com.apple.MobileAsset.MetalToolchain-v17.3.7003.10.GzPOjD/Metal.xctoolchain/usr/metal/current/bin",
  "source_sha256": {
    "benchmarks/megakernels/qwen3.py": "d54bd5fd5f9973c8fdd98a6c656c0ca79d424457db242061ce88b8601b9d91f4",
    "benchmarks/megakernels/qwen3_runtime.py": "7076d3413e6df90e577288874de55d5f1af0c22dd5d8c9e6706be238f01e5304"
  },
  "compiler_and_kernels_sha256": "d5ef2e16bb64f10ecc082e42fb3027b25a113439e21dbca6f0bc4c46a6d2c1a2",
  "geometry": {
    "model_type": "qwen3",
    "hidden_size": 1024,
    "num_hidden_layers": 28,
    "intermediate_size": 3072,
    "num_attention_heads": 16,
    "rms_norm_eps": 1e-06,
    "vocab_size": 151936,
    "num_key_value_heads": 8,
    "max_position_embeddings": 40960,
    "rope_theta": 1000000,
    "head_dim": 128,
    "tie_word_embeddings": true,
    "rope_scaling": null,
    "capacity": 67,
    "threads": 256,
    "storage_dtype": "float16",
    "dtype": "float16"
  },
  "parameter_count": 596049920,
  "precision_comparison": {
    "class": "same_storage_precision",
    "same_weight_representation": true,
    "bitwise_exact": false,
    "storage_dtype": "float16",
    "accumulation_dtype": "float32",
    "policy": "Identical converted checkpoint weights and storage dtype; reduction order may differ. Full logits, greedy tokens and KV caches must pass the recorded tolerances before timing."
  },
  "weights": {
    "source_dtype": "bfloat16",
    "both_backends": "float16",
    "same_values": true,
    "quantized": false
  },
  "workload": {
    "batch": 1,
    "query_tokens": 1,
    "contexts": [
      0,
      16,
      64
    ],
    "capacity": 67,
    "seed": 0,
    "synthetic_tokens": [
      129240,
      96777,
      77660,
      40991,
      46771,
      6226,
      11432,
      2512,
      26630,
      123565,
      98669,
      138680,
      76519,
      92170,
      147490,
      110837,
      96065,
      82596,
      85072,
      142071,
      42139,
      123957,
      101930,
      417,
      59886,
      130270,
      84220,
      5103,
      116214,
      110861,
      128625,
      26689,
      13566,
      131148,
      3359,
      82267,
      12216,
      45537,
      73091,
      64221,
      61267,
      4303,
      814,
      18883,
      1259,
      101892,
      79860,
      98331,
      39093,
      93499,
      116087,
      58295,
      70031,
      151512,
      122307,
      149024,
      57663,
      104158,
      144354,
      98828,
      127673,
      104600,
      106963,
      59091,
      132967,
      20526,
      87956
    ],
    "trials": 9,
    "validation_steps_per_context": 3
  },
  "execution": {
    "metile_dispatches_per_forward": 1,
    "metile_threadgroups": 1,
    "native_backend": "unpatched MLX-LM Qwen3",
    "scope": "embedding, every decoder layer, final RMSNorm and full vocabulary logits"
  },
  "timing": {
    "metric": "synchronized wall seconds per fixed-context, one-token full forward",
    "included": [
      "native model graph construction/evaluation and synchronization",
      "prepared meTile dispatch and synchronization",
      "KV update at the same fixed position",
      "complete vocabulary logits"
    ],
    "excluded": [
      "checkpoint loading and float16 conversion",
      "weight packing and compilation",
      "prefill and cache preparation",
      "input token allocation",
      "cache offset reset",
      "argmax, sampling and tokenization"
    ],
    "allocation_difference": "meTile reuses prepared buffers; native MLX manages intermediate/output allocations. Native KV capacity is warmed and does not grow during timing.",
    "aggregation": "median of within-trial MLX/meTile wall-time ratios",
    "limitation": "same-position replay, not autoregressive generation throughput or end-to-end latency; correctness checks separately advance three tokens per context"
  },
  "results": [
    {
      "context_tokens": 0,
      "input_token": 129240,
      "status": "validation_failed",
      "correctness": [
        {
          "position": 0,
          "input_token": 129240,
          "logits": {
            "passed": true,
            "max_absolute_error": 0.0078125,
            "mean_absolute_error": 0.001609255326911807,
            "rtol": 0.02,
            "atol": 0.05,
            "mlx_next_token": 40864,
            "metile_next_token": 40864,
            "greedy_token_equal": true
          },
          "cache": {
            "passed": true,
            "max_absolute_error": 0.015625,
            "layers_checked": 28,
            "valid_tokens_checked": 1,
            "keys_and_values_checked": true,
            "rtol": 0.02,
            "atol": 0.02
          }
        },
        {
          "position": 1,
          "input_token": 40864,
          "logits": {
            "passed": true,
            "max_absolute_error": 0.03125,
            "mean_absolute_error": 0.0050816903822124004,
            "rtol": 0.02,
            "atol": 0.05,
            "mlx_next_token": 198,
            "metile_next_token": 198,
            "greedy_token_equal": true
          },
          "cache": {
            "passed": false,
            "max_absolute_error": 0.125,
            "layers_checked": 28,
            "valid_tokens_checked": 2,
            "keys_and_values_checked": true,
            "rtol": 0.02,
            "atol": 0.02
          }
        }
      ],
      "error": "correctness failed at context 0: {'position': 1, 'input_token': 40864, 'logits': {'passed': True, 'max_absolute_error': 0.03125, 'mean_absolute_error': 0.0050816903822124004, 'rtol': 0.02, 'atol': 0.05, 'mlx_next_token': 198, 'metile_next_token': 198, 'greedy_token_equal': True}, 'cache': {'passed': False, 'max_absolute_error': 0.125, 'layers_checked': 28, 'valid_tokens_checked': 2, 'keys_and_values_checked': True, 'rtol': 0.02, 'atol': 0.02}}"
    }
  ]
}