{
 "name": "Nano LM Index",
 "version": "V1",
 "generated": "2026-10-30T06:48:05Z",
 "param_limit": 600000000,
 "generation_budget": 32768,
 "samples_per_item": 4,
 "items": 33485,
 "categories": [
  {
   "id": "knowledge",
   "name": "Knowledge",
   "benchmarks": [
    "mmlu_cf",
    "arc_easy"
   ]
  },
  {
   "id": "language",
   "name": "Language Understanding",
   "benchmarks": [
    "hellaswag_pro",
    "global_piqa"
   ]
  },
  {
   "id": "math",
   "name": "Math & Reasoning",
   "benchmarks": [
    "gsm8k_platinum",
    "math_500"
   ]
  },
  {
   "id": "tools",
   "name": "Tools & Automation",
   "benchmarks": [
    "when2call",
    "toolalpaca"
   ]
  },
  {
   "id": "coding",
   "name": "Coding",
   "benchmarks": [
    "mbpp_plus",
    "evoeval"
   ]
  },
  {
   "id": "classification",
   "name": "Classification & Extraction",
   "benchmarks": [
    "banking77",
    "clinc150"
   ]
  }
 ],
 "benchmarks": {
  "mmlu_cf": {
   "name": "MMLU-CF",
   "answer_type": "letter",
   "licence": "CDLA-Permissive-2.0",
   "items": 10000,
   "chance": 0.2563,
   "about": "Contamination-free MMLU: multiple-choice questions across 14 academic subjects."
  },
  "arc_easy": {
   "name": "ARC-Easy",
   "answer_type": "letter",
   "licence": "CC-BY-SA-4.0",
   "items": 2376,
   "chance": 0.2617845117845118,
   "about": "Grade-school science questions, the easy partition of the AI2 Reasoning Challenge."
  },
  "hellaswag_pro": {
   "name": "HellaSwag-Pro (English)",
   "answer_type": "letter",
   "licence": "none stated (repository has no licence file)",
   "items": 5600,
   "chance": 0.2428779761904762,
   "about": "Pick the sensible continuation of an everyday scenario, in seven robustness variants."
  },
  "global_piqa": {
   "name": "Global PIQA (English)",
   "answer_type": "letter",
   "licence": "CC-BY-SA-4.0",
   "items": 203,
   "chance": 0.4187192118226601,
   "about": "Physical common sense: choose the solution that actually works."
  },
  "gsm8k_platinum": {
   "name": "GSM8K-Platinum",
   "answer_type": "number",
   "licence": "MIT",
   "items": 1209,
   "chance": 0.0,
   "about": "Grade-school word problems, with the label errors of GSM8K cleaned out."
  },
  "math_500": {
   "name": "MATH-500",
   "answer_type": "math",
   "licence": "MIT (MATH)",
   "items": 500,
   "chance": 0.0,
   "about": "Competition mathematics, judged by symbolic equivalence of the final answer."
  },
  "when2call": {
   "name": "When2Call",
   "answer_type": "letter",
   "licence": "CC-BY-4.0",
   "items": 3646,
   "chance": 0.2679648930334613,
   "about": "Decide whether to call a tool, ask a follow-up question, or say it cannot help."
  },
  "toolalpaca": {
   "name": "ToolAlpaca",
   "answer_type": "tool",
   "licence": "Apache-2.0",
   "items": 170,
   "chance": 0.0,
   "about": "Write the correct tool calls, with arguments, for real and simulated APIs."
  },
  "mbpp_plus": {
   "name": "MBPP+",
   "answer_type": "code",
   "licence": "Apache-2.0 (EvalPlus) / CC-BY-4.0 (MBPP)",
   "items": 377,
   "chance": 0.0,
   "about": "Short Python programming tasks, checked against EvalPlus's extended tests."
  },
  "evoeval": {
   "name": "EvoEval",
   "answer_type": "code",
   "licence": "Apache-2.0",
   "items": 824,
   "chance": 0.0,
   "about": "Programming problems evolved from HumanEval to be harder, subtler or combined."
  },
  "banking77": {
   "name": "Banking77",
   "answer_type": "label",
   "licence": "CC-BY-4.0",
   "items": 3080,
   "chance": 0.012987012987012988,
   "about": "Assign one of 77 fine-grained intents to a customer message sent to a bank."
  },
  "clinc150": {
   "name": "CLINC150 (+OOS)",
   "answer_type": "label",
   "licence": "CC-BY-3.0",
   "items": 5500,
   "chance": 0.18181818181818182,
   "about": "Assign one of 150 intents across 10 domains, or recognise that none applies."
  }
 },
 "entries": [
  {
   "entry": "qwen3-0.6b:think",
   "model": "qwen3-0.6b",
   "name": "Qwen3-0.6B",
   "repo": "Qwen/Qwen3-0.6B",
   "org": "Qwen",
   "revision": "c1899de289a04d12100db370d81485cdf75e47ca",
   "mode": "think",
   "status": "ranked",
   "why": null,
   "params": 596049920,
   "context": 40960,
   "architecture": "Qwen3ForCausalLM",
   "licence": "apache-2.0",
   "rank": 1,
   "index": 48.3666,
   "index_ci95": [
    47.2588,
    49.4783
   ],
   "raw_index": 57.0421,
   "categories": {
    "knowledge": {
     "skill": 0.536741,
     "ci95": [
      0.5271,
      0.5467
     ]
    },
    "language": {
     "skill": 0.271652,
     "ci95": [
      0.2258,
      0.3169
     ]
    },
    "math": {
     "skill": 0.773076,
     "ci95": [
      0.7548,
      0.7916
     ]
    },
    "tools": {
     "skill": 0.393883,
     "ci95": [
      0.3608,
      0.4272
     ]
    },
    "coding": {
     "skill": 0.45223,
     "ci95": [
      0.4268,
      0.4782
     ]
    },
    "classification": {
     "skill": 0.474414,
     "ci95": [
      0.464,
      0.485
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.283717,
     "raw": 0.4673,
     "chance": 0.2563,
     "skill_ci95": [
      0.2727,
      0.2946
     ],
     "truncation_rate": 0.001075,
     "mean_output_tokens": 941.3,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 477,
      "truncated": 43
     }
    },
    "arc_easy": {
     "skill": 0.789766,
     "raw": 0.844802,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.7734,
      0.8063
     ],
     "truncation_rate": 0.000316,
     "mean_output_tokens": 507.9,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 17,
      "truncated": 3
     }
    },
    "hellaswag_pro": {
     "skill": 0.316609,
     "raw": 0.482589,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.3023,
      0.3312
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 1037.6,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 1
     }
    },
    "global_piqa": {
     "skill": 0.226695,
     "raw": 0.550493,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.1356,
      0.3178
     ],
     "truncation_rate": 0.001232,
     "mean_output_tokens": 671.3,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 1
     }
    },
    "gsm8k_platinum": {
     "skill": 0.818652,
     "raw": 0.818652,
     "chance": 0.0,
     "skill_ci95": [
      0.8007,
      0.8362
     ],
     "truncation_rate": 0.001654,
     "mean_output_tokens": 1903.8,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 8
     }
    },
    "math_500": {
     "skill": 0.7275,
     "raw": 0.7275,
     "chance": 0.0,
     "skill_ci95": [
      0.696,
      0.759
     ],
     "truncation_rate": 0.009,
     "mean_output_tokens": 5735.5,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 18
     }
    },
    "when2call": {
     "skill": 0.205414,
     "raw": 0.418335,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.1856,
      0.2247
     ],
     "truncation_rate": 6.9e-05,
     "mean_output_tokens": 401.0,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 1
     }
    },
    "toolalpaca": {
     "skill": 0.582353,
     "raw": 0.582353,
     "chance": 0.0,
     "skill_ci95": [
      0.5191,
      0.6456
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 537.3,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 1
     }
    },
    "mbpp_plus": {
     "skill": 0.532493,
     "raw": 0.532493,
     "chance": 0.0,
     "skill_ci95": [
      0.4894,
      0.5763
     ],
     "truncation_rate": 0.005305,
     "mean_output_tokens": 2802.8,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 8
     }
    },
    "evoeval": {
     "skill": 0.371966,
     "raw": 0.371966,
     "chance": 0.0,
     "skill_ci95": [
      0.3441,
      0.4002
     ],
     "truncation_rate": 0.011226,
     "mean_output_tokens": 4934.1,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 37
     }
    },
    "banking77": {
     "skill": 0.458717,
     "raw": 0.465747,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.4429,
      0.4743
     ],
     "truncation_rate": 0.015422,
     "mean_output_tokens": 749.7,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 337,
      "truncated": 190
     }
    },
    "clinc150": {
     "skill": 0.490111,
     "raw": 0.582818,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.4763,
      0.5043
     ],
     "truncation_rate": 0.000364,
     "mean_output_tokens": 195.5,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 782,
      "truncated": 8
     }
    }
   },
   "decoding": {
    "temperature": 0.6,
    "top_p": 0.95,
    "top_k": 20,
    "min_p": 0.0,
    "source": "model card",
    "quote": "For thinking mode, use `Temperature=0.6`, `TopP=0.95`, `TopK=20`, and `MinP=0` (the default setting in `generation_config.json`).",
    "url": "https://huggingface.co/Qwen/Qwen3-0.6B/blob/c1899de289a04d12100db370d81485cdf75e47ca/README.md"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "qwen3-0.6b:no-think",
   "model": "qwen3-0.6b",
   "name": "Qwen3-0.6B",
   "repo": "Qwen/Qwen3-0.6B",
   "org": "Qwen",
   "revision": "c1899de289a04d12100db370d81485cdf75e47ca",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 596049920,
   "context": 40960,
   "architecture": "Qwen3ForCausalLM",
   "licence": "apache-2.0",
   "rank": 2,
   "index": 34.2539,
   "index_ci95": [
    32.996,
    35.5647
   ],
   "raw_index": 44.4423,
   "categories": {
    "knowledge": {
     "skill": 0.390457,
     "ci95": [
      0.3774,
      0.4034
     ]
    },
    "language": {
     "skill": 0.225164,
     "ci95": [
      0.1668,
      0.2835
     ]
    },
    "math": {
     "skill": 0.572497,
     "ci95": [
      0.5509,
      0.5938
     ]
    },
    "tools": {
     "skill": 0.216752,
     "ci95": [
      0.1836,
      0.2505
     ]
    },
    "coding": {
     "skill": 0.332475,
     "ci95": [
      0.3068,
      0.358
     ]
    },
    "classification": {
     "skill": 0.317888,
     "ci95": [
      0.3069,
      0.329
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.150498,
     "raw": 0.368225,
     "chance": 0.2563,
     "skill_ci95": [
      0.1384,
      0.1627
     ],
     "truncation_rate": 0.001275,
     "mean_output_tokens": 70.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 415,
      "truncated": 51
     }
    },
    "arc_easy": {
     "skill": 0.630416,
     "raw": 0.727168,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.6068,
      0.6535
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 17.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 178
     }
    },
    "hellaswag_pro": {
     "skill": 0.291431,
     "raw": 0.463527,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.2755,
      0.3073
     ],
     "truncation_rate": 4.5e-05,
     "mean_output_tokens": 28.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 512,
      "truncated": 1
     }
    },
    "global_piqa": {
     "skill": 0.158898,
     "raw": 0.511084,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0445,
      0.2733
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 17.3,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 10
     }
    },
    "gsm8k_platinum": {
     "skill": 0.633995,
     "raw": 0.633995,
     "chance": 0.0,
     "skill_ci95": [
      0.6117,
      0.6563
     ],
     "truncation_rate": 0.000207,
     "mean_output_tokens": 238.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1,
      "truncated": 1
     }
    },
    "math_500": {
     "skill": 0.511,
     "raw": 0.511,
     "chance": 0.0,
     "skill_ci95": [
      0.4745,
      0.5465
     ],
     "truncation_rate": 0.0065,
     "mean_output_tokens": 855.8,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 13
     }
    },
    "when2call": {
     "skill": 0.061446,
     "raw": 0.312946,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0413,
      0.0812
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 8.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 21
     }
    },
    "toolalpaca": {
     "skill": 0.372059,
     "raw": 0.372059,
     "chance": 0.0,
     "skill_ci95": [
      0.3088,
      0.4368
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 80.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2
     }
    },
    "mbpp_plus": {
     "skill": 0.41313,
     "raw": 0.41313,
     "chance": 0.0,
     "skill_ci95": [
      0.3693,
      0.4582
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 60.4,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.25182,
     "raw": 0.25182,
     "chance": 0.0,
     "skill_ci95": [
      0.226,
      0.2779
     ],
     "truncation_rate": 0.00182,
     "mean_output_tokens": 342.6,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 6
     }
    },
    "banking77": {
     "skill": 0.31472,
     "raw": 0.32362,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.2986,
      0.3307
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 7.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 213
     }
    },
    "clinc150": {
     "skill": 0.321056,
     "raw": 0.4445,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.3054,
      0.3369
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 198
     }
    }
   },
   "decoding": {
    "temperature": 0.7,
    "top_p": 0.8,
    "top_k": 20,
    "min_p": 0.0,
    "source": "model card",
    "quote": "For non-thinking mode, we suggest using `Temperature=0.7`, `TopP=0.8`, `TopK=20`, and `MinP=0`.",
    "url": "https://huggingface.co/Qwen/Qwen3-0.6B/blob/c1899de289a04d12100db370d81485cdf75e47ca/README.md"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "falcon-h1-0.5b:no-think",
   "model": "falcon-h1-0.5b",
   "name": "Falcon-H1-0.5B-Instruct",
   "repo": "tiiuae/Falcon-H1-0.5B-Instruct",
   "org": "tiiuae",
   "revision": "8f2587ca06bff78d8fa1adfccbe8c24d5f86b368",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 521411104,
   "context": 16384,
   "architecture": "FalconH1ForCausalLM",
   "licence": "falcon-llm-license",
   "rank": 3,
   "index": 32.9562,
   "index_ci95": [
    31.6368,
    34.3162
   ],
   "raw_index": 43.0155,
   "categories": {
    "knowledge": {
     "skill": 0.450485,
     "ci95": [
      0.4372,
      0.4637
     ]
    },
    "language": {
     "skill": 0.263987,
     "ci95": [
      0.2056,
      0.3233
     ]
    },
    "math": {
     "skill": 0.721534,
     "ci95": [
      0.6981,
      0.7454
     ]
    },
    "tools": {
     "skill": 0.126471,
     "ci95": [
      0.0941,
      0.1618
     ]
    },
    "coding": {
     "skill": 0.380379,
     "ci95": [
      0.351,
      0.41
     ]
    },
    "classification": {
     "skill": 0.034516,
     "ci95": [
      0.027,
      0.0425
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.215678,
     "raw": 0.4167,
     "chance": 0.2563,
     "skill_ci95": [
      0.2028,
      0.229
     ],
     "truncation_rate": 0.0002,
     "mean_output_tokens": 238.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 695,
      "truncated": 2
     }
    },
    "arc_easy": {
     "skill": 0.685291,
     "raw": 0.767677,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.6625,
      0.7081
     ],
     "truncation_rate": 0.000421,
     "mean_output_tokens": 204.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 72,
      "truncated": 1
     }
    },
    "hellaswag_pro": {
     "skill": 0.239838,
     "raw": 0.424464,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.2226,
      0.2568
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 253.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 69
     }
    },
    "global_piqa": {
     "skill": 0.288136,
     "raw": 0.586207,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.1695,
      0.4068
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 120.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 9
     }
    },
    "gsm8k_platinum": {
     "skill": 0.813069,
     "raw": 0.813069,
     "chance": 0.0,
     "skill_ci95": [
      0.7899,
      0.8346
     ],
     "truncation_rate": 0.001654,
     "mean_output_tokens": 336.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2,
      "truncated": 2
     }
    },
    "math_500": {
     "skill": 0.63,
     "raw": 0.63,
     "chance": 0.0,
     "skill_ci95": [
      0.588,
      0.672
     ],
     "truncation_rate": 0.006,
     "mean_output_tokens": 788.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1,
      "truncated": 3
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.249314,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.005211,
     "mean_output_tokens": 304.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 14,
      "truncated": 19
     }
    },
    "toolalpaca": {
     "skill": 0.252941,
     "raw": 0.252941,
     "chance": 0.0,
     "skill_ci95": [
      0.1882,
      0.3235
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 107.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 20
     }
    },
    "mbpp_plus": {
     "skill": 0.469496,
     "raw": 0.469496,
     "chance": 0.0,
     "skill_ci95": [
      0.4218,
      0.5199
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 482.7,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.291262,
     "raw": 0.291262,
     "chance": 0.0,
     "skill_ci95": [
      0.2597,
      0.3228
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 510.9,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "banking77": {
     "skill": 0.055921,
     "raw": 0.068182,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.047,
      0.0648
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 7.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 254
     }
    },
    "clinc150": {
     "skill": 0.013111,
     "raw": 0.192545,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0004,
      0.0262
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 5.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 396
     }
    }
   },
   "decoding": {
    "temperature": 0.0,
    "source": "greedy (no sampling settings published)"
   },
   "samples": 1,
   "budget": 32768,
   "engine": "vllm",
   "requests": 33485
  },
  {
   "entry": "minicpm4-0.5b:no-think",
   "model": "minicpm4-0.5b",
   "name": "MiniCPM4-0.5B",
   "repo": "openbmb/MiniCPM4-0.5B",
   "org": "openbmb",
   "revision": "5253c7fcc5e29e1cf3eacb59a58adf1ba4df8630",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 433873920,
   "context": 32768,
   "architecture": "MiniCPMForCausalLM",
   "licence": "apache-2.0",
   "rank": 4,
   "index": 23.3446,
   "index_ci95": [
    22.3161,
    24.4137
   ],
   "raw_index": 33.0608,
   "categories": {
    "knowledge": {
     "skill": 0.328627,
     "ci95": [
      0.3163,
      0.341
     ]
    },
    "language": {
     "skill": 0.225478,
     "ci95": [
      0.1747,
      0.2787
     ]
    },
    "math": {
     "skill": 0.40167,
     "ci95": [
      0.3819,
      0.4216
     ]
    },
    "tools": {
     "skill": 0.009559,
     "ci95": [
      0.0044,
      0.0162
     ]
    },
    "coding": {
     "skill": 0.412112,
     "ci95": [
      0.3859,
      0.4385
     ]
    },
    "classification": {
     "skill": 0.023232,
     "ci95": [
      0.0198,
      0.0268
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.103805,
     "raw": 0.3335,
     "chance": 0.2563,
     "skill_ci95": [
      0.0932,
      0.1143
     ],
     "truncation_rate": 0.0119,
     "mean_output_tokens": 585.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1989,
      "truncated": 476
     }
    },
    "arc_easy": {
     "skill": 0.553449,
     "raw": 0.670349,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.5309,
      0.5758
     ],
     "truncation_rate": 0.000947,
     "mean_output_tokens": 162.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 154,
      "truncated": 9
     }
    },
    "hellaswag_pro": {
     "skill": 0.342906,
     "raw": 0.5025,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.3278,
      0.3587
     ],
     "truncation_rate": 0.000313,
     "mean_output_tokens": 81.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 125,
      "truncated": 7
     }
    },
    "global_piqa": {
     "skill": 0.108051,
     "raw": 0.481527,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0064,
      0.214
     ],
     "truncation_rate": 0.004926,
     "mean_output_tokens": 204.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 21,
      "truncated": 4
     }
    },
    "gsm8k_platinum": {
     "skill": 0.510339,
     "raw": 0.510339,
     "chance": 0.0,
     "skill_ci95": [
      0.4876,
      0.5337
     ],
     "truncation_rate": 0.0122,
     "mean_output_tokens": 757.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 5,
      "truncated": 59
     }
    },
    "math_500": {
     "skill": 0.293,
     "raw": 0.293,
     "chance": 0.0,
     "skill_ci95": [
      0.26,
      0.3255
     ],
     "truncation_rate": 0.1245,
     "mean_output_tokens": 4912.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 7,
      "truncated": 249
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.25576,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0007
     ],
     "truncation_rate": 0.00144,
     "mean_output_tokens": 66.8,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 21,
      "unparseable": 2
     }
    },
    "toolalpaca": {
     "skill": 0.019118,
     "raw": 0.019118,
     "chance": 0.0,
     "skill_ci95": [
      0.0088,
      0.0324
     ],
     "truncation_rate": 0.063235,
     "mean_output_tokens": 2456.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 450,
      "truncated": 43
     }
    },
    "mbpp_plus": {
     "skill": 0.516578,
     "raw": 0.516578,
     "chance": 0.0,
     "skill_ci95": [
      0.4715,
      0.561
     ],
     "truncation_rate": 0.001326,
     "mean_output_tokens": 127.2,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 2
     }
    },
    "evoeval": {
     "skill": 0.307646,
     "raw": 0.307646,
     "chance": 0.0,
     "skill_ci95": [
      0.2806,
      0.3353
     ],
     "truncation_rate": 0.003337,
     "mean_output_tokens": 403.4,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 11
     }
    },
    "banking77": {
     "skill": 0.046464,
     "raw": 0.058847,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0396,
      0.0535
     ],
     "truncation_rate": 0.000731,
     "mean_output_tokens": 40.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1306,
      "truncated": 9
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.018136,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 4.5e-05,
     "mean_output_tokens": 7.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2390,
      "truncated": 1
     }
    }
   },
   "decoding": {
    "temperature": 0.8,
    "top_p": 0.8,
    "top_k": 50,
    "source": "generation_config.json"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "qwen2.5-0.5b:no-think",
   "model": "qwen2.5-0.5b",
   "name": "Qwen2.5-0.5B-Instruct",
   "repo": "Qwen/Qwen2.5-0.5B-Instruct",
   "org": "Qwen",
   "revision": "7ae557604adf67be50417f59c2c2f167def9a775",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 494032768,
   "context": 32768,
   "architecture": "Qwen2ForCausalLM",
   "licence": "apache-2.0",
   "rank": 5,
   "index": 22.1792,
   "index_ci95": [
    21.4477,
    23.2019
   ],
   "raw_index": 32.7811,
   "categories": {
    "knowledge": {
     "skill": 0.35929,
     "ci95": [
      0.3465,
      0.3721
     ]
    },
    "language": {
     "skill": 0.116764,
     "ci95": [
      0.1105,
      0.1579
     ]
    },
    "math": {
     "skill": 0.363317,
     "ci95": [
      0.3436,
      0.3831
     ]
    },
    "tools": {
     "skill": 0.241163,
     "ci95": [
      0.2065,
      0.2763
     ]
    },
    "coding": {
     "skill": 0.23883,
     "ci95": [
      0.2178,
      0.2597
     ]
    },
    "classification": {
     "skill": 0.01139,
     "ci95": [
      0.0084,
      0.0145
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.173255,
     "raw": 0.38515,
     "chance": 0.2563,
     "skill_ci95": [
      0.1622,
      0.1845
     ],
     "truncation_rate": 0.0002,
     "mean_output_tokens": 34.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 189,
      "truncated": 8
     }
    },
    "arc_easy": {
     "skill": 0.545325,
     "raw": 0.664352,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.5219,
      0.5683
     ],
     "truncation_rate": 0.000105,
     "mean_output_tokens": 15.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 9,
      "truncated": 1
     }
    },
    "hellaswag_pro": {
     "skill": 0.233528,
     "raw": 0.419687,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.2199,
      0.2476
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 29.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 3
     }
    },
    "global_piqa": {
     "skill": 0.0,
     "raw": 0.410099,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.0806
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 15.9,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "gsm8k_platinum": {
     "skill": 0.443135,
     "raw": 0.443135,
     "chance": 0.0,
     "skill_ci95": [
      0.4212,
      0.4657
     ],
     "truncation_rate": 0.001241,
     "mean_output_tokens": 346.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 5,
      "truncated": 6
     }
    },
    "math_500": {
     "skill": 0.2835,
     "raw": 0.2835,
     "chance": 0.0,
     "skill_ci95": [
      0.252,
      0.3145
     ],
     "truncation_rate": 0.0085,
     "mean_output_tokens": 884.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 17,
      "truncated": 17
     }
    },
    "when2call": {
     "skill": 0.042619,
     "raw": 0.299163,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0244,
      0.0609
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1
     }
    },
    "toolalpaca": {
     "skill": 0.439706,
     "raw": 0.439706,
     "chance": 0.0,
     "skill_ci95": [
      0.3735,
      0.5074
     ],
     "truncation_rate": 0.001471,
     "mean_output_tokens": 92.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 14,
      "truncated": 1
     }
    },
    "mbpp_plus": {
     "skill": 0.297745,
     "raw": 0.297745,
     "chance": 0.0,
     "skill_ci95": [
      0.2626,
      0.3329
     ],
     "truncation_rate": 0.000663,
     "mean_output_tokens": 214.8,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 1
     }
    },
    "evoeval": {
     "skill": 0.179915,
     "raw": 0.179915,
     "chance": 0.0,
     "skill_ci95": [
      0.1584,
      0.2012
     ],
     "truncation_rate": 0.000607,
     "mean_output_tokens": 294.2,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 2
     }
    },
    "banking77": {
     "skill": 0.02278,
     "raw": 0.035471,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0168,
      0.0289
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 7.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 282
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.075818,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.3,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2369
     }
    }
   },
   "decoding": {
    "temperature": 0.7,
    "top_p": 0.8,
    "top_k": 20,
    "repetition_penalty": 1.1,
    "source": "generation_config.json"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "granite-4.0-h-350m:no-think",
   "model": "granite-4.0-h-350m",
   "name": "Granite-4.0-H-350M",
   "repo": "ibm-granite/granite-4.0-h-350m",
   "org": "ibm-granite",
   "revision": "3b17b717b8f2f5d305b0a92c1491e239aeda19c8",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 340332224,
   "context": 32768,
   "architecture": "GraniteMoeHybridForCausalLM",
   "licence": "apache-2.0",
   "rank": 6,
   "index": 19.8534,
   "index_ci95": [
    18.5667,
    21.1807
   ],
   "raw_index": 31.7491,
   "categories": {
    "knowledge": {
     "skill": 0.207609,
     "ci95": [
      0.1926,
      0.2228
     ]
    },
    "language": {
     "skill": 0.130427,
     "ci95": [
      0.0762,
      0.1895
     ]
    },
    "math": {
     "skill": 0.237154,
     "ci95": [
      0.2165,
      0.2581
     ]
    },
    "tools": {
     "skill": 0.205882,
     "ci95": [
      0.1676,
      0.2441
     ]
    },
    "coding": {
     "skill": 0.335971,
     "ci95": [
      0.307,
      0.3643
     ]
    },
    "classification": {
     "skill": 0.074159,
     "ci95": [
      0.0653,
      0.0832
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.085115,
     "raw": 0.3196,
     "chance": 0.2563,
     "skill_ci95": [
      0.0729,
      0.0974
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 3.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 5
     }
    },
    "arc_easy": {
     "skill": 0.330103,
     "raw": 0.505471,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.3027,
      0.3575
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "hellaswag_pro": {
     "skill": 0.150684,
     "raw": 0.356964,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.1344,
      0.1672
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 3.9,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "global_piqa": {
     "skill": 0.110169,
     "raw": 0.482759,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.2288
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.0,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "gsm8k_platinum": {
     "skill": 0.314309,
     "raw": 0.314309,
     "chance": 0.0,
     "skill_ci95": [
      0.2887,
      0.3408
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 119.3,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "math_500": {
     "skill": 0.16,
     "raw": 0.16,
     "chance": 0.0,
     "skill_ci95": [
      0.128,
      0.192
     ],
     "truncation_rate": 0.004,
     "mean_output_tokens": 309.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 103,
      "truncated": 2
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.248217,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2
     }
    },
    "toolalpaca": {
     "skill": 0.411765,
     "raw": 0.411765,
     "chance": 0.0,
     "skill_ci95": [
      0.3353,
      0.4882
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 52.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 31
     }
    },
    "mbpp_plus": {
     "skill": 0.466844,
     "raw": 0.466844,
     "chance": 0.0,
     "skill_ci95": [
      0.4164,
      0.5172
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 68.2,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.205097,
     "raw": 0.205097,
     "chance": 0.0,
     "skill_ci95": [
      0.1772,
      0.233
     ],
     "truncation_rate": 0.001214,
     "mean_output_tokens": 305.5,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 1
     }
    },
    "banking77": {
     "skill": 0.134539,
     "raw": 0.145779,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.122,
      0.1474
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 6.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 530
     }
    },
    "clinc150": {
     "skill": 0.013778,
     "raw": 0.193091,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0009,
      0.0267
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 5.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 408
     }
    }
   },
   "decoding": {
    "temperature": 0.0,
    "source": "greedy (no sampling settings published)"
   },
   "samples": 1,
   "budget": 32768,
   "engine": "vllm",
   "requests": 33485
  },
  {
   "entry": "granite-4.0-350m:no-think",
   "model": "granite-4.0-350m",
   "name": "Granite-4.0-350M",
   "repo": "ibm-granite/granite-4.0-350m",
   "org": "ibm-granite",
   "revision": "bd8a1497065c0d6ba1ef19af6b0d2b14bacf71c2",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 352379904,
   "context": 32768,
   "architecture": "GraniteMoeHybridForCausalLM",
   "licence": "apache-2.0",
   "rank": 7,
   "index": 15.0144,
   "index_ci95": [
    13.9183,
    16.2439
   ],
   "raw_index": 26.4334,
   "categories": {
    "knowledge": {
     "skill": 0.135652,
     "ci95": [
      0.1209,
      0.1503
     ]
    },
    "language": {
     "skill": 0.099437,
     "ci95": [
      0.0545,
      0.1605
     ]
    },
    "math": {
     "skill": 0.266752,
     "ci95": [
      0.2469,
      0.2874
     ]
    },
    "tools": {
     "skill": 0.023529,
     "ci95": [
      0.0088,
      0.0412
     ]
    },
    "coding": {
     "skill": 0.347205,
     "ci95": [
      0.3184,
      0.376
     ]
    },
    "classification": {
     "skill": 0.028289,
     "ci95": [
      0.0238,
      0.0327
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.053516,
     "raw": 0.2961,
     "chance": 0.2563,
     "skill_ci95": [
      0.0413,
      0.0655
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 61
     }
    },
    "arc_easy": {
     "skill": 0.217788,
     "raw": 0.422559,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.191,
      0.2446
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 6
     }
    },
    "hellaswag_pro": {
     "skill": 0.114127,
     "raw": 0.329286,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.0979,
      0.1306
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 117
     }
    },
    "global_piqa": {
     "skill": 0.084746,
     "raw": 0.46798,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.2034
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "gsm8k_platinum": {
     "skill": 0.399504,
     "raw": 0.399504,
     "chance": 0.0,
     "skill_ci95": [
      0.3722,
      0.4268
     ],
     "truncation_rate": 0.004136,
     "mean_output_tokens": 290.7,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 5
     }
    },
    "math_500": {
     "skill": 0.134,
     "raw": 0.134,
     "chance": 0.0,
     "skill_ci95": [
      0.106,
      0.164
     ],
     "truncation_rate": 0.086,
     "mean_output_tokens": 3115.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 162,
      "truncated": 43
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.21119,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 126
     }
    },
    "toolalpaca": {
     "skill": 0.047059,
     "raw": 0.047059,
     "chance": 0.0,
     "skill_ci95": [
      0.0176,
      0.0824
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 65.3,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 153
     }
    },
    "mbpp_plus": {
     "skill": 0.445623,
     "raw": 0.445623,
     "chance": 0.0,
     "skill_ci95": [
      0.3952,
      0.496
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 84.4,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.248786,
     "raw": 0.248786,
     "chance": 0.0,
     "skill_ci95": [
      0.2197,
      0.2779
     ],
     "truncation_rate": 0.010922,
     "mean_output_tokens": 636.1,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 9
     }
    },
    "banking77": {
     "skill": 0.056579,
     "raw": 0.068831,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0477,
      0.0655
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 5.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 547
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.101091,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1973
     }
    }
   },
   "decoding": {
    "temperature": 0.0,
    "source": "greedy (no sampling settings published)"
   },
   "samples": 1,
   "budget": 32768,
   "engine": "vllm",
   "requests": 33485
  },
  {
   "entry": "ernie-4.5-0.3b:no-think",
   "model": "ernie-4.5-0.3b",
   "name": "ERNIE-4.5-0.3B-PT",
   "repo": "baidu/ERNIE-4.5-0.3B-PT",
   "org": "baidu",
   "revision": "b565cf6caebdb7a1eadf00100857b1ed5e044f12",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 360748032,
   "context": 131072,
   "architecture": "Ernie4_5ForCausalLM",
   "licence": "apache-2.0",
   "rank": 8,
   "index": 14.8774,
   "index_ci95": [
    14.0816,
    15.7962
   ],
   "raw_index": 25.8842,
   "categories": {
    "knowledge": {
     "skill": 0.185201,
     "ci95": [
      0.1747,
      0.1956
     ]
    },
    "language": {
     "skill": 0.10451,
     "ci95": [
      0.0763,
      0.1468
     ]
    },
    "math": {
     "skill": 0.252352,
     "ci95": [
      0.2377,
      0.2675
     ]
    },
    "tools": {
     "skill": 0.047794,
     "ci95": [
      0.0309,
      0.0662
     ]
    },
    "coding": {
     "skill": 0.30279,
     "ci95": [
      0.2776,
      0.3268
     ]
    },
    "classification": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.077215,
     "raw": 0.313725,
     "chance": 0.2563,
     "skill_ci95": [
      0.0686,
      0.0857
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 80.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1162
     }
    },
    "arc_easy": {
     "skill": 0.293187,
     "raw": 0.47822,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.2741,
      0.3121
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 67.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 185
     }
    },
    "hellaswag_pro": {
     "skill": 0.158173,
     "raw": 0.362634,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.1458,
      0.1703
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 9.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 278
     }
    },
    "global_piqa": {
     "skill": 0.050847,
     "raw": 0.448276,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.1356
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1
     }
    },
    "gsm8k_platinum": {
     "skill": 0.350703,
     "raw": 0.350703,
     "chance": 0.0,
     "skill_ci95": [
      0.3302,
      0.3718
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 278.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 3
     }
    },
    "math_500": {
     "skill": 0.154,
     "raw": 0.154,
     "chance": 0.0,
     "skill_ci95": [
      0.1335,
      0.1755
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 673.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 351
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.234915,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 26.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 269
     }
    },
    "toolalpaca": {
     "skill": 0.095588,
     "raw": 0.095588,
     "chance": 0.0,
     "skill_ci95": [
      0.0618,
      0.1324
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 152.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 104
     }
    },
    "mbpp_plus": {
     "skill": 0.389257,
     "raw": 0.389257,
     "chance": 0.0,
     "skill_ci95": [
      0.3455,
      0.4324
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 263.1,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.216323,
     "raw": 0.216323,
     "chance": 0.0,
     "skill_ci95": [
      0.1914,
      0.2409
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 442.5,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "banking77": {
     "skill": 0.0,
     "raw": 0.010958,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 67.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 11539
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.0515,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 4.5e-05,
     "mean_output_tokens": 49.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 15656,
      "truncated": 1
     }
    }
   },
   "decoding": {
    "temperature": 0.8,
    "top_p": 0.8,
    "top_k": 50,
    "repetition_penalty": 1.0,
    "source": "generation_config.json"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "lfm2.5-350m:no-think",
   "model": "lfm2.5-350m",
   "name": "LFM2.5-350M",
   "repo": "LiquidAI/LFM2.5-350M",
   "org": "LiquidAI",
   "revision": "9e6c6ccf47cd318696e137d381a7ded8fe4df09f",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 354483968,
   "context": 128000,
   "architecture": "Lfm2ForCausalLM",
   "licence": "lfm1.0",
   "rank": 9,
   "index": 14.3093,
   "index_ci95": [
    13.5071,
    15.5008
   ],
   "raw_index": 24.9513,
   "categories": {
    "knowledge": {
     "skill": 0.346775,
     "ci95": [
      0.3328,
      0.3611
     ]
    },
    "language": {
     "skill": 0.109597,
     "ci95": [
      0.0866,
      0.1691
     ]
    },
    "math": {
     "skill": 0.155696,
     "ci95": [
      0.1392,
      0.1722
     ]
    },
    "tools": {
     "skill": 0.058088,
     "ci95": [
      0.036,
      0.0831
     ]
    },
    "coding": {
     "skill": 0.167885,
     "ci95": [
      0.1449,
      0.1911
     ]
    },
    "classification": {
     "skill": 0.020518,
     "ci95": [
      0.0167,
      0.0245
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.15749,
     "raw": 0.373425,
     "chance": 0.2563,
     "skill_ci95": [
      0.1447,
      0.17
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 50
     }
    },
    "arc_easy": {
     "skill": 0.53606,
     "raw": 0.657513,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.5104,
      0.5617
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 30
     }
    },
    "hellaswag_pro": {
     "skill": 0.185296,
     "raw": 0.38317,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.1687,
      0.2022
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 8
     }
    },
    "global_piqa": {
     "skill": 0.033898,
     "raw": 0.438424,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.1525
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.0,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "gsm8k_platinum": {
     "skill": 0.164392,
     "raw": 0.164392,
     "chance": 0.0,
     "skill_ci95": [
      0.1458,
      0.1834
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 46.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 5
     }
    },
    "math_500": {
     "skill": 0.147,
     "raw": 0.147,
     "chance": 0.0,
     "skill_ci95": [
      0.121,
      0.174
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 191.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.190208,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 3.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 97
     }
    },
    "toolalpaca": {
     "skill": 0.116176,
     "raw": 0.116176,
     "chance": 0.0,
     "skill_ci95": [
      0.0721,
      0.1662
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 155.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 85
     }
    },
    "mbpp_plus": {
     "skill": 0.245358,
     "raw": 0.245358,
     "chance": 0.0,
     "skill_ci95": [
      0.2042,
      0.2871
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 52.0,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.090413,
     "raw": 0.090413,
     "chance": 0.0,
     "skill_ci95": [
      0.0722,
      0.1098
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 185.1,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "banking77": {
     "skill": 0.041036,
     "raw": 0.05349,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0334,
      0.0489
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 8.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 4765
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.134591,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 3.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 24
     }
    }
   },
   "decoding": {
    "temperature": 0.1,
    "top_k": 50,
    "repetition_penalty": 1.05,
    "source": "model card",
    "quote": "Generation parameters: temperature: 0.1, top_k: 50, repetition_penalty: 1.05",
    "url": "https://huggingface.co/LiquidAI/LFM2.5-350M/blob/9e6c6ccf47cd318696e137d381a7ded8fe4df09f/README.md"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "lfm2-350m:no-think",
   "model": "lfm2-350m",
   "name": "LFM2-350M",
   "repo": "LiquidAI/LFM2-350M",
   "org": "LiquidAI",
   "revision": "f37d3f5c8c5484bc01dad379a595cf4c68c4e70e",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 354483968,
   "context": 128000,
   "architecture": "Lfm2ForCausalLM",
   "licence": "lfm1.0",
   "rank": 10,
   "index": 13.5232,
   "index_ci95": [
    13.0096,
    14.2582
   ],
   "raw_index": 24.0675,
   "categories": {
    "knowledge": {
     "skill": 0.301601,
     "ci95": [
      0.2879,
      0.3153
     ]
    },
    "language": {
     "skill": 0.079529,
     "ci95": [
      0.0726,
      0.112
     ]
    },
    "math": {
     "skill": 0.213909,
     "ci95": [
      0.1983,
      0.2301
     ]
    },
    "tools": {
     "skill": 0.025,
     "ci95": [
      0.014,
      0.0382
     ]
    },
    "coding": {
     "skill": 0.165039,
     "ci95": [
      0.1447,
      0.1857
     ]
    },
    "classification": {
     "skill": 0.026316,
     "ci95": [
      0.0232,
      0.0295
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.107335,
     "raw": 0.336125,
     "chance": 0.2563,
     "skill_ci95": [
      0.0962,
      0.1186
     ],
     "truncation_rate": 0.000175,
     "mean_output_tokens": 79.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 3582,
      "truncated": 7
     }
    },
    "arc_easy": {
     "skill": 0.495867,
     "raw": 0.627841,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.4711,
      0.5202
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 31.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 435
     }
    },
    "hellaswag_pro": {
     "skill": 0.159057,
     "raw": 0.363304,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.1441,
      0.1743
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 47.3,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 3395
     }
    },
    "global_piqa": {
     "skill": 0.0,
     "raw": 0.397783,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.0636
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 41.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 108
     }
    },
    "gsm8k_platinum": {
     "skill": 0.235318,
     "raw": 0.235318,
     "chance": 0.0,
     "skill_ci95": [
      0.2177,
      0.2541
     ],
     "truncation_rate": 0.001654,
     "mean_output_tokens": 412.4,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 8
     }
    },
    "math_500": {
     "skill": 0.1925,
     "raw": 0.1925,
     "chance": 0.0,
     "skill_ci95": [
      0.1665,
      0.219
     ],
     "truncation_rate": 0.0035,
     "mean_output_tokens": 731.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 3,
      "truncated": 7
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.167032,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.000823,
     "mean_output_tokens": 73.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 909,
      "truncated": 12
     }
    },
    "toolalpaca": {
     "skill": 0.05,
     "raw": 0.05,
     "chance": 0.0,
     "skill_ci95": [
      0.0279,
      0.0765
     ],
     "truncation_rate": 0.029412,
     "mean_output_tokens": 1450.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 58,
      "truncated": 20
     }
    },
    "mbpp_plus": {
     "skill": 0.206897,
     "raw": 0.206897,
     "chance": 0.0,
     "skill_ci95": [
      0.1724,
      0.2434
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 283.1,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "evoeval": {
     "skill": 0.12318,
     "raw": 0.12318,
     "chance": 0.0,
     "skill_ci95": [
      0.1029,
      0.1438
     ],
     "truncation_rate": 0.001517,
     "mean_output_tokens": 404.5,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 5
     }
    },
    "banking77": {
     "skill": 0.052632,
     "raw": 0.064935,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0464,
      0.059
     ],
     "truncation_rate": 0.012013,
     "mean_output_tokens": 631.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 7400,
      "truncated": 148
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.123182,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.002136,
     "mean_output_tokens": 114.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 7537,
      "truncated": 47
     }
    }
   },
   "decoding": {
    "temperature": 0.3,
    "min_p": 0.15,
    "repetition_penalty": 1.05,
    "source": "model card",
    "quote": "Generation parameters: We recommend the following parameters: temperature=0.3, min_p=0.15, repetition_penalty=1.05",
    "url": "https://huggingface.co/LiquidAI/LFM2-350M/blob/f37d3f5c8c5484bc01dad379a595cf4c68c4e70e/README.md"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  },
  {
   "entry": "baguettotron:think",
   "model": "baguettotron",
   "name": "Baguettotron",
   "repo": "PleIAs/Baguettotron",
   "org": "PleIAs",
   "revision": "504b4c11c8a48c168ce3cbae8afa4ed759eb5b25",
   "mode": "think",
   "status": "ranked",
   "why": null,
   "params": 320956992,
   "context": 4096,
   "architecture": "LlamaForCausalLM",
   "licence": "apache-2.0",
   "rank": 11,
   "index": 6.5857,
   "index_ci95": [
    6.2331,
    6.9426
   ],
   "raw_index": 13.4879,
   "categories": {
    "knowledge": {
     "skill": 0.182257,
     "ci95": [
      0.1676,
      0.197
     ]
    },
    "language": {
     "skill": 0.054233,
     "ci95": [
      0.0463,
      0.0623
     ]
    },
    "math": {
     "skill": 0.157328,
     "ci95": [
      0.1438,
      0.1709
     ]
    },
    "tools": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    },
    "coding": {
     "skill": 0.001326,
     "ci95": [
      0.0,
      0.004
     ]
    },
    "classification": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.044104,
     "raw": 0.2891,
     "chance": 0.2563,
     "skill_ci95": [
      0.0321,
      0.0559
     ],
     "truncation_rate": 0.063,
     "mean_output_tokens": 775.9,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 591,
      "truncated": 630
     }
    },
    "arc_easy": {
     "skill": 0.32041,
     "raw": 0.498316,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.293,
      0.3478
     ],
     "truncation_rate": 0.02904,
     "mean_output_tokens": 629.2,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 43,
      "truncated": 69
     }
    },
    "hellaswag_pro": {
     "skill": 0.108466,
     "raw": 0.325,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.0927,
      0.1245
     ],
     "truncation_rate": 0.053929,
     "mean_output_tokens": 746.5,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 878,
      "truncated": 302
     }
    },
    "global_piqa": {
     "skill": 0.0,
     "raw": 0.147783,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.226601,
     "mean_output_tokens": 1331.8,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 56,
      "truncated": 46
     }
    },
    "gsm8k_platinum": {
     "skill": 0.312655,
     "raw": 0.312655,
     "chance": 0.0,
     "skill_ci95": [
      0.2862,
      0.3391
     ],
     "truncation_rate": 0.022333,
     "mean_output_tokens": 837.8,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 27,
      "unparseable": 14
     }
    },
    "math_500": {
     "skill": 0.002,
     "raw": 0.002,
     "chance": 0.0,
     "skill_ci95": [
      0.0,
      0.006
     ],
     "truncation_rate": 0.328,
     "mean_output_tokens": 2109.4,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 38,
      "truncated": 164
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.039221,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.506034,
     "mean_output_tokens": 1799.3,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 1129,
      "truncated": 1845
     }
    },
    "toolalpaca": {
     "skill": 0.0,
     "raw": 0.0,
     "chance": 0.0,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.652941,
     "mean_output_tokens": 2658.9,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 111,
      "unparseable": 22
     }
    },
    "mbpp_plus": {
     "skill": 0.002653,
     "raw": 0.002653,
     "chance": 0.0,
     "skill_ci95": [
      0.0,
      0.008
     ],
     "truncation_rate": 0.71618,
     "mean_output_tokens": 3005.1,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 270
     }
    },
    "evoeval": {
     "skill": 0.0,
     "raw": 0.0,
     "chance": 0.0,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.879854,
     "mean_output_tokens": 3424.6,
     "reasoning_rate": 1.0,
     "failures": {
      "truncated": 725
     }
    },
    "banking77": {
     "skill": 0.0,
     "raw": 0.0,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.726948,
     "mean_output_tokens": 3007.1,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 841,
      "truncated": 2239
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.001818,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.882364,
     "mean_output_tokens": 2997.4,
     "reasoning_rate": 1.0,
     "failures": {
      "unparseable": 603,
      "truncated": 4853
     }
    }
   },
   "decoding": {
    "temperature": 0.0,
    "source": "greedy (no sampling settings published)"
   },
   "samples": 1,
   "budget": 32768,
   "engine": "vllm",
   "requests": 33485
  },
  {
   "entry": "smollm2-360m:no-think",
   "model": "smollm2-360m",
   "name": "SmolLM2-360M-Instruct",
   "repo": "HuggingFaceTB/SmolLM2-360M-Instruct",
   "org": "HuggingFaceTB",
   "revision": "a10cc1512eabd3dde888204e902eca88bddb4951",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 361821120,
   "context": 8192,
   "architecture": "LlamaForCausalLM",
   "licence": "apache-2.0",
   "rank": 12,
   "index": 4.8382,
   "index_ci95": [
    4.4009,
    5.8698
   ],
   "raw_index": 16.2748,
   "categories": {
    "knowledge": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0003
     ]
    },
    "language": {
     "skill": 0.030176,
     "ci95": [
      0.0232,
      0.0854
     ]
    },
    "math": {
     "skill": 0.04553,
     "ci95": [
      0.0369,
      0.0546
     ]
    },
    "tools": {
     "skill": 0.002941,
     "ci95": [
      0.0,
      0.0088
     ]
    },
    "coding": {
     "skill": 0.211648,
     "ci95": [
      0.1856,
      0.2378
     ]
    },
    "classification": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.0,
     "raw": 0.2471,
     "chance": 0.2563,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 9.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 21
     }
    },
    "arc_easy": {
     "skill": 0.0,
     "raw": 0.242424,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 7.3,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2
     }
    },
    "hellaswag_pro": {
     "skill": 0.060352,
     "raw": 0.288571,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.0448,
      0.0757
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.9,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "global_piqa": {
     "skill": 0.0,
     "raw": 0.413793,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.1102
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 6.7,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "gsm8k_platinum": {
     "skill": 0.081059,
     "raw": 0.081059,
     "chance": 0.0,
     "skill_ci95": [
      0.0662,
      0.0968
     ],
     "truncation_rate": 0.033085,
     "mean_output_tokens": 431.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 20,
      "truncated": 40
     }
    },
    "math_500": {
     "skill": 0.01,
     "raw": 0.01,
     "chance": 0.0,
     "skill_ci95": [
      0.002,
      0.02
     ],
     "truncation_rate": 0.336,
     "mean_output_tokens": 2882.3,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 263,
      "truncated": 168
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.227372,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.3,
     "reasoning_rate": 0.0,
     "failures": {}
    },
    "toolalpaca": {
     "skill": 0.005882,
     "raw": 0.005882,
     "chance": 0.0,
     "skill_ci95": [
      0.0,
      0.0176
     ],
     "truncation_rate": 0.205882,
     "mean_output_tokens": 1816.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 63,
      "truncated": 35
     }
    },
    "mbpp_plus": {
     "skill": 0.297082,
     "raw": 0.297082,
     "chance": 0.0,
     "skill_ci95": [
      0.2519,
      0.3448
     ],
     "truncation_rate": 0.018568,
     "mean_output_tokens": 263.3,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 7
     }
    },
    "evoeval": {
     "skill": 0.126214,
     "raw": 0.126214,
     "chance": 0.0,
     "skill_ci95": [
      0.1032,
      0.1493
     ],
     "truncation_rate": 0.054612,
     "mean_output_tokens": 674.2,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 45
     }
    },
    "banking77": {
     "skill": 0.0,
     "raw": 0.005844,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.000325,
     "mean_output_tokens": 13.7,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 257,
      "truncated": 1
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.007636,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 5.5,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 3463
     }
    }
   },
   "decoding": {
    "temperature": 0.0,
    "source": "greedy (no sampling settings published)"
   },
   "samples": 1,
   "budget": 32768,
   "engine": "vllm",
   "requests": 33485
  },
  {
   "entry": "gemma-3-270m:no-think",
   "model": "gemma-3-270m",
   "name": "Gemma 3 270M (it)",
   "repo": "google/gemma-3-270m-it",
   "org": "google",
   "revision": "ac82b4e820549b854eebf28ce6dedaf9fdfa17b3",
   "mode": "no-think",
   "status": "ranked",
   "why": null,
   "params": 268098176,
   "context": 32768,
   "architecture": "Gemma3ForCausalLM",
   "licence": "gemma",
   "rank": 13,
   "index": 3.3418,
   "index_ci95": [
    3.0375,
    3.9548
   ],
   "raw_index": 14.5119,
   "categories": {
    "knowledge": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    },
    "language": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0328
     ]
    },
    "math": {
     "skill": 0.108032,
     "ci95": [
      0.0966,
      0.1201
     ]
    },
    "tools": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    },
    "coding": {
     "skill": 0.092477,
     "ci95": [
      0.0771,
      0.1085
     ]
    },
    "classification": {
     "skill": 0.0,
     "ci95": [
      0.0,
      0.0
     ]
    }
   },
   "benchmarks": {
    "mmlu_cf": {
     "skill": 0.0,
     "raw": 0.235825,
     "chance": 0.2563,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0001,
     "mean_output_tokens": 9.2,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2098,
      "truncated": 4
     }
    },
    "arc_easy": {
     "skill": 0.0,
     "raw": 0.233691,
     "chance": 0.2617845117845118,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 5.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 550
     }
    },
    "hellaswag_pro": {
     "skill": 0.0,
     "raw": 0.224241,
     "chance": 0.2428779761904762,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 8.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 2023
     }
    },
    "global_piqa": {
     "skill": 0.0,
     "raw": 0.403941,
     "chance": 0.4187192118226601,
     "skill_ci95": [
      0.0,
      0.0657
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 4.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 37
     }
    },
    "gsm8k_platinum": {
     "skill": 0.136063,
     "raw": 0.136063,
     "chance": 0.0,
     "skill_ci95": [
      0.1224,
      0.1503
     ],
     "truncation_rate": 0.017783,
     "mean_output_tokens": 937.0,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 23,
      "truncated": 86
     }
    },
    "math_500": {
     "skill": 0.08,
     "raw": 0.08,
     "chance": 0.0,
     "skill_ci95": [
      0.0615,
      0.1
     ],
     "truncation_rate": 0.039,
     "mean_output_tokens": 2093.6,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 13,
      "truncated": 78
     }
    },
    "when2call": {
     "skill": 0.0,
     "raw": 0.213179,
     "chance": 0.2679648930334613,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.0,
     "mean_output_tokens": 3.8,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 1135
     }
    },
    "toolalpaca": {
     "skill": 0.0,
     "raw": 0.0,
     "chance": 0.0,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.027941,
     "mean_output_tokens": 1462.1,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 454,
      "truncated": 19
     }
    },
    "mbpp_plus": {
     "skill": 0.120027,
     "raw": 0.120027,
     "chance": 0.0,
     "skill_ci95": [
      0.0935,
      0.1485
     ],
     "truncation_rate": 0.013263,
     "mean_output_tokens": 565.1,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 20
     }
    },
    "evoeval": {
     "skill": 0.064927,
     "raw": 0.064927,
     "chance": 0.0,
     "skill_ci95": [
      0.0498,
      0.0807
     ],
     "truncation_rate": 0.002427,
     "mean_output_tokens": 404.3,
     "reasoning_rate": 0.0,
     "failures": {
      "truncated": 8
     }
    },
    "banking77": {
     "skill": 0.0,
     "raw": 0.004627,
     "chance": 0.012987012987012988,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.01737,
     "mean_output_tokens": 589.9,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 7985,
      "truncated": 214
     }
    },
    "clinc150": {
     "skill": 0.0,
     "raw": 0.024909,
     "chance": 0.18181818181818182,
     "skill_ci95": [
      0.0,
      0.0
     ],
     "truncation_rate": 0.000864,
     "mean_output_tokens": 37.4,
     "reasoning_rate": 0.0,
     "failures": {
      "unparseable": 15731,
      "truncated": 19
     }
    }
   },
   "decoding": {
    "temperature": 1.0,
    "top_p": 0.95,
    "top_k": 64,
    "source": "generation_config.json"
   },
   "samples": 4,
   "budget": 32768,
   "engine": "vllm",
   "requests": 133940
  }
 ]
}